New ConnMonitor to Track "Stalled" Connection State for SSH Conns

cmd/server/main-server.go

@@ -600,7 +600,7 @@ func main() {
 		// use fmt instead of log here to make sure it goes directly to stderr
 		fmt.Fprintf(os.Stderr, "WAVESRV-ESTART ws:%s web:%s version:%s buildtime:%s\n", wsListener.Addr(), webListener.Addr(), WaveVersion, BuildTime)
 	}()
-	go wshutil.RunWshRpcOverListener(unixListener)
+	go wshutil.RunWshRpcOverListener(unixListener, nil)
 	web.RunWebServer(webListener) // blocking
 	runtime.KeepAlive(waveLock)
 }

cmd/wsh/cmd/wshcmd-connserver.go

@@ -97,7 +97,7 @@ func handleNewListenerConn(conn net.Conn, router *wshutil.WshRouter) {
 				router.UnregisterLink(baseds.LinkId(linkId))
 			}
 		}()
-		wshutil.AdaptStreamToMsgCh(conn, proxy.FromRemoteCh)
+		wshutil.AdaptStreamToMsgCh(conn, proxy.FromRemoteCh, nil)
 	}()
 	linkId := router.RegisterUntrustedLink(proxy)
 	linkIdContainer.Store(int32(linkId))
@@ -265,7 +265,7 @@ func serverRunRouterDomainSocket(jwtToken string) error {
 			log.Printf("upstream domain socket closed, shutting down")
 			wshutil.DoShutdown("", 0, true)
 		}()
-		wshutil.AdaptStreamToMsgCh(conn, upstreamProxy.FromRemoteCh)
+		wshutil.AdaptStreamToMsgCh(conn, upstreamProxy.FromRemoteCh, nil)
 	}()
 
 	// register the domain socket connection as upstream

emain/emain.ts

@@ -412,6 +412,16 @@ async function appMain() {
             fireAndForget(createNewWaveWindow);
         }
     });
+    electron.powerMonitor.on("resume", () => {
+        console.log("system resumed from sleep, notifying server");
+        fireAndForget(async () => {
+            try {
+                await RpcApi.NotifySystemResumeCommand(ElectronWshClient, { noresponse: true });
+            } catch (e) {
+                console.log("error calling NotifySystemResumeCommand", e);
+            }
+        });
+    });
     const rawGlobalHotKey = launchSettings?.["app:globalhotkey"];
     if (rawGlobalHotKey) {
         registerGlobalHotkey(rawGlobalHotKey);

frontend/app/block/block.scss

@@ -284,8 +284,8 @@
         .connstatus-overlay {
             position: absolute;
             top: calc(var(--header-height) + 6px);
-            left: 6px;
-            right: 6px;
+            left: 8px;
+            right: 8px;
             z-index: var(--zindex-block-mask-inner);
             display: flex;
             align-items: center;
@@ -296,6 +296,7 @@
             backdrop-filter: blur(50px);
             border-radius: 6px;
             box-shadow: 0px 13px 16px 0px rgb(from var(--block-bg-color) r g b / 40%);
+            opacity: 0.85;
 
             .connstatus-content {
                 display: flex;

frontend/app/block/connectionbutton.tsx

@@ -80,6 +80,14 @@ export const ConnectionButton = React.memo(
                     color = "var(--grey-text-color)";
                     titleText = "Disconnected from " + connection;
                     showDisconnectedSlash = true;
+                } else if (connStatus?.connhealthstatus === "degraded" || connStatus?.connhealthstatus === "stalled") {
+                    color = "var(--warning-color)";
+                    iconName = "signal-bars-slash";
+                    if (connStatus.connhealthstatus === "degraded") {
+                        titleText = "Connection degraded: " + connection;
+                    } else {
+                        titleText = "Connection stalled: " + connection;
+                    }
                 }
                 if (iconSvg != null) {
                     connIconElem = iconSvg;

frontend/app/block/connstatusoverlay.tsx

@@ -14,6 +14,100 @@ import * as jotai from "jotai";
 import { OverlayScrollbarsComponent } from "overlayscrollbars-react";
 import * as React from "react";
 
+function formatElapsedTime(elapsedMs: number): string {
+    if (elapsedMs <= 0) {
+        return "";
+    }
+
+    const elapsedSeconds = Math.floor(elapsedMs / 1000);
+
+    if (elapsedSeconds < 60) {
+        return `${elapsedSeconds}s`;
+    }
+
+    const elapsedMinutes = Math.floor(elapsedSeconds / 60);
+    if (elapsedMinutes < 60) {
+        return `${elapsedMinutes}m`;
+    }
+
+    const elapsedHours = Math.floor(elapsedMinutes / 60);
+    const remainingMinutes = elapsedMinutes % 60;
+
+    if (elapsedHours < 24) {
+        if (remainingMinutes === 0) {
+            return `${elapsedHours}h`;
+        }
+        return `${elapsedHours}h${remainingMinutes}m`;
+    }
+
+    return "more than a day";
+}
+
+const StalledOverlay = React.memo(
+    ({
+        connName,
+        connStatus,
+        overlayRefCallback,
+    }: {
+        connName: string;
+        connStatus: ConnStatus;
+        overlayRefCallback: (el: HTMLDivElement | null) => void;
+    }) => {
+        const [elapsedTime, setElapsedTime] = React.useState<string>("");
+
+        const handleDisconnect = React.useCallback(() => {
+            const prtn = RpcApi.ConnDisconnectCommand(TabRpcClient, connName, { timeout: 5000 });
+            prtn.catch((e) => console.log("error disconnecting", connName, e));
+        }, [connName]);
+
+        React.useEffect(() => {
+            if (!connStatus.lastactivitybeforestalledtime) {
+                return;
+            }
+
+            const updateElapsed = () => {
+                const now = Date.now();
+                const lastActivity = connStatus.lastactivitybeforestalledtime!;
+                const elapsed = now - lastActivity;
+                setElapsedTime(formatElapsedTime(elapsed));
+            };
+
+            updateElapsed();
+            const interval = setInterval(updateElapsed, 1000);
+
+            return () => clearInterval(interval);
+        }, [connStatus.lastactivitybeforestalledtime]);
+
+        return (
+            <div
+                className="@container absolute top-[calc(var(--header-height)+6px)] left-1.5 right-1.5 z-[var(--zindex-block-mask-inner)] overflow-hidden rounded-md bg-[var(--conn-status-overlay-bg-color)] backdrop-blur-[50px] shadow-lg opacity-85"
+                ref={overlayRefCallback}
+            >
+                <div className="flex items-center gap-3 w-full pt-2.5 pb-2.5 pr-2 pl-3">
+                    <i
+                        className="fa-solid fa-triangle-exclamation text-warning text-base shrink-0"
+                        title="Connection Stalled"
+                    ></i>
+                    <div className="text-[11px] font-semibold leading-4 tracking-[0.11px] text-white min-w-0 flex-1 break-words @max-xxs:hidden">
+                        Connection to "{connName}" is stalled
+                        {elapsedTime && ` (no activity for ${elapsedTime})`}
+                    </div>
+                    <div className="flex-1 hidden @max-xxs:block"></div>
+                    <Button
+                        className="outlined grey text-[11px] py-[3px] px-[7px] @max-w350:text-[12px] @max-w350:py-[5px] @max-w350:px-[6px]"
+                        onClick={handleDisconnect}
+                        title="Disconnect"
+                    >
+                        <span className="@max-w350:hidden!">Disconnect</span>
+                        <i className="fa-solid fa-link-slash hidden! @max-w350:inline!"></i>
+                    </Button>
+                </div>
+            </div>
+        );
+    }
+);
+StalledOverlay.displayName = "StalledOverlay";
+
 export const ConnStatusOverlay = React.memo(
     ({
         nodeModel,
@@ -121,10 +215,17 @@ export const ConnStatusOverlay = React.memo(
             [showError, showWshError, connStatus.error, connStatus.wsherror]
         );
 
-        if (!showWshError && (isLayoutMode || connStatus.status == "connected" || connModalOpen)) {
+        let showStalled = connStatus.status == "connected" && connStatus.connhealthstatus == "stalled";
+        if (!showWshError && !showStalled && (isLayoutMode || connStatus.status == "connected" || connModalOpen)) {
             return null;
         }
 
+        if (showStalled && !showWshError) {
+            return (
+                <StalledOverlay connName={connName} connStatus={connStatus} overlayRefCallback={overlayRefCallback} />
+            );
+        }
+
         return (
             <div className="connstatus-overlay" ref={overlayRefCallback}>
                 <div className="connstatus-content">

frontend/app/store/wshclientapi.ts

@@ -512,6 +512,11 @@ class RpcApiType {
         return client.wshRpcCall("notify", data, opts);
     }
 
+    // command "notifysystemresume" [call]
+    NotifySystemResumeCommand(client: WshClient, opts?: RpcOpts): Promise<void> {
+        return client.wshRpcCall("notifysystemresume", null, opts);
+    }
+
     // command "path" [call]
     PathCommand(client: WshClient, data: PathCommandData, opts?: RpcOpts): Promise<string> {
         return client.wshRpcCall("path", data, opts);

frontend/tailwindsetup.css

@@ -67,6 +67,7 @@
 
     --container-w600: 600px;
     --container-w450: 450px;
+    --container-w350: 350px;
     --container-xs: 300px;
     --container-xxs: 200px;
     --container-tiny: 120px;

frontend/types/gotypes.d.ts

@@ -785,6 +785,7 @@ declare global {
     // wshrpc.ConnStatus
     type ConnStatus = {
         status: string;
+        connhealthstatus?: string;
         wshenabled: boolean;
         connection: string;
         connected: boolean;
@@ -794,6 +795,8 @@ declare global {
         wsherror?: string;
         nowshreason?: string;
         wshversion?: string;
+        lastactivitybeforestalledtime?: number;
+        keepalivesenttime?: number;
     };
 
     // wshrpc.CpuDataRequest

pkg/blockcontroller/blockcontroller.go

@@ -290,11 +290,31 @@ func DestroyBlockController(blockId string) {
 	deleteController(blockId)
 }
 
+func sendConnMonitorInputNotification(controller Controller) {
+	connName := controller.GetConnName()
+	if connName == "" || conncontroller.IsLocalConnName(connName) || conncontroller.IsWslConnName(connName) {
+		return
+	}
+
+	connOpts, parseErr := remote.ParseOpts(connName)
+	if parseErr != nil {
+		return
+	}
+	sshConn := conncontroller.MaybeGetConn(connOpts)
+	if sshConn != nil {
+		monitor := sshConn.GetMonitor()
+		if monitor != nil {
+			monitor.NotifyInput()
+		}
+	}
+}
+
 func SendInput(blockId string, inputUnion *BlockInputUnion) error {
 	controller := getController(blockId)
 	if controller == nil {
 		return fmt.Errorf("no controller found for block %s", blockId)
 	}
+	sendConnMonitorInputNotification(controller)
 	return controller.SendInput(inputUnion)
 }
 
@@ -413,7 +433,10 @@ func CheckConnStatus(blockId string) error {
 	if err != nil {
 		return fmt.Errorf("error parsing connection name: %w", err)
 	}
-	conn := conncontroller.GetConn(opts)
+	conn := conncontroller.MaybeGetConn(opts)
+	if conn == nil {
+		return fmt.Errorf("no connection found")
+	}
 	connStatus := conn.DeriveConnStatus()
 	if connStatus.Status != conncontroller.Status_Connected {
 		return fmt.Errorf("not connected: %s", connStatus.Status)

pkg/blockcontroller/durableshellcontroller.go

@@ -229,7 +229,7 @@ func (dsc *DurableShellController) startNewJob(ctx context.Context, blockMeta wa
 	if err != nil {
 		return "", fmt.Errorf("invalid ssh remote name (%s): %w", connName, err)
 	}
-	conn := conncontroller.GetConn(opts)
+	conn := conncontroller.MaybeGetConn(opts)
 	if conn == nil {
 		return "", fmt.Errorf("connection %q not found", connName)
 	}

pkg/blockcontroller/shellcontroller.go

@@ -355,7 +355,7 @@ func (bc *ShellController) getConnUnion(logCtx context.Context, remoteName strin
 		if err != nil {
 			return ConnUnion{}, fmt.Errorf("invalid ssh remote name (%s): %w", remoteName, err)
 		}
-		conn := conncontroller.GetConn(opts)
+		conn := conncontroller.MaybeGetConn(opts)
 		if conn == nil {
 			return ConnUnion{}, fmt.Errorf("ssh connection not found: %s", remoteName)
 		}

pkg/jobmanager/jobmanager.go

@@ -440,7 +440,7 @@ func handleJobDomainSocketClient(conn net.Conn) {
 			panichandler.PanicHandler("handleJobDomainSocketClient:AdaptStreamToMsgCh", recover())
 		}()
 		defer serverImpl.Close()
-		wshutil.AdaptStreamToMsgCh(conn, inputCh)
+		wshutil.AdaptStreamToMsgCh(conn, inputCh, nil)
 	}()
 
 	_ = wshRpc