Compare commits

...
13 changed files with 495 additions and 432 deletions

No files matched your search

+5 -5
View File
@@ -695,11 +695,11 @@ jobs:
shell: bash
run: |
set -euo pipefail
# Security CDN endpoints can lack files advertised in their indexes.
# Fall back per file, with the final Bullseye security snapshot for packages only.
# Live signed indexes still select versions and verify every download.
printf 'http://deb.debian.org/debian-security\tpriority:1\nhttp://security.debian.org/debian-security\tpriority:2\nhttp://snapshot.debian.org/archive/debian-security/20260901T000000Z/\tpriority:3 type:deb\n' > /etc/apt/security-mirrors.txt
sed -i 's|http://deb.debian.org/debian-security|mirror+file:/etc/apt/security-mirrors.txt|g' /etc/apt/sources.list
# Bullseye LTS ended on 2026-08-31; its security index is now frozen.
# Pin the final snapshot and waive expiry only for this archived source.
# APT still verifies its signature and every package's checksum.
sed -i '/ bullseye-security /d' /etc/apt/sources.list
printf 'deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/20260901T000000Z/ bullseye-security main\n' >> /etc/apt/sources.list
apt-get -o Acquire::Retries=4 update
apt-get -o Acquire::Retries=4 install -y curl build-essential python3 git libfuse2 file rpm \
libarchive-tools \
+1 -1
View File
@@ -47,7 +47,7 @@ export interface ServerStats {
disks: DiskInfo[]; // All mounted disks
netRxSpeed: number; // Total network receive speed (bytes/sec)
netTxSpeed: number; // Total network transmit speed (bytes/sec)
latencyMs: number | null; // TCP connection establishment latency to the SSH endpoint
latencyMs: number | null; // SSH transport ping round-trip to the SSH endpoint
netInterfaces: NetInterfaceInfo[]; // Per-interface network stats
lastUpdated: number | null; // Timestamp of last successful update
}
+3 -3
View File
@@ -55,7 +55,7 @@ const {
_resetAlgorithmSupportCacheForTests,
} = require("./sshAlgorithms.cjs");
const { enableSshNoDelay, enableTcpNoDelay } = require("./tcpNoDelay.cjs");
const { createTcpConnectLatencyProbe } = require("./tcpConnectLatency.cjs");
const { createSshPingLatencyProbe } = require("./sshBridge/sshPingLatency.cjs");
const {
configureTerminalSessionDataEmitter,
} = require("./emitTerminalSessionData.cjs");
@@ -1395,7 +1395,7 @@ const { ensureMoshStatsConnection, ensureEtStatsConnection } = createMoshStatsCo
});
const { createSessionOpsApi } = require("./sshBridge/sessionOps.cjs");
const measureTcpConnectLatency = createTcpConnectLatencyProbe({ net });
const measureSshPingLatency = createSshPingLatencyProbe();
const sessionOpsApi = createSessionOpsApi({
get sessions() { return sessions; },
get electronModule() { return electronModule; },
@@ -1403,7 +1403,7 @@ const sessionOpsApi = createSessionOpsApi({
getSessionDecoder, resetSessionDecoders, sessionEncodings, normalizeTerminalEncoding,
safeSend,
quoteShellArg, log, ensureMoshStatsConnection, ensureEtStatsConnection,
measureTcpConnectLatency,
measureSshPingLatency,
execOnEtSession: (...args) => require("./terminalBridge.cjs").execOnEtSession(...args),
getServerStats: undefined,
});
@@ -50,7 +50,7 @@ function makeApi(session, execOnEtSession, extra = {}) {
iconv: { encodingExists: () => true },
sessionEncodings: new Map(),
resetSessionDecoders: () => {},
measureTcpConnectLatency: async () => 3,
measureSshPingLatency: async () => 3,
...extra,
});
}
@@ -78,12 +78,11 @@ test("getServerStats opens an ET stats companion connection for direct ET sessio
sshUserHost: "alice@example.test",
sshOptions: [],
sshEnv: {},
tcpLatencyTarget: { hostname: "example.test", port: 2022 },
etStatsAuth: { hostname: "example.test", username: "alice" },
};
let ensureCalls = 0;
let execFallbackCalls = 0;
const latencyTargets = [];
const pingedConns = [];
const api = makeApi(
session,
async () => {
@@ -98,8 +97,8 @@ test("getServerStats opens an ET stats companion connection for direct ET sessio
s.etStatsConn = fakeConn(LINUX_STATS);
return s.etStatsConn;
},
measureTcpConnectLatency: async (target) => {
latencyTargets.push(target);
measureSshPingLatency: async (conn) => {
pingedConns.push(conn);
return 3;
},
},
@@ -114,7 +113,7 @@ test("getServerStats opens an ET stats companion connection for direct ET sessio
assert.equal(result.stats.memTotal, 8000);
assert.equal(result.stats.cpuCores, 4);
assert.equal(typeof result.stats.latencyMs, "number");
assert.deepEqual(latencyTargets, [{ hostname: "example.test", port: 2022 }]);
assert.deepEqual(pingedConns, [session.etStatsConn]);
});
test("getServerStats falls back to execOnEtSession for jumped ET sessions", async () => {
@@ -188,7 +187,9 @@ test("getServerStats falls back to execOnEtSession when the direct ET companion
assert.equal(execFallbackCalls, 2);
assert.equal(result.success, true);
assert.equal(result.stats.memTotal, 8000);
assert.equal(result.stats.latencyMs, 3);
// No SSH connection to ping in exec-fallback mode — latency stays unknown
// rather than opening an unauthenticated TCP probe to the SSH port (#3320).
assert.equal(result.stats.latencyMs, null);
});
test("readRemoteHistory probes ET sessions and parses the detected shell", async () => {
+12 -17
View File
@@ -121,19 +121,6 @@ function createSessionOpsApi(ctx) {
// POSIX sh's $PPID is sshd (needed by the cwd probe).
return `exec sh -c ${quoteShellArg(withRemoteWatchdog(command, timeoutMs))}`;
}
function getTcpLatencyTarget(session) {
if (session.tcpLatencyDirect === false) return null;
const auth = session.tcpLatencyTarget || session.moshStatsAuth || session.etStatsAuth || session._reuseEndpoint || null;
if (auth?.hasJumpHost || auth?.hasProxy) return null;
const hostname = auth?.hostname || session.hostname;
const rawPort = auth?.port ?? 22;
const port = Number(rawPort);
if (!hostname || !Number.isInteger(port) || port < 1 || port > 65535) return null;
return { hostname, port };
}
async function getSessionRemoteInfo(_event, payload) {
const { sessionId } = payload || {};
const session = sessions.get(sessionId);
@@ -1274,9 +1261,17 @@ function createSessionOpsApi(ctx) {
}
return executeBoundedSshCommand(conn, command, options);
};
const tcpLatencyTarget = getTcpLatencyTarget(session);
const tcpLatencyPromise = tcpLatencyTarget && typeof measureTcpConnectLatency === 'function'
? Promise.resolve(measureTcpConnectLatency(tcpLatencyTarget)).catch(() => null)
// Measure latency with an SSH transport ping on the connection already
// serving stats. A separate TCP connect to the SSH port would land in
// sshd logs as a failed pre-auth login with no username (issue #3320).
// A shared interactive connection must honor the user's keepalive opt-out
// (some network devices ignore these requests and would poison the FIFO).
// Dedicated Mosh/ET companions disable periodic keepalives internally;
// they are not shared with forwarding and can still carry stats pings.
const pingDisabled = sshStatsConn === session.conn
&& sshStatsConn?.config?.keepaliveInterval === 0;
const pingLatencyPromise = sshStatsConn && !pingDisabled && typeof measureSshPingLatency === 'function'
? Promise.resolve(measureSshPingLatency(sshStatsConn)).catch(() => null)
: Promise.resolve(null);
const formatStatsError = (error) => (
error?.code === "SSH_EXEC_OPEN_TIMEOUT" || error?.code === "SSH_EXEC_RUN_TIMEOUT"
@@ -1303,7 +1298,7 @@ function createSessionOpsApi(ctx) {
.filter(Boolean)
.join('|');
}
const measuredLatency = await tcpLatencyPromise;
const measuredLatency = await pingLatencyPromise;
if (settled) return;
const latencyMs = Number.isFinite(measuredLatency) ? measuredLatency : null;
@@ -59,7 +59,7 @@ const LINUX_STATS =
const MACOS_STATS =
"NC_LATENCY_MARK|CPU:27|CORES:10|MEMINFO:32768 4096 0 8192 2048 1536|PROCS:123;1.2;Finder|DISKS:/:120:460:26:apfs:/dev/disk3|NET:en0:1000:3000";
function makeSessionOps(sessions) {
function makeSessionOps(sessions, extra = {}) {
return createSessionOpsApi({
get sessions() {
return sessions;
@@ -68,8 +68,9 @@ function makeSessionOps(sessions) {
clearTimeout,
Buffer,
quoteShellArg,
measureTcpConnectLatency: async () => 3,
measureSshPingLatency: async () => 3,
// The rest of the sessionOps surface isn't exercised by getServerStats.
...extra,
});
}
@@ -556,7 +557,7 @@ test("getServerStats opens a Mosh stats companion connection when session.conn i
s.moshStatsConn = fakeConn(LINUX_STATS);
return s.moshStatsConn;
},
measureTcpConnectLatency: async () => 3,
measureSshPingLatency: async () => 3,
});
const result = await api.getServerStats({ sender: {} }, { sessionId: "sid" });
@@ -618,25 +619,20 @@ test("getServerStats does not touch the companion path for a normal SSH session"
assert.equal(result.success, true);
});
test("getServerStats measures TCP connectivity instead of SSH protocol latency", async () => {
test("getServerStats measures latency by pinging the stats connection", async () => {
const sessions = new Map();
const session = {
type: "mosh",
hostname: "vm.example.test",
moshStatsAuth: { hostname: "vm.example.test", port: 2222 },
moshStatsConn: fakeConn(LINUX_STATS),
moshStatsConn: Object.assign(fakeConn(LINUX_STATS), { config: { keepaliveInterval: 0 } }),
};
sessions.set("sid", session);
const probes = [];
const api = createSessionOpsApi({
sessions,
setTimeout,
clearTimeout,
Buffer,
quoteShellArg,
measureTcpConnectLatency: async (target) => {
probes.push(target);
const pingedConns = [];
const api = makeSessionOps(sessions, {
measureSshPingLatency: async (conn) => {
pingedConns.push(conn);
return 2;
},
});
@@ -645,24 +641,19 @@ test("getServerStats measures TCP connectivity instead of SSH protocol latency",
assert.equal(result.success, true);
assert.equal(result.stats.latencyMs, 2);
assert.deepEqual(probes, [{ hostname: "vm.example.test", port: 2222 }]);
assert.deepEqual(pingedConns, [session.moshStatsConn]);
});
test("getServerStats skips a misleading direct probe for jump-host sessions", async () => {
test("getServerStats pings the companion connection for jump-host sessions", async () => {
const sessions = new Map([["sid", {
type: "mosh",
moshStatsAuth: { hostname: "private.example.test", port: 22, hasJumpHost: true },
moshStatsConn: fakeConn(LINUX_STATS),
}]]);
let probeCalls = 0;
const api = createSessionOpsApi({
sessions,
setTimeout,
clearTimeout,
Buffer,
quoteShellArg,
measureTcpConnectLatency: async () => {
probeCalls += 1;
let pingCalls = 0;
const api = makeSessionOps(sessions, {
measureSshPingLatency: async () => {
pingCalls += 1;
return 2;
},
});
@@ -670,8 +661,37 @@ test("getServerStats skips a misleading direct probe for jump-host sessions", as
const result = await api.getServerStats({ sender: {} }, { sessionId: "sid" });
assert.equal(result.success, true);
assert.equal(result.stats.latencyMs, 2);
assert.equal(pingCalls, 1);
});
test("getServerStats reports no latency when only an exec fallback is available", async () => {
const sessions = new Map([["sid", {
type: "et",
sshUserHost: "alice@example.test",
sshOptions: [],
sshEnv: {},
etStatsAuth: { hostname: "private.example.test", hasJumpHost: true },
}]]);
let pingCalls = 0;
const api = makeSessionOps(sessions, {
execOnEtSession: async (_session, command) => ({
success: true,
stdout: command.includes('echo "DISKS:$disks"') ? "DISKS:" : LINUX_STATS,
stderr: "",
}),
measureSshPingLatency: async () => {
pingCalls += 1;
return 2;
},
});
const result = await api.getServerStats({ sender: {} }, { sessionId: "sid" });
assert.equal(result.success, true);
assert.equal(result.stats.memTotal, 8000);
assert.equal(result.stats.latencyMs, null);
assert.equal(probeCalls, 0);
assert.equal(pingCalls, 0);
});
test("getServerStats closes a blocked probe channel when stats time out", async () => {
@@ -1164,3 +1184,19 @@ for (const blocked of [false, true]) {
}
});
}
for (const keepaliveInterval of [0, 10000]) {
test(`getServerStats honors shared SSH keepalive policy (${keepaliveInterval})`, async () => {
const conn = fakeConn(LINUX_STATS);
conn.config = { keepaliveInterval };
const sessions = new Map([["sid", { type: "ssh", conn }]]);
let pingCalls = 0;
const api = makeSessionOps(sessions, {
measureSshPingLatency: async () => { pingCalls++; return 2; },
});
const result = await api.getServerStats({ sender: {} }, { sessionId: "sid" });
assert.equal(result.success, true);
assert.equal(pingCalls, keepaliveInterval === 0 ? 0 : 1);
assert.equal(result.stats.latencyMs, keepaliveInterval === 0 ? null : 2);
});
}
@@ -0,0 +1,118 @@
"use strict";
const DEFAULT_PING_TIMEOUT_MS = 3000;
/**
* Measures SSH round-trip latency with a transport-level ping
* (GLOBAL_REQUEST keepalive@openssh.com, want_reply) on an already
* authenticated ssh2 connection.
*
* This deliberately does NOT open a second TCP connection to the SSH port:
* a raw TCP probe is indistinguishable from a pre-auth connection attempt
* on the server and shows up as a failed login with no username in sshd
* logs / login-audit panels (see issue #3320).
*
* The ssh2 client dispatches global-request replies through the FIFO
* `conn._callbacks` queue (REQUEST_SUCCESS/REQUEST_FAILURE handlers shift
* the next entry), so pushing a callback immediately before sending the
* ping pairs it with the matching reply. On connection teardown the client
* flushes the queue with an error argument, which resolves `null`.
*
* At most one ping may be unresolved per connection: while a previous ping is
* in flight or its timeout tombstone still occupies a FIFO slot, subsequent
* calls resolve `null` without queueing another callback.
*/
// Serializes probes per connection: ssh2 correlates global-request replies
// with queued callbacks purely by FIFO order, so at most one unresolved ping
// (in-flight or timed-out tombstone) may occupy a slot in `conn._callbacks`
// at any time. Otherwise the next reply is delivered to the already-settled
// tombstone and the queue desynchronizes (e.g. swallowing forwardIn results),
// and repeated polls would grow the queue indefinitely.
const lastQueuedCallbackByConn = new WeakMap();
function createSshPingLatencyProbe({
setTimeoutFn = setTimeout,
clearTimeoutFn = clearTimeout,
now = () => performance.now(),
defaultTimeoutMs = DEFAULT_PING_TIMEOUT_MS,
} = {}) {
return function measureSshPingLatency(conn, timeoutMs = defaultTimeoutMs) {
return new Promise((resolve) => {
const proto = conn?._protocol;
const callbacks = conn?._callbacks;
if (!proto || typeof proto.ping !== "function" || !Array.isArray(callbacks)) {
resolve(null);
return;
}
// If a previous ping on this connection is still unresolved (in-flight
// or left as a timeout tombstone) and its callback still occupies a
// queue slot, skip this poll entirely: queueing another callback would
// misalign the FIFO once the outstanding reply (or the tombstone's late
// reply) arrives.
const previous = lastQueuedCallbackByConn.get(conn);
if (previous && callbacks.includes(previous)) {
resolve(null);
return;
}
const startedAt = now();
let settled = false;
let timer = null;
const finish = (value, consumeSlot, keepTracking) => {
if (settled) return;
settled = true;
if (timer !== null) clearTimeoutFn(timer);
if (consumeSlot) {
const index = callbacks.indexOf(onReply);
if (index >= 0) callbacks.splice(index, 1);
}
if (!keepTracking) lastQueuedCallbackByConn.delete(conn);
resolve(value);
};
const onReply = (hadErr) => {
// ssh2 reports REQUEST_SUCCESS as `false` and REQUEST_FAILURE as
// `true`; the latter is OpenSSH's normal reply to the unsupported
// keepalive@openssh.com request, so both count as a completed ping.
// Only an Error instance (transport teardown/flush) is a failure.
if (hadErr instanceof Error) {
// The client is flushing the whole queue with an Error while
// iterating it by index; splicing here would shift later
// callbacks into already-visited slots, so ssh2 would skip
// them (e.g. a pending forwardIn reply would hang until its
// own timeout). Leave the array untouched instead — it is
// being discarded by the client anyway.
finish(null, false, false);
return;
}
// A normal reply has already been shifted out of the FIFO by ssh2
// before reaching this callback, so the splice below is a no-op;
// consuming this slot also frees the connection for the next ping.
finish(Math.max(0, Math.round(now() - startedAt)), true, false);
};
callbacks.push(onReply);
lastQueuedCallbackByConn.set(conn, onReply);
try {
proto.ping();
} catch {
// The request was never sent, so the queued callback can never be
// consumed; remove it to keep the FIFO intact.
finish(null, true, false);
return;
}
// On timeout, deliberately leave `onReply` in the queue as a tombstone:
// global-request replies carry no request ID and ssh2 correlates them
// by FIFO order, so a late reply must still consume this slot. The
// tracking entry stays set so later polls skip this connection until
// the tombstone is actually consumed by a reply.
timer = setTimeoutFn(() => finish(null, false, true), timeoutMs);
});
};
}
module.exports = {
createSshPingLatencyProbe,
DEFAULT_PING_TIMEOUT_MS,
};
@@ -0,0 +1,95 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const { generateKeyPairSync } = require("node:crypto");
const { once } = require("node:events");
const net = require("node:net");
const { Client, Server } = require("ssh2");
const { createSshPingLatencyProbe } = require("./sshPingLatency.cjs");
const { createSessionOpsApi } = require("./sessionOps.cjs");
const hostKey = generateKeyPairSync("rsa", { modulusLength: 2048 }).privateKey
.export({ type: "pkcs1", format: "pem" });
for (const keepaliveInterval of [10000, 0]) {
test(`stats reuse authenticated SSH and preserve forwarding (keepalive=${keepaliveInterval})`,
{ timeout: 15000 }, async (t) => {
let connections = 0;
let authenticated = 0;
let pings = 0;
const peers = new Set();
const server = new Server({ hostKeys: [hostKey] }, (peer) => {
peers.add(peer);
peer.on("close", () => peers.delete(peer));
peer.on("error", () => {}); // The baseline raw probe closes before handshake.
const handlers = peer._protocol._handlers;
const original = handlers.GLOBAL_REQUEST;
handlers.GLOBAL_REQUEST = (protocol, name, wantReply, data) => {
if (name === "keepalive@openssh.com") {
pings++;
// Model a router that silently ignores keepalive packets.
if (keepaliveInterval === 0) return;
}
original(protocol, name, wantReply, data);
};
peer.on("authentication", (ctx) => ctx.accept());
peer.on("ready", () => {
authenticated++;
peer.on("request", (accept, reject, name) => {
if (name === "tcpip-forward") accept(43210);
else if (name === "cancel-tcpip-forward") accept();
else reject();
});
peer.on("session", (accept) => {
accept().on("exec", (acceptExec) => {
const stream = acceptExec();
stream.exit(0);
stream.end("NC_LATENCY_MARK|CPU:5|CORES:2|MEMINFO:8000 4000 100 900 0 0|DISKS:|NET:");
});
});
});
});
server._srv.on("connection", () => { connections++; });
const conn = new Client();
t.after(async () => {
conn.destroy();
for (const peer of peers) peer.end();
await new Promise((resolve) => server.close(resolve));
});
server.listen(0, "127.0.0.1");
await once(server, "listening");
const port = server.address().port;
// Reproduce the old probe: a second connection with no authentication.
const raw = net.connect(port, "127.0.0.1");
await once(raw, "connect");
raw.destroy();
await once(raw, "close");
conn.connect({ host: "127.0.0.1", port, username: "test", keepaliveInterval });
await once(conn, "ready");
const baseline = connections;
assert.equal(baseline, 2);
assert.equal(authenticated, 1);
const sessions = new Map([["sid", { type: "ssh", conn }]]);
const api = createSessionOpsApi({
sessions, setTimeout, clearTimeout, Buffer,
quoteShellArg: (value) => "'" + value.replace(/'/g, "'\\''") + "'",
measureSshPingLatency: createSshPingLatencyProbe(),
});
for (let i = 0; i < 3; i++) {
const [stats, forwardPort] = await Promise.all([
api.getServerStats({}, { sessionId: "sid" }),
new Promise((resolve, reject) => conn.forwardIn("127.0.0.1", 0,
(err, value) => err ? reject(err) : resolve(value))),
]);
assert.equal(stats.success, true);
assert.equal(forwardPort, 43210);
if (keepaliveInterval > 0) assert.equal(Number.isFinite(stats.stats.latencyMs), true);
else assert.equal(stats.stats.latencyMs, null);
await new Promise((resolve, reject) => conn.unforwardIn("127.0.0.1", forwardPort,
(err) => err ? reject(err) : resolve()));
}
assert.equal(connections, baseline, "stats must not create unauthenticated connections");
assert.equal(authenticated, 1);
assert.equal(pings, keepaliveInterval > 0 ? 3 : 0);
assert.equal(conn._callbacks.length, 0);
});
}
@@ -0,0 +1,191 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const { createSshPingLatencyProbe } = require("./sshPingLatency.cjs");
// Minimal ssh2-client stand-in: a FIFO global-request callback queue plus a
// _protocol.ping() that answers after `delayMs`, like the real REQUEST_SUCCESS
// handler shifting the next queued callback.
function createFakeSshClient({ delayMs = 5, hadErr = false, failPing = false } = {}) {
const callbacks = [];
const proto = {
ping() {
if (failPing) throw new Error("send failed");
setTimeout(() => {
const next = callbacks.shift();
if (next) next(hadErr);
}, delayMs);
},
};
return { _protocol: proto, _callbacks: callbacks };
}
function createProbe(overrides = {}) {
const timers = [];
return {
timers,
measure: createSshPingLatencyProbe({
now: () => fakeNow,
setTimeoutFn: (fn, ms) => {
const timer = { fn, ms, fired: false };
timers.push(timer);
return timer;
},
clearTimeoutFn: (timer) => {
if (timer) timer.fired = true;
},
...overrides,
}),
};
}
let fakeNow = 0;
test("measures round-trip time between ping and reply", async () => {
const { measure } = createProbe();
const conn = createFakeSshClient({ delayMs: 25 });
fakeNow = 1000;
const pending = measure(conn);
fakeNow = 1042;
const latency = await pending;
assert.equal(latency, 42);
});
test("resolves null for connections that cannot carry a transport ping", async () => {
const { measure } = createProbe();
assert.equal(await measure(null), null);
assert.equal(await measure({}), null);
assert.equal(await measure({ _protocol: { ping() {} }, _callbacks: "nope" }), null);
// ET exec-fallback conns expose only exec()
assert.equal(await measure({ exec() {} }), null);
});
test("treats REQUEST_FAILURE (`true`) as a completed ping", async () => {
// OpenSSH answers the unsupported keepalive@openssh.com global request with
// SSH_MSG_REQUEST_FAILURE, which ssh2 reports as boolean `true`.
const { measure } = createProbe();
const conn = createFakeSshClient({ delayMs: 25, hadErr: true });
fakeNow = 1000;
const pending = measure(conn);
fakeNow = 1042;
assert.equal(await pending, 42);
assert.equal(conn._callbacks.length, 0);
});
test("resolves null when the reply reports an error", async () => {
const { measure } = createProbe();
const conn = createFakeSshClient({ hadErr: new Error("connection closed") });
assert.equal(await measure(conn), null);
});
test("disconnect flush does not splice the queue being iterated", async () => {
// Like ssh2's close handler, flush the queue by index so that splicing the
// currently executing entry would shift later callbacks into already-visited
// slots and make ssh2 skip them (e.g. a pending forwardIn reply).
const { measure } = createProbe();
const conn = createFakeSshClient({ delayMs: 10_000 });
let laterCalled = false;
const later = () => { laterCalled = true; };
const pending = measure(conn);
conn._callbacks.push(later);
assert.equal(conn._callbacks.length, 2);
const flushed = conn._callbacks;
for (let i = 0; i < flushed.length; ++i) flushed[i](new Error("No response from server"));
assert.equal(await pending, null);
// The queue must be untouched during the flush so the later callback is
// still visited and receives the disconnect error.
assert.equal(flushed.length, 2);
assert.ok(laterCalled, "later callback received the disconnect error");
});
test("resolves null when ping() throws", async () => {
const { measure } = createProbe();
const conn = createFakeSshClient({ failPing: true });
assert.equal(await measure(conn), null);
});
test("times out but keeps its tombstone callback in the FIFO queue", async () => {
const { measure, timers } = createProbe();
const conn = createFakeSshClient({ delayMs: 10_000 });
fakeNow = 0;
const pending = measure(conn, 500);
assert.equal(conn._callbacks.length, 1);
const timeoutEntry = timers[0];
fakeNow = 500;
timeoutEntry.fn();
assert.equal(await pending, null);
assert.equal(timeoutEntry.fired, true);
// The slot is preserved so a late reply still consumes it and cannot be
// misattributed to a subsequent queued callback.
assert.equal(conn._callbacks.length, 1);
assert.equal(conn._callbacks[0].name, "onReply");
});
test("a late reply after a real timeout consumes its tombstone slot without resolving twice", async () => {
const measure = createSshPingLatencyProbe({ defaultTimeoutMs: 50 });
const conn = createFakeSshClient({ delayMs: 500 });
fakeNow = 0;
const latency = await measure(conn);
assert.equal(latency, null);
assert.equal(conn._callbacks.length, 1);
// The delayed fake reply fires into the preserved tombstone slot (the fake
// shifts it, like the real REQUEST_SUCCESS/REQUEST_FAILURE handlers); the
// settled promise must stay null (no unhandled rejection / double resolve).
await new Promise((resolve) => setTimeout(resolve, 550));
assert.equal(conn._callbacks.length, 0);
});
test("skips polls while a previous ping is in flight", async () => {
const { measure } = createProbe();
const conn = createFakeSshClient({ delayMs: 10_000 });
fakeNow = 0;
const pending = measure(conn, 500);
// A second poll while the first ping is still in flight must not queue
// another callback or send another ping.
assert.equal(await measure(conn), null);
assert.equal(conn._callbacks.length, 1);
});
test("skips polls after a timeout until the tombstone is consumed, then resumes", async () => {
const { measure, timers } = createProbe();
const conn = createFakeSshClient({ delayMs: 10_000 });
fakeNow = 0;
const first = measure(conn, 100);
assert.equal(conn._callbacks.length, 1);
// Later stats polls must not queue additional callbacks while the
// tombstone is unconsumed; ssh2 would otherwise deliver the next reply
// to the already-settled tombstone and desynchronize the FIFO.
assert.equal(await measure(conn), null);
assert.equal(await measure(conn), null);
assert.equal(conn._callbacks.length, 1);
fakeNow = 100;
timers[0].fn();
assert.equal(await first, null);
assert.equal(conn._callbacks.length, 1);
// A late reply consumes the tombstone slot (like ssh2 shifting it), after
// which new pings are allowed again.
const tombstone = conn._callbacks.shift();
tombstone(false);
assert.equal(conn._callbacks.length, 0);
fakeNow = 1000;
const pending = measure(conn);
fakeNow = 1042;
conn._callbacks.shift()(false);
assert.equal(await pending, 42);
});
@@ -339,10 +339,6 @@ function createStartSessionApi(ctx) {
_reuseEndpoint: normalizeEndpoint(buildConnectionReuseEndpoint(options, {
agentForwarding: options._actualAgentForwarding ?? options.agentForwarding,
})),
tcpLatencyDirect:
!Array.isArray(options.jumpHosts) || options.jumpHosts.length === 0
? !options.proxy
: false,
cols: options.cols || 80,
rows: options.rows || 24,
};
-161
View File
@@ -1,161 +0,0 @@
"use strict";
const dns = require("node:dns");
const DEFAULT_TIMEOUT_MS = 3000;
const DEFAULT_CACHE_TTL_MS = 30000;
const DEFAULT_FAILURE_CACHE_TTL_MS = 5000;
const MAX_CACHE_ENTRIES = 256;
const MAX_CONCURRENT_PROBES = 8;
const MAX_QUEUED_PROBES = 64;
function createTcpConnectLatencyProbe({
net,
lookup = dns.lookup,
now = () => performance.now(),
cacheNow = () => Date.now(),
cacheTtlMs = DEFAULT_CACHE_TTL_MS,
failureCacheTtlMs = DEFAULT_FAILURE_CACHE_TTL_MS,
maxCacheEntries = MAX_CACHE_ENTRIES,
maxConcurrentProbes = MAX_CONCURRENT_PROBES,
maxQueuedProbes = MAX_QUEUED_PROBES,
setTimer = setTimeout,
clearTimer = clearTimeout,
}) {
const resultCache = new Map();
const inFlight = new Map();
const queue = [];
let activeProbeCount = 0;
const concurrentLimit = Math.max(1, Math.floor(maxConcurrentProbes));
const queueLimit = Math.max(0, Math.floor(maxQueuedProbes));
function runProbe({ hostname, port, timeoutMs }) {
return new Promise((resolve) => {
let startedAt = net.isIP?.(hostname) ? now() : null;
let settled = false;
let socket = null;
const finish = (latencyMs) => {
if (settled) return;
settled = true;
try { socket?.destroy(); } catch { /* best-effort cleanup */ }
resolve(latencyMs);
};
try {
const connectOptions = { host: hostname, port };
if (startedAt === null) {
connectOptions.lookup = (host, options, callback) => {
lookup(host, options, (err, address, family) => {
startedAt = now();
callback(err, address, family);
});
};
}
socket = net.createConnection(connectOptions, () => {
const connectedAt = now();
finish(Math.max(0, Math.round(connectedAt - (startedAt ?? connectedAt))));
});
socket.once("error", () => finish(null));
socket.setTimeout(timeoutMs, () => finish(null));
} catch {
finish(null);
}
});
}
function startScheduledProbe(task) {
activeProbeCount += 1;
let result;
try {
result = Promise.resolve(task());
} catch (err) {
result = Promise.reject(err);
}
return result.finally(() => {
activeProbeCount -= 1;
while (activeProbeCount < concurrentLimit && queue.length > 0) {
const next = queue.shift();
clearTimer(next.timer);
const remainingMs = next.deadline - cacheNow();
if (remainingMs <= 0) {
next.resolve(null);
continue;
}
startScheduledProbe(() => next.task(remainingMs)).then(next.resolve, next.reject);
}
});
}
function scheduleProbe(task, timeoutMs) {
if (activeProbeCount < concurrentLimit) return startScheduledProbe(() => task(timeoutMs));
if (queue.length >= queueLimit) return Promise.resolve(null);
return new Promise((resolve, reject) => {
const entry = {
task,
resolve,
reject,
deadline: cacheNow() + timeoutMs,
timer: null,
};
entry.timer = setTimer(() => {
const index = queue.indexOf(entry);
if (index >= 0) queue.splice(index, 1);
resolve(null);
}, timeoutMs);
queue.push(entry);
});
}
function cacheResult(key, value) {
resultCache.delete(key);
resultCache.set(key, {
value,
expiresAt: cacheNow() + (value === null ? failureCacheTtlMs : cacheTtlMs),
});
while (resultCache.size > maxCacheEntries) {
resultCache.delete(resultCache.keys().next().value);
}
}
return function measureTcpConnectLatency({ hostname, port, timeoutMs = DEFAULT_TIMEOUT_MS }) {
if (!hostname || !Number.isInteger(port) || port < 1 || port > 65535) {
return Promise.resolve(null);
}
const key = `${String(hostname).toLowerCase()}\u0000${port}`;
const pending = inFlight.get(key);
if (pending) return pending;
const cached = resultCache.get(key);
const checkedAt = cacheNow();
if (cached && cached.expiresAt > checkedAt) {
resultCache.delete(key);
resultCache.set(key, cached);
return Promise.resolve(cached.value);
}
resultCache.delete(key);
const promise = scheduleProbe(
(remainingMs) => runProbe({ hostname, port, timeoutMs: remainingMs }),
timeoutMs,
)
.catch(() => null)
.then((value) => {
inFlight.delete(key);
cacheResult(key, value);
return value;
});
inFlight.set(key, promise);
return promise;
};
}
module.exports = {
createTcpConnectLatencyProbe,
DEFAULT_TIMEOUT_MS,
DEFAULT_CACHE_TTL_MS,
MAX_CONCURRENT_PROBES,
MAX_QUEUED_PROBES,
};
-202
View File
@@ -1,202 +0,0 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const { EventEmitter } = require("node:events");
const { createTcpConnectLatencyProbe } = require("./tcpConnectLatency.cjs");
function createSocket() {
const socket = new EventEmitter();
socket.setTimeout = (_ms, callback) => { socket.timeoutCallback = callback; };
socket.destroy = () => { socket.destroyedByProbe = true; };
return socket;
}
test("TCP latency stops timing as soon as the socket connects", async () => {
const socket = createSocket();
let connected;
const times = [100, 102.4];
const measure = createTcpConnectLatencyProbe({
net: {
createConnection(options, callback) {
assert.equal(options.host, "vm.test");
assert.equal(options.port, 22);
options.lookup("vm.test", {}, () => {});
connected = callback;
return socket;
},
},
lookup: (_host, _options, callback) => callback(null, "192.0.2.10", 4),
now: () => times.shift(),
});
const pending = measure({ hostname: "vm.test", port: 22 });
connected();
assert.equal(await pending, 2);
assert.equal(socket.destroyedByProbe, true);
});
test("TCP latency excludes hostname lookup time", async () => {
const socket = createSocket();
let connected;
let finishLookup;
const times = [500, 503];
const measure = createTcpConnectLatencyProbe({
net: {
createConnection(options, callback) {
options.lookup("slow-dns.test", {}, () => {});
connected = callback;
return socket;
},
},
lookup: (_host, _options, callback) => { finishLookup = callback; },
now: () => times.shift(),
});
const pending = measure({ hostname: "slow-dns.test", port: 22 });
finishLookup(null, "192.0.2.20", 4);
connected();
assert.equal(await pending, 3);
});
test("TCP latency returns no value when the endpoint cannot be reached", async () => {
const socket = createSocket();
const measure = createTcpConnectLatencyProbe({
net: { createConnection: () => socket },
now: () => 100,
});
const pending = measure({ hostname: "offline.test", port: 22 });
socket.emit("error", new Error("unreachable"));
assert.equal(await pending, null);
assert.equal(socket.destroyedByProbe, true);
});
test("TCP latency deduplicates concurrent probes and reuses a recent result", async () => {
const sockets = [];
const connectCallbacks = [];
const probeTimes = [100, 104, 200, 205];
let clock = 1_000;
const measure = createTcpConnectLatencyProbe({
net: {
isIP: () => 4,
createConnection(_options, callback) {
const socket = createSocket();
sockets.push(socket);
connectCallbacks.push(callback);
return socket;
},
},
now: () => probeTimes.shift(),
cacheNow: () => clock,
cacheTtlMs: 30_000,
});
const first = measure({ hostname: "192.0.2.30", port: 22 });
const concurrent = measure({ hostname: "192.0.2.30", port: 22 });
assert.equal(first, concurrent);
assert.equal(sockets.length, 1);
connectCallbacks[0]();
assert.equal(await first, 4);
assert.equal(await measure({ hostname: "192.0.2.30", port: 22 }), 4);
assert.equal(sockets.length, 1);
clock += 30_001;
const refreshed = measure({ hostname: "192.0.2.30", port: 22 });
assert.equal(sockets.length, 2);
connectCallbacks[1]();
assert.equal(await refreshed, 5);
});
test("TCP latency caps concurrent probes without evicting pending deduplication", async () => {
const connectCallbacks = [];
let activeSockets = 0;
let maxActiveSockets = 0;
const measure = createTcpConnectLatencyProbe({
net: {
isIP: () => 4,
createConnection(_options, callback) {
const socket = createSocket();
activeSockets += 1;
maxActiveSockets = Math.max(maxActiveSockets, activeSockets);
socket.destroy = () => {
if (!socket.destroyedByProbe) activeSockets -= 1;
socket.destroyedByProbe = true;
};
connectCallbacks.push(callback);
return socket;
},
},
now: () => 100,
cacheNow: () => 1_000,
maxCacheEntries: 2,
maxConcurrentProbes: 2,
});
const first = measure({ hostname: "192.0.2.1", port: 22 });
const duplicate = measure({ hostname: "192.0.2.1", port: 22 });
const second = measure({ hostname: "192.0.2.2", port: 22 });
const third = measure({ hostname: "192.0.2.3", port: 22 });
assert.equal(first, duplicate);
assert.equal(connectCallbacks.length, 2);
assert.equal(maxActiveSockets, 2);
connectCallbacks[0]();
await first;
assert.equal(connectCallbacks.length, 3);
assert.equal(maxActiveSockets, 2);
connectCallbacks[1]();
connectCallbacks[2]();
await Promise.all([second, third]);
const firstAgain = measure({ hostname: "192.0.2.1", port: 22 });
assert.equal(connectCallbacks.length, 4, "oldest settled result should be evicted");
connectCallbacks[3]();
await firstAgain;
});
test("TCP latency bounds queued work and includes queue wait in the timeout", async () => {
const connectCallbacks = [];
const timeoutCallbacks = [];
let clock = 1_000;
const measure = createTcpConnectLatencyProbe({
net: {
isIP: () => 4,
createConnection(_options, callback) {
connectCallbacks.push(callback);
return createSocket();
},
},
now: () => 100,
cacheNow: () => clock,
maxConcurrentProbes: 1,
maxQueuedProbes: 1,
setTimer: (callback) => {
timeoutCallbacks.push(callback);
return callback;
},
clearTimer: () => {},
});
const active = measure({ hostname: "192.0.2.1", port: 22, timeoutMs: 3_000 });
const queued = measure({ hostname: "192.0.2.2", port: 22, timeoutMs: 3_000 });
const overflow = measure({ hostname: "192.0.2.3", port: 22, timeoutMs: 3_000 });
assert.equal(await overflow, null);
assert.equal(connectCallbacks.length, 1);
timeoutCallbacks[0]();
assert.equal(await queued, null);
connectCallbacks[0]();
await active;
assert.equal(connectCallbacks.length, 1, "expired queued probe must not start later");
clock += 5_001;
const retried = measure({ hostname: "192.0.2.2", port: 22, timeoutMs: 3_000 });
assert.equal(connectCallbacks.length, 2);
connectCallbacks[1]();
await retried;
});
@@ -1244,12 +1244,6 @@ main();
sshEnv: sshEnvironment?.env || {},
sshOptions: sshEnvironment?.sshOptions || [],
sshUserHost: sshEnvironment?.userHost || "",
tcpLatencyTarget: {
hostname: options.hostname,
port: options.etPort || 2022,
},
tcpLatencyDirect:
(!Array.isArray(options.jumpHosts) || options.jumpHosts.length === 0) && !options.proxy,
etStatsAuth: {
hostname: options.hostname,
port: options.port || 22,