Compare commits

...
3 changed files with 169 additions and 2 deletions

No files matched your search

+36 -2
View File
@@ -23,6 +23,24 @@ function isSuccessfulCommandResult(result) {
return result?.success && (result.code === 0 || result.code === null || result.code === undefined);
}
// Docker list commands on a slow host (lazy daemon, snap docker, many images,
// slow SSH link) can legitimately take longer than the previous fixed 12 s
// window, so the lists get more headroom (#3387).
const DOCKER_LIST_TIMEOUT_MS = 30000;
function isSshExecTimeoutResult(result) {
if (result?.timedOut === true) return true;
const text = `${result?.error || ""}\n${result?.stderr || ""}`;
// Only command-execution timeouts mean docker never finished; the daemon
// hint is appropriate there. Channel-open timeouts are SSH transport
// failures and keep their specific error instead of the Docker hint.
return text.includes("SSH command execution timed out");
}
function buildDockerTimeoutError(timeoutMs) {
return `Docker command timed out after ${timeoutMs} ms; the Docker daemon may be stopped or unresponsive on the remote host`;
}
function dockerCommandError(result, fallback) {
return (result?.stderr || result?.error || "").trim() || fallback;
}
@@ -173,10 +191,24 @@ function createDockerOpsApi({ execOnSession, getSession }) {
const result = await execOnSession(event, sessionId, cmd, timeoutMs);
if (isSuccessfulCommandResult(result)) return result;
if (isSshExecTimeoutResult(result)) {
return {
success: false,
error: buildDockerTimeoutError(timeoutMs),
stderr: result?.stderr,
};
}
if (isDockerSocketPermissionError(result)) {
const sudoPassword = getSessionSudoPassword(getSession?.(sessionId));
let lastSudoResult = null;
const sudoTimeoutResult = () => ({
success: false,
error: buildDockerTimeoutError(timeoutMs),
stderr: lastSudoResult?.stderr,
});
const nopasswdResult = await execOnSession(
event,
sessionId,
@@ -185,6 +217,7 @@ function createDockerOpsApi({ execOnSession, getSession }) {
);
if (isSuccessfulCommandResult(nopasswdResult)) return nopasswdResult;
lastSudoResult = nopasswdResult;
if (isSshExecTimeoutResult(nopasswdResult)) return sudoTimeoutResult();
if (sudoPassword) {
const sudoResult = await execOnSession(
@@ -196,6 +229,7 @@ function createDockerOpsApi({ execOnSession, getSession }) {
);
if (isSuccessfulCommandResult(sudoResult)) return sudoResult;
lastSudoResult = sudoResult;
if (isSshExecTimeoutResult(sudoResult)) return sudoTimeoutResult();
}
return {
@@ -217,13 +251,13 @@ function createDockerOpsApi({ execOnSession, getSession }) {
}
async function listContainers(event, sessionId) {
const result = await runDocker(event, sessionId, "ps -a --format '{{json .}}'", 12000);
const result = await runDocker(event, sessionId, "ps -a --format '{{json .}}'", DOCKER_LIST_TIMEOUT_MS);
if (!result.success) return { success: false, error: result.error };
return { success: true, containers: parseDockerContainers(result.stdout) };
}
async function listImages(event, sessionId) {
const result = await runDocker(event, sessionId, "images --format '{{json .}}'", 12000);
const result = await runDocker(event, sessionId, "images --format '{{json .}}'", DOCKER_LIST_TIMEOUT_MS);
if (!result.success) return { success: false, error: result.error };
return { success: true, images: parseDockerImages(result.stdout) };
}
@@ -192,6 +192,126 @@ test("listContainers retries with explicit sudo autofill password on mosh or et
assert.deepEqual(calls[2].execOptions, { stdin: "saved-secret\n" });
});
test("listContainers and listImages allow 30s for slow docker daemons", async () => {
const calls = [];
const dockerOps = createDockerOpsApi({
getSession: () => ({}),
execOnSession: async (_event, sessionId, command, timeoutMs) => {
calls.push({ command, timeoutMs });
return {
success: true,
stdout: '{"ID":"abc123","Names":"web","Image":"nginx","State":"running"}\n',
stderr: "",
code: 0,
};
},
});
await dockerOps.listContainers(null, "s1");
await dockerOps.listImages(null, "s1");
assert.equal(calls.length, 2);
assert.equal(calls[0].timeoutMs, 30000);
assert.equal(calls[1].timeoutMs, 30000);
});
test("listContainers maps ssh exec timeouts to a docker daemon hint without retrying", async () => {
const calls = [];
const dockerOps = createDockerOpsApi({
getSession: () => ({ systemManagerSudoPassword: "host-secret" }),
execOnSession: async (_event, sessionId, command, timeoutMs) => {
calls.push({ command, timeoutMs });
return {
success: false,
stdout: "",
stderr: "",
error: "SSH command execution timed out after 30000 ms",
code: 1,
};
},
});
const result = await dockerOps.listContainers(null, "s1");
assert.equal(result.success, false);
assert.match(result.error, /Docker command timed out after 30000 ms/);
assert.match(result.error, /daemon may be stopped or unresponsive/i);
assert.equal(calls.length, 1);
});
test("listImages maps ssh exec timeouts to a docker daemon hint without retrying", async () => {
const calls = [];
const dockerOps = createDockerOpsApi({
getSession: () => ({}),
execOnSession: async (_event, sessionId, command, timeoutMs) => {
calls.push({ command, timeoutMs });
return {
success: false,
stdout: "",
stderr: "",
error: "SSH command execution timed out after 30000 ms",
code: 1,
};
},
});
const result = await dockerOps.listImages(null, "s1");
assert.equal(result.success, false);
assert.match(result.error, /Docker command timed out after 30000 ms/);
assert.match(result.error, /daemon may be stopped or unresponsive/i);
assert.equal(calls.length, 1);
});
test("listContainers keeps the SSH-specific error for exec channel-open timeouts", async () => {
const calls = [];
const dockerOps = createDockerOpsApi({
getSession: () => ({}),
execOnSession: async (_event, sessionId, command, timeoutMs) => {
calls.push({ command, timeoutMs });
return {
success: false,
stdout: "",
stderr: "",
error: "SSH exec channel open timed out after 30000 ms",
code: 1,
};
},
});
const result = await dockerOps.listContainers(null, "s1");
assert.equal(result.success, false);
assert.match(result.error, /SSH exec channel open timed out/);
assert.doesNotMatch(result.error, /daemon may be stopped/i);
assert.equal(calls.length, 1);
});
test("listContainers maps an explicit timedOut result flag to the docker daemon hint", async () => {
const calls = [];
const dockerOps = createDockerOpsApi({
getSession: () => ({}),
execOnSession: async (_event, sessionId, command, timeoutMs) => {
calls.push({ command, timeoutMs });
return {
success: false,
stdout: "",
stderr: "",
error: "Command failed: ssh -o BatchMode=no docker ps\n",
timedOut: true,
code: 1,
};
},
});
const result = await dockerOps.listContainers(null, "s1");
assert.equal(result.success, false);
assert.match(result.error, /Docker command timed out after 30000 ms/);
assert.match(result.error, /daemon may be stopped or unresponsive/i);
assert.equal(calls.length, 1);
});
test("docker image actions retry with sudo and send saved passwords through stdin", async () => {
const calls = [];
const dockerOps = createDockerOpsApi({
@@ -1116,11 +1116,24 @@ main();
if (Number.isFinite(maxBuffer) && maxBuffer > 0) {
execFileOptions.maxBuffer = Math.floor(maxBuffer);
}
// Node does not expose a `timedOut` flag on execFile callback errors
// (only `killed`/`signal`), so track the deadline ourselves. The
// deadline here matches `execFileOptions.timeout`, which is what makes
// Node terminate the ssh child.
let timedOut = false;
const timeoutTimer = setTimeout(() => {
timedOut = true;
}, timeoutMs);
const child = execFile(sshCmd, args, execFileOptions, (err, stdout, stderr) => {
clearTimeout(timeoutTimer);
if (err) {
resolve({
success: false,
error: err.message,
// Expose whether the ssh process was terminated because
// options.timeout elapsed so callers (e.g. dockerOps) can
// recognize subprocess timeouts without parsing messages.
timedOut,
stdout: stdout || "",
stderr: stderr || "",
code: typeof err.code === "number" && err.code !== 0 ? err.code : 1,