Compare commits

...
3 changed files with 441 additions and 5 deletions

No files matched your search

+7
View File
@@ -44,6 +44,13 @@ handle, reopen the file at zero, and compare every byte sequentially with the
local output. Uploads hash the local file and sequentially reread the complete
remote stage before promotion. A size-only acknowledgement is insufficient.
Other SSH software retains its existing concurrency and chunk sizes.
Transfers sharing one TERM-SSHD channel, including pooled aliases, serialize
their complete body and verification phases across uploads and downloads.
Restored transfers serialize remote source fingerprints and checkpoint-prefix
checks too, acquiring the relevant source or target channel before each check.
Cancelling a queued transfer does not close the shared channel. Active transfer
cleanup waits for CLOSE; a failed or timed-out CLOSE discards all aliases of
the uncertain channel before another transfer can use it.
This compatibility policy trades throughput and resume efficiency for correct
contents. Allow sufficient MCP command time for both the transfer and the
+101 -5
View File
@@ -2474,6 +2474,41 @@ async function acquireSessionFastDownloadSlot(client, transfer) {
}
}
// TERM-SSHD compatibility requires one file handle per channel. Per-file request
// concurrency is not enough: pooled SFTP aliases share that physical channel,
// and a second body or verification OPEN would fail even for tiny files.
// Reuse the cancelable one-slot queue, keyed by the stream rather than its
// wrapper client, across both directions and the complete verification phase.
async function withSequentialSftpTransferSlot(client, transfer, operation) {
if (isScpModeClient(client)) return operation();
await requireSftpChannel(client);
const channel = client.sftp;
if (getSftpTransferConcurrency(channel, DOWNLOAD_TRANSFER_CONCURRENCY) !== 1) {
return operation();
}
await acquireSessionFastDownloadSlot(channel, transfer);
try {
return await operation();
} catch (error) {
if (error?.uncleanSftpChannel && client.sftp === channel) {
abandonWedgedVerificationSftpChannel(client);
}
throw error;
} finally {
releaseSessionFastDownloadSlot(channel);
}
}
async function closeSequentialSftpTransferHandle(sftp, handle) {
const result = await closeSftpHandleBestEffort(sftp, handle);
if (result.unclean) {
const error = new Error("SFTP body handle could not be closed");
error.noTransferFallback = true;
error.uncleanSftpChannel = true;
throw error;
}
}
async function acquireIsolatedDownloadChannel(client, transfer) {
const pool = getIsolatedDownloadChannelPool(client);
if (transfer?.cancelled) return null;
@@ -2651,6 +2686,22 @@ async function uploadFile(
encoding = "utf-8",
onBytesCommitted = null,
options = {},
) {
return withSequentialSftpTransferSlot(client, transfer, () => uploadFileOnChannel(
localPath, remotePath, client, fileSize, transfer, sendProgress, encoding, onBytesCommitted, options,
));
}
async function uploadFileOnChannel(
localPath,
remotePath,
client,
fileSize,
transfer,
sendProgress,
encoding = "utf-8",
onBytesCommitted = null,
options = {},
) {
const generatedStagePath = options.generatedStagePath === true;
const existingRemoteMode = Number.isFinite(options.existingMode)
@@ -4704,6 +4755,9 @@ function abandonWedgedVerificationSftpChannel(client) {
// object back to later browse/transfer work (hasSftpChannelApi only checks
// method presence). Non-sudo sessions can reopen; sudo recovery stays disabled.
client.sftp = null;
for (const alias of sftpClients?.values?.() || []) {
if (alias?.sftp === sftp) alias.sftp = null;
}
try { sftp.end?.(); } catch { /* ignore */ }
try { sftp.destroy?.(); } catch { /* ignore */ }
}
@@ -5279,7 +5333,14 @@ async function uploadFileConcurrent(
// A closed channel never answers CLOSE. Skip it; the 2s race is only for
// a live shared session. Falling through to another strategy on a dead
// channel is handled by noTransferFallback on the close error.
if (!skipClose && sftp?.readable !== false) {
if (!skipClose && sftp?.readable !== false
&& getSftpTransferConcurrency(sftp, UPLOAD_TRANSFER_CONCURRENCY) === 1) {
try {
await closeSequentialSftpTransferHandle(sftp, remoteHandle);
} catch (error) {
remoteCloseError = error;
}
} else if (!skipClose && sftp?.readable !== false) {
let closeTimeout = null;
try {
await Promise.race([
@@ -5433,7 +5494,13 @@ async function downloadFileResumableFast(
if (remoteHandle) {
const skipClose = disposeChannel && (failed || transfer.cancelled);
if (!skipClose) {
if (failed || transfer.cancelled) {
if (sequentialPeer && (failed || transfer.cancelled)) {
try {
await closeSequentialSftpTransferHandle(sftp, remoteHandle);
} catch (error) {
remoteCloseError = error;
}
} else if (failed || transfer.cancelled) {
// Shared cancel/failure already settled the transfer; do not await
// CLOSE (adds up to 2s on a dead channel). Best-effort close only.
closeSftpHandle(sftp, remoteHandle).catch(() => {});
@@ -5484,6 +5551,7 @@ async function downloadFileResumableFast(
error.noTransferFallback = true;
throw error;
}
if (remoteCloseError?.uncleanSftpChannel) throw remoteCloseError;
if (!failed && !transfer.cancelled && remoteCloseError) {
if (disposeChannel) {
const error = new Error("The isolated SFTP channel failed while closing", { cause: remoteCloseError });
@@ -5517,6 +5585,21 @@ async function downloadFile(
sendProgress,
encoding = "utf-8",
runCancelablePreflight = async (operation) => operation(),
) {
return withSequentialSftpTransferSlot(client, transfer, () => downloadFileOnChannel(
remotePath, localPath, client, fileSize, transfer, sendProgress, encoding, runCancelablePreflight,
));
}
async function downloadFileOnChannel(
remotePath,
localPath,
client,
fileSize,
transfer,
sendProgress,
encoding = "utf-8",
runCancelablePreflight = async (operation) => operation(),
) {
if (isScpModeClient(client)) {
transfer.pauseSupported = false;
@@ -6412,7 +6495,7 @@ async function startTransferNow(event, payload, onProgress) {
return transfer.sourceFingerprintPromise;
};
const verifyResumeContent = async (bytes, createSourceHash, createStagedHash) => {
const verifyResumeContent = async (bytes, createSourceHash, createStagedHash, remoteClient = null) => {
if (!bytes) return;
const previousPhase = transfer.phase;
const verificationStartedAt = Date.now();
@@ -6442,7 +6525,7 @@ async function startTransferNow(event, payload, onProgress) {
transfer.phase = "verifying";
publishVerificationProgress(true);
try {
await runTransferAbortableOperation(transfer, (signal) => assertMatchingResumeContent(
const verify = () => runTransferAbortableOperation(transfer, (signal) => assertMatchingResumeContent(
createSourceHash({
signal,
onProgress(value) {
@@ -6458,6 +6541,11 @@ async function startTransferNow(event, payload, onProgress) {
},
}),
));
// Resume prefix reads run before the body wrapper. Acquire separately,
// before installing the I/O abort hook, so queued cancellation works and
// no source/target channel is held while acquiring another channel.
if (remoteClient) await withSequentialSftpTransferSlot(remoteClient, transfer, verify);
else await verify();
publishVerificationProgress(true);
} finally {
transfer.phase = previousPhase || "transferring";
@@ -6654,7 +6742,12 @@ async function startTransferNow(event, payload, onProgress) {
}
if (transfer.resumable && transfer.sourceFingerprint) {
await transfer.verifySourceFingerprint(transfer.sourceFingerprint);
const verify = () => transfer.verifySourceFingerprint(transfer.sourceFingerprint);
// Initial remote identity reads also precede the download/upload body.
// Do not put this lock inside shared hash helpers: body verification and
// soft-resume paths can already own the channel.
if (sourceClient) await withSequentialSftpTransferSlot(sourceClient, transfer, verify);
else await verify();
}
sendProgress(transfer.checkpointBytes, fileSize, { force: true });
@@ -6812,6 +6905,7 @@ async function startTransferNow(event, payload, onProgress) {
options,
),
(options) => hashLocalPrefix(downloadTargetPath, verifyBytes, options),
client,
);
}
const download = stageLocalDownload
@@ -7079,6 +7173,7 @@ async function startTransferNow(event, payload, onProgress) {
options,
),
(options) => hashLocalPrefix(tempPath, verifyBytes, options),
sourceClient,
);
}
const downloadProgress = (transferred, reportedTotal, options = {}) => {
@@ -7228,6 +7323,7 @@ async function startTransferNow(event, payload, onProgress) {
verifyBytes,
options,
),
targetClient,
);
}
await uploadFile(
@@ -168,3 +168,336 @@ for (const corrupt of [false, true]) {
}
});
}
async function createSequentialTransferFixture(t) {
const payload = crypto.randomBytes(3 * 32768 + 17);
const files = new Map([["/source.bin", payload]]);
const handles = new Map();
const paths = [];
const staged = [];
let rejectedOpens = 0;
let opens = 0;
const missing = () => Object.assign(new Error("ENOENT"), { code: 2 });
const stats = (p) => {
const data = files.get(String(p));
if (!data) throw missing();
return { size: data.length, mode: 0o100644, mtime: 1, isDirectory: () => false, isSymbolicLink: () => false };
};
const sftp = new EventEmitter();
Object.assign(sftp, {
_client: { _remoteVer: "TERM-SSHD" },
readdir(_p, cb) { cb(null, []); }, mkdir(_p, cb) { cb(null); },
stat(p, cb) { try { cb(null, stats(p)); } catch (err) { cb(err); } },
lstat(p, cb) { try { cb(null, stats(p)); } catch (err) { cb(err); } },
unlink(p, cb) { files.delete(String(p)); cb(null); },
chmod(_p, _mode, cb) { cb(null); },
open(p, flags, cb) {
if (handles.size) {
rejectedOpens++;
return cb(new Error("General failure: one open handle per channel"));
}
const name = String(p);
if (flags === "w") files.set(name, Buffer.alloc(payload.length));
if (!files.has(name)) return cb(missing());
const handle = Buffer.from(String(++opens));
handles.set(handle, { name, cursor: 0 }); cb(null, handle);
},
write(handle, buffer, offset, length, _position, cb) {
const h = handles.get(handle);
buffer.copy(files.get(h.name), h.cursor, offset, offset + length);
h.cursor += length;
setTimeout(() => cb(null), 5);
},
read(handle, buffer, offset, length, _position, cb) {
const h = handles.get(handle); const data = files.get(h.name);
const count = Math.min(length, data.length - h.cursor);
data.copy(buffer, offset, h.cursor, h.cursor + count); h.cursor += count;
setTimeout(() => cb(null, count), 5);
},
close(handle, cb) { handles.delete(handle); cb(null); },
end() {},
});
const createClient = () => ({
sftp, __netcattySingleChannelSsh: true,
stat: async (p) => stats(p),
rename: async (from, to) => { files.set(String(to), files.get(String(from))); files.delete(String(from)); },
delete: async (p) => { files.delete(String(p)); },
client: { _remoteVer: "TERM-SSHD", sftp() { throw new Error("No terminal subsystem reuse"); } },
});
// Real pooled single-channel clients are distinct aliases of one SFTP stream.
const clients = new Map([["first", createClient()], ["alias", createClient()]]);
transferBridge.init({ sftpClients: clients });
t.after(async () => {
for (const p of [...paths, ...staged]) await fs.rm(p, { force: true });
});
return {
payload, files, sftp, handles, clients,
rejectedOpens: () => rejectedOpens,
async start(direction, sftpId = "first", options = {}) {
const transferId = options.transferId || crypto.randomUUID();
const localPath = tempDirBridge.getTempFilePath(`term-concurrent-${transferId}.bin`);
paths.push(localPath);
const stagePath = tempDirBridge.getTransferTempFilePath(transferId,
direction === "copy" ? "source.bin" : path.basename(localPath));
staged.push(stagePath);
if (direction === "upload") await fs.writeFile(localPath, payload);
const targetPath = direction === "download" ? localPath : `/uploaded-${transferId}.bin`;
const checkpoint = options.checkpointBytes || options.downloadCheckpointBytes || 0;
if (checkpoint > 0) await fs.writeFile(stagePath, payload.subarray(0, checkpoint));
if (direction === "copy" && options.resumeStage === "upload") {
await fs.writeFile(stagePath, payload);
const remoteStage = `/.${path.basename(targetPath)}.netcatty-${transferId}.part`;
files.set(remoteStage, payload.subarray(0, options.uploadCheckpointBytes || 0));
}
const result = await transferBridge.startTransfer({ sender: { send() {} } }, {
transferId,
sourcePath: direction === "upload" ? localPath : "/source.bin", targetPath,
sourceType: direction === "upload" ? "local" : "sftp",
targetType: direction === "download" ? "local" : "sftp",
sourceSftpId: sftpId, targetSftpId: sftpId,
totalBytes: payload.length, resumable: true,
...options,
});
return { result, targetPath, direction };
},
};
}
for (const secondId of ["first", "alias"]) {
test(`TERM-SSHD small downloads serialize through body and verification (${secondId})`, async (t) => {
const fixture = await createSequentialTransferFixture(t);
const downloads = await Promise.all([fixture.start("download"), fixture.start("download", secondId)]);
for (const download of downloads) {
assert.equal(download.result.error, undefined);
assert.deepEqual(await fs.readFile(download.targetPath), fixture.payload);
}
assert.equal(fixture.rejectedOpens(), 0, "the channel must never hold two file handles");
});
}
for (const directions of [["upload", "upload"], ["upload", "download"]]) {
test(`TERM-SSHD aliases serialize ${directions.join(" and ")} including verification`, async (t) => {
const fixture = await createSequentialTransferFixture(t);
const transfers = await Promise.all(directions.map((direction, index) => fixture.start(direction, index ? "alias" : "first")));
for (const transfer of transfers) {
assert.equal(transfer.result.error, undefined);
assert.deepEqual(transfer.direction === "upload"
? fixture.files.get(transfer.targetPath)
: await fs.readFile(transfer.targetPath), fixture.payload);
}
assert.equal(fixture.rejectedOpens(), 0, "the channel must never hold two file handles");
});
}
test("TERM-SSHD queued cancellation releases its turn without closing the owner channel", async (t) => {
const fixture = await createSequentialTransferFixture(t);
const first = fixture.start("download");
const transferId = crypto.randomUUID();
const cancelled = fixture.start("download", "alias", { transferId });
const deadline = Date.now() + 1_000;
while (transferBridge._getSessionFastDownloadWaiterCountForTests(fixture.sftp) === 0 && Date.now() < deadline) {
await new Promise((resolve) => setImmediate(resolve));
}
assert.equal(transferBridge._getSessionFastDownloadWaiterCountForTests(fixture.sftp), 1);
await transferBridge.cancelTransfer(null, { transferId });
assert.match((await cancelled).result.error || "", /cancelled/i);
const last = fixture.start("download", "alias");
for (const download of await Promise.all([first, last])) {
assert.equal(download.result.error, undefined);
assert.deepEqual(await fs.readFile(download.targetPath), fixture.payload);
}
assert.equal(fixture.rejectedOpens(), 0);
assert.equal(transferBridge._getSessionFastDownloadWaiterCountForTests(fixture.sftp), 0);
});
test("TERM-SSHD failed transfer releases the physical channel for the next alias", async (t) => {
const fixture = await createSequentialTransferFixture(t);
const originalOpen = fixture.sftp.open;
let rejectFirst = true;
fixture.sftp.open = (...args) => {
if (rejectFirst) {
rejectFirst = false;
args.at(-1)(Object.assign(new Error("Permission denied"), { code: 3 }));
} else originalOpen(...args);
};
const [first, next] = await Promise.all([fixture.start("download"), fixture.start("download", "alias")]);
assert.match(first.result.error || "", /Permission denied/);
assert.equal(next.result.error, undefined);
assert.deepEqual(await fs.readFile(next.targetPath), fixture.payload);
assert.equal(fixture.rejectedOpens(), 0);
});
for (const direction of ["download", "upload"]) {
for (const closeMode of ["delayed", "error", "timeout"]) {
test(`TERM-SSHD active ${direction} cancel handles ${closeMode} CLOSE before releasing an alias`, { timeout: 5_000 }, async (t) => {
const fixture = await createSequentialTransferFixture(t);
const originalClose = fixture.sftp.close;
fixture.sftp.close = (handle, callback) => {
if (closeMode === "timeout") return;
if (closeMode === "error") return callback(new Error("General failure"));
setTimeout(() => originalClose(handle, callback), 100);
};
const transferId = crypto.randomUUID();
const first = fixture.start(direction, "first", { transferId });
const deadline = Date.now() + 1_000;
while (fixture.handles.size === 0 && Date.now() < deadline) {
await new Promise((resolve) => setImmediate(resolve));
}
assert.equal(fixture.handles.size, 1);
const next = fixture.start("download", "alias");
await transferBridge.cancelTransfer(null, { transferId });
const [cancelled, download] = await Promise.all([first, next]);
assert.ok(cancelled.result.error);
if (closeMode === "delayed") {
assert.equal(download.result.error, undefined);
assert.deepEqual(await fs.readFile(download.targetPath), fixture.payload);
} else {
assert.match(download.result.error || "", /Please reconnect/);
}
assert.equal(fixture.rejectedOpens(), 0, "an unresolved handle must never be reused");
});
}
}
for (const direction of ["download", "upload"]) {
test(`TERM-SSHD active ${direction} error awaits CLOSE before releasing an alias`, async (t) => {
const fixture = await createSequentialTransferFixture(t);
const method = direction === "upload" ? "write" : "read";
const originalIo = fixture.sftp[method];
let rejectFirst = true;
fixture.sftp[method] = (...args) => {
if (rejectFirst) {
rejectFirst = false;
setTimeout(() => args.at(-1)(new Error("Permission denied")), 5);
} else originalIo(...args);
};
const originalClose = fixture.sftp.close;
fixture.sftp.close = (handle, callback) => setTimeout(() => originalClose(handle, callback), 100);
const first = fixture.start(direction);
const deadline = Date.now() + 1_000;
while (fixture.handles.size === 0 && Date.now() < deadline) {
await new Promise((resolve) => setImmediate(resolve));
}
const next = fixture.start("download", "alias");
const [failed, download] = await Promise.all([first, next]);
assert.match(failed.result.error || "", /Permission denied/);
assert.equal(download.result.error, undefined);
assert.deepEqual(await fs.readFile(download.targetPath), fixture.payload);
assert.equal(fixture.rejectedOpens(), 0);
});
}
for (const direction of ["download", "copy"]) {
test(`TERM-SSHD resumed ${direction} source fingerprint waits behind an active alias`, { timeout: 5_000 }, async (t) => {
const fixture = await createSequentialTransferFixture(t);
const first = fixture.start("download");
const deadline = Date.now() + 1_000;
while (fixture.handles.size === 0 && Date.now() < deadline) {
await new Promise((resolve) => setImmediate(resolve));
}
const sourceFingerprint = `sha256:p${fixture.payload.length}:${crypto.createHash("sha256").update(fixture.payload).digest("hex")}`;
const resumed = fixture.start(direction, "alias", { sourceFingerprint, checkpointBytes: 32768, downloadCheckpointBytes: 32768 });
for (const transfer of await Promise.all([first, resumed])) {
assert.equal(transfer.result.error, undefined);
assert.deepEqual(transfer.direction === "copy" ? fixture.files.get(transfer.targetPath)
: await fs.readFile(transfer.targetPath), fixture.payload);
}
assert.equal(fixture.rejectedOpens(), 0);
});
}
for (const mode of ["download", "copy-download", "copy-upload"]) {
test(`TERM-SSHD ${mode} checkpoint verification waits for an alias admitted after fingerprint`, { timeout: 5_000 }, async (t) => {
const fixture = await createSequentialTransferFixture(t);
const sourceFingerprint = `sha256:p${fixture.payload.length}:${crypto.createHash("sha256").update(fixture.payload).digest("hex")}`;
const originalClose = fixture.sftp.close;
let competing;
fixture.sftp.close = (handle, callback) => {
originalClose(handle, callback);
// The first close completes the full-source fingerprint. Admit an alias
// before the later checkpoint-prefix proof asks for another handle.
if (!competing) competing = fixture.start("download", "first");
};
const result = await fixture.start(mode === "download" ? "download" : "copy", "alias", {
sourceFingerprint,
checkpointBytes: 32768, downloadCheckpointBytes: 32768,
...(mode === "copy-upload" ? { resumeStage: "upload", uploadCheckpointBytes: 32768 } : {}),
});
const other = await competing;
assert.equal(result.result.error, undefined);
assert.equal(other.result.error, undefined);
assert.deepEqual(result.direction === "copy" ? fixture.files.get(result.targetPath)
: await fs.readFile(result.targetPath), fixture.payload);
assert.deepEqual(await fs.readFile(other.targetPath), fixture.payload);
assert.equal(fixture.rejectedOpens(), 0);
});
}
for (const phase of ["fingerprint", "checkpoint"]) {
test(`TERM-SSHD queued ${phase} verification cancels without opening another handle`, { timeout: 5_000 }, async (t) => {
const fixture = await createSequentialTransferFixture(t);
const sourceFingerprint = `sha256:p${fixture.payload.length}:${crypto.createHash("sha256").update(fixture.payload).digest("hex")}`;
let competing;
if (phase === "checkpoint") {
const originalClose = fixture.sftp.close;
fixture.sftp.close = (handle, callback) => {
originalClose(handle, callback);
if (!competing) competing = fixture.start("download", "first");
};
} else {
competing = fixture.start("download", "first");
const deadline = Date.now() + 1_000;
while (fixture.handles.size === 0 && Date.now() < deadline) {
await new Promise((resolve) => setImmediate(resolve));
}
}
const transferId = crypto.randomUUID();
const resumed = fixture.start("download", "alias", { transferId, sourceFingerprint, checkpointBytes: 32768 });
const deadline = Date.now() + 1_000;
while ((!competing || transferBridge._getSessionFastDownloadWaiterCountForTests(fixture.sftp) === 0) && Date.now() < deadline) {
await new Promise((resolve) => setImmediate(resolve));
}
assert.equal(transferBridge._getSessionFastDownloadWaiterCountForTests(fixture.sftp), 1);
await transferBridge.cancelTransfer(null, { transferId });
assert.match((await resumed).result.error || "", /cancelled/i);
const other = await competing;
assert.equal(other.result.error, undefined);
assert.deepEqual(await fs.readFile(other.targetPath), fixture.payload);
assert.equal(fixture.rejectedOpens(), 0);
assert.equal(transferBridge._getSessionFastDownloadWaiterCountForTests(fixture.sftp), 0);
});
}
test("TERM-SSHD remote-copy upload checkpoint locks its target rather than its distinct source", { timeout: 5_000 }, async (t) => {
const source = await createSequentialTransferFixture(t);
const target = await createSequentialTransferFixture(t);
transferBridge.init({ sftpClients: new Map([
...source.clients,
["target-first", target.clients.get("first")],
["target-alias", target.clients.get("alias")],
]) });
const competing = target.start("download", "target-first");
const deadline = Date.now() + 1_000;
while (target.handles.size === 0 && Date.now() < deadline) {
await new Promise((resolve) => setImmediate(resolve));
}
const transferId = crypto.randomUUID();
const targetPath = `/uploaded-${transferId}.bin`;
target.files.set(`/.${path.basename(targetPath)}.netcatty-${transferId}.part`, source.payload.subarray(0, 32768));
const sourceFingerprint = `sha256:p${source.payload.length}:${crypto.createHash("sha256").update(source.payload).digest("hex")}`;
const copy = source.start("copy", "first", {
transferId, targetSftpId: "target-alias", sourceFingerprint,
resumeStage: "upload", uploadCheckpointBytes: 32768,
});
const [copied, other] = await Promise.all([copy, competing]);
assert.equal(copied.result.error, undefined);
assert.equal(other.result.error, undefined);
assert.deepEqual(target.files.get(copied.targetPath), source.payload);
assert.deepEqual(await fs.readFile(other.targetPath), target.payload);
assert.equal(source.rejectedOpens(), 0);
assert.equal(target.rejectedOpens(), 0);
});