fix(sftp): keep bulk queues and large-file resume responsive (#3226)

* fix(sftp): keep bulk queues and large-file resume responsive

* fix(sftp): preserve resume compatibility and timeout cleanup

* test(sftp): verify fixture cleanup in fresh environments

* fix: address Codex review on PR #3226

* fix(sftp): remove blocking periodic history persistence

---------

Co-authored-by: netcatty-bot <308658023+netcatty-bot@users.noreply.github.com>
This commit is contained in:
陈大猫andnetcatty-bot authored and GitHub committed 2026-08-31 19:27:41 +08:00
1 parent 39d7c38a6a
commit fb034f08bb
8 files changed
+714 -31

No files matched your search

@@ -84,3 +84,49 @@ test("queued work stays paused until resumed and can be cancelled", async () =>
await paused;
assert.deepEqual(order, ["b1"]);
});
test("enqueuing a large blocked batch does not repeatedly inspect the existing queue", async () => {
const scheduler = createGlobalSftpTransferScheduler();
let release: (() => void) | undefined;
let limitReads = 0;
const readLimit = () => { limitReads += 1; return 1; };
const blocker = scheduler.run("panel", "active", ["host"], readLimit, () => (
new Promise<void>((resolve) => { release = resolve; })
));
const count = 2_000;
const completed: number[] = [];
const jobs = Array.from({ length: count }, (_, index) => scheduler.run(
"panel", `queued-${index}`, ["host"], readLimit,
async () => { completed.push(index); },
));
await new Promise((resolve) => setImmediate(resolve));
const readsWhileBlocked = limitReads;
release?.();
await Promise.all([blocker, ...jobs]);
assert.deepEqual(completed, Array.from({ length: count }, (_, index) => index));
assert.ok(readsWhileBlocked <= count * 3,
`a blocked batch should be inspected linearly, got ${readsWhileBlocked} limit reads for ${count} files`);
});
test("large batches of immediately completed files yield to user input", async () => {
const scheduler = createGlobalSftpTransferScheduler();
let completed = 0;
const count = 1_000;
const inputTurn = new Promise<number>((resolve) => setTimeout(() => resolve(completed), 0));
const jobs = Array.from({ length: count }, (_, index) => scheduler.run(
"panel", `tiny-${index}`, ["host"], () => 2, async () => { completed += 1; },
));
const completedAtInput = await inputTurn;
await Promise.all(jobs);
assert.ok(completedAtInput < count, "input must run before the entire batch drains");
assert.equal(completed, count);
});
test("a synchronous job failure releases its slot for queued work", async () => {
const scheduler = createGlobalSftpTransferScheduler();
const failed = scheduler.run("panel", "failed", ["host"], () => 1, () => { throw new Error("read failed"); });
const next = scheduler.run("panel", "next", ["host"], () => 1, async () => "completed");
await assert.rejects(failed, /read failed/);
assert.equal(await next, "completed");
});
@@ -44,6 +44,25 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
let lastOwnerId: string | null = null;
let prioritySequence = 0;
const pausedJobs = new Map<string, ScheduledJob<unknown>>();
let pumpScheduled = false;
let startsSinceYield = 0;
const schedulePump = () => {
if (pumpScheduled || queue.length === 0) return;
pumpScheduled = true;
const run = () => {
pumpScheduled = false;
pump();
};
// Coalesce a discovery/enqueue burst into one scan. Periodically yield to
// input/paint as well, including when many tiny jobs resolve immediately.
if (startsSinceYield >= 64) {
startsSinceYield = 0;
setTimeout(run, 0);
} else {
queueMicrotask(run);
}
};
const normalizeResourceKeys = (keys: readonly string[]) => [...new Set(keys.length > 0 ? keys : ["local"])];
const canRun = (job: ScheduledJob<unknown>) => {
@@ -60,24 +79,31 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
const pump = () => {
while (queue.length > 0) {
const runnable = queue.map((job, index) => ({ job, index })).filter(({ job }) => canRun(job));
if (runnable.length === 0) return;
const highestPriority = runnable.reduce((max, { job }) => Math.max(max, job.priority), 0);
const prioritizedIndexes = queue
.map((job, index) => ({ job, index }))
.filter(({ job }) => job.priority === highestPriority && canRun(job));
const alternate = lastOwnerId === null
? undefined
: prioritizedIndexes.find(({ job }) => job.ownerId !== lastOwnerId);
const index = alternate?.index ?? prioritizedIndexes[0]?.index ?? 0;
let index = -1;
for (let candidateIndex = 0; candidateIndex < queue.length; candidateIndex += 1) {
const candidate = queue[candidateIndex];
if (!canRun(candidate)) continue;
const selected = queue[index];
if (!selected || candidate.priority > selected.priority || (
candidate.priority === selected.priority
&& selected.ownerId === lastOwnerId
&& candidate.ownerId !== lastOwnerId
)) index = candidateIndex;
}
if (index < 0) return;
const [job] = queue.splice(index, 1);
if (!job) return;
adjustActive(job, 1);
lastOwnerId = job.ownerId;
void job.work().then(job.resolve, job.reject).finally(() => {
startsSinceYield += 1;
void (async () => job.work())().then(job.resolve, job.reject).finally(() => {
adjustActive(job, -1);
pump();
schedulePump();
});
if (startsSinceYield >= 64) {
schedulePump();
return;
}
}
};
@@ -85,7 +111,8 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
run<T>(ownerId: string, taskId: string, resourceKeys: readonly string[], readLimit: LimitReader, work: () => Promise<T>): Promise<T> {
return new Promise<T>((resolve, reject) => {
queue.push({ ownerId, taskId, resourceKeys: normalizeResourceKeys(resourceKeys), priority: 0, readLimit, work, resolve, reject } as ScheduledJob<unknown>);
pump();
if (queue.length === 1 && !pumpScheduled && startsSinceYield < 64) pump();
else schedulePump();
});
},
prioritize(taskId: string) {
@@ -93,7 +120,7 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
if (!job) return;
prioritySequence += 1;
job.priority = prioritySequence;
pump();
schedulePump();
},
pause(taskId: string) {
const index = queue.findIndex((job) => job.taskId === taskId);
@@ -108,7 +135,7 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
if (!job) return false;
pausedJobs.delete(taskId);
queue.push(job);
pump();
schedulePump();
return true;
},
cancel(taskId: string) {
@@ -117,7 +144,7 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
if (!job) return false;
pausedJobs.delete(taskId);
job.reject(new Error("Transfer cancelled"));
pump();
schedulePump();
return true;
},
};