mirror of
https://github.com/binaricat/Netcatty.git
synced 2026-11-20 23:20:46 +00:00
fix(sftp): keep bulk queues and large-file resume responsive (#3226)
* fix(sftp): keep bulk queues and large-file resume responsive * fix(sftp): preserve resume compatibility and timeout cleanup * test(sftp): verify fixture cleanup in fresh environments * fix: address Codex review on PR #3226 * fix(sftp): remove blocking periodic history persistence --------- Co-authored-by: netcatty-bot <308658023+netcatty-bot@users.noreply.github.com>
This commit is contained in:
1 parent
39d7c38a6a
commit
fb034f08bb
8 files changed
+714
-31
No files matched your search
@@ -84,3 +84,49 @@ test("queued work stays paused until resumed and can be cancelled", async () =>
|
||||
await paused;
|
||||
assert.deepEqual(order, ["b1"]);
|
||||
});
|
||||
|
||||
test("enqueuing a large blocked batch does not repeatedly inspect the existing queue", async () => {
|
||||
const scheduler = createGlobalSftpTransferScheduler();
|
||||
let release: (() => void) | undefined;
|
||||
let limitReads = 0;
|
||||
const readLimit = () => { limitReads += 1; return 1; };
|
||||
const blocker = scheduler.run("panel", "active", ["host"], readLimit, () => (
|
||||
new Promise<void>((resolve) => { release = resolve; })
|
||||
));
|
||||
const count = 2_000;
|
||||
const completed: number[] = [];
|
||||
const jobs = Array.from({ length: count }, (_, index) => scheduler.run(
|
||||
"panel", `queued-${index}`, ["host"], readLimit,
|
||||
async () => { completed.push(index); },
|
||||
));
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
const readsWhileBlocked = limitReads;
|
||||
release?.();
|
||||
await Promise.all([blocker, ...jobs]);
|
||||
|
||||
assert.deepEqual(completed, Array.from({ length: count }, (_, index) => index));
|
||||
assert.ok(readsWhileBlocked <= count * 3,
|
||||
`a blocked batch should be inspected linearly, got ${readsWhileBlocked} limit reads for ${count} files`);
|
||||
});
|
||||
|
||||
test("large batches of immediately completed files yield to user input", async () => {
|
||||
const scheduler = createGlobalSftpTransferScheduler();
|
||||
let completed = 0;
|
||||
const count = 1_000;
|
||||
const inputTurn = new Promise<number>((resolve) => setTimeout(() => resolve(completed), 0));
|
||||
const jobs = Array.from({ length: count }, (_, index) => scheduler.run(
|
||||
"panel", `tiny-${index}`, ["host"], () => 2, async () => { completed += 1; },
|
||||
));
|
||||
const completedAtInput = await inputTurn;
|
||||
await Promise.all(jobs);
|
||||
assert.ok(completedAtInput < count, "input must run before the entire batch drains");
|
||||
assert.equal(completed, count);
|
||||
});
|
||||
|
||||
test("a synchronous job failure releases its slot for queued work", async () => {
|
||||
const scheduler = createGlobalSftpTransferScheduler();
|
||||
const failed = scheduler.run("panel", "failed", ["host"], () => 1, () => { throw new Error("read failed"); });
|
||||
const next = scheduler.run("panel", "next", ["host"], () => 1, async () => "completed");
|
||||
await assert.rejects(failed, /read failed/);
|
||||
assert.equal(await next, "completed");
|
||||
});
|
||||
@@ -44,6 +44,25 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
|
||||
let lastOwnerId: string | null = null;
|
||||
let prioritySequence = 0;
|
||||
const pausedJobs = new Map<string, ScheduledJob<unknown>>();
|
||||
let pumpScheduled = false;
|
||||
let startsSinceYield = 0;
|
||||
|
||||
const schedulePump = () => {
|
||||
if (pumpScheduled || queue.length === 0) return;
|
||||
pumpScheduled = true;
|
||||
const run = () => {
|
||||
pumpScheduled = false;
|
||||
pump();
|
||||
};
|
||||
// Coalesce a discovery/enqueue burst into one scan. Periodically yield to
|
||||
// input/paint as well, including when many tiny jobs resolve immediately.
|
||||
if (startsSinceYield >= 64) {
|
||||
startsSinceYield = 0;
|
||||
setTimeout(run, 0);
|
||||
} else {
|
||||
queueMicrotask(run);
|
||||
}
|
||||
};
|
||||
|
||||
const normalizeResourceKeys = (keys: readonly string[]) => [...new Set(keys.length > 0 ? keys : ["local"])];
|
||||
const canRun = (job: ScheduledJob<unknown>) => {
|
||||
@@ -60,24 +79,31 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
|
||||
|
||||
const pump = () => {
|
||||
while (queue.length > 0) {
|
||||
const runnable = queue.map((job, index) => ({ job, index })).filter(({ job }) => canRun(job));
|
||||
if (runnable.length === 0) return;
|
||||
const highestPriority = runnable.reduce((max, { job }) => Math.max(max, job.priority), 0);
|
||||
const prioritizedIndexes = queue
|
||||
.map((job, index) => ({ job, index }))
|
||||
.filter(({ job }) => job.priority === highestPriority && canRun(job));
|
||||
const alternate = lastOwnerId === null
|
||||
? undefined
|
||||
: prioritizedIndexes.find(({ job }) => job.ownerId !== lastOwnerId);
|
||||
const index = alternate?.index ?? prioritizedIndexes[0]?.index ?? 0;
|
||||
let index = -1;
|
||||
for (let candidateIndex = 0; candidateIndex < queue.length; candidateIndex += 1) {
|
||||
const candidate = queue[candidateIndex];
|
||||
if (!canRun(candidate)) continue;
|
||||
const selected = queue[index];
|
||||
if (!selected || candidate.priority > selected.priority || (
|
||||
candidate.priority === selected.priority
|
||||
&& selected.ownerId === lastOwnerId
|
||||
&& candidate.ownerId !== lastOwnerId
|
||||
)) index = candidateIndex;
|
||||
}
|
||||
if (index < 0) return;
|
||||
const [job] = queue.splice(index, 1);
|
||||
if (!job) return;
|
||||
adjustActive(job, 1);
|
||||
lastOwnerId = job.ownerId;
|
||||
void job.work().then(job.resolve, job.reject).finally(() => {
|
||||
startsSinceYield += 1;
|
||||
void (async () => job.work())().then(job.resolve, job.reject).finally(() => {
|
||||
adjustActive(job, -1);
|
||||
pump();
|
||||
schedulePump();
|
||||
});
|
||||
if (startsSinceYield >= 64) {
|
||||
schedulePump();
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -85,7 +111,8 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
|
||||
run<T>(ownerId: string, taskId: string, resourceKeys: readonly string[], readLimit: LimitReader, work: () => Promise<T>): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
queue.push({ ownerId, taskId, resourceKeys: normalizeResourceKeys(resourceKeys), priority: 0, readLimit, work, resolve, reject } as ScheduledJob<unknown>);
|
||||
pump();
|
||||
if (queue.length === 1 && !pumpScheduled && startsSinceYield < 64) pump();
|
||||
else schedulePump();
|
||||
});
|
||||
},
|
||||
prioritize(taskId: string) {
|
||||
@@ -93,7 +120,7 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
|
||||
if (!job) return;
|
||||
prioritySequence += 1;
|
||||
job.priority = prioritySequence;
|
||||
pump();
|
||||
schedulePump();
|
||||
},
|
||||
pause(taskId: string) {
|
||||
const index = queue.findIndex((job) => job.taskId === taskId);
|
||||
@@ -108,7 +135,7 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
|
||||
if (!job) return false;
|
||||
pausedJobs.delete(taskId);
|
||||
queue.push(job);
|
||||
pump();
|
||||
schedulePump();
|
||||
return true;
|
||||
},
|
||||
cancel(taskId: string) {
|
||||
@@ -117,7 +144,7 @@ export function createGlobalSftpTransferScheduler(): GlobalSftpTransferScheduler
|
||||
if (!job) return false;
|
||||
pausedJobs.delete(taskId);
|
||||
job.reject(new Error("Transfer cancelled"));
|
||||
pump();
|
||||
schedulePump();
|
||||
return true;
|
||||
},
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user