From e2cc824152fa1d3cc63d9ac5d5d095962239d29b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 6 Aug 2026 07:54:15 +0000 Subject: [PATCH] fix(sftp): bound multi-select downloads and preserve dest mtime MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Codex review on PR #2772: serialize local blob reads, bound multi-select root downloads before pre-scan, replace remaining Unicode em dashes, and stamp destination mtimes after successful transfers so skip-unchanged can match on a later pass. Co-authored-by: 陈大猫 --- .../state/sftp/transferDirectoryFileBatch.ts | 2 +- .../state/sftp/transferDirectoryOps.ts | 2 +- components/sftp/hooks/useSftpViewFileOps.ts | 75 +++++++++++++------ electron/bridges/transferBridge.cjs | 60 +++++++++++++++ electron/bridges/transferBridge.test.cjs | 47 ++++++++++++ 5 files changed, 161 insertions(+), 25 deletions(-) diff --git a/application/state/sftp/transferDirectoryFileBatch.ts b/application/state/sftp/transferDirectoryFileBatch.ts index ad4e97eb5..4dfabc316 100644 --- a/application/state/sftp/transferDirectoryFileBatch.ts +++ b/application/state/sftp/transferDirectoryFileBatch.ts @@ -289,7 +289,7 @@ export async function transferDiscoveredFiles( ? { ...t, status: "failed" as TransferStatus, - error: "SFTP session lost — reconnect and resume remaining files", + error: "SFTP session lost - reconnect and resume remaining files", speed: 0, endTime: Date.now(), } diff --git a/application/state/sftp/transferDirectoryOps.ts b/application/state/sftp/transferDirectoryOps.ts index 3a049399d..5a28bee68 100644 --- a/application/state/sftp/transferDirectoryOps.ts +++ b/application/state/sftp/transferDirectoryOps.ts @@ -1048,7 +1048,7 @@ export function useSftpDirectoryTransferOps({ ? { ...t, status: "failed" as TransferStatus, - error: "SFTP session lost — reconnect and resume remaining files", + error: "SFTP session lost - reconnect and resume remaining files", speed: 0, endTime: Date.now(), } diff --git a/components/sftp/hooks/useSftpViewFileOps.ts b/components/sftp/hooks/useSftpViewFileOps.ts index e9af597db..21e25d1ce 100644 --- a/components/sftp/hooks/useSftpViewFileOps.ts +++ b/components/sftp/hooks/useSftpViewFileOps.ts @@ -1,6 +1,11 @@ import { useCallback, useRef, useState } from "react"; import type { SftpFileEntry } from "../../../types"; +import type { TransferStatus } from "../../../domain/models"; import { getParentPath, joinPath as joinFsPath } from "../../../application/state/sftp/utils"; +import { + DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY, + runBoundedConcurrency, +} from "../../../application/state/sftp/transferConcurrency"; import { logger } from "../../../lib/logger"; import { toast } from "../../ui/toast"; import { getFileExtension, getLanguageId, FileOpenerType, SystemAppInfo } from "../../../lib/sftpFileUtils"; @@ -12,6 +17,14 @@ import type { TextEditorModalSnapshot } from "../../TextEditorModal"; import type { UseSftpViewFileOpsParams, UseSftpViewFileOpsResult } from "./useSftpViewFileOps.types"; import { assertSftpFileFitsBuiltinEditor } from "../sftpEditorFileLimits"; +/** Local multi-select blob downloads read whole files into ArrayBuffers. */ +const LOCAL_BLOB_DOWNLOAD_CONCURRENCY = 1; +/** + * Multi-select roots each run folder pre-scan before the transfer scheduler. + * Bound them so listing fan-out cannot multiply by selection size. + */ +const MULTI_SELECT_ROOT_DOWNLOAD_CONCURRENCY = DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY; + export const useSftpViewFileOps = ({ sftpRef, behaviorRef, @@ -538,7 +551,14 @@ export const useSftpViewFileOps = ({ if (!pane.connection) return; if (pane.connection.isLocal) { - await Promise.allSettled(files.map((file) => handleDownloadFileForSide(side, file))); + // Sequential: each local download materializes a full ArrayBuffer. + await runBoundedConcurrency( + files, + LOCAL_BLOB_DOWNLOAD_CONCURRENCY, + async (file) => { + await handleDownloadFileForSide(side, file); + }, + ); return; } @@ -556,31 +576,40 @@ export const useSftpViewFileOps = ({ const selectedDirectory = await selectDirectory(t("sftp.context.download")); if (!selectedDirectory) return; - // Enqueue in parallel — each downloadToLocal uses the dedicated transfer - // pool / host scheduler. Awaiting one-by-one forced serial multi-select - // throughput even though per-file SFTP I/O is already pipelined. - const results = await Promise.allSettled(files.map(async (file) => { - const sourcePath = sftpRef.current.joinPath(pane.connection.currentPath, file.name); - const targetPath = joinFsPath(selectedDirectory, file.name); - const isDirectory = isNavigableDirectory(file); - const fileSize = typeof file.size === "string" ? parseInt(file.size, 10) || 0 : (file.size || 0); + // Bound root jobs: directory roots run full pre-scan before the scheduler, + // so unbounded Promise.allSettled would multiply LIST concurrency. + const results: Array> = []; + await runBoundedConcurrency( + files, + MULTI_SELECT_ROOT_DOWNLOAD_CONCURRENCY, + async (file, index) => { + try { + const sourcePath = sftpRef.current.joinPath(pane.connection.currentPath, file.name); + const targetPath = joinFsPath(selectedDirectory, file.name); + const isDirectory = isNavigableDirectory(file); + const fileSize = typeof file.size === "string" ? parseInt(file.size, 10) || 0 : (file.size || 0); - const status = await sftpRef.current.downloadToLocal({ - fileName: file.name, - sourcePath, - targetPath, - sftpId, - connectionId: pane.connection.id, - sourceHostId: pane.connection.hostId, - sourceHostLabel: pane.connection.hostLabel, - sourceEncoding: pane.filenameEncoding, - isDirectory, - totalBytes: isDirectory ? undefined : fileSize, - }); - return { file, status }; - })); + const status = await sftpRef.current.downloadToLocal({ + fileName: file.name, + sourcePath, + targetPath, + sftpId, + connectionId: pane.connection.id, + sourceHostId: pane.connection.hostId, + sourceHostLabel: pane.connection.hostLabel, + sourceEncoding: pane.filenameEncoding, + isDirectory, + totalBytes: isDirectory ? undefined : fileSize, + }); + results[index] = { status: "fulfilled", value: { file, status } }; + } catch (reason) { + results[index] = { status: "rejected", reason }; + } + }, + ); for (const result of results) { + if (!result) continue; if (result.status === "fulfilled") { const { file, status } = result.value; if (status === "completed") { diff --git a/electron/bridges/transferBridge.cjs b/electron/bridges/transferBridge.cjs index 368cf45b8..7c078e64c 100644 --- a/electron/bridges/transferBridge.cjs +++ b/electron/bridges/transferBridge.cjs @@ -862,6 +862,63 @@ async function promoteLocalTransfer(stagedPath, targetPath, options = {}) { } } +/** + * Apply the source mtime to the committed destination so skip-unchanged + * (size + mtime) can match on a later folder transfer. Best-effort: never + * fails the transfer if utimes/setstat is unsupported. + */ +async function preserveTransferredDestinationMtime(transfer) { + try { + let mtimeMs = Number(transfer?.sourceSoftIdentity?.mtimeMs); + if ((!Number.isFinite(mtimeMs) || mtimeMs <= 0) + && typeof transfer?.captureSourceSoftIdentity === "function") { + await transfer.captureSourceSoftIdentity(); + mtimeMs = Number(transfer?.sourceSoftIdentity?.mtimeMs); + } + if (!Number.isFinite(mtimeMs) || mtimeMs <= 0) return; + + // Compare / skip logic uses whole seconds; stamp the destination the same way. + const mtimeSec = Math.floor(mtimeMs / 1000); + if (mtimeSec <= 0) return; + const when = new Date(mtimeSec * 1000); + + if (transfer.targetType === "local" && transfer.targetPath) { + await fs.promises.utimes(transfer.targetPath, when, when); + return; + } + + if (transfer.targetType !== "sftp" || !transfer.targetSftpId || !transfer.targetPath) { + return; + } + const client = sftpClients.get(transfer.targetSftpId); + // SCP has no SETSTAT; same-host `cp -a` already preserves mtime. + if (!client || isScpModeClient(client)) return; + + await requireSftpChannel(client); + const encoded = encodePathForSession( + transfer.targetSftpId, + transfer.targetPath, + transfer.targetEncoding, + ); + if (typeof client.setStat === "function") { + await client.setStat(encoded, { mtime: mtimeSec, atime: mtimeSec }); + return; + } + const sftp = client.sftp; + if (!sftp || typeof sftp.setstat !== "function") return; + await new Promise((resolve, reject) => { + sftp.setstat(encoded, { mtime: mtimeSec, atime: mtimeSec }, (err) => ( + err ? reject(err) : resolve() + )); + }); + } catch (err) { + console.warn( + "[transferBridge] failed to preserve destination mtime:", + err?.message || String(err), + ); + } +} + async function inspectLocalPromotionTarget(targetPath) { const absoluteTargetPath = path.resolve(targetPath); const visitedLinkStates = new Set(); @@ -5604,6 +5661,8 @@ async function startTransferNow(event, payload, onProgress) { } sendProgress(fileSize, fileSize); + // Stamp destination mtime from the source so skip-unchanged can match later. + await preserveTransferredDestinationMtime(transfer); logTransferDiag(transfer, "done", { transferred: fileSize, total: fileSize, @@ -6585,6 +6644,7 @@ module.exports = { releaseSftpTransferSession, listTransferSftpIds, _promoteLocalTransferForTests: promoteLocalTransfer, + _preserveTransferredDestinationMtimeForTests: preserveTransferredDestinationMtime, _stableLocalFileIdentityForTests: stableLocalFileIdentity, _getWorkerTransferLifecycleEpochCountForTests: () => workerTransferLifecycleEpochs.size, _setWorkerTransferLifecycleEpochForTests: (transferId, epoch) => { diff --git a/electron/bridges/transferBridge.test.cjs b/electron/bridges/transferBridge.test.cjs index 5be5bc7d8..5b5efeef7 100644 --- a/electron/bridges/transferBridge.test.cjs +++ b/electron/bridges/transferBridge.test.cjs @@ -10008,3 +10008,50 @@ test("local promotion succeeds when backup still matches validated identity", as transferBridge._stableLocalFileIdentityForTests(originalStat), ); }); + +test("preserveTransferredDestinationMtime stamps local targets from sourceSoftIdentity", async (t) => { + const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-preserve-mtime-")); + t.after(async () => fs.promises.rm(tempDir, { recursive: true, force: true })); + + const targetPath = path.join(tempDir, "copied.bin"); + await fs.promises.writeFile(targetPath, Buffer.from("payload")); + const before = await fs.promises.stat(targetPath); + const sourceMtimeMs = 1_700_000_000_000; // 2023-11-14T22:13:20.000Z + + await transferBridge._preserveTransferredDestinationMtimeForTests({ + targetType: "local", + targetPath, + sourceSoftIdentity: { size: 7, mtimeMs: sourceMtimeMs }, + }); + + const after = await fs.promises.stat(targetPath); + assert.equal(Math.floor(after.mtimeMs / 1000), Math.floor(sourceMtimeMs / 1000)); + assert.notEqual(Math.floor(after.mtimeMs / 1000), Math.floor(before.mtimeMs / 1000)); +}); + +test("local-to-local transfer preserves source mtime on the destination", async (t) => { + const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-local-mtime-")); + t.after(async () => fs.promises.rm(tempDir, { recursive: true, force: true })); + + const sourcePath = path.join(tempDir, "source.bin"); + const targetPath = path.join(tempDir, "target.bin"); + await fs.promises.writeFile(sourcePath, Buffer.from("hello-mtime")); + const sourceMtimeMs = 1_600_000_000_000; // 2020-09-13T12:26:40.000Z + await fs.promises.utimes(sourcePath, new Date(sourceMtimeMs), new Date(sourceMtimeMs)); + + transferBridge.init({ sftpClients: new Map() }); + const result = await transferBridge.startTransfer({ sender: createSender() }, { + transferId: "local-mtime-preserve", + sourcePath, + targetPath, + sourceType: "local", + targetType: "local", + totalBytes: 11, + resumable: false, + }); + + assert.equal(result.error, undefined); + const targetStat = await fs.promises.stat(targetPath); + assert.equal(Math.floor(targetStat.mtimeMs / 1000), Math.floor(sourceMtimeMs / 1000)); + assert.deepEqual(await fs.promises.readFile(targetPath), Buffer.from("hello-mtime")); +});