Compare commits

...
Author SHA1 Message Date
bincxz 9fe24fa461 fix(sftp): address Codex progressive conflict and hang findings
Add root-level conflict preflight to progressive folder uploads, wake all
backpressure waiters, preserve cancel listeners across uploadEntriesDirect,
and surface empty-directory mkdir failures.
2026-08-07 01:58:58 +08:00
bincxz 96378f6fca fix(sftp): finish multi-root pause HOL and parent rollup guards
Race-wait on all latched parents so resuming a non-head folder can proceed,
and only roll progressive parents to completed after terminal failed status.
2026-08-07 01:43:39 +08:00
bincxz a35945ae8e fix(sftp): address multi-agent review on progressive upload and retry
Gate progressive drops to path-backed folders only, prefer pool sessions on
drag-drop retry with in-flight and cancel guards, soft-check upload mtime
without fail-open contentVerifiedSeparately, skip multi-root pause HOL,
and fail closed when the local tree stream misses tree-end.
2026-08-07 01:36:18 +08:00
bincxz fee1953267 fix(sftp): harden progressive folder UX, retry, and size-based uploads
Align the SFTP panel with transfer-center progress, keep nested tree batches
until tree-end, honor pause during progressive enqueue, add low-cost retry for
external drag-drop files, and drop whole-file upload digests for industry
size-based resume (download live-log prefix checks unchanged).
2026-08-07 01:25:00 +08:00
bincxz 12471c1723 fix(sftp): stream folder discovery into concurrent uploads
Local path-backed folder drops now edge-scan and edge-transfer: listLocalTree
streams entry batches while upload workers consume them, so the first files
start before a 100k-file walk finishes. Cancel aborts the walk, clears the
queue, and stops creating child tasks. Compressed uploads still materialize
fully then pack.
2026-08-07 00:02:53 +08:00
bincxz d43960902c fix(sftp): finish Codex follow-ups on folder scan cancel UX
- Abort sibling listLocalTree walks when one multi-root scan fails
- Prefer ERR_*_CANCELLED over matching "cancel" in error text
- Clear scanning phase on settle; only label live rows as Scanning
- Omit directory metadata sizes from drop entries used for totals
2026-08-06 23:53:23 +08:00
bincxz 8ed5777a63 fix(sftp): harden folder scan cancellation and conflicts 2026-08-06 23:39:00 +08:00
bincxz b0e198f836 fix(sftp): preserve cancellation in final scan cleanup 2026-08-06 23:30:59 +08:00
bincxz 908d9a544b fix(sftp): settle failed drop upload retry 2026-08-06 23:27:43 +08:00
bincxz 14d5c4301d fix(sftp): keep cancellation active through session setup 2026-08-06 23:21:30 +08:00
bincxz f87e4fddef fix(sftp): cancel scan task immediately 2026-08-06 23:17:22 +08:00
bincxz 96eaf837f4 fix(sftp): keep folder scan task across retries 2026-08-06 23:10:59 +08:00
bincxz 375c14be68 fix(sftp): preserve scan failure state during setup errors 2026-08-06 23:02:09 +08:00
bincxz bfc821d7da fix(sftp): keep scan task cancellable during setup 2026-08-06 22:54:47 +08:00
bincxz bb725765d3 fix(terminal): accept native folder drop paths 2026-08-06 22:42:09 +08:00
bincxz 163af7176f fix(sftp): preserve path-only terminal drops 2026-08-06 22:34:49 +08:00
bincxz 938c4e4675 fix(sftp): make folder scan cancellation bridge-safe 2026-08-06 22:27:21 +08:00
bincxz b8ecf1adfd fix(sftp): instant folder-drop scan UI and drop full-tree pre-scan
Restore interleaved folder transfers as the only non-compressed path, remove
the folder pre-scan setting, and make local folder drops show a scanning task
with live discovered counts via native listLocalTree (with cancel + progress).

UI shows "done · found" instead of a fake grand-total percentage while
discovery is still growing.
2026-08-06 21:56:14 +08:00
52 changed files with 4306 additions and 2438 deletions

No files matched your search

+2 -2
View File
@@ -268,6 +268,8 @@ export const enVaultMessages: Messages = {
'sftp.transfers.calculatingTotal': 'Calculating total size...',
'sftp.transfers.filesCount': '{count} files',
'sftp.transfers.filesProgress': '{current}/{total} files',
// Progressive folder walk: total is "found so far", not a fixed grand total.
'sftp.transfers.filesDiscoveredProgress': '{completed} done · {discovered} found',
'sftp.transfers.expandChildren': 'Show files',
'sftp.transfers.collapseChildren': 'Hide files',
'sftp.transfers.expandChildList': 'Show detail',
@@ -560,8 +562,6 @@ export const enVaultMessages: Messages = {
'settings.sftp.compressedUpload.enable': 'Enable folder compression',
'settings.sftp.compressedUpload.enableDesc': 'Automatically compress folders using tar before transfer. Requires tar support on the server. Falls back to regular transfer if not available.',
'settings.sftp.folderPrescan.enable': 'Pre-scan folders before transfer',
'settings.sftp.folderPrescan.enableDesc': 'Walk the full folder tree first so the transfer center shows a stable file total before bytes move. Slightly delays the first transfer on very large trees.',
'settings.sftp.skipUnchanged.enable': 'Skip unchanged files',
'settings.sftp.skipUnchanged.enableDesc': 'Skip files whose size and modification time already match the destination (rsync-style). Disabled when replacing a folder into a staging path.',
+1 -2
View File
@@ -305,6 +305,7 @@ export const ruVaultMessages: Messages = {
'sftp.transfers.calculatingTotal': 'Вычисление общего размера...',
'sftp.transfers.filesCount': '{count} файл(ов)',
'sftp.transfers.filesProgress': '{current}/{total} файл(ов)',
'sftp.transfers.filesDiscoveredProgress': '{completed} готово · {discovered} найдено',
'sftp.transfers.expandChildren': 'Показать файлы',
'sftp.transfers.collapseChildren': 'Скрыть файлы',
'sftp.transfers.expandChildList': 'Показать детали',
@@ -597,8 +598,6 @@ export const ruVaultMessages: Messages = {
'settings.sftp.compressedUpload.enable': 'Включить сжатие папок',
'settings.sftp.compressedUpload.enableDesc': 'Автоматически сжимать папки с помощью tar перед передачей. Требует поддержки tar на сервере. Если она недоступна, будет использована обычная передача.',
'settings.sftp.folderPrescan.enable': 'Предварительное сканирование папок',
'settings.sftp.folderPrescan.enableDesc': 'Сначала обойти всё дерево папок, чтобы центр передачи показал стабильное число файлов до начала передачи байтов. На очень больших деревьях немного откладывает первую передачу.',
'settings.sftp.skipUnchanged.enable': 'Пропускать неизменённые файлы',
'settings.sftp.skipUnchanged.enableDesc': 'Пропускать файлы, у которых размер и время изменения уже совпадают с назначением (в стиле rsync). Отключается при замене папки во временный путь.',
+1
View File
@@ -863,6 +863,7 @@ export const zhCNCoreMessages: Messages = {
'sftp.transfers.calculatingTotal': '正在统计总大小...',
'sftp.transfers.filesCount': '{count} 个文件',
'sftp.transfers.filesProgress': '{current}/{total} 个文件',
'sftp.transfers.filesDiscoveredProgress': '已完成 {completed} · 已发现 {discovered}',
'sftp.transfers.expandChildren': '展开文件',
'sftp.transfers.collapseChildren': '收起文件',
'sftp.transfers.expandChildList': '展开详情',
@@ -206,8 +206,6 @@ export const zhCNTerminalMessages: Messages = {
'settings.sftp.compressedUpload.enable': '启用文件夹压缩',
'settings.sftp.compressedUpload.enableDesc': '自动使用 tar 压缩文件夹后再传输。需要服务器支持 tar 命令,不支持时自动回退到普通传输。',
'settings.sftp.folderPrescan.enable': '传输前预扫描文件夹',
'settings.sftp.folderPrescan.enableDesc': '先完整遍历文件夹树,传输中心在开始传字节前就能显示稳定的文件总数。超大目录会略微推迟首个文件开传。',
'settings.sftp.skipUnchanged.enable': '跳过未改文件',
'settings.sftp.skipUnchanged.enableDesc': '当目标文件的大小与修改时间已与源一致时跳过传输(类似 rsync)。替换到暂存路径时自动禁用。',
+1
View File
@@ -863,6 +863,7 @@ export const zhTWCoreMessages: Messages = {
'sftp.transfers.calculatingTotal': '正在統計總大小...',
'sftp.transfers.filesCount': '{count} 個檔案',
'sftp.transfers.filesProgress': '{current}/{total} 個檔案',
'sftp.transfers.filesDiscoveredProgress': '已完成 {completed} · 已發現 {discovered}',
'sftp.transfers.expandChildren': '展開檔案',
'sftp.transfers.collapseChildren': '收起檔案',
'sftp.transfers.expandChildList': '展開詳情',
@@ -206,8 +206,6 @@ export const zhTWTerminalMessages: Messages = {
'settings.sftp.compressedUpload.enable': '啟用資料夾壓縮',
'settings.sftp.compressedUpload.enableDesc': '自動使用 tar 壓縮資料夾後再傳輸。需要伺服器支援 tar 指令,不支援時自動回退到普通傳輸。',
'settings.sftp.folderPrescan.enable': '傳輸前預掃描資料夾',
'settings.sftp.folderPrescan.enableDesc': '先完整遍歷資料夾樹,傳輸中心在開始傳位元組前就能顯示穩定的檔案總數。超大目錄會略微推遲首個檔案開傳。',
'settings.sftp.skipUnchanged.enable': '跳過未改檔案',
'settings.sftp.skipUnchanged.enableDesc': '當目標檔案的大小與修改時間已與來源一致時跳過傳輸(類似 rsync)。替換到暫存路徑時自動停用。',
+4 -14
View File
@@ -28,7 +28,6 @@ import {
STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY,
STORAGE_KEY_SSH_TRANSPORT_IDLE_TTL_MS,
STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD,
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
STORAGE_KEY_SHOW_ONLY_UNGROUPED_HOSTS_IN_ROOT,
STORAGE_KEY_SHOW_RECENT_HOSTS,
@@ -81,7 +80,6 @@ interface UseSettingsStorageSyncParams {
sftpAutoSync: boolean;
sftpShowHiddenFiles: boolean;
sftpUseCompressedUpload: boolean;
sftpFolderPrescan: boolean;
sftpSkipUnchanged: boolean;
sftpAutoOpenSidebar: boolean;
sftpFollowTerminalCwd: boolean;
@@ -129,7 +127,6 @@ interface UseSettingsStorageSyncParams {
setSftpAutoSync: Dispatch<SetStateAction<boolean>>;
setSftpShowHiddenFiles: Dispatch<SetStateAction<boolean>>;
setSftpUseCompressedUpload: Dispatch<SetStateAction<boolean>>;
setSftpFolderPrescan: Dispatch<SetStateAction<boolean>>;
setSftpSkipUnchanged: Dispatch<SetStateAction<boolean>>;
setSftpAutoOpenSidebar: Dispatch<SetStateAction<boolean>>;
setSftpFollowTerminalCwd: Dispatch<SetStateAction<boolean>>;
@@ -171,7 +168,7 @@ export function useSettingsStorageSync({
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
@@ -180,7 +177,7 @@ export function useSettingsStorageSync({
setTerminalThemeId, setTerminalThemeDarkId, setTerminalThemeLightId,
setFollowAppTerminalThemeState, setTerminalFontFamilyId, setTerminalFontSize,
setSftpDoubleClickBehavior, setSftpAutoSync, setSftpShowHiddenFiles,
setSftpUseCompressedUpload, setSftpFolderPrescan, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
setSftpUseCompressedUpload, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
setShowRecentHostsState, setHostClickBehaviorState, setShowOnlyUngroupedHostsInRootState, setShowSftpTabState, setShowHostTreeSidebarState, setTerminalSidePanelAutoOpenState, setTerminalSidePanelAutoOpenTabState, setShellOnlyTabNumberShortcutsState, setDisableTerminalFontZoomState, setRestorePreviousSessionState, setRestoreTerminalCwdState,
setEditorWordWrapState, setSessionLogsEnabled, setSessionLogsDir, setSessionLogsFormat, setSessionLogsTimestampsEnabled, setSshDebugLogsEnabled, setSshDeepLinkEnabledState, setJmsDeepLinkEnabledState, setExplorerContextMenuEnabledState,
setGlobalHotkeyEnabled, setWindowOpacity, setAppIconVariant, setAutoUpdateEnabled, setWorkspaceFocusStyleState,
@@ -195,7 +192,7 @@ export function useSettingsStorageSync({
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
@@ -205,7 +202,7 @@ export function useSettingsStorageSync({
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
@@ -385,12 +382,6 @@ export function useSettingsStorageSync({
setSftpUseCompressedUpload(newValue);
}
}
if (e.key === STORAGE_KEY_SFTP_FOLDER_PRESCAN && e.newValue !== null) {
const newValue = e.newValue === 'true';
if (newValue !== s.sftpFolderPrescan) {
setSftpFolderPrescan(newValue);
}
}
if (e.key === STORAGE_KEY_SFTP_SKIP_UNCHANGED && e.newValue !== null) {
const newValue = e.newValue === 'true';
if (newValue !== s.sftpSkipUnchanged) {
@@ -560,7 +551,6 @@ export function useSettingsStorageSync({
setSftpTransferConcurrencyState,
setSshTransportIdleTtlMsState,
setSftpUseCompressedUpload,
setSftpFolderPrescan,
setSftpSkipUnchanged,
setShowOnlyUngroupedHostsInRootState,
setShowHostTreeSidebarState,
@@ -0,0 +1,186 @@
import assert from "node:assert/strict";
import test from "node:test";
import type { TransferTask } from "../../../domain/models";
import {
isExternalDragDropFileUpload,
retryExternalDragDropFileUpload,
} from "./externalDragDropRetry";
const baseTask = (overrides: Partial<TransferTask> = {}): TransferTask => ({
id: "child-1",
fileName: "a.txt",
sourcePath: "/tmp/docs/a.txt",
targetPath: "/remote/docs/a.txt",
sourceConnectionId: "external",
targetConnectionId: "conn-1",
targetHostId: "host-1",
direction: "upload",
status: "failed",
totalBytes: 12,
transferredBytes: 0,
speed: 0,
startTime: 1,
isDirectory: false,
origin: "drag-drop",
parentTaskId: "folder-1",
error: "Network error",
...overrides,
});
test("detects progressive drag-drop file children as retryable", () => {
assert.equal(isExternalDragDropFileUpload(baseTask()), true);
assert.equal(isExternalDragDropFileUpload(baseTask({ isDirectory: true })), false);
assert.equal(isExternalDragDropFileUpload(baseTask({ origin: "manual" })), false);
assert.equal(isExternalDragDropFileUpload(baseTask({ sourceConnectionId: "local" })), false);
assert.equal(isExternalDragDropFileUpload(baseTask({ status: "completed" })), false);
assert.equal(isExternalDragDropFileUpload(baseTask({ retryable: false })), false);
});
test("retry reuses the same transfer id and starts a stream upload", async () => {
const patches: Array<{ id: string; updates: Partial<TransferTask> }> = [];
const streams: Array<Record<string, unknown>> = [];
// No pool acquire → falls back to browse sftp id.
const result = await retryExternalDragDropFileUpload(baseTask(), {
getBrowseSftpId: () => "sftp-live",
startStreamTransfer: async (options) => {
streams.push(options as unknown as Record<string, unknown>);
return { transferId: options.transferId };
},
onPatch: (taskId, updates) => patches.push({ id: taskId, updates }),
});
assert.equal(result.success, true);
assert.equal(streams.length, 1);
assert.equal(streams[0].transferId, "child-1");
assert.equal(streams[0].sourcePath, "/tmp/docs/a.txt");
assert.equal(streams[0].targetPath, "/remote/docs/a.txt");
assert.equal(streams[0].targetSftpId, "sftp-live");
assert.equal(patches[0].updates.status, "transferring");
assert.equal(patches.at(-1)?.updates.status, "completed");
});
test("retry prefers a dedicated pool session over the browse sftp id", async () => {
let released = false;
let acquired = false;
const result = await retryExternalDragDropFileUpload(baseTask(), {
getBrowseSftpId: () => "sftp-live",
acquireTransferSession: async () => {
acquired = true;
return {
sftpId: "sftp-pool",
poolKey: "host-1",
release: () => { released = true; },
discard: () => {},
};
},
startStreamTransfer: async (options) => {
assert.equal(options.targetSftpId, "sftp-pool");
return {};
},
onPatch: () => {},
});
assert.equal(result.success, true);
assert.equal(acquired, true);
assert.equal(released, true);
});
test("retry falls back to browse when pool acquire is unavailable", async () => {
const result = await retryExternalDragDropFileUpload(baseTask(), {
getBrowseSftpId: () => "sftp-live",
startStreamTransfer: async (options) => {
assert.equal(options.targetSftpId, "sftp-live");
return {};
},
onPatch: () => {},
});
assert.equal(result.success, true);
});
test("retry rolls parent to completed when all siblings succeed", async () => {
const patches: Array<{ id: string; updates: Partial<TransferTask> }> = [];
const result = await retryExternalDragDropFileUpload(baseTask(), {
getBrowseSftpId: () => "sftp-live",
startStreamTransfer: async () => ({}),
getTask: (id) => (
id === "folder-1"
? baseTask({
id: "folder-1",
isDirectory: true,
status: "failed",
sourcePath: "/tmp/docs",
targetPath: "/remote/docs",
error: "1 of 2 files failed",
totalBytes: 2,
transferredBytes: 1,
parentTaskId: undefined,
})
: undefined
),
getChildTasks: () => [
baseTask({ id: "child-1", status: "transferring" }),
baseTask({ id: "child-2", status: "completed", sourcePath: "/tmp/docs/b.txt" }),
],
onPatch: (taskId, updates) => patches.push({ id: taskId, updates }),
});
assert.equal(result.success, true);
const parentPatch = patches.find((p) => p.id === "folder-1");
assert.ok(parentPatch);
assert.equal(parentPatch?.updates.status, "completed");
});
test("retry does not rollup a still-transferring progressive parent", async () => {
const patches: Array<{ id: string; updates: Partial<TransferTask> }> = [];
await retryExternalDragDropFileUpload(baseTask(), {
getBrowseSftpId: () => "sftp-live",
startStreamTransfer: async () => ({}),
getTask: (id) => (
id === "folder-1"
? baseTask({
id: "folder-1",
isDirectory: true,
status: "transferring",
sourcePath: "/tmp/docs",
parentTaskId: undefined,
})
: undefined
),
getChildTasks: () => [
baseTask({ id: "child-1", status: "transferring" }),
baseTask({ id: "child-2", status: "completed", sourcePath: "/tmp/docs/b.txt" }),
],
onPatch: (taskId, updates) => patches.push({ id: taskId, updates }),
});
assert.equal(patches.some((p) => p.id === "folder-1"), false);
});
test("retry fails clearly when no sftp session can be opened", async () => {
const patches: Array<Partial<TransferTask>> = [];
const result = await retryExternalDragDropFileUpload(baseTask(), {
getBrowseSftpId: () => undefined,
startStreamTransfer: async () => {
throw new Error("should not start");
},
onPatch: (_id, updates) => patches.push(updates),
});
assert.equal(result.success, false);
assert.match(result.error || "", /No SFTP session/);
assert.equal(patches.at(-1)?.status, "failed");
});
test("stream error keeps the row failed with the backend message", async () => {
const patches: Array<Partial<TransferTask>> = [];
const result = await retryExternalDragDropFileUpload(baseTask(), {
getBrowseSftpId: () => "sftp-live",
startStreamTransfer: async () => ({ error: "Permission denied" }),
onPatch: (_id, updates) => patches.push(updates),
});
assert.equal(result.success, false);
assert.equal(result.error, "Permission denied");
assert.equal(patches.at(-1)?.status, "failed");
assert.equal(patches.at(-1)?.error, "Permission denied");
});
@@ -0,0 +1,267 @@
/**
* Low-cost retry for progressive / external drag-drop file uploads.
*
* These rows use sourceConnectionId "external" and never have dual-pane
* endpoints, so the generic processTransfer retry path silently no-ops.
* Retry re-opens a single startStreamTransfer with the stored local path.
*/
import type { TransferTask } from "../../../domain/models";
import type { TransferConnectionLease } from "./transferConnectionPool";
import { isTransferCancelledFlag } from "./transferCancelLatch";
export function isExternalDragDropFileUpload(
task: Pick<
TransferTask,
| "origin"
| "direction"
| "sourceConnectionId"
| "isDirectory"
| "sourcePath"
| "targetPath"
| "retryable"
| "status"
>,
): boolean {
if (task.retryable === false) return false;
if (task.origin !== "drag-drop") return false;
if (task.direction !== "upload") return false;
if (task.sourceConnectionId !== "external") return false;
if (task.isDirectory) return false;
if (!task.sourcePath || task.sourcePath === "local") return false;
if (!task.targetPath) return false;
return task.status === "failed" || task.status === "cancelled" || task.status === "attention";
}
export type ExternalDragDropRetryDeps = {
getBrowseSftpId: (connectionId: string) => string | undefined;
acquireTransferSession?: (
hostId: string,
transferId: string,
) => Promise<TransferConnectionLease>;
startStreamTransfer: (options: {
transferId: string;
sourcePath: string;
targetPath: string;
sourceType: "local";
targetType: "local" | "sftp";
targetSftpId?: string;
targetHostId?: string;
totalBytes?: number;
resumable?: boolean;
checkpointBytes?: number;
}) => Promise<{ error?: string; cancelled?: boolean } | undefined>;
clearPendingCancel?: (transferId: string) => Promise<unknown>;
cleanupArtifacts?: (task: TransferTask) => Promise<void>;
onPatch: (taskId: string, updates: Partial<TransferTask>) => void;
/** Live store lookup for terminal cancel races and completion bytes. */
getTask?: (taskId: string) => TransferTask | undefined;
/** Children of a progressive parent (for rollup after child success). */
getChildTasks?: (parentTaskId: string) => TransferTask[];
};
/**
* Restart a failed/cancelled external drag-drop file upload in place (same id).
* Returns true when the stream completed successfully.
*/
export async function retryExternalDragDropFileUpload(
task: TransferTask,
deps: ExternalDragDropRetryDeps,
): Promise<{ success: boolean; error?: string }> {
if (!isExternalDragDropFileUpload(task)) {
return { success: false, error: "Not an external drag-drop file upload" };
}
try {
await deps.clearPendingCancel?.(task.id);
} catch {
// best-effort
}
try {
await deps.cleanupArtifacts?.(task);
} catch {
// best-effort
}
const targetIsLocal = task.targetConnectionId === "local" || !task.targetHostId;
let lease: TransferConnectionLease | null = null;
let targetSftpId: string | undefined;
try {
if (!targetIsLocal) {
// Prefer a dedicated pool session when hostId is known — same policy as
// progressive upload (do not pin the browse/tab session).
if (task.targetHostId && deps.acquireTransferSession) {
lease = await deps.acquireTransferSession(task.targetHostId, `${task.id}:retry`);
targetSftpId = lease.sftpId;
}
if (!targetSftpId) {
targetSftpId = deps.getBrowseSftpId(task.targetConnectionId);
}
if (!targetSftpId) {
const error = "No SFTP session available to retry this upload. Reconnect and try again.";
deps.onPatch(task.id, {
status: "failed",
error,
endTime: Date.now(),
speed: 0,
});
return { success: false, error };
}
}
deps.onPatch(task.id, {
status: "transferring",
error: undefined,
transferredBytes: 0,
checkpointBytes: 0,
speed: 0,
endTime: undefined,
phase: "transferring",
reconnectRequired: false,
pauseUnavailableReason: undefined,
startTime: Date.now(),
});
const result = await deps.startStreamTransfer({
transferId: task.id,
sourcePath: task.sourcePath,
targetPath: task.targetPath,
sourceType: "local",
targetType: targetIsLocal ? "local" : "sftp",
targetSftpId: targetIsLocal ? undefined : targetSftpId,
targetHostId: targetIsLocal ? undefined : task.targetHostId,
totalBytes: task.totalBytes > 0 ? task.totalBytes : undefined,
resumable: true,
checkpointBytes: 0,
});
// Late cancel can settle the row before this return; never resurrect it.
if (isTransferCancelledFlag(task.id) || result?.cancelled) {
deps.onPatch(task.id, {
status: "cancelled",
error: undefined,
endTime: Date.now(),
speed: 0,
phase: undefined,
});
return { success: false, error: "Transfer cancelled" };
}
if (result?.error) {
deps.onPatch(task.id, {
status: "failed",
error: result.error,
endTime: Date.now(),
speed: 0,
phase: undefined,
});
return { success: false, error: result.error };
}
const live = deps.getTask?.(task.id);
const completedBytes = Math.max(
live?.totalBytes ?? 0,
live?.transferredBytes ?? 0,
task.totalBytes,
0,
);
deps.onPatch(task.id, {
status: "completed",
error: undefined,
transferredBytes: completedBytes,
totalBytes: Math.max(live?.totalBytes ?? 0, task.totalBytes, completedBytes),
endTime: Date.now(),
speed: 0,
phase: undefined,
});
// Progressive parents finalize once with "N of M failed". After a child
// retry succeeds, re-roll the parent when no failed children remain.
if (task.parentTaskId && deps.getChildTasks) {
rollupParentAfterChildSuccess(task.parentTaskId, task.id, deps);
}
return { success: true };
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
if (isTransferCancelledFlag(task.id) || /cancel/i.test(message)) {
deps.onPatch(task.id, {
status: "cancelled",
error: undefined,
endTime: Date.now(),
speed: 0,
phase: undefined,
});
return { success: false, error: "Transfer cancelled" };
}
deps.onPatch(task.id, {
status: "failed",
error: message,
endTime: Date.now(),
speed: 0,
phase: undefined,
});
if (lease && /session|sftp|disconnect|not found/i.test(message)) {
try { lease.discard(); } catch { /* best-effort */ }
lease = null;
}
return { success: false, error: message };
} finally {
try { lease?.release(); } catch { /* best-effort */ }
}
}
function rollupParentAfterChildSuccess(
parentTaskId: string,
completedChildId: string,
deps: ExternalDragDropRetryDeps,
): void {
// Only re-roll parents that already finished the progressive walk as failed.
// Never promote a still-scanning/transferring parent mid-walk.
const parent = deps.getTask?.(parentTaskId);
if (
parent
&& parent.status !== "failed"
&& parent.status !== "attention"
&& parent.status !== "cancelled"
) {
return;
}
const children = deps.getChildTasks?.(parentTaskId) ?? [];
if (children.length === 0) return;
const stillFailed = children.some(
(child) =>
child.id !== completedChildId
&& (child.status === "failed" || child.status === "attention"),
);
if (stillFailed) return;
const active = children.some(
(child) =>
child.id !== completedChildId
&& (
child.status === "transferring"
|| child.status === "pending"
|| child.status === "queued"
|| child.status === "pausing"
|| child.status === "paused"
),
);
if (active) return;
const completedCount = children.filter(
(child) => child.id === completedChildId || child.status === "completed",
).length;
const total = Math.max(
Number(parent?.totalBytes) || 0,
children.length,
completedCount,
);
deps.onPatch(parentTaskId, {
status: "completed",
error: undefined,
transferredBytes: completedCount,
totalBytes: total,
speed: 0,
endTime: Date.now(),
phase: undefined,
});
}
@@ -4,10 +4,8 @@ import assert from "node:assert/strict";
import {
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY,
DEFAULT_SFTP_FOLDER_PRESCAN,
DEFAULT_SFTP_SKIP_UNCHANGED,
resolveSftpDirectoryListingConcurrency,
resolveSftpFolderPrescanEnabled,
resolveSftpSkipUnchangedEnabled,
resolveSftpTransferConcurrency,
runBoundedConcurrency,
@@ -27,12 +25,9 @@ test("defaults directory listing fanout to four concurrent readdirs", () => {
assert.equal(DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY, 4);
});
test("defaults folder pre-scan and skip-unchanged to enabled", () => {
assert.equal(resolveSftpFolderPrescanEnabled(() => null), DEFAULT_SFTP_FOLDER_PRESCAN);
test("defaults skip-unchanged to enabled", () => {
assert.equal(resolveSftpSkipUnchangedEnabled(() => null), DEFAULT_SFTP_SKIP_UNCHANGED);
assert.equal(DEFAULT_SFTP_FOLDER_PRESCAN, true);
assert.equal(DEFAULT_SFTP_SKIP_UNCHANGED, true);
assert.equal(resolveSftpFolderPrescanEnabled(() => false), false);
assert.equal(resolveSftpSkipUnchangedEnabled(() => false), false);
});
@@ -13,8 +13,6 @@ export const DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY = 4;
export const MIN_SFTP_DIRECTORY_LISTING_CONCURRENCY = 1;
export const MAX_SFTP_DIRECTORY_LISTING_CONCURRENCY = 8;
/** Default on: accurate folder totals before first byte (rsync --no-i-r style). */
export const DEFAULT_SFTP_FOLDER_PRESCAN = true;
/** Default on: skip size+mtime matches like rsync's generator. */
export const DEFAULT_SFTP_SKIP_UNCHANGED = true;
@@ -38,13 +36,6 @@ export function resolveSftpDirectoryListingConcurrency(
: DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY;
}
export function resolveSftpFolderPrescanEnabled(
readStoredValue: () => boolean | null | undefined,
): boolean {
const stored = readStoredValue();
return stored == null ? DEFAULT_SFTP_FOLDER_PRESCAN : stored;
}
export function resolveSftpSkipUnchangedEnabled(
readStoredValue: () => boolean | null | undefined,
): boolean {
@@ -1,311 +0,0 @@
import assert from "node:assert/strict";
import test from "node:test";
import type { SftpFileEntry } from "../../../domain/models";
import { compareDirectoryTraversalPaths } from "../../../domain/sftpDirectoryCheckpoint";
import { createSftpDirectoryTraversalBudget } from "../../../domain/sftpDirectoryCheckpoint";
import { discoverTransferTree } from "./transferDirectoryDiscovery";
const directoryEntry = (name: string): SftpFileEntry => ({
name,
type: "directory",
size: 0,
sizeFormatted: "0 B",
lastModified: 0,
lastModifiedFormatted: "",
});
const fileEntry = (name: string, size = 1, lastModified = 1_700_000_000): SftpFileEntry => ({
name,
type: "file",
size,
sizeFormatted: `${size} B`,
lastModified,
lastModifiedFormatted: "",
});
test("discoverTransferTree returns a stable flat file list with bounded listings", async () => {
const previousWindow = (globalThis as { window?: unknown }).window;
(globalThis as { window?: unknown }).window = {
netcatty: {
realpathSftp: async (_sftpId: string, remotePath: string) => remotePath,
},
};
let active = 0;
let maxActive = 0;
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
active += 1;
maxActive = Math.max(maxActive, active);
await new Promise((resolve) => setTimeout(resolve, 1));
active -= 1;
if (path === "/source") {
return [directoryEntry("a"), directoryEntry("b"), fileEntry("root.txt")];
}
if (path === "/source/a") return [fileEntry("a1.txt"), fileEntry("a2.txt")];
if (path === "/source/b") return [fileEntry("b1.txt")];
return [];
};
try {
const counts: number[] = [];
const result = await discoverTransferTree({
sourcePath: "/source",
targetPath: "/target",
sourceIsLocal: false,
sourceSftpId: "sftp-1",
sourceEncoding: "auto",
listLocalFiles: async () => [],
listRemoteFiles,
listingConcurrency: 2,
onDiscoveredFiles: (total) => counts.push(total),
});
assert.deepEqual(
result.files.map((file) => file.sourcePath),
[...result.files.map((file) => file.sourcePath)].sort((left, right) =>
compareDirectoryTraversalPaths(left, right)
),
"pre-scan file order must match resume comparator",
);
assert.deepEqual(
result.files.map((file) => file.sourcePath).sort(),
["/source/a/a1.txt", "/source/a/a2.txt", "/source/b/b1.txt", "/source/root.txt"],
);
assert.deepEqual(
result.directories.map((dir) => dir.sourcePath).sort(),
["/source", "/source/a", "/source/b"],
);
assert.equal(counts.at(-1), 4);
assert.equal(result.omittedSymlinkDirectoryErrors, 0);
assert.ok(maxActive >= 2, `expected parallel listings, got ${maxActive}`);
assert.ok(maxActive <= 2, `listing concurrency exceeded: ${maxActive}`);
} finally {
(globalThis as { window?: unknown }).window = previousWindow;
}
});
test("discoverTransferTree skips symlink directory cycles across BFS waves", async () => {
const previousWindow = (globalThis as { window?: unknown }).window;
const listCalls = new Map<string, number>();
(globalThis as { window?: unknown }).window = {
netcatty: {
realpathSftp: async (_sftpId: string, remotePath: string) => (
remotePath === "/source/loop" ? "/source" : remotePath
),
},
};
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
listCalls.set(path, (listCalls.get(path) ?? 0) + 1);
if (path === "/source") {
return [
directoryEntry("child"),
{ ...directoryEntry("loop"), type: "symlink", linkTarget: "directory" },
fileEntry("root.txt"),
];
}
if (path === "/source/child") return [fileEntry("nested.txt")];
throw new Error(`unexpected list of ${path}`);
};
try {
const result = await discoverTransferTree({
sourcePath: "/source",
targetPath: "/target",
sourceIsLocal: false,
sourceSftpId: "sftp-1",
sourceEncoding: "auto",
followSymlinks: true,
listLocalFiles: async () => [],
listRemoteFiles,
listingConcurrency: 4,
});
assert.equal(listCalls.has("/source/loop"), false);
assert.equal(listCalls.get("/source"), 1);
assert.equal(listCalls.get("/source/child"), 1);
assert.deepEqual(
result.files.map((file) => file.sourcePath).sort(),
["/source/child/nested.txt", "/source/root.txt"],
);
assert.deepEqual(
result.directories.map((dir) => dir.sourcePath).sort(),
["/source", "/source/child"],
"cyclic symlink dirs must not appear in the creation plan",
);
assert.equal(result.omittedSymlinkDirectoryErrors, 0);
} finally {
(globalThis as { window?: unknown }).window = previousWindow;
}
});
test("discoverTransferTree copies sibling symlink aliases to distinct targets", async () => {
const previousWindow = (globalThis as { window?: unknown }).window;
const listCalls = new Map<string, number>();
(globalThis as { window?: unknown }).window = {
netcatty: {
realpathSftp: async (_sftpId: string, remotePath: string) => {
if (remotePath === "/source/alias-a" || remotePath === "/source/alias-b") {
return "/shared";
}
return remotePath;
},
},
};
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
listCalls.set(path, (listCalls.get(path) ?? 0) + 1);
if (path === "/source") {
return [
{ ...directoryEntry("alias-a"), type: "symlink", linkTarget: "directory" },
{ ...directoryEntry("alias-b"), type: "symlink", linkTarget: "directory" },
];
}
if (path === "/source/alias-a" || path === "/source/alias-b") {
return [fileEntry("shared.txt")];
}
throw new Error(`unexpected list of ${path}`);
};
try {
const result = await discoverTransferTree({
sourcePath: "/source",
targetPath: "/target",
sourceIsLocal: false,
sourceSftpId: "sftp-1",
sourceEncoding: "auto",
followSymlinks: true,
listLocalFiles: async () => [],
listRemoteFiles,
listingConcurrency: 4,
});
assert.equal(listCalls.get("/source/alias-a"), 1);
assert.equal(listCalls.get("/source/alias-b"), 1);
assert.deepEqual(
result.files.map((file) => file.targetPath).sort(),
["/target/alias-a/shared.txt", "/target/alias-b/shared.txt"],
);
} finally {
(globalThis as { window?: unknown }).window = previousWindow;
}
});
test("discoverTransferTree counts max-depth symlink directory omissions as errors", async () => {
const previousWindow = (globalThis as { window?: unknown }).window;
(globalThis as { window?: unknown }).window = {
netcatty: {
realpathSftp: async (_sftpId: string, remotePath: string) => remotePath,
},
};
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
const depth = path === "/source"
? 0
: path.startsWith("/source/")
? path.slice("/source/".length).split("/").filter(Boolean).length
: -1;
if (depth < 0 || depth > 32) throw new Error(`unexpected list of ${path}`);
const nextLink = { ...directoryEntry("deep"), type: "symlink" as const, linkTarget: "directory" as const };
if (depth === 0) {
return [
nextLink,
{
name: "link.txt",
type: "symlink",
linkTarget: "file",
size: 3,
sizeFormatted: "3 B",
lastModified: 1_700_000_000,
lastModifiedFormatted: "",
},
];
}
return [nextLink];
};
try {
const result = await discoverTransferTree({
sourcePath: "/source",
targetPath: "/target",
sourceIsLocal: false,
sourceSftpId: "sftp-1",
sourceEncoding: "auto",
followSymlinks: true,
listLocalFiles: async () => [],
listRemoteFiles,
listingConcurrency: 2,
});
assert.equal(result.omittedSymlinkDirectoryErrors, 1);
assert.equal(result.files.length, 1);
assert.equal(result.files[0]?.isSymlink, true);
assert.equal(result.files[0]?.name, "link.txt");
} finally {
(globalThis as { window?: unknown }).window = previousWindow;
}
});
test("discoverTransferTree bounds local folder pre-scan memory", async () => {
const listLocalFiles = async (path: string): Promise<SftpFileEntry[]> => {
if (path === "/local") return [directoryEntry("a"), fileEntry("root.txt")];
if (path === "/local/a") return [fileEntry("nested.txt")];
return [];
};
await assert.rejects(
() => discoverTransferTree({
sourcePath: "/local",
targetPath: "/target",
sourceIsLocal: true,
sourceSftpId: null,
sourceEncoding: "auto",
listLocalFiles,
listRemoteFiles: async () => [],
listingConcurrency: 2,
traversalBudget: createSftpDirectoryTraversalBudget({ maxDirectories: 1, maxEntries: 200_000 }),
}),
/Directory traversal directory limit exceeded/,
);
});
test("discoverTransferTree waits while paused between BFS waves", async () => {
let paused = true;
let listCalls = 0;
let pauseWaits = 0;
const listLocalFiles = async (path: string): Promise<SftpFileEntry[]> => {
listCalls += 1;
if (path === "/local") return [directoryEntry("a"), fileEntry("root.txt")];
if (path === "/local/a") return [fileEntry("nested.txt")];
return [];
};
const discovery = discoverTransferTree({
sourcePath: "/local",
targetPath: "/target",
sourceIsLocal: true,
sourceSftpId: null,
sourceEncoding: "auto",
listLocalFiles,
listRemoteFiles: async () => [],
listingConcurrency: 1,
waitWhilePaused: async () => {
pauseWaits += 1;
while (paused) {
await new Promise((resolve) => setTimeout(resolve, 5));
}
},
});
// First waitWhilePaused (wave start) should block before any listing.
await new Promise((resolve) => setTimeout(resolve, 30));
assert.equal(listCalls, 0, "pause must block pre-scan listings");
assert.ok(pauseWaits >= 1, "expected waitWhilePaused to be invoked");
paused = false;
const result = await discovery;
assert.equal(result.files.length, 2);
assert.ok(listCalls >= 2);
assert.ok(pauseWaits >= 2, "expected pause checks across waves/listings");
});
@@ -1,279 +0,0 @@
import type { SftpFileEntry, SftpFilenameEncoding } from "../../../domain/models";
import {
accountSftpDirectoryEntries,
claimSftpDirectoryVisit,
compareDirectoryTraversalPaths,
createSftpDirectoryBranchAncestors,
createSftpDirectoryTraversalBudget,
releaseSftpDirectoryVisit,
shouldFollowSftpSymlinkDirectory,
type SftpDirectoryTraversalBudget,
} from "../../../domain/sftpDirectoryCheckpoint";
import { netcattyBridge } from "../../../infrastructure/services/netcattyBridge";
import { logger } from "../../../lib/logger";
import {
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
resolveSftpDirectoryListingConcurrency,
runBoundedConcurrency,
} from "./transferConcurrency";
import { joinPath, joinTransferTargetPath } from "./utils";
export type DiscoveredTransferFile = {
name: string;
sourcePath: string;
targetPath: string;
size: number;
lastModified: number;
/**
* True when the listing entry was a symlink. Link-node size/mtime must not
* drive skip-unchanged; transfer follows the target bytes.
*/
isSymlink?: boolean;
};
export type DiscoveredTransferDirectory = {
sourcePath: string;
targetPath: string;
};
export type DiscoverTransferTreeResult = {
files: DiscoveredTransferFile[];
directories: DiscoveredTransferDirectory[];
/** Symlink directories omitted at max follow depth (incomplete tree). */
omittedSymlinkDirectoryErrors: number;
};
type ListingGate = {
run: <T>(fn: () => Promise<T>) => Promise<T>;
};
function createListingGate(concurrency = DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY): ListingGate {
const limit = Math.max(1, Math.floor(concurrency) || 1);
let active = 0;
const waiters: Array<() => void> = [];
const acquire = async () => {
if (active < limit) {
active += 1;
return;
}
await new Promise<void>((resolve) => {
waiters.push(() => {
active += 1;
resolve();
});
});
};
const release = () => {
active = Math.max(0, active - 1);
const next = waiters.shift();
if (next) next();
};
return {
async run<T>(fn: () => Promise<T>): Promise<T> {
await acquire();
try {
return await fn();
} finally {
release();
}
},
};
}
function getEntrySize(entry: SftpFileEntry): number {
if (typeof entry.size === "string") {
const parsed = parseInt(entry.size, 10);
return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
}
return typeof entry.size === "number" && entry.size > 0 ? entry.size : 0;
}
export type DiscoverTransferTreeOptions = {
sourcePath: string;
targetPath: string;
sourceIsLocal: boolean;
sourceSftpId: string | null;
sourceEncoding: SftpFilenameEncoding;
followSymlinks?: boolean;
listLocalFiles: (path: string) => Promise<SftpFileEntry[]>;
listRemoteFiles: (sftpId: string, path: string, encoding?: SftpFilenameEncoding) => Promise<SftpFileEntry[]>;
shouldAbort?: () => boolean;
/** Honor transfer-center pause between BFS waves / listings. */
waitWhilePaused?: () => Promise<void>;
onDiscoveredFiles?: (totalFiles: number) => void;
listingConcurrency?: number;
traversalBudget?: SftpDirectoryTraversalBudget;
};
/**
* Full-tree discovery (rsync --no-inc-recursive style): walk every directory
* before transfer starts so the UI can show a stable file total.
* Listings are bounded-parallel; nested walks share one semaphore.
*/
export async function discoverTransferTree(
options: DiscoverTransferTreeOptions,
): Promise<DiscoverTransferTreeResult> {
const followSymlinks = options.followSymlinks === true;
const listingConcurrency = options.listingConcurrency
?? resolveSftpDirectoryListingConcurrency();
const listingGate = createListingGate(listingConcurrency);
const traversal = options.traversalBudget ?? createSftpDirectoryTraversalBudget();
const files: DiscoveredTransferFile[] = [];
const directories: DiscoveredTransferDirectory[] = [
{ sourcePath: options.sourcePath, targetPath: options.targetPath },
];
let omittedSymlinkDirectoryErrors = 0;
type DirJob = {
sourcePath: string;
targetPath: string;
symlinkDepth: number;
/** Ancestors on this BFS branch (copied for parallel sibling aliases). */
branchAncestors: Set<string>;
/** False for the root (already in `directories`); true for accepted children. */
includeInCreationPlan: boolean;
};
const queue: DirJob[] = [{
sourcePath: options.sourcePath,
targetPath: options.targetPath,
symlinkDepth: 0,
branchAncestors: createSftpDirectoryBranchAncestors(),
includeInCreationPlan: false,
}];
const publishCount = () => {
options.onDiscoveredFiles?.(files.length);
};
const processDir = async (job: DirJob): Promise<DirJob[]> => {
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
await options.waitWhilePaused?.();
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
let claimedCanonicalPath: string | null = null;
try {
let canonicalPath = job.sourcePath;
if (!options.sourceIsLocal && options.sourceSftpId) {
const bridge = netcattyBridge.get();
canonicalPath = await bridge?.realpathSftp?.(
options.sourceSftpId,
job.sourcePath,
options.sourceEncoding,
).catch(() => job.sourcePath) ?? job.sourcePath;
}
// Bound both remote and local pre-scan trees (Codex P1 on 86c64f48).
claimedCanonicalPath = claimSftpDirectoryVisit(
traversal,
canonicalPath,
job.branchAncestors,
);
// Cycle / already visited: do not create an empty target dir for this path.
if (!claimedCanonicalPath) return [];
// Only record directories after the visit is accepted so cyclic symlink
// children (e.g. /source/loop -> /source) are omitted from mkdir plans.
if (job.includeInCreationPlan) {
directories.push({ sourcePath: job.sourcePath, targetPath: job.targetPath });
}
const listed = await listingGate.run(async () => {
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
await options.waitWhilePaused?.();
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
if (options.sourceIsLocal) {
return options.listLocalFiles(job.sourcePath);
}
if (!options.sourceSftpId) throw new Error("No source connection");
return options.listRemoteFiles(
options.sourceSftpId,
job.sourcePath,
options.sourceEncoding,
);
});
const filtered = listed.filter((entry) => entry.name !== "." && entry.name !== "..");
accountSftpDirectoryEntries(traversal, filtered.length);
const childDirs: DirJob[] = [];
const regularFiles: SftpFileEntry[] = [];
for (const entry of filtered) {
if (entry.type === "directory") {
childDirs.push({
sourcePath: joinPath(job.sourcePath, entry.name),
targetPath: joinTransferTargetPath(job.targetPath, entry.name),
symlinkDepth: job.symlinkDepth,
branchAncestors: createSftpDirectoryBranchAncestors(job.branchAncestors),
includeInCreationPlan: true,
});
} else if (
followSymlinks
&& entry.type === "symlink"
&& entry.linkTarget === "directory"
) {
if (shouldFollowSftpSymlinkDirectory(job.symlinkDepth)) {
childDirs.push({
sourcePath: joinPath(job.sourcePath, entry.name),
targetPath: joinTransferTargetPath(job.targetPath, entry.name),
symlinkDepth: job.symlinkDepth + 1,
branchAncestors: createSftpDirectoryBranchAncestors(job.branchAncestors),
includeInCreationPlan: true,
});
} else {
omittedSymlinkDirectoryErrors += 1;
logger.warn(
`[SFTP] Skipping symlink directory at max depth during pre-scan: ${joinPath(job.sourcePath, entry.name)}`,
);
}
} else {
regularFiles.push(entry);
}
}
childDirs.sort((left, right) => left.sourcePath.localeCompare(right.sourcePath));
regularFiles.sort((left, right) => left.name.localeCompare(right.name));
for (const entry of regularFiles) {
files.push({
name: entry.name,
sourcePath: joinPath(job.sourcePath, entry.name),
targetPath: joinTransferTargetPath(job.targetPath, entry.name),
size: getEntrySize(entry),
lastModified: typeof entry.lastModified === "number" ? entry.lastModified : 0,
isSymlink: entry.type === "symlink",
});
}
publishCount();
return childDirs;
} finally {
if (claimedCanonicalPath) {
releaseSftpDirectoryVisit(traversal, claimedCanonicalPath, job.branchAncestors);
}
}
};
// BFS waves: each wave lists up to listingConcurrency directories in parallel.
while (queue.length > 0) {
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
await options.waitWhilePaused?.();
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
const wave = queue.splice(0, queue.length);
const nested: DirJob[][] = new Array(wave.length);
await runBoundedConcurrency(
wave,
listingConcurrency,
async (job, index) => {
nested[index] = await processDir(job);
},
);
for (const children of nested) {
if (!children) continue;
for (const child of children) queue.push(child);
}
}
// Match dedicated-resume / transfer-center ordering so directoryEntryIndex and
// manifest checkpoints stay valid across interrupt + rebuild.
files.sort((left, right) => compareDirectoryTraversalPaths(left.sourcePath, right.sourcePath));
return { files, directories, omittedSymlinkDirectoryErrors };
}
@@ -1,223 +0,0 @@
import assert from "node:assert/strict";
import test from "node:test";
import type { MutableRefObject } from "react";
import type { TransferTask } from "../../../domain/models";
import { STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY } from "../../../infrastructure/config/storageKeys";
import { transferDiscoveredFiles } from "./transferDirectoryFileBatch";
const memoryStore = new Map<string, string>();
(globalThis as { localStorage?: Storage }).localStorage = {
getItem: (key: string) => memoryStore.get(key) ?? null,
setItem: (key: string, value: string) => {
memoryStore.set(key, String(value));
},
removeItem: (key: string) => {
memoryStore.delete(key);
},
clear: () => memoryStore.clear(),
key: () => null,
length: 0,
} as Storage;
memoryStore.set(STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY, "1");
const rootTask = (): TransferTask => ({
id: "root",
fileName: "source",
sourcePath: "/source",
targetPath: "/target",
sourceConnectionId: "local",
targetConnectionId: "local",
direction: "upload",
status: "transferring",
totalBytes: 1,
transferredBytes: 0,
speed: 0,
startTime: 0,
isDirectory: true,
});
test("transferDiscoveredFiles re-stats source before skip-unchanged", async () => {
const transfersRef = { current: [rootTask()] } as MutableRefObject<TransferTask[]>;
const cancelledTasksRef = { current: new Set<string>() } as MutableRefObject<Set<string>>;
const activeChildIdsRef = {
current: new Map<string, Set<string>>(),
} as MutableRefObject<Map<string, Set<string>>>;
let transfers = transfersRef.current;
let transferCalls = 0;
let sourceStatCalls = 0;
const staleMtime = 1_700_000_000_000;
const freshMtime = 1_700_000_100_000;
await transferDiscoveredFiles({
rootTask: rootTask(),
files: [{
name: "a.txt",
sourcePath: "/source/a.txt",
targetPath: "/target/a.txt",
size: 10,
lastModified: staleMtime,
}],
sourceSftpId: null,
targetSftpId: null,
sourceIsLocal: true,
targetIsLocal: true,
sourceEncoding: "auto",
targetEncoding: "auto",
rootTaskId: "root",
skipUnchanged: true,
startEntryIndex: 0,
cancelledTasksRef,
activeChildIdsRef,
transfersRef,
setTransfers: (updater) => {
transfers = typeof updater === "function" ? updater(transfers) : updater;
transfersRef.current = transfers;
},
waitWhileTransferPaused: async () => {},
isPauseLatched: () => false,
transferFile: async () => {
transferCalls += 1;
},
tryStatTarget: async () => ({
size: 10,
lastModified: staleMtime,
type: "file",
}),
tryStatSource: async () => {
sourceStatCalls += 1;
return {
size: 10,
lastModified: freshMtime,
type: "file",
};
},
});
assert.equal(sourceStatCalls, 1, "must re-stat source before skip check");
assert.equal(transferCalls, 1, "stale listing must not skip a changed source");
const child = transfers.find((row) => row.parentTaskId === "root");
assert.ok(child);
assert.equal(child?.status, "completed");
assert.equal(child?.sourceLastModified, freshMtime);
});
test("transferDiscoveredFiles does not skip when source re-stat fails", async () => {
const transfersRef = { current: [rootTask()] } as MutableRefObject<TransferTask[]>;
const cancelledTasksRef = { current: new Set<string>() } as MutableRefObject<Set<string>>;
const activeChildIdsRef = {
current: new Map<string, Set<string>>(),
} as MutableRefObject<Map<string, Set<string>>>;
let transfers = transfersRef.current;
let transferCalls = 0;
let transferredTotalBytes: number | undefined;
let transferredSourceLastModified: number | undefined;
const mtime = 1_700_000_000_000;
await transferDiscoveredFiles({
rootTask: rootTask(),
files: [{
name: "a.txt",
sourcePath: "/source/a.txt",
targetPath: "/target/a.txt",
size: 10,
lastModified: mtime,
}],
sourceSftpId: null,
targetSftpId: null,
sourceIsLocal: true,
targetIsLocal: true,
sourceEncoding: "auto",
targetEncoding: "auto",
rootTaskId: "root",
skipUnchanged: true,
startEntryIndex: 0,
cancelledTasksRef,
activeChildIdsRef,
transfersRef,
setTransfers: (updater) => {
transfers = typeof updater === "function" ? updater(transfers) : updater;
transfersRef.current = transfers;
},
waitWhileTransferPaused: async () => {},
isPauseLatched: () => false,
transferFile: async (task) => {
transferCalls += 1;
transferredTotalBytes = task.totalBytes;
transferredSourceLastModified = task.sourceLastModified;
},
tryStatTarget: async () => ({
size: 10,
lastModified: mtime,
type: "file",
}),
tryStatSource: async () => null,
});
assert.equal(transferCalls, 1, "missing fresh source meta must not skip");
assert.equal(transferredTotalBytes, 0, "must not pass stale listing size after re-stat failure");
assert.equal(
transferredSourceLastModified,
undefined,
"must clear stale listing mtime after re-stat failure",
);
});
test("transferDiscoveredFiles skips only when fresh source still matches", async () => {
const transfersRef = { current: [rootTask()] } as MutableRefObject<TransferTask[]>;
const cancelledTasksRef = { current: new Set<string>() } as MutableRefObject<Set<string>>;
const activeChildIdsRef = {
current: new Map<string, Set<string>>(),
} as MutableRefObject<Map<string, Set<string>>>;
let transfers = transfersRef.current;
let transferCalls = 0;
const mtime = 1_700_000_000_000;
await transferDiscoveredFiles({
rootTask: rootTask(),
files: [{
name: "a.txt",
sourcePath: "/source/a.txt",
targetPath: "/target/a.txt",
size: 10,
lastModified: mtime - 60_000,
}],
sourceSftpId: null,
targetSftpId: null,
sourceIsLocal: true,
targetIsLocal: true,
sourceEncoding: "auto",
targetEncoding: "auto",
rootTaskId: "root",
skipUnchanged: true,
startEntryIndex: 0,
cancelledTasksRef,
activeChildIdsRef,
transfersRef,
setTransfers: (updater) => {
transfers = typeof updater === "function" ? updater(transfers) : updater;
transfersRef.current = transfers;
},
waitWhileTransferPaused: async () => {},
isPauseLatched: () => false,
transferFile: async () => {
transferCalls += 1;
},
tryStatTarget: async () => ({
size: 10,
lastModified: mtime,
type: "file",
}),
tryStatSource: async () => ({
size: 10,
lastModified: mtime,
type: "file",
}),
});
assert.equal(transferCalls, 0, "matching fresh identity must skip transfer");
const child = transfers.find((row) => row.parentTaskId === "root");
assert.equal(child?.status, "completed");
assert.equal(child?.transferredBytes, 10);
});
@@ -1,354 +0,0 @@
import type { Dispatch, MutableRefObject, SetStateAction } from "react";
import type {
SftpFilenameEncoding,
TransferStatus,
TransferTask,
} from "../../../domain/models";
import { createDirectoryEntryIdentity } from "../../../domain/sftpDirectoryCheckpoint";
import { isUnchangedTransferCandidate } from "../../../domain/sftpTransferSkip";
import { STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY } from "../../../infrastructure/config/storageKeys";
import { localStorageAdapter } from "../../../infrastructure/persistence/localStorageAdapter";
import { logger } from "../../../lib/logger";
import { isSessionError } from "./errors";
import type { DiscoveredTransferFile } from "./transferDirectoryDiscovery";
import { runSftpTransferWorkers } from "./transferConcurrency";
import { isTransferCancelledError } from "./transferRetry";
type TransferOneFile = (
task: TransferTask,
sourceSftpId: string | null,
targetSftpId: string | null,
sourceIsLocal: boolean,
targetIsLocal: boolean,
sourceEncoding: SftpFilenameEncoding,
targetEncoding: SftpFilenameEncoding,
rootTaskId: string,
sameHost?: boolean,
) => Promise<void>;
export type TransferDiscoveredFilesParams = {
rootTask: TransferTask;
files: DiscoveredTransferFile[];
sourceSftpId: string | null;
targetSftpId: string | null;
sourceIsLocal: boolean;
targetIsLocal: boolean;
sourceEncoding: SftpFilenameEncoding;
targetEncoding: SftpFilenameEncoding;
rootTaskId: string;
sameHost?: boolean;
skipUnchanged: boolean;
startEntryIndex: number;
cancelledTasksRef: MutableRefObject<Set<string>>;
activeChildIdsRef: MutableRefObject<Map<string, Set<string>>>;
transfersRef: MutableRefObject<TransferTask[]>;
setTransfers: Dispatch<SetStateAction<TransferTask[]>>;
waitWhileTransferPaused: (rootTaskId: string, taskId?: string) => Promise<void>;
isPauseLatched: (rootTaskId: string, taskId?: string) => boolean;
transferFile: TransferOneFile;
tryStatTarget: (
targetPath: string,
) => Promise<{ size: number; lastModified: number; type: string } | null>;
/** Fresh source identity immediately before skip-unchanged (Codex P1). */
tryStatSource: (
sourcePath: string,
) => Promise<{ size: number; lastModified: number; type: string } | null>;
};
/**
* Transfer a flat pre-scanned file list with the same child-task bookkeeping as
* the interleaved directory walk.
*/
export async function transferDiscoveredFiles(
params: TransferDiscoveredFilesParams,
): Promise<number> {
const {
rootTask,
files,
sourceSftpId,
targetSftpId,
sourceIsLocal,
targetIsLocal,
sourceEncoding,
targetEncoding,
rootTaskId,
sameHost,
skipUnchanged,
startEntryIndex,
cancelledTasksRef,
activeChildIdsRef,
transfersRef,
setTransfers,
waitWhileTransferPaused,
isPauseLatched,
transferFile,
tryStatTarget,
tryStatSource,
} = params;
if (files.length === 0) return 0;
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
? { ...candidate, phase: "transferring" }
: candidate));
const errors: Error[] = [];
let sessionLostError: Error | null = null;
await runSftpTransferWorkers(
files,
() => localStorageAdapter.readNumber(STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY),
async (file, fileIndex) => {
if (sessionLostError) throw sessionLostError;
if (cancelledTasksRef.current.has(rootTask.id) || cancelledTasksRef.current.has(rootTaskId)) {
throw new Error("Transfer cancelled");
}
const fileSize = file.size;
const sourcePath = file.sourcePath;
const targetPath = file.targetPath;
let skipSourceSize = fileSize;
let skipSourceLastModified = file.lastModified;
const directoryEntryIndex = startEntryIndex + fileIndex;
let directoryEntryIdentity = createDirectoryEntryIdentity({
sourcePath,
targetPath,
size: fileSize,
lastModified: file.lastModified,
});
const persistedChild = transfersRef.current.find((candidate) => (
candidate.parentTaskId === rootTaskId
&& candidate.sourcePath === sourcePath
&& candidate.targetPath === targetPath
));
if (persistedChild?.status === "completed") return;
await waitWhileTransferPaused(rootTaskId);
if (cancelledTasksRef.current.has(rootTask.id) || cancelledTasksRef.current.has(rootTaskId)) {
throw new Error("Transfer cancelled");
}
if (isPauseLatched(rootTaskId)) {
await waitWhileTransferPaused(rootTaskId);
}
// Symlink listing attrs are the link node; transfer follows target bytes.
// Re-stat regular files so skip and transfer both use current size/mtime
// (Codex P1: pre-scan listing attrs can go stale before transfer starts).
let freshSourceOk = false;
if (!file.isSymlink) {
const freshSource = await tryStatSource(sourcePath);
if (freshSource && freshSource.type !== "directory") {
skipSourceSize = freshSource.size;
skipSourceLastModified = freshSource.lastModified;
directoryEntryIdentity = createDirectoryEntryIdentity({
sourcePath,
targetPath,
size: skipSourceSize,
lastModified: skipSourceLastModified,
});
freshSourceOk = true;
} else {
// Do not pass stale pre-scan size as an explicit snapshot (Codex P2).
// Child totalBytes 0 is omitted at startStreamTransfer via `|| undefined`,
// so the dedicated transfer session re-stats the source itself.
skipSourceSize = 0;
skipSourceLastModified = 0;
}
}
if (skipUnchanged && !file.isSymlink && freshSourceOk) {
const existing = await tryStatTarget(targetPath);
if (
existing
&& existing.type !== "directory"
&& isUnchangedTransferCandidate(
{ size: skipSourceSize, lastModified: skipSourceLastModified, mtimeUnit: "ms" },
{ size: existing.size, lastModified: existing.lastModified, mtimeUnit: "ms" },
)
) {
const skippedId = persistedChild?.id ?? crypto.randomUUID();
setTransfers((prev) => {
const hasChild = prev.some((row) => row.id === skippedId);
const next = hasChild
? prev.map((row) => row.id === skippedId
? {
...row,
status: "completed" as TransferStatus,
transferredBytes: skipSourceSize,
totalBytes: skipSourceSize,
endTime: Date.now(),
speed: 0,
error: undefined,
}
: row)
: [...prev, {
...rootTask,
id: skippedId,
fileName: file.name,
originalFileName: file.name,
sourcePath,
targetPath,
isDirectory: false,
progressMode: "bytes" as const,
parentTaskId: rootTaskId,
totalBytes: skipSourceSize,
transferredBytes: skipSourceSize,
sourceLastModified: skipSourceLastModified,
directoryEntryIndex,
directoryEntryIdentity,
status: "completed" as TransferStatus,
speed: 0,
startTime: Date.now(),
endTime: Date.now(),
}];
return next.map((row) => {
if (row.id !== rootTaskId) return row;
if (row.status === "paused" || row.status === "pausing" || isPauseLatched(rootTaskId)) {
return { ...row, speed: 0 };
}
return { ...row, transferredBytes: row.transferredBytes + 1 };
});
});
return;
}
}
const fileId = persistedChild?.id ?? crypto.randomUUID();
if (!activeChildIdsRef.current.has(rootTaskId)) {
activeChildIdsRef.current.set(rootTaskId, new Set());
}
activeChildIdsRef.current.get(rootTaskId)!.add(fileId);
const childTask: TransferTask = {
...rootTask,
...persistedChild,
id: fileId,
fileName: file.name,
originalFileName: file.name,
sourcePath,
targetPath,
isDirectory: false,
progressMode: "bytes",
parentTaskId: rootTaskId,
totalBytes: skipSourceSize,
sourceLastModified: skipSourceLastModified || undefined,
directoryEntryIndex,
directoryEntryIdentity,
retryable: rootTask.retryable,
lifecycleEpoch: undefined,
phase: undefined,
pauseUnavailableReason: undefined,
};
setTransfers((prev) => persistedChild
? prev.map((candidate) => candidate.id === fileId ? {
...childTask,
status: "queued" as TransferStatus,
speed: 0,
error: undefined,
endTime: undefined,
lifecycleEpoch: undefined,
} : candidate)
: [...prev, {
...childTask,
status: "queued" as TransferStatus,
transferredBytes: 0,
speed: 0,
startTime: Date.now(),
lifecycleEpoch: undefined,
}]);
try {
await transferFile(
childTask,
sourceSftpId,
targetSftpId,
sourceIsLocal,
targetIsLocal,
sourceEncoding,
targetEncoding,
rootTaskId,
sameHost,
);
activeChildIdsRef.current.get(rootTaskId)?.delete(fileId);
setTransfers((prev) => {
const parentRow = prev.find((row) => row.id === rootTaskId);
const parentFrozen = !!parentRow && (
parentRow.status === "paused"
|| parentRow.status === "pausing"
|| isPauseLatched(rootTaskId)
);
return prev.map((t) => {
if (t.id === fileId) {
return {
...t,
status: "completed" as TransferStatus,
endTime: Date.now(),
transferredBytes: t.totalBytes,
};
}
if (t.id === rootTaskId) {
if (parentFrozen) return { ...t, speed: 0 };
return { ...t, transferredBytes: t.transferredBytes + 1, speed: t.speed };
}
return t;
});
});
await waitWhileTransferPaused(rootTaskId);
} catch (err) {
activeChildIdsRef.current.get(rootTaskId)?.delete(fileId);
const message = err instanceof Error ? err.message : String(err);
if (isTransferCancelledError(err)) {
setTransfers((prev) => prev.map((t) => (
t.id === fileId
? { ...t, status: "cancelled" as TransferStatus, error: undefined, endTime: Date.now() }
: t
)));
errors.push(err instanceof Error ? err : new Error(message));
return;
}
setTransfers((prev) => prev.map((t) => (
t.id === fileId
? { ...t, status: "failed" as TransferStatus, error: message }
: t
)));
if (isSessionError(err) && !sessionLostError) {
sessionLostError = err instanceof Error ? err : new Error(message);
setTransfers((prev) => prev.map((t) => (
t.parentTaskId === rootTaskId
&& (t.status === "queued" || t.status === "pending")
? {
...t,
status: "failed" as TransferStatus,
error: "SFTP session lost - reconnect and resume remaining files",
speed: 0,
endTime: Date.now(),
}
: t
)));
}
errors.push(err instanceof Error ? err : new Error(message));
if (sessionLostError) throw sessionLostError;
if (isPauseLatched(rootTaskId)) {
await waitWhileTransferPaused(rootTaskId);
}
}
},
{
beforeClaim: async () => {
if (sessionLostError) return;
await waitWhileTransferPaused(rootTaskId);
},
},
).catch((err) => {
if (sessionLostError || isTransferCancelledError(err)) return;
throw err;
});
if (sessionLostError) {
logger.warn("[SFTP] Directory transfer stopped early: session lost", sessionLostError.message);
} else if (errors.length > 0) {
logger.debug?.("[SFTP] Some files in directory transfer failed", errors);
}
return errors.length;
}
@@ -146,9 +146,9 @@ test("directory transfer discovers each directory once with bounded listing conc
assert.equal(listCalls.size, directoryCount + 1);
assert.ok(Array.from(listCalls.values()).every((count) => count === 1));
assert.equal(listCalls.has("/source/loop"), false, "canonical symlink cycles must not be listed");
// Bounded parallel discovery (default 4) - never unbounded fan-out.
assert.ok(maxActiveListings >= 2, `expected parallel listings, got ${maxActiveListings}`);
assert.ok(maxActiveListings <= 4, `listing concurrency exceeded budget: ${maxActiveListings}`);
// Interleaved walk processes sibling subdirectories sequentially so resume
// manifests stay deterministic (no full-tree pre-scan fan-out).
assert.equal(maxActiveListings, 1, `expected sequential listings, got ${maxActiveListings}`);
assert.equal(tasks.find((task) => task.id === root.id)?.totalBytes, directoryCount);
} finally {
await act(async () => {
+7 -131
View File
@@ -11,7 +11,6 @@ import {
} from "../../../domain/sftpDirectoryCheckpoint";
import { isUnchangedTransferCandidate } from "../../../domain/sftpTransferSkip";
import {
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY,
} from "../../../infrastructure/config/storageKeys";
@@ -21,14 +20,9 @@ import { logger } from "../../../lib/logger";
import {
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
resolveSftpDirectoryListingConcurrency,
resolveSftpFolderPrescanEnabled,
resolveSftpSkipUnchangedEnabled,
runSftpTransferWorkers,
} from "./transferConcurrency";
import {
discoverTransferTree,
} from "./transferDirectoryDiscovery";
import { transferDiscoveredFiles } from "./transferDirectoryFileBatch";
import { getSftpTransferResourceKeys, globalSftpTransferScheduler } from "./globalTransferScheduler";
import { resolveDedicatedStreamEndpointIds } from "../../../domain/sftpDedicatedStreamPolicy";
import { isSessionError } from "./errors";
@@ -103,38 +97,10 @@ function isCancelledLocalOrGlobal(
return false;
}
function readFolderPrescanEnabled(): boolean {
return resolveSftpFolderPrescanEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN));
}
function readSkipUnchangedEnabled(): boolean {
return resolveSftpSkipUnchangedEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED));
}
async function ensureTransferDirectory(
targetPath: string,
targetIsLocal: boolean,
targetSftpId: string | null,
targetEncoding: SftpFilenameEncoding,
): Promise<void> {
if (targetIsLocal) {
try {
await netcattyBridge.get()?.mkdirLocal?.(targetPath);
} catch (mkdirErr: unknown) {
const isEEXIST = mkdirErr instanceof Error && mkdirErr.message.includes("EEXIST");
if (!isEEXIST) throw mkdirErr;
const stat = await netcattyBridge.get()?.statLocal?.(targetPath);
if (stat && stat.type !== "directory") {
throw new Error(`Target path exists as a file: ${targetPath}`);
}
}
return;
}
if (targetSftpId) {
await netcattyBridge.get()?.mkdirSftp(targetSftpId, targetPath, targetEncoding);
}
}
async function tryStatTransferPath(
filePath: string,
isLocal: boolean,
@@ -577,102 +543,10 @@ export function useSftpDirectoryTransferOps({
throw new Error("Transfer cancelled");
}
// Root-only pre-scan (rsync --no-inc-recursive): discover the full tree first
// so the transfer center can show a stable file total before bytes move.
if (!discoveryProgress && readFolderPrescanEnabled()) {
const skipUnchanged = readSkipUnchangedEnabled()
&& !task.replaceExistingTarget
&& !String(task.targetPath).includes(".netcatty-");
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
? {
...candidate,
phase: "scanning",
totalBytes: Math.max(candidate.transferredBytes, candidate.totalBytes || 0),
}
: candidate));
const traversal = traversalBudget ?? createSftpDirectoryTraversalBudget();
const discovered = await discoverTransferTree({
sourcePath: task.sourcePath,
targetPath: task.targetPath,
sourceIsLocal,
sourceSftpId,
sourceEncoding,
followSymlinks,
listLocalFiles,
listRemoteFiles,
traversalBudget: traversal,
shouldAbort: () => (
cancelledTasksRef.current.has(task.id)
|| cancelledTasksRef.current.has(rootTaskId)
),
waitWhilePaused: async () => {
await waitWhileTransferPaused(rootTaskId);
},
onDiscoveredFiles: (totalFiles) => {
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
? {
...candidate,
phase: "scanning",
totalBytes: Math.max(totalFiles, candidate.transferredBytes),
}
: candidate));
},
});
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
? {
...candidate,
totalBytes: Math.max(discovered.files.length, candidate.transferredBytes),
}
: candidate));
// Create directories breadth-first so parents exist before children.
for (const dir of discovered.directories) {
if (cancelledTasksRef.current.has(task.id) || cancelledTasksRef.current.has(rootTaskId)) {
throw new Error("Transfer cancelled");
}
await waitWhileTransferPaused(rootTaskId);
await ensureTransferDirectory(dir.targetPath, targetIsLocal, targetSftpId, targetEncoding);
}
const fileErrors = await transferDiscoveredFiles({
rootTask: task,
files: discovered.files,
sourceSftpId,
targetSftpId,
sourceIsLocal,
targetIsLocal,
sourceEncoding,
targetEncoding,
rootTaskId,
sameHost,
skipUnchanged,
startEntryIndex: 0,
cancelledTasksRef,
activeChildIdsRef,
transfersRef,
setTransfers,
waitWhileTransferPaused,
isPauseLatched,
transferFile,
tryStatTarget: (targetPath) => tryStatTransferTarget(
targetPath,
targetIsLocal,
targetSftpId,
targetEncoding,
),
tryStatSource: (sourcePath) => tryStatTransferPath(
sourcePath,
sourceIsLocal,
sourceSftpId,
sourceEncoding,
),
});
// Match interleaved walk: max-depth symlink omissions count as failures.
return fileErrors + discovered.omittedSymlinkDirectoryErrors;
}
// Always interleave discovery with transfer (rsync --inc-recursive style).
// Full-tree pre-scan was removed: non-compressed folder uploads start bytes
// as soon as each directory is listed; compressed upload is a separate path.
// UI should treat totalBytes as "discovered so far", not a fixed grand total.
let totalErrors = 0;
const progress = discoveryProgress ?? {
discoveredFiles: 0,
@@ -772,7 +646,9 @@ export function useSftpDirectoryTransferOps({
? {
...candidate,
totalBytes: Math.max(progress.discoveredFiles, candidate.transferredBytes),
phase: "scanning",
// Keep scanning only until the first file completes; later lists must
// not flip an in-progress folder bar back to indeterminate thrash.
...(candidate.transferredBytes <= 0 ? { phase: "scanning" as const } : null),
}
: candidate));
@@ -59,3 +59,96 @@ test("upload task callbacks write through the transfer store without page callba
assert.equal(patches[1].updates.transferredBytes, 100);
assert.deepEqual(dismissed, []);
});
test("progress promotes pending scanning folder rows into transferring", () => {
const patches: Array<{ taskId: string; updates: Partial<TransferTask> }> = [];
const liveTask: TransferTask = {
id: "folder-1",
fileName: "docs",
sourcePath: "local",
targetPath: "/remote/docs",
sourceConnectionId: "external",
targetConnectionId: "connection-1",
direction: "upload",
status: "pending",
totalBytes: 0,
transferredBytes: 0,
speed: 0,
startTime: 1,
isDirectory: true,
progressMode: "files",
phase: "scanning",
};
const store = {
upsertTasks: () => {},
patchTask: (taskId: string, updates: Partial<TransferTask>) => patches.push({ taskId, updates }),
dismiss: () => {},
getTask: (taskId: string) => (taskId === "folder-1" ? liveTask : undefined),
};
const callbacks = createUploadTaskCallbacks({
ownerId: "owner-1",
connectionId: "connection-1",
targetPath: "/remote",
store,
});
callbacks.onTaskProgress?.("folder-1", {
transferred: 12,
total: 400,
speed: 0,
percent: 3,
phase: "transferring",
});
assert.equal(patches[0].taskId, "folder-1");
assert.equal(patches[0].updates.status, "transferring");
assert.equal(patches[0].updates.transferredBytes, 12);
assert.equal(patches[0].updates.totalBytes, 400);
assert.equal(patches[0].updates.phase, "transferring");
});
test("scanning callbacks expose live file counts in files progress mode", () => {
const upserts: TransferTask[][] = [];
const patches: Array<{ taskId: string; updates: Partial<TransferTask> }> = [];
const store = {
upsertTasks: (tasks: readonly TransferTask[]) => upserts.push([...tasks]),
patchTask: (taskId: string, updates: Partial<TransferTask>) => patches.push({ taskId, updates }),
dismiss: () => {},
};
const callbacks = createUploadTaskCallbacks({
ownerId: "owner-1",
connectionId: "connection-1",
targetPath: "/remote",
store,
});
callbacks.onScanningStart?.("scan-1", { label: "docs" });
callbacks.onScanningProgress?.("scan-1", {
fileCount: 1284,
directoryCount: 40,
entryCount: 1324,
label: "docs",
});
assert.equal(upserts[0][0].fileName, "docs");
assert.equal(upserts[0][0].phase, "scanning");
assert.equal(upserts[0][0].progressMode, "files");
assert.equal(upserts[0][0].status, "pending");
assert.deepEqual(patches[0], {
taskId: "scan-1",
updates: {
// Found files live in totalBytes; completed stays 0 during scan.
transferredBytes: 0,
totalBytes: 1284,
progressMode: "files",
phase: "scanning",
fileName: "docs",
},
});
// Display contract: 0 done · N found (not N done · N found).
const discovered = Math.max(patches[0].updates.totalBytes ?? 0, patches[0].updates.transferredBytes ?? 0);
const completed = patches[0].updates.transferredBytes ?? 0;
assert.equal(completed, 0);
assert.equal(discovered, 1284);
});
+48 -4
View File
@@ -6,7 +6,7 @@ import { joinPath } from "./utils";
type UploadTransferStore = Pick<
typeof sftpTransferCenterStore,
"upsertTasks" | "patchTask" | "dismiss"
>;
> & Partial<Pick<typeof sftpTransferCenterStore, "getTask">>;
interface UploadTaskCallbacksParams {
ownerId: string;
@@ -27,11 +27,11 @@ export const createUploadTaskCallbacks = ({
targetConnectionKey,
store = sftpTransferCenterStore,
}: UploadTaskCallbacksParams): UploadCallbacks => ({
onScanningStart: (taskId: string) => {
onScanningStart: (taskId: string, info?: { label?: string }) => {
store.upsertTasks([{
id: taskId,
ownerId,
fileName: "Scanning files...",
fileName: info?.label?.trim() || "Scanning files...",
sourcePath: "local",
targetPath,
sourceConnectionId: "external",
@@ -42,18 +42,30 @@ export const createUploadTaskCallbacks = ({
targetConnectionKey,
direction: "upload",
status: "pending" as TransferStatus,
// Progressive discovery: totalBytes = found so far, completed stays 0
// until real transfers start (UI: "0 done · N found").
totalBytes: 0,
transferredBytes: 0,
speed: 0,
startTime: Date.now(),
isDirectory: true,
progressMode: "bytes",
progressMode: "files",
origin: "drag-drop",
background: false,
resumable: true,
phase: "scanning",
}]);
},
onScanningProgress: (taskId: string, progress) => {
store.patchTask(taskId, {
// Found count is the growing total; nothing is completed during scan.
totalBytes: Math.max(0, progress.fileCount),
transferredBytes: 0,
progressMode: "files",
phase: "scanning",
...(progress.label?.trim() ? { fileName: progress.label.trim() } : null),
});
},
onScanningEnd: (taskId: string) => {
store.dismiss(taskId);
},
@@ -90,8 +102,36 @@ export const createUploadTaskCallbacks = ({
const durableCheckpoint = Number.isFinite(Number(progress.checkpointBytes))
? Math.max(0, Math.trunc(Number(progress.checkpointBytes)))
: progress.transferred;
// Progressive folder walks keep the scanning row as pending until real
// progress arrives. Promote pending/queued → transferring so the panel
// leaves "Waiting..." and matches the transfer-center live state.
const current = "getTask" in store && typeof store.getTask === "function"
? store.getTask(taskId)
: undefined;
const shouldPromote =
!!current
&& (current.status === "pending" || current.status === "queued")
&& current.reconnectRequired !== true
&& (
progress.phase === "scanning"
|| progress.phase === "transferring"
|| progress.transferred > 0
|| progress.total > 0
);
// Only patch fingerprint/checkpoint while paused — do not keep animating
// high-water transferred after the user hit Pause.
const isPausedLike = current?.status === "paused" || current?.status === "pausing";
if (isPausedLike) {
store.patchTask(taskId, {
checkpointBytes: durableCheckpoint,
resumable: progress.resumable,
pauseUnavailableReason: progress.pauseUnavailableReason,
...("sourceFingerprint" in progress && progress.sourceFingerprint
? { sourceFingerprint: progress.sourceFingerprint as string }
: null),
});
return;
}
store.patchTask(taskId, {
transferredBytes: progress.transferred,
totalBytes: progress.total,
@@ -101,6 +141,7 @@ export const createUploadTaskCallbacks = ({
phase: progress.phase,
resumable: progress.resumable,
pauseUnavailableReason: progress.pauseUnavailableReason,
...(shouldPromote ? { status: "transferring" as TransferStatus } : null),
// Durable pause identity may arrive on a forced progress event while
// status is already paused — keep it for restart/resume safety.
...("sourceFingerprint" in progress && progress.sourceFingerprint
@@ -121,6 +162,7 @@ export const createUploadTaskCallbacks = ({
endTime: Date.now(),
transferredBytes: totalBytes,
speed: 0,
phase: undefined,
});
},
onTaskFailed: (taskId: string, error: string) => {
@@ -129,6 +171,7 @@ export const createUploadTaskCallbacks = ({
endTime: Date.now(),
error,
speed: 0,
phase: undefined,
});
},
onTaskCancelled: (taskId: string) => {
@@ -136,6 +179,7 @@ export const createUploadTaskCallbacks = ({
status: "cancelled" as TransferStatus,
endTime: Date.now(),
speed: 0,
phase: undefined,
});
},
});
@@ -15,11 +15,63 @@ import {
UploadResult,
startUploadScanningTask,
} from "../../../lib/uploadService";
import { extractDropEntries, type DropEntry } from "../../../lib/sftpFileUtils";
import { uploadLocalFoldersProgressively } from "../../../lib/progressiveFolderUpload";
import {
captureDropPayload,
formatDropScanLabel,
isDropScanCancelledError,
localTreeToDropEntries,
materializeDropEntries,
type DropEntry,
type LocalTreeListEntry,
} from "../../../lib/sftpFileUtils";
function isUploadScanCancelled(controller: UploadController, error?: unknown): boolean {
return controller.isCancelled() || isDropScanCancelledError(error);
}
// Re-export UploadResult for external usage
export type { UploadResult };
type LocalTreeScanOptions = {
onProgress?: (progress: { fileCount: number; directoryCount: number; entryCount: number }) => void;
onEntries?: (entries: LocalTreeListEntry[]) => void;
abortSignal?: AbortSignal;
};
function createDropScanCancelledError(): Error {
const error = new Error("Drop scan cancelled");
(error as Error & { code?: string }).code = "ERR_DROP_SCAN_CANCELLED";
return error;
}
async function listLocalTreeWithAbort(
bridge: NetcattyBridge,
localPath: string,
options: LocalTreeScanOptions = {},
): Promise<LocalTreeListEntry[]> {
const scanId = `drop-scan-${crypto.randomUUID()}`;
const abortSignal = options.abortSignal;
const requestCancel = () => {
void bridge.cancelLocalTreeScan?.(scanId);
};
if (abortSignal?.aborted) {
throw createDropScanCancelledError();
}
abortSignal?.addEventListener("abort", requestCancel, { once: true });
const { abortSignal: _abortSignal, ...bridgeOptions } = options;
try {
return await bridge.listLocalTree!(localPath, { ...bridgeOptions, scanId });
} catch (error) {
if (abortSignal?.aborted || isDropScanCancelledError(error)) {
throw createDropScanCancelledError();
}
throw error;
} finally {
abortSignal?.removeEventListener("abort", requestCancel);
}
}
import type { UseSftpExternalOperationsParams, SftpExternalOperationsResult } from "./useSftpExternalOperations.types";
import { getSftpTransferResourceKeys, globalSftpTransferScheduler } from "./globalTransferScheduler";
import { localStorageAdapter } from "../../../infrastructure/persistence/localStorageAdapter";
@@ -48,6 +100,10 @@ import {
registerExternalUploadController,
unregisterExternalUploadController,
} from "./externalUploadRuntime";
import {
isTransferPauseLatched,
waitWhileTransferOrRootPaused,
} from "./transferPauseLatch";
type UploadConflictResolver = {
resolve: (action: FileConflictAction) => void;
@@ -151,9 +207,12 @@ export const useSftpExternalOperations = (
callbacks: UploadCallbacks,
): UploadCallbacks => ({
...callbacks,
onScanningStart: (taskId) => {
onScanningStart: (taskId, info) => {
registerUploadController(taskId, controller);
callbacks.onScanningStart?.(taskId);
callbacks.onScanningStart?.(taskId, info);
},
onScanningProgress: (taskId, progress) => {
callbacks.onScanningProgress?.(taskId, progress);
},
onTaskCreated: (task) => {
registerUploadController(task.id, controller);
@@ -764,9 +823,12 @@ export const useSftpExternalOperations = (
conflictDefaults.set(conflictType, action);
},
});
controller.addCancelListener(() => {
cancelPendingUploadConflicts(controller);
});
});
};
}, []);
}, [cancelPendingUploadConflicts]);
// Create upload bridge that wraps netcattyBridge.
// Pass connect-time Host so pooled stream uploads open the pinned endpoint
@@ -888,52 +950,271 @@ export const useSftpExternalOperations = (
const uploadExternalFiles = useCallback(
async (side: "left" | "right", dataTransfer: DataTransfer, targetPath?: string): Promise<UploadResult[]> => {
// DataTransfer is only valid during the drop event. Capture entries BEFORE
// DataTransfer is only valid during the drop event. Capture roots BEFORE
// any await (session reconnect can take seconds while a transfer is busy).
// Otherwise extractDropEntries returns [] and the UI toasts "Uploaded files: 0".
let capturedEntries: DropEntry[];
try {
capturedEntries = await extractDropEntries(dataTransfer);
} catch (error) {
logger.error("[SFTP] Failed to read dropped files:", error);
throw error;
}
if (capturedEntries.length === 0) {
// Native tree expansion (listLocalTree) happens after the scanning UI is up.
const dropPayload = captureDropPayload(dataTransfer);
if (dropPayload.roots.length === 0 && dropPayload.filesFallback.length === 0) {
return [];
}
const pane = getActivePane(side);
if (!pane?.connection) {
throw new Error("No active connection");
}
const bridge = netcattyBridge.get();
if (!bridge) {
throw new Error("Bridge not available");
}
const uploadTargetPath = targetPath || pane.connection.currentPath;
const controller = new UploadController();
const callbacks = bindUploadControllerCallbacks(
controller,
createUploadCallbacks(
pane.connection.id,
uploadTargetPath,
pane.connection.isLocal ? undefined : pane.connection.hostId,
pane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(pane.connection.id),
pane.connection.isLocal ? undefined : pane.connection.hostLabel,
),
);
const pathBackedFolderRoots = dropPayload.roots
.filter((root) => root.isDirectory && !!root.localPath)
.map((root) => ({ name: root.name, localPath: root.localPath! }));
// Progressive only when *every* root is a path-backed folder. Mixed drops
// (folder + files, path-less dirs) must fall through to materialize so
// sibling items are not silently dropped.
const canProgressiveUpload = pathBackedFolderRoots.length > 0
&& pathBackedFolderRoots.length === dropPayload.roots.length
&& !!bridge.listLocalTree
&& !useCompressedUpload;
const needsDeepScan = dropPayload.roots.some((root) => root.isDirectory);
const scanLabel = formatDropScanLabel(dropPayload.roots);
// Instant feedback: scanning / parent row appears before work begins.
const scanningTask = needsDeepScan
? startUploadScanningTask(callbacks, crypto.randomUUID(), { label: scanLabel })
: null;
if (scanningTask && typeof window !== "undefined") {
window.dispatchEvent(new CustomEvent("netcatty:open-sftp-transfer-center"));
}
const scanAbort = new AbortController();
const detachScanCancel = controller.addCancelListener(() => {
scanAbort.abort();
if (scanningTask?.isOpen()) scanningTask.cancel();
});
// Path-backed folders (no compressed upload): edge-scan + edge-transfer.
// Open the SFTP session first, then stream discovery batches into upload
// workers so the first files start while the rest of the tree is still walking.
if (canProgressiveUpload) {
const runProgressive = async (forceReconnect = false): Promise<UploadResult[]> => {
const { sftpId, release } = await resolveRemoteSftpId(side, { forceReconnect });
const livePane = getActivePane(side) ?? pane;
if (!livePane.connection) {
release();
throw new Error("No active connection");
}
const uploadPaneId = livePane.id;
const liveTargetPath = targetPath || livePane.connection.currentPath;
const connectHost = resolveUploadConnectHost(uploadPaneId, livePane.connection.isLocal);
const uploadBridge = createUploadBridge(connectHost);
const liveCallbacks = livePane.connection.id === pane.connection.id
&& liveTargetPath === uploadTargetPath
? callbacks
: bindUploadControllerCallbacks(
controller,
createUploadCallbacks(
livePane.connection.id,
liveTargetPath,
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
livePane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(livePane.connection.id),
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
),
);
const parentTaskIds = new Map<string, string>();
if (scanningTask && pathBackedFolderRoots.length === 1) {
parentTaskIds.set(pathBackedFolderRoots[0].name, scanningTask.taskId);
// Keep the scanning row as the folder parent (do not dismiss it).
scanningTask.complete = () => {
/* no-op: progressive path settles via onTaskCompleted/Cancelled */
};
} else if (scanningTask?.isOpen()) {
scanningTask.complete();
}
try {
if (controller.isCancelled()) {
scanningTask?.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
const results = await uploadLocalFoldersProgressively(
pathBackedFolderRoots,
{
targetPath: liveTargetPath,
sftpId,
targetHostId: livePane.connection.isLocal ? undefined : livePane.connection.hostId,
isLocal: livePane.connection.isLocal,
bridge: uploadBridge,
joinPath,
callbacks: liveCallbacks,
parentTaskIds,
abortSignal: scanAbort.signal,
// Soft-pause must freeze discovery enqueue + child UI rows, not
// only the open streams (otherwise Pause still floods the queue).
waitWhilePaused: (parentTaskId) => waitWhileTransferOrRootPaused(parentTaskId),
isPaused: (parentTaskId) => isTransferPauseLatched(parentTaskId),
resolveConflict: createUploadConflictResolver(controller),
listLocalTree: (localPath, treeOptions) => listLocalTreeWithAbort(bridge, localPath, {
...treeOptions,
abortSignal: scanAbort.signal,
}),
},
controller,
);
if (clearDirCacheEntry && targetPath) {
clearDirCacheEntry(livePane.connection.id, liveTargetPath);
}
if (liveTargetPath === livePane.connection.currentPath) {
await refresh(side, { tabId: uploadPaneId });
}
return results;
} finally {
release();
}
};
try {
return await runProgressive(false);
} catch (error) {
if (isSessionError(error) && ensureRemoteSftpId) {
logger.warn("[SFTP] Progressive upload session lost; reconnecting once", error);
try {
return await runProgressive(true);
} catch (retryError) {
if (isUploadScanCancelled(controller, retryError)) {
scanningTask?.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
if (scanningTask?.isOpen()) scanningTask.fail(retryError);
throw retryError;
}
}
if (isUploadScanCancelled(controller, error)) {
scanningTask?.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
if (scanningTask?.isOpen()) scanningTask.fail(error);
logger.error("[SFTP] Progressive folder upload failed:", error);
throw error;
} finally {
detachScanCancel();
unregisterUploadController(controller);
}
}
let capturedEntries: DropEntry[] = [];
try {
const scanT0 = performance.now();
capturedEntries = await materializeDropEntries(dropPayload, {
abortSignal: scanAbort.signal,
isCancelled: () => controller.isCancelled(),
listLocalTree: bridge.listLocalTree
? (localPath, treeOptions) => listLocalTreeWithAbort(bridge, localPath, {
...treeOptions,
abortSignal: scanAbort.signal,
})
: undefined,
onProgress: scanningTask
? (progress) => {
callbacks.onScanningProgress?.(scanningTask.taskId, {
...progress,
label: progress.label || scanLabel,
});
}
: undefined,
});
logger.debug(
`[SFTP:perf] materializeDropEntries — ${capturedEntries.length} entries — ${(performance.now() - scanT0).toFixed(0)}ms`,
);
} catch (error) {
if (scanningTask?.isOpen()) {
if (isUploadScanCancelled(controller, error)) {
scanningTask.cancel();
} else {
scanningTask.fail(error);
}
}
detachScanCancel();
unregisterUploadController(controller);
if (isUploadScanCancelled(controller, error)) {
return [{ fileName: "", success: false, cancelled: true }];
}
logger.error("[SFTP] Failed to read dropped files:", error);
throw error;
}
if (controller.isCancelled()) {
scanningTask?.cancel();
detachScanCancel();
unregisterUploadController(controller);
return [{ fileName: "", success: false, cancelled: true }];
}
if (capturedEntries.length === 0) {
scanningTask?.complete();
detachScanCancel();
unregisterUploadController(controller);
return [];
}
// Keep scanning visible through session open so the UI never goes blank
// between "N files found" and the real transfer rows.
const run = async (forceReconnect = false): Promise<UploadResult[]> => {
const pane = getActivePane(side);
if (!pane?.connection) {
const { sftpId, release } = await resolveRemoteSftpId(side, { forceReconnect });
const livePane = getActivePane(side) ?? pane;
if (!livePane.connection) {
release();
throw new Error("No active connection");
}
const bridge = netcattyBridge.get();
if (!bridge) {
throw new Error("Bridge not available");
}
const { sftpId, release } = await resolveRemoteSftpId(side, { forceReconnect });
const livePane = getActivePane(side) ?? pane;
if (!livePane.connection) throw new Error("No active connection");
const uploadPaneId = livePane.id;
const uploadTargetPath = targetPath || livePane.connection.currentPath;
const controller = new UploadController();
const callbacks = bindUploadControllerCallbacks(
controller,
createUploadCallbacks(
livePane.connection.id,
uploadTargetPath,
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
livePane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(livePane.connection.id),
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
),
);
const liveTargetPath = targetPath || livePane.connection.currentPath;
const connectHost = resolveUploadConnectHost(uploadPaneId, livePane.connection.isLocal);
const uploadBridge = createUploadBridge(connectHost);
// Retarget callbacks if the pane path/id changed after reconnect.
const liveCallbacks = livePane.connection.id === pane.connection.id
&& liveTargetPath === uploadTargetPath
? callbacks
: bindUploadControllerCallbacks(
controller,
createUploadCallbacks(
livePane.connection.id,
liveTargetPath,
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
livePane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(livePane.connection.id),
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
),
);
const transferCallbacks = scanningTask
? {
...liveCallbacks,
onTaskCreated: (task: Parameters<NonNullable<UploadCallbacks["onTaskCreated"]>>[0]) => {
scanningTask.complete();
liveCallbacks.onTaskCreated?.(task);
},
}
: liveCallbacks;
let transferSucceeded = false;
try {
if (controller.isCancelled()) {
scanningTask?.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
const hasDirectory = capturedEntries.some((entry) => (
entry.isDirectory || entry.relativePath.replace(/\\/g, "/").includes("/")
));
@@ -951,13 +1232,13 @@ export const useSftpExternalOperations = (
run: async (uploadSftpId) => uploadEntriesDirect(
capturedEntries,
{
targetPath: uploadTargetPath,
targetPath: liveTargetPath,
sftpId: uploadSftpId,
targetHostId: livePane.connection!.isLocal ? undefined : livePane.connection!.hostId,
isLocal: livePane.connection!.isLocal,
bridge: uploadBridge,
joinPath,
callbacks,
callbacks: transferCallbacks,
useCompressedUpload,
resolveConflict: createUploadConflictResolver(controller),
},
@@ -966,15 +1247,19 @@ export const useSftpExternalOperations = (
});
if (clearDirCacheEntry && targetPath) {
clearDirCacheEntry(livePane.connection.id, uploadTargetPath);
clearDirCacheEntry(livePane.connection.id, liveTargetPath);
}
if (uploadTargetPath === livePane.connection.currentPath) {
if (liveTargetPath === livePane.connection.currentPath) {
await refresh(side, { tabId: uploadPaneId });
}
transferSucceeded = true;
return results;
} finally {
if (scanningTask?.isOpen()) {
if (controller.isCancelled()) scanningTask.cancel();
else if (transferSucceeded) scanningTask.complete();
}
release();
unregisterUploadController(controller);
}
};
@@ -983,10 +1268,26 @@ export const useSftpExternalOperations = (
} catch (error) {
if (isSessionError(error) && ensureRemoteSftpId) {
logger.warn("[SFTP] Upload session lost; reconnecting and retrying once", error);
return await run(true);
try {
return await run(true);
} catch (retryError) {
if (isUploadScanCancelled(controller, retryError)) {
scanningTask?.cancel();
} else if (scanningTask?.isOpen()) {
scanningTask.fail(retryError);
}
throw retryError;
}
}
if (scanningTask?.isOpen()) {
if (controller.isCancelled()) scanningTask.cancel();
else scanningTask.fail(error);
}
logger.error("[SFTP] Upload failed:", error);
throw error;
} finally {
detachScanCancel();
unregisterUploadController(controller);
}
},
[
@@ -1146,105 +1447,125 @@ export const useSftpExternalOperations = (
connectionCacheKeyMapRef.current,
);
const initialUploadTargetPath = targetPath || originatingPane.connection.currentPath;
const controller = new UploadController();
const callbacks = bindUploadControllerCallbacks(
controller,
createUploadCallbacks(
originatingPane.connection.id,
initialUploadTargetPath,
originatingPane.connection.isLocal ? undefined : originatingPane.connection.hostId,
originatingPane.connection.isLocal
? undefined
: connectionCacheKeyMapRef.current.get(originatingPane.connection.id),
originatingPane.connection.isLocal ? undefined : originatingPane.connection.hostLabel,
),
);
const folderName = folderPath.replace(/\\/g, "/").split("/").filter(Boolean).pop()
|| folderPath;
const scanningTask = startUploadScanningTask(callbacks, crypto.randomUUID(), {
label: folderName,
});
if (typeof window !== "undefined") {
window.dispatchEvent(new CustomEvent("netcatty:open-sftp-transfer-center"));
}
const scanAbort = new AbortController();
const detachScanCancel = controller.addCancelListener(() => {
scanAbort.abort();
if (scanningTask.isOpen()) scanningTask.cancel();
});
let capturedEntries: DropEntry[] | null = null;
const run = async (forceReconnect = false): Promise<UploadResult[]> => {
const pane = resolveUploadTargetPane({
side,
tabId: originatingTabId,
getActivePane,
getPaneByTabId,
getPaneByConnectionId,
});
assertUploadEndpointUnchanged(
pane.connection,
originatingEndpoint,
connectionCacheKeyMapRef.current,
);
const bridge = netcattyBridge.get();
if (!bridge) throw new Error("Bridge not available");
if (!bridge.listLocalTree) throw new Error("Folder upload not supported");
const { sftpId, release } = await resolveRemoteSftpId(side, {
forceReconnect,
connectionId: pane.connection.id,
tabId: originatingTabId,
});
// Never re-resolve via getActivePane after awaits — focus may have moved.
const livePane = resolveUploadTargetPane({
side,
tabId: originatingTabId,
getActivePane,
getPaneByTabId,
getPaneByConnectionId,
});
assertUploadEndpointUnchanged(
livePane.connection,
originatingEndpoint,
connectionCacheKeyMapRef.current,
);
const uploadPaneId = livePane.id;
const uploadTargetPath = targetPath || livePane.connection.currentPath;
const controller = new UploadController();
const callbacks = bindUploadControllerCallbacks(
controller,
createUploadCallbacks(
livePane.connection.id,
uploadTargetPath,
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
livePane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(livePane.connection.id),
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
),
);
// Pin connect-time Host before listLocalTree: a slow folder scan can
// outlive a same-hostId tab rebind, and resolveUploadConnectHost would
// otherwise open the pooled stream bridge on the newly selected endpoint.
const uploadConnectHost = resolveUploadConnectHost(
uploadPaneId,
livePane.connection.isLocal,
);
const uploadBridge = createUploadBridge(uploadConnectHost);
const scanningTask = startUploadScanningTask(callbacks);
if (controller.isCancelled()) {
scanningTask.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
let release = () => {};
try {
const localEntries = await bridge.listLocalTree(folderPath);
const pane = resolveUploadTargetPane({
side,
tabId: originatingTabId,
getActivePane,
getPaneByTabId,
getPaneByConnectionId,
});
assertUploadEndpointUnchanged(
pane.connection,
originatingEndpoint,
connectionCacheKeyMapRef.current,
);
const bridge = netcattyBridge.get();
if (!bridge) throw new Error("Bridge not available");
if (!bridge.listLocalTree) throw new Error("Folder upload not supported");
const resolved = await resolveRemoteSftpId(side, {
forceReconnect,
connectionId: pane.connection.id,
tabId: originatingTabId,
});
const sftpId = resolved.sftpId;
release = resolved.release;
// Never re-resolve via getActivePane after awaits — focus may have moved.
const livePane = resolveUploadTargetPane({
side,
tabId: originatingTabId,
getActivePane,
getPaneByTabId,
getPaneByConnectionId,
});
assertUploadEndpointUnchanged(
livePane.connection,
originatingEndpoint,
connectionCacheKeyMapRef.current,
);
const uploadPaneId = livePane.id;
const uploadTargetPath = targetPath || livePane.connection.currentPath;
// Pin connect-time Host before listLocalTree: a slow folder scan can
// outlive a same-hostId tab rebind, and resolveUploadConnectHost would
// otherwise open the pooled stream bridge on the newly selected endpoint.
const uploadConnectHost = resolveUploadConnectHost(
uploadPaneId,
livePane.connection.isLocal,
);
const uploadBridge = createUploadBridge(uploadConnectHost);
const liveCallbacks = bindUploadControllerCallbacks(
controller,
createUploadCallbacks(
livePane.connection.id,
uploadTargetPath,
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
livePane.connection.isLocal
? undefined
: connectionCacheKeyMapRef.current.get(livePane.connection.id),
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
),
);
const transferCallbacks = {
...liveCallbacks,
onTaskCreated: (task: Parameters<NonNullable<UploadCallbacks["onTaskCreated"]>>[0]) => {
scanningTask.complete();
liveCallbacks.onTaskCreated?.(task);
},
};
if (capturedEntries === null) {
const localEntries = await listLocalTreeWithAbort(bridge, folderPath, {
abortSignal: scanAbort.signal,
onProgress: (progress) => {
callbacks.onScanningProgress?.(scanningTask.taskId, {
...progress,
label: folderName,
});
},
});
capturedEntries = localTreeToDropEntries(localEntries);
}
if (controller.isCancelled()) {
scanningTask.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
scanningTask.complete();
const entries: DropEntry[] = localEntries.map((entry) => {
if (entry.type === "directory") {
return {
file: null,
relativePath: entry.relativePath,
isDirectory: true,
};
}
const file = {
name: entry.relativePath.split("/").pop() || entry.relativePath,
size: entry.size,
lastModified: entry.lastModified,
type: "",
path: entry.localPath,
arrayBuffer: async () => {
const currentBridge = netcattyBridge.get();
if (!currentBridge?.readLocalFile) {
throw new Error("Local file reading not supported");
}
return currentBridge.readLocalFile(entry.localPath);
},
} as File & { path?: string };
return {
file,
relativePath: entry.relativePath,
isDirectory: false,
};
});
const results = await runWithCompressedUploadSession({
enabled: useCompressedUpload,
@@ -1258,7 +1579,7 @@ export const useSftpExternalOperations = (
: undefined,
shouldDiscard: isSessionError,
run: async (uploadSftpId) => uploadEntriesDirect(
entries,
capturedEntries,
{
targetPath: uploadTargetPath,
sftpId: uploadSftpId,
@@ -1266,7 +1587,7 @@ export const useSftpExternalOperations = (
isLocal: livePane.connection!.isLocal,
bridge: uploadBridge,
joinPath,
callbacks,
callbacks: transferCallbacks,
useCompressedUpload,
resolveConflict: createUploadConflictResolver(controller),
},
@@ -1282,18 +1603,8 @@ export const useSftpExternalOperations = (
await refresh(refreshSide, { tabId: uploadPaneId });
}
return results;
} catch (error) {
if (controller.isCancelled()) {
scanningTask.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
if (scanningTask.isOpen()) {
scanningTask.fail(error);
}
throw error;
} finally {
release();
unregisterUploadController(controller);
}
};
@@ -1302,11 +1613,33 @@ export const useSftpExternalOperations = (
} catch (error) {
if (isSessionError(error) && ensureRemoteSftpId) {
logger.warn("[SFTP] Folder upload session lost; reconnecting and retrying once", error);
return await run(true);
try {
return await run(true);
} catch (retryError) {
if (isUploadScanCancelled(controller, retryError)) {
scanningTask.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
if (scanningTask.isOpen()) scanningTask.fail(retryError);
throw retryError;
}
}
if (isUploadScanCancelled(controller, error)) {
scanningTask.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
if (scanningTask.isOpen()) scanningTask.fail(error);
logger.error("[SFTP] Folder picker upload failed:", error);
throw error;
}
finally {
detachScanCancel();
unregisterUploadController(controller);
if (scanningTask.isOpen()) {
if (controller.isCancelled()) scanningTask.cancel();
else scanningTask.complete();
}
}
},
[
acquireTransferSession,
+67 -3
View File
@@ -60,6 +60,10 @@ import {
} from "./transferConflictLifecycle";
import { getParentPath, joinPath } from "./utils";
import { promoteDirectoryReplaceStage as promoteDirectoryReplacePaths } from "./directoryReplacePromotion";
import {
isExternalDragDropFileUpload,
retryExternalDragDropFileUpload,
} from "./externalDragDropRetry";
/** Keep the MutableRefObject mirror in sync with the process-global latch set. */
function syncPausedTasksRef(ref: { current: Set<string> }, taskId: string, latched: boolean) {
@@ -1294,8 +1298,62 @@ export const useSftpTransfers = ({
const retryTransfer = useCallback(
async (transferId: string) => {
const task = transfersRef.current.find((t) => t.id === transferId);
// Prefer the live owner list; fall back to the center store (drag-drop
// children are upserted there and mirrored via subscribe).
const task = transfersRef.current.find((t) => t.id === transferId)
?? sftpTransferCenterStore.getTask(transferId);
if (!task || task.retryable === false) return;
// Progressive / external drag-drop children never have dual-pane endpoints
// (sourceConnectionId is "external"). Re-run startStreamTransfer in place.
if (isExternalDragDropFileUpload(task)) {
if (inFlightTransferIdsRef.current.has(transferId)) return;
inFlightTransferIdsRef.current.add(transferId);
clearTransferCancelled(transferId);
cancelledTasksRef.current.delete(transferId);
try {
const result = await retryExternalDragDropFileUpload(task, {
getBrowseSftpId: (connectionId) => sftpSessionsRef.current.get(connectionId),
acquireTransferSession: acquireTransferSession
? (hostId, id) => acquireTransferSession(hostId, id)
: undefined,
startStreamTransfer: async (options) => {
const bridge = netcattyBridge.get();
if (!bridge?.startStreamTransfer) {
return { error: "Stream transfer is unavailable" };
}
return bridge.startStreamTransfer(options);
},
clearPendingCancel: (id) => netcattyBridge.get()?.clearPendingTransferCancel?.(id),
cleanupArtifacts: cleanupTaskArtifacts,
getTask: (id) => sftpTransferCenterStore.getTask(id)
?? transfersRef.current.find((row) => row.id === id),
getChildTasks: (parentId) => {
const fromStore = sftpTransferCenterStore.getOwnerTasks(ownerId)
.filter((row) => row.parentTaskId === parentId);
if (fromStore.length > 0) return fromStore;
return transfersRef.current.filter((row) => row.parentTaskId === parentId);
},
onPatch: (taskId, updates) => {
transferRuntime.patchTask(taskId, updates);
setTransfers((prev) => {
if (!prev.some((row) => row.id === taskId)) {
// Store-only child: mirror the patch by re-pulling owner rows.
return sftpTransferCenterStore.getOwnerTasks(ownerId);
}
return prev.map((row) => (row.id === taskId ? { ...row, ...updates } : row));
});
},
});
if (!result.success && result.error && !/cancelled/i.test(result.error)) {
notify.warning(result.error, "SFTP");
}
} finally {
inFlightTransferIdsRef.current.delete(transferId);
}
return;
}
await cleanupTaskArtifacts(task);
const retriedTask: TransferTask = {
@@ -1316,7 +1374,13 @@ export const useSftpTransfers = ({
};
const endpoints = resolveTaskEndpoints(task);
if (!endpoints) return;
if (!endpoints) {
notify.warning(
"Could not resolve transfer endpoints for retry. Reconnect the target and try again.",
"SFTP",
);
return;
}
const { targetSide, sourcePane, targetPane } = endpoints;
const completionHandler = completionHandlersRef.current.get(transferId);
@@ -1335,7 +1399,7 @@ export const useSftpTransfers = ({
await processTransfer(retriedTask, sourcePane, targetPane, targetSide);
},
// eslint-disable-next-line react-hooks/exhaustive-deps -- processTransfer is defined inline
[cleanupTaskArtifacts, resolveTaskEndpoints, setTransfers],
[acquireTransferSession, cleanupTaskArtifacts, ownerId, resolveTaskEndpoints, setTransfers, sftpSessionsRef],
);
const clearCompletedTransfers = useCallback(() => {
+3 -18
View File
@@ -35,7 +35,6 @@ import {
STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR,
STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD,
STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY,
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
STORAGE_KEY_SSH_TRANSPORT_IDLE_TTL_MS,
STORAGE_KEY_SFTP_TRANSFER_POOL_IDLE_TTL_MS,
@@ -88,7 +87,6 @@ import { localStorageAdapter } from '../../infrastructure/persistence/localStora
import { netcattyBridge } from '../../infrastructure/services/netcattyBridge';
import {
resolveSftpTransferConcurrency,
resolveSftpFolderPrescanEnabled,
resolveSftpSkipUnchangedEnabled,
} from './sftp/transferConcurrency';
import { resolveSshTransportIdleTtlMs } from '../../infrastructure/config/sshTransportIdleTtl';
@@ -426,9 +424,6 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
// Folder transfer concurrency is renderer-only (runSftpTransferWorkers).
// Do not push it into main-process host admission - that made multi-select
// top-level files queue against each other and against folder children.
const [sftpFolderPrescan, setSftpFolderPrescan] = useState<boolean>(() =>
resolveSftpFolderPrescanEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN)),
);
const [sftpSkipUnchanged, setSftpSkipUnchanged] = useState<boolean>(() =>
resolveSftpSkipUnchangedEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED)),
);
@@ -902,8 +897,6 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
if (storedHidden === 'true' || storedHidden === 'false') setSftpShowHiddenFiles(storedHidden === 'true');
const storedCompress = readStoredString(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD);
if (storedCompress === 'true' || storedCompress === 'false') setSftpUseCompressedUpload(storedCompress === 'true');
const storedFolderPrescan = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN);
if (storedFolderPrescan != null) setSftpFolderPrescan(storedFolderPrescan);
const storedSkipUnchanged = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED);
if (storedSkipUnchanged != null) setSftpSkipUnchanged(storedSkipUnchanged);
const storedAutoOpenSidebar = readStoredString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR);
@@ -1137,7 +1130,7 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
@@ -1147,7 +1140,7 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
setTerminalThemeId, setTerminalThemeDarkId, setTerminalThemeLightId,
setFollowAppTerminalThemeState, setTerminalFontFamilyId, setTerminalFontSize: applyIncomingTerminalFontSize,
setSftpDoubleClickBehavior, setSftpAutoSync, setSftpShowHiddenFiles,
setSftpUseCompressedUpload, setSftpFolderPrescan, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
setSftpUseCompressedUpload, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
setShowRecentHostsState, setHostClickBehaviorState, setShowOnlyUngroupedHostsInRootState, setShowSftpTabState, setShowHostTreeSidebarState, setTerminalSidePanelAutoOpenState, setTerminalSidePanelAutoOpenTabState, setShellOnlyTabNumberShortcutsState, setDisableTerminalFontZoomState, setRestorePreviousSessionState, setRestoreTerminalCwdState,
setEditorWordWrapState, setSessionLogsEnabled, setSessionLogsDir, setSessionLogsFormat, setSessionLogsTimestampsEnabled, setSshDebugLogsEnabled, setSshDeepLinkEnabledState: applyIncomingSshDeepLinkEnabled, setJmsDeepLinkEnabledState: applyIncomingJmsDeepLinkEnabled, setExplorerContextMenuEnabledState: applyIncomingExplorerContextMenuEnabled,
setGlobalHotkeyEnabled, setWindowOpacity: applyIncomingWindowOpacity, setAppIconVariant, setAutoUpdateEnabled, setWorkspaceFocusStyleState,
@@ -1375,12 +1368,6 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
notifySettingsChanged(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD, sftpUseCompressedUpload);
}, [sftpUseCompressedUpload, notifySettingsChanged]);
useEffect(() => {
localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN, sftpFolderPrescan);
if (!persistMountedRef.current) return;
notifySettingsChanged(STORAGE_KEY_SFTP_FOLDER_PRESCAN, sftpFolderPrescan);
}, [sftpFolderPrescan, notifySettingsChanged]);
useEffect(() => {
localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED, sftpSkipUnchanged);
if (!persistMountedRef.current) return;
@@ -1772,8 +1759,6 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
setSftpShowHiddenFiles,
sftpUseCompressedUpload,
setSftpUseCompressedUpload,
sftpFolderPrescan,
setSftpFolderPrescan,
sftpSkipUnchanged,
setSftpSkipUnchanged,
sftpAutoOpenSidebar,
@@ -1860,7 +1845,7 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
uiFontFamilyId, uiLanguage, customCSS,
terminalThemeId, terminalFontFamilyId, terminalFontSize, terminalSettings,
customKeyBindings, editorWordWrap,
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles, sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles, sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom,
customThemes, workspaceFocusStyle, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
]),
-5
View File
@@ -74,7 +74,6 @@ import {
STORAGE_KEY_SFTP_AUTO_SYNC,
STORAGE_KEY_SFTP_SHOW_HIDDEN_FILES,
STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD,
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR,
STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD,
@@ -288,7 +287,6 @@ export const SYNCABLE_SETTING_STORAGE_KEYS = [
STORAGE_KEY_SFTP_AUTO_SYNC,
STORAGE_KEY_SFTP_SHOW_HIDDEN_FILES,
STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD,
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR,
STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD,
@@ -494,8 +492,6 @@ export function collectSyncableSettings(): SyncPayload['settings'] {
if (hidden === 'true' || hidden === 'false') settings.sftpShowHiddenFiles = hidden === 'true';
const compress = localStorageAdapter.readString(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD);
if (compress === 'true' || compress === 'false') settings.sftpUseCompressedUpload = compress === 'true';
const folderPrescan = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN);
if (folderPrescan != null) settings.sftpFolderPrescan = folderPrescan;
const skipUnchanged = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED);
if (skipUnchanged != null) settings.sftpSkipUnchanged = skipUnchanged;
const autoOpenSidebar = localStorageAdapter.readString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR);
@@ -734,7 +730,6 @@ async function applySyncableSettings(settings: NonNullable<SyncPayload['settings
if (settings.sftpAutoSync != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_AUTO_SYNC, String(settings.sftpAutoSync));
if (settings.sftpShowHiddenFiles != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_SHOW_HIDDEN_FILES, String(settings.sftpShowHiddenFiles));
if (settings.sftpUseCompressedUpload != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD, String(settings.sftpUseCompressedUpload));
if (settings.sftpFolderPrescan != null) localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN, settings.sftpFolderPrescan);
if (settings.sftpSkipUnchanged != null) localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED, settings.sftpSkipUnchanged);
if (settings.sftpAutoOpenSidebar != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR, String(settings.sftpAutoOpenSidebar));
if (settings.sftpFollowTerminalCwd != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD, String(settings.sftpFollowTerminalCwd));
+26 -2
View File
@@ -129,8 +129,11 @@ test("directory parent detection matches side-queue file-count semantics", () =>
}), false);
});
test("folder parent progress uses file counts not byte formatting", () => {
test("folder parent progress uses progressive discovery labels not byte formatting", () => {
const t = (key: string, params?: Record<string, string | number>) => {
if (key === "sftp.transfers.filesDiscoveredProgress") {
return `${params?.completed} done · ${params?.discovered} found`;
}
if (key === "sftp.transfers.filesProgress") {
return `${params?.current}/${params?.total} files`;
}
@@ -146,14 +149,35 @@ test("folder parent progress uses file counts not byte formatting", () => {
totalBytes: 12,
transferredBytes: 1,
speed: 0,
phase: "transferring",
}, t);
assert.equal(display.percent, (1 / 12) * 100);
assert.equal(display.detail, "1/12 files");
assert.equal(display.detail, "1 done · 12 found");
assert.equal(display.indeterminate, false);
// Must not look like "1 Bytes / 12 Bytes"
assert.doesNotMatch(display.detail, /Bytes/i);
});
test("folder parent scanning stays indeterminate while nothing has completed", () => {
const t = (key: string, params?: Record<string, string | number>) => {
if (key === "sftp.transfers.filesDiscoveredProgress") {
return `${params?.completed} done · ${params?.discovered} found`;
}
return key;
};
const display = buildGlobalTransferProgressDisplay({
status: "transferring",
isDirectory: true,
progressMode: "files",
totalBytes: 40,
transferredBytes: 0,
speed: 0,
phase: "scanning",
}, t);
assert.equal(display.detail, "0 done · 40 found");
assert.equal(display.indeterminate, true);
});
test("single-file progress still shows byte totals", () => {
const t = (key: string) => key;
const display = buildGlobalTransferProgressDisplay({
+52 -9
View File
@@ -181,28 +181,56 @@ export type GlobalTransferProgressDisplay = {
/**
* Build progress labels for a top-level row. Directory parents use file counts
* so we never show "1 Bytes / 12 Bytes" for n/m files.
*
* Folder walks interleave discovery with transfer: totalBytes is "found so far",
* not a fixed grand total. Active rows use progressive copy so the UI does not
* look like a lying overall percentage.
*/
export function buildGlobalTransferProgressDisplay(
task: Pick<TransferTask, "status" | "isDirectory" | "parentTaskId" | "progressMode" | "totalBytes" | "transferredBytes" | "speed">,
task: Pick<TransferTask, "status" | "isDirectory" | "parentTaskId" | "progressMode" | "totalBytes" | "transferredBytes" | "speed" | "phase">,
t: (key: string, params?: Record<string, string | number>) => string,
): GlobalTransferProgressDisplay {
const isDirParent = isDirectoryParentTask(task);
const percent = getGlobalTransferProgressPercent(task);
const hasTotal = task.totalBytes > 0;
const discovered = Math.max(task.totalBytes, task.transferredBytes);
if (isDirParent) {
const indeterminate = task.status === "transferring" && !hasTotal;
const isActive = task.status === "transferring"
|| task.status === "pausing"
|| task.status === "queued"
|| task.status === "pending";
// Still discovering, or no completed files yet — pulse rather than freeze a fake %.
const indeterminate = isActive && (
task.phase === "scanning"
|| !hasTotal
|| task.transferredBytes <= 0
);
let detail = "";
if (task.status === "transferring" || task.status === "pausing" || task.status === "queued" || task.status === "pending") {
detail = hasTotal
? t("sftp.transfers.filesProgress", { current: task.transferredBytes, total: task.totalBytes })
: t("sftp.transfers.filesCount", { count: task.transferredBytes });
if (isActive) {
if (discovered > 0) {
detail = t("sftp.transfers.filesDiscoveredProgress", {
completed: task.transferredBytes,
discovered,
});
}
} else if (task.status === "completed" && hasTotal) {
detail = t("sftp.transfers.filesCount", { count: task.totalBytes });
} else if (hasTotal) {
detail = t("sftp.transfers.filesProgress", { current: task.transferredBytes, total: task.totalBytes });
} else if (discovered > 0) {
detail = t("sftp.transfers.filesDiscoveredProgress", {
completed: task.transferredBytes,
discovered,
});
}
return { percent, detail, indeterminate };
// Soft percent of files found so far (discovered may still grow).
const softPercent = discovered > 0
? Math.min(100, (task.transferredBytes / discovered) * 100)
: 0;
return {
percent: task.status === "completed" ? 100 : softPercent,
detail,
indeterminate,
};
}
const detailParts: string[] = [];
@@ -394,6 +422,14 @@ function TransferRow({
if (task.status === "pausing" || task.status === "paused" || task.status === "interrupted") {
return t(statusLabelKey(task.status));
}
// Scanning runs as pending/transferring; never override terminal statuses
// (cancelled/failed/completed) that still carry a stale phase value.
if (
task.phase === "scanning"
&& (task.status === "pending" || task.status === "queued" || task.status === "transferring")
) {
return t("sftp.transferCenter.phase.scanning");
}
if (task.phase && task.status === "transferring") {
return t(`sftp.transferCenter.phase.${task.phase}`);
}
@@ -687,6 +723,13 @@ export function GlobalSftpTransferCenter() {
// progress ticks used to re-render this TopTabs child on every byte.
const badge = useSftpTransferCenterBadge();
const [open, setOpen] = useState(false);
// Folder drops fire this so the scanning row is visible immediately without
// requiring the user to notice the badge first.
useEffect(() => {
const openCenter = () => setOpen(true);
window.addEventListener("netcatty:open-sftp-transfer-center", openCenter);
return () => window.removeEventListener("netcatty:open-sftp-transfer-center", openCenter);
}, []);
const snapshot = useSftpTransferCenterWhenOpen(open);
const [bucket, setBucket] = useState<GlobalTransferBucket>("all");
const [showBackground, setShowBackground] = useState(false);
+64
View File
@@ -64,6 +64,70 @@ test("renders active transfer cancel action with an item-specific label", () =>
assert.match(markup, /aria-label="Pause: archive\.tar\.gz"/);
});
test("directory parents show transfer-center done/found progress instead of Waiting", () => {
const markup = renderTransferItem({
...baseTask,
id: "folder-1",
fileName: "big-tree",
isDirectory: true,
progressMode: "files",
status: "pending",
phase: "transferring",
transferredBytes: 65,
totalBytes: 108202,
error: undefined,
speed: 0,
});
assert.match(markup, /65 done · 108202 found/);
assert.doesNotMatch(markup, />Waiting\.\.\.</);
});
test("scanning directory parents keep scanning label with live discovery counts", () => {
const markup = renderTransferItem({
...baseTask,
id: "folder-scan",
fileName: "docs",
isDirectory: true,
progressMode: "files",
status: "pending",
phase: "scanning",
transferredBytes: 0,
totalBytes: 1284,
error: undefined,
speed: 0,
});
assert.match(markup, /Scanning/);
assert.match(markup, /0 done · 1284 found/);
});
test("child rows reserve an action column wide enough for pause and cancel", () => {
const markup = renderTransferItem(
{
...baseTask,
id: "child-transfer-actions",
parentTaskId: "transfer-1",
status: "transferring",
error: undefined,
transferredBytes: 256,
totalBytes: 1024,
speed: 128,
resumable: true,
},
{
isChild: true,
childNameColumnWidth: 260,
onResizeNameColumn: () => {},
onSetNameColumnWidth: () => {},
},
);
assert.match(markup, /grid-template-columns:24px 260px 10px minmax\(0, 1fr\) 56px/);
assert.match(markup, /aria-label="Pause: archive\.tar\.gz"/);
assert.match(markup, /aria-label="Cancel: archive\.tar\.gz"/);
});
test("renders paused transfer resume action", () => {
const markup = renderTransferItem({
...baseTask,
@@ -40,7 +40,7 @@ export default function SettingsFileAssociationsTab() {
sftpAutoSync, setSftpAutoSync,
sftpShowHiddenFiles, setSftpShowHiddenFiles,
sftpUseCompressedUpload, setSftpUseCompressedUpload,
sftpFolderPrescan, setSftpFolderPrescan,
sftpSkipUnchanged, setSftpSkipUnchanged,
sftpAutoOpenSidebar, setSftpAutoOpenSidebar,
sftpFollowTerminalCwd, setSftpFollowTerminalCwd,
@@ -211,12 +211,6 @@ export default function SettingsFileAssociationsTab() {
</span>
</div>
</SettingRow>
<SettingRow
label={t('settings.sftp.folderPrescan.enable')}
description={t('settings.sftp.folderPrescan.enableDesc')}
>
<Toggle checked={sftpFolderPrescan} onChange={setSftpFolderPrescan} />
</SettingRow>
<SettingRow
label={t('settings.sftp.skipUnchanged.enable')}
description={t('settings.sftp.skipUnchanged.enableDesc')}
+76 -52
View File
@@ -26,10 +26,17 @@ import { getParentPath } from '../../application/state/sftp/utils';
import { useSftpTransferTask } from '../../application/state/sftpTransferCenterStore';
import { cn } from '../../lib/utils';
import { TransferTask } from '../../types';
import {
buildGlobalTransferProgressDisplay,
isDirectoryParentTask,
} from '../GlobalSftpTransferCenter';
import { Button } from '../ui/button';
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '../ui/tooltip';
import { formatSpeed, formatTransferBytes } from './utils';
/** Child rows need room for Pause + Cancel (2×24px icons + gap). */
const CHILD_ACTIONS_COLUMN_PX = 56;
interface SftpTransferItemProps {
task: TransferTask;
isChild?: boolean;
@@ -132,16 +139,20 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
// Optimistic spinner from click until store status moves off paused/interrupted.
const [resumeClicked, setResumeClicked] = useState(false);
// Align with global transfer center: directory parents default to file-count
// progress unless explicitly compressed/byte mode.
const progressMode = task.progressMode
?? (task.isDirectory && !task.parentTaskId ? 'files' : 'bytes');
const isDirParent = task.isDirectory && !task.parentTaskId && progressMode === 'files';
const hasKnownTotal = task.totalBytes > 0 || (!isDirParent && !!task.sourceLastModified);
const progress = hasKnownTotal
? Math.min((task.transferredBytes / task.totalBytes) * 100, 100)
: 0;
const isIndeterminate = task.status === 'transferring' && !hasKnownTotal;
// Same progress model as the global transfer center (done · found for folders).
const isDirParent = isDirectoryParentTask(task);
const centerProgress = buildGlobalTransferProgressDisplay(task, t);
const hasKnownTotal = isDirParent
? task.totalBytes > 0 && task.transferredBytes > 0 && task.phase !== 'scanning'
: task.totalBytes > 0 || !!task.sourceLastModified;
const progress = isDirParent
? centerProgress.percent
: hasKnownTotal
? Math.min((task.transferredBytes / task.totalBytes) * 100, 100)
: 0;
const isIndeterminate = isDirParent
? centerProgress.indeterminate && (task.status === 'transferring' || task.status === 'pending' || task.status === 'queued' || task.status === 'pausing')
: task.status === 'transferring' && !hasKnownTotal;
const isActiveTransfer = task.status === 'transferring' || task.status === 'pausing';
// Reconnect / dedicated resume window — keep the action slot as a spinner
// until the first real progress clears reconnectRequired.
@@ -178,13 +189,9 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
? formatTransferBytes(task.totalBytes)
: '';
const fileCountDisplay = isDirParent && (isActiveTransfer || isPausedLike)
? (task.totalBytes > 0
? t('sftp.transfers.filesProgress', { current: task.transferredBytes, total: task.totalBytes })
: t('sftp.transfers.filesCount', { count: task.transferredBytes }))
: isDirParent && task.status === 'completed' && task.totalBytes > 0
? t('sftp.transfers.filesCount', { count: task.totalBytes })
: '';
// Prefer the transfer-center detail string so the panel never lags behind
// "N done · M found" while status is still pending during progressive walks.
const fileCountDisplay = isDirParent ? centerProgress.detail : '';
const speedFormatted = effectiveSpeed > 0 ? formatSpeed(effectiveSpeed) : '';
const targetDirectoryPath = task.isDirectory ? task.targetPath : getParentPath(task.targetPath);
@@ -193,29 +200,42 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
// the backend drained in-flight chunks ("finish current step").
const pausingLabel = t('sftp.transferCenter.status.pausing');
const resumingLabel = t('sftp.transferCenter.status.resuming');
const isLiveScanning = task.phase === 'scanning'
&& (task.status === 'pending' || task.status === 'queued' || task.status === 'transferring');
const progressOverlayText = isResuming
? resumingLabel
: task.status === 'pending'
? t('sftp.task.waiting')
: isLiveScanning
? (fileCountDisplay
? `${t('sftp.transferCenter.phase.scanning')} · ${fileCountDisplay}`
: t('sftp.transferCenter.phase.scanning'))
: task.status === 'pausing'
? pausingLabel
: isIndeterminate
? t('sftp.transfer.preparing')
: isDirParent
? (fileCountDisplay
? `${fileCountDisplay}${hasKnownTotal ? ` • ${Math.round(progress)}%` : ''}`
: hasKnownTotal
? `${Math.round(progress)}%`
: '...')
: bytesDisplay
? `${bytesDisplay}${hasKnownTotal ? ` • ${Math.round(progress)}%` : ''}`
: hasKnownTotal
? `${Math.round(progress)}%`
: '...';
: isDirParent
? (fileCountDisplay
|| (task.status === 'pending' || task.status === 'queued'
? t('sftp.task.waiting')
: isIndeterminate
? '...'
: `${Math.round(progress)}%`))
: task.status === 'pending'
? t('sftp.task.waiting')
: isIndeterminate
? t('sftp.transfer.preparing')
: bytesDisplay
? `${bytesDisplay}${hasKnownTotal ? ` • ${Math.round(progress)}%` : ''}`
: hasKnownTotal
? `${Math.round(progress)}%`
: '...';
const progressBarWidth = task.status === 'pending' || (task.status === 'transferring' && !hasKnownTotal) || isIndeterminate
? (task.status === 'pending' || !hasKnownTotal ? '100%' : `${progress}%`)
: `${progress}%`;
const progressBarWidth = isDirParent
? (centerProgress.indeterminate || isLiveScanning
? '100%'
: `${progress}%`)
: task.status === 'pending'
|| (task.status === 'transferring' && !hasKnownTotal)
|| isIndeterminate
? (task.status === 'pending' || !hasKnownTotal ? '100%' : `${progress}%`)
: `${progress}%`;
const statusIcon = isResuming
? <Loader2 size={12} className="animate-spin text-primary" />
@@ -269,7 +289,12 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
</div>
);
const progressSummaryText = isResuming || isActiveTransfer || isPausedLike || task.status === 'pending'
const progressSummaryText = isResuming
|| isActiveTransfer
|| isPausedLike
|| task.status === 'pending'
|| task.status === 'queued'
|| (isDirParent && !!fileCountDisplay)
? [speedFormatted, progressOverlayText].filter(Boolean).join(' • ')
: '';
const showTransferSizeCalculation = task.status === 'transferring' && !hasKnownTotal && !isDirParent;
@@ -437,7 +462,9 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
data-transfer-status={task.status}
data-transfer-direction={task.direction}
style={{
gridTemplateColumns: `24px ${childNameColumnWidth}px 10px minmax(0, 1fr) 24px`,
// Last column reserves space for Pause + Cancel so the
// progress bar never paints under the action buttons.
gridTemplateColumns: `24px ${childNameColumnWidth}px 10px minmax(0, 1fr) ${CHILD_ACTIONS_COLUMN_PX}px`,
}}
>
<div className="flex h-full items-center justify-center text-muted-foreground">
@@ -468,10 +495,10 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
</TooltipTrigger>
<TooltipContent side="top">{resizeNameColumnLabel}</TooltipContent>
</Tooltip>
<div className="min-w-0">
<div className="min-w-0 overflow-hidden">
{childProgressBar}
</div>
<div className="flex h-full items-center justify-center">
<div className="flex h-full min-w-0 items-center justify-end gap-0.5 pl-1">
{actionButtons}
</div>
</div>
@@ -551,12 +578,16 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
{toggleChildrenButton}
{progressSummaryText && (
<span className="ml-auto shrink-0 whitespace-nowrap text-[10px] text-muted-foreground font-mono">
<span className="ml-auto min-w-0 max-w-[50%] truncate whitespace-nowrap text-right text-[10px] text-muted-foreground font-mono">
{progressSummaryText}
</span>
)}
{actionButtons}
{/* Keep pause/cancel outside the progress summary so long
"N done · M found" labels never crowd the action buttons. */}
<div className="ml-1 shrink-0">
{actionButtons}
</div>
</div>
{showBelowParentProgress && (
@@ -622,6 +653,9 @@ const arePropsEqual = (
if (prev.fileName !== next.fileName) return false;
if (prev.targetPath !== next.targetPath) return false;
if (prev.totalBytes !== next.totalBytes) return false;
if (prev.transferredBytes !== next.transferredBytes) return false;
if (prev.phase !== next.phase) return false;
if (prev.progressMode !== next.progressMode) return false;
if ((prevProps.canRevealTarget ?? false) !== (nextProps.canRevealTarget ?? false)) return false;
if ((prevProps.canCopyTargetPath ?? false) !== (nextProps.canCopyTargetPath ?? false)) return false;
if ((prevProps.isChild ?? false) !== (nextProps.isChild ?? false)) return false;
@@ -634,20 +668,10 @@ const arePropsEqual = (
if ((prevProps.childListId ?? '') !== (nextProps.childListId ?? '')) return false;
if ((prevProps.resizeHandleTabIndex ?? 0) !== (nextProps.resizeHandleTabIndex ?? 0)) return false;
if (next.status === 'transferring' || next.status === 'pausing') {
if (next.totalBytes <= 0 && prev.transferredBytes !== next.transferredBytes) return false;
const prevProgress = prev.totalBytes > 0 ? (prev.transferredBytes / prev.totalBytes) * 100 : 0;
const nextProgress = next.totalBytes > 0 ? (next.transferredBytes / next.totalBytes) * 100 : 0;
if (Math.abs(nextProgress - prevProgress) >= 0.1) return false;
if (next.status === 'transferring' || next.status === 'pausing' || next.status === 'pending' || next.status === 'queued') {
if (next.speed !== prev.speed) return false;
}
if (next.status === 'pending') {
return true;
}
return true;
};
@@ -463,7 +463,7 @@ export const useSftpKeyboardShortcuts = ({
const hasClipboardItems = (dataTransfer?.items?.length ?? 0) > 0;
// webkitGetAsEntry must be invoked synchronously during the paste event.
const dropEntriesPromise = dataTransfer?.items?.length
? extractDropEntries(dataTransfer)
? extractDropEntries(dataTransfer).catch(() => [])
: null;
const pastedFileSnapshot = dataTransfer?.files?.length
? Array.from(dataTransfer.files).filter((file) => file.name)
+4 -4
View File
@@ -20,8 +20,8 @@ import { assertSftpFileFitsBuiltinEditor } from "../sftpEditorFileLimits";
/** Local multi-select blob downloads read whole files into ArrayBuffers. */
const LOCAL_BLOB_DOWNLOAD_CONCURRENCY = 1;
/**
* Multi-select roots each run folder pre-scan before the transfer scheduler.
* Bound them so listing fan-out cannot multiply by selection size.
* Multi-select roots each start their own interleaved folder walk / session
* work. Bound them so many selected directories cannot stampede the scheduler.
*/
const MULTI_SELECT_ROOT_DOWNLOAD_CONCURRENCY = DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY;
@@ -576,8 +576,8 @@ export const useSftpViewFileOps = ({
const selectedDirectory = await selectDirectory(t("sftp.context.download"));
if (!selectedDirectory) return;
// Bound root jobs: directory roots run full pre-scan before the scheduler,
// so unbounded Promise.allSettled would multiply LIST concurrency.
// Bound root jobs: each directory root walks and transfers independently,
// so unbounded Promise.allSettled would multiply session / LIST pressure.
const results: Array<PromiseSettledResult<{ file: SftpFileEntry; status: TransferStatus }>> = [];
await runBoundedConcurrency(
files,
@@ -111,4 +111,16 @@ describe('extractRootPathsFromDropEntries', () => {
['/home/user/folder'],
);
});
it('uses native path-only entries from a folder scan', () => {
assert.deepEqual(
extractRootPathsFromDropEntries([{
file: null,
localPath: '/home/user/folder/src/main.ts',
relativePath: 'folder/src/main.ts',
isDirectory: false,
}]),
['/home/user/folder'],
);
});
});
-2
View File
@@ -44,8 +44,6 @@ export function extractRootPathsFromDropEntries(dropEntries: DropEntry[]): strin
const seenPaths = new Set<string>();
for (const entry of dropEntries) {
if (!entry.file) continue;
const fullPath = getDropEntryLocalPath(entry);
if (!fullPath) continue;
-1
View File
@@ -324,7 +324,6 @@ export interface SyncPayload {
sftpAutoSync?: boolean;
sftpShowHiddenFiles?: boolean;
sftpUseCompressedUpload?: boolean;
sftpFolderPrescan?: boolean;
sftpSkipUnchanged?: boolean;
sftpAutoOpenSidebar?: boolean;
sftpFollowTerminalCwd?: boolean;
+156 -33
View File
@@ -307,31 +307,69 @@ async function statLocal(event, payload) {
};
}
async function collectLocalTreeEntries(rootPath, limits = {}) {
function throwIfLocalTreeCancelled(isCancelled) {
if (typeof isCancelled === "function" && isCancelled()) {
const error = new Error("Local directory traversal cancelled");
error.code = "ERR_LOCAL_TREE_CANCELLED";
throw error;
}
}
const LOCAL_TREE_ENTRY_BATCH_SIZE = 64;
async function collectLocalTreeEntries(rootPath, limits = {}, onProgress, isCancelled, onEntries) {
const rootStat = await fs.promises.stat(rootPath);
if (!rootStat.isDirectory()) {
throw new Error("Selected path is not a directory");
}
throwIfLocalTreeCancelled(isCancelled);
const traversalBudget = createLocalTreeTraversalBudget(limits);
claimLocalTreeDirectory(traversalBudget);
const rootName = path.basename(rootPath);
const rootRealPath = await fs.promises.realpath(rootPath);
const entries = [{
const rootEntry = {
localPath: rootPath,
relativePath: rootName,
type: "directory",
size: rootStat.size,
lastModified: rootStat.mtime.getTime(),
}];
};
// When streaming batches, avoid retaining the full tree in memory for 100k+
// drops. Callers that only need the complete array still get it below.
const retainAll = typeof onEntries !== "function";
const entries = retainAll ? [rootEntry] : null;
let pendingBatch = [rootEntry];
const flushBatch = (force = false) => {
if (typeof onEntries !== "function") return;
if (!force && pendingBatch.length < LOCAL_TREE_ENTRY_BATCH_SIZE) return;
if (pendingBatch.length === 0) return;
const batch = pendingBatch;
pendingBatch = [];
onEntries(batch);
};
flushBatch(true);
const queue = [{
localPath: rootPath,
relativePath: rootName,
ancestorRealPaths: new Set([rootRealPath]),
}];
let queueIndex = 0;
let fileCount = 0;
let directoryCount = 1;
let lastReportedTotal = 0;
const reportProgress = (force = false) => {
if (typeof onProgress !== "function") return;
const entryCount = fileCount + directoryCount;
if (!force && entryCount - lastReportedTotal < 32) return;
lastReportedTotal = entryCount;
onProgress({ fileCount, directoryCount, entryCount });
};
reportProgress(true);
while (queueIndex < queue.length) {
throwIfLocalTreeCancelled(isCancelled);
const current = queue[queueIndex++];
const children = await fs.promises.readdir(current.localPath, { withFileTypes: true });
accountLocalTreeEntries(traversalBudget, children.length);
@@ -339,35 +377,44 @@ async function collectLocalTreeEntries(rootPath, limits = {}) {
const metadataConcurrency = 32;
for (let start = 0; start < children.length; start += metadataConcurrency) {
const inspected = await Promise.all(
throwIfLocalTreeCancelled(isCancelled);
const inspected = (await Promise.all(
children.slice(start, start + metadataConcurrency).map(async (child) => {
const childPath = path.join(current.localPath, child.name);
const childRelativePath = `${current.relativePath}/${child.name}`;
// Use lstat to distinguish links, then stat the target. Directory
// links retain the established folder-upload behavior, while the
// real-path ancestor chain prevents junction/symlink cycles.
const linkStat = await fs.promises.lstat(childPath);
const stat = linkStat.isSymbolicLink()
? await fs.promises.stat(childPath).catch(() => linkStat)
: linkStat;
const isDirectory = stat.isDirectory();
const realPath = isDirectory
? await fs.promises.realpath(childPath)
: null;
const isCycle = !!realPath && current.ancestorRealPaths.has(realPath);
const ancestorRealPaths = realPath
? new Set([...current.ancestorRealPaths, realPath])
: current.ancestorRealPaths;
return {
childPath,
childRelativePath,
stat,
isDirectory,
isCycle,
ancestorRealPaths,
};
try {
// Use lstat to distinguish links, then stat the target. Directory
// links retain the established folder-upload behavior, while the
// real-path ancestor chain prevents junction/symlink cycles.
const linkStat = await fs.promises.lstat(childPath);
const stat = linkStat.isSymbolicLink()
? await fs.promises.stat(childPath).catch(() => linkStat)
: linkStat;
const isDirectory = stat.isDirectory();
const realPath = isDirectory
? await fs.promises.realpath(childPath)
: null;
const isCycle = !!realPath && current.ancestorRealPaths.has(realPath);
const ancestorRealPaths = realPath
? new Set([...current.ancestorRealPaths, realPath])
: current.ancestorRealPaths;
return {
childPath,
childRelativePath,
stat,
isDirectory,
isCycle,
ancestorRealPaths,
};
} catch (error) {
// A folder can change while it is being scanned. Match the
// tolerant browser traversal and skip entries that disappear or
// become inaccessible instead of aborting the whole drop.
console.warn(`Could not inspect ${childPath}:`, error.message);
return null;
}
}),
);
)).filter(Boolean);
// Promise.all preserves the sorted child order, so restart manifests stay
// deterministic while metadata I/O is parallelized.
@@ -379,30 +426,106 @@ async function collectLocalTreeEntries(rootPath, limits = {}) {
// Cyclic links cannot be represented by a finite copied tree. Skip the
// loop itself instead of misreporting it as a file that later fails.
if (child.isCycle) continue;
entries.push({
const row = {
localPath: child.childPath,
relativePath: child.childRelativePath,
type: child.isDirectory ? "directory" : "file",
size: child.stat.size,
lastModified: child.stat.mtime.getTime(),
});
};
if (retainAll) entries.push(row);
pendingBatch.push(row);
if (child.isDirectory) {
directoryCount += 1;
queue.push({
localPath: child.childPath,
relativePath: child.childRelativePath,
ancestorRealPaths: child.ancestorRealPaths,
});
} else {
fileCount += 1;
}
}
flushBatch();
reportProgress();
}
}
return entries;
throwIfLocalTreeCancelled(isCancelled);
flushBatch(true);
reportProgress(true);
return retainAll ? entries : [];
}
async function listLocalTree(event, payload) {
return collectLocalTreeEntries(payload.path);
const progressChannel = typeof payload?.progressChannel === "string" && payload.progressChannel
? payload.progressChannel
: null;
const cancelChannel = typeof payload?.cancelChannel === "string" && payload.cancelChannel
? payload.cancelChannel
: null;
let cancelled = false;
const onCancel = () => {
cancelled = true;
};
// Lazy-require so unit tests can import collectLocalTreeEntries without a
// full Electron binary (top-level require("electron") breaks node --test).
let electronIpcMain = null;
if (cancelChannel) {
try {
electronIpcMain = require("electron").ipcMain;
electronIpcMain.on(cancelChannel, onCancel);
} catch {
electronIpcMain = null;
}
}
const onProgress = progressChannel
? (stats) => {
try {
event.sender.send(progressChannel, stats);
} catch {
// Renderer may have gone away mid-scan.
}
}
: undefined;
const entriesChannel = typeof payload?.entriesChannel === "string" && payload.entriesChannel
? payload.entriesChannel
: null;
const onEntries = entriesChannel
? (batch) => {
try {
// Always send plain arrays for entry batches. The stream end marker is
// a separate object so the preload can keep its listener until every
// nested batch has been delivered (invoke reply races with send).
event.sender.send(entriesChannel, batch);
} catch {
// Renderer may have gone away mid-scan.
}
}
: undefined;
try {
return await collectLocalTreeEntries(
payload.path,
payload.limits || {},
onProgress,
() => cancelled,
onEntries,
);
} finally {
// Must be sent after the last entry batch and before the invoke resolves
// is not enough alone — preload must wait for this marker before removing
// its listener, otherwise deep nested files (discovered late) are dropped.
if (entriesChannel) {
try {
event.sender.send(entriesChannel, { type: "tree-end" });
} catch {
// Renderer may have gone away mid-scan.
}
}
if (cancelChannel && electronIpcMain) {
electronIpcMain.removeListener(cancelChannel, onCancel);
}
}
}
/**
+78
View File
@@ -183,6 +183,56 @@ test("collectLocalTreeEntries preserves empty directories in selected folders",
}
});
test("collectLocalTreeEntries reports progressive file counts during scan", async () => {
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-progress-"));
const selected = path.join(root, "project");
await fs.promises.mkdir(path.join(selected, "src"), { recursive: true });
await fs.promises.writeFile(path.join(selected, "readme.txt"), "hi");
await fs.promises.writeFile(path.join(selected, "src", "main.txt"), "hello");
const progress = [];
try {
const entries = await collectLocalTreeEntries(selected, {}, (stats) => {
progress.push({ ...stats });
});
assert.equal(entries.filter((entry) => entry.type === "file").length, 2);
assert.ok(progress.length >= 1, "expected at least one progress event");
const final = progress[progress.length - 1];
assert.equal(final.fileCount, 2);
assert.equal(final.directoryCount, 2);
assert.equal(final.entryCount, 4);
} finally {
await fs.promises.rm(root, { recursive: true, force: true });
}
});
test("collectLocalTreeEntries aborts when cancelled mid-scan", async () => {
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-cancel-"));
const selected = path.join(root, "project");
await fs.promises.mkdir(path.join(selected, "a"), { recursive: true });
await fs.promises.mkdir(path.join(selected, "b"), { recursive: true });
await fs.promises.writeFile(path.join(selected, "a", "1.txt"), "1");
await fs.promises.writeFile(path.join(selected, "b", "2.txt"), "2");
let cancel = false;
try {
await assert.rejects(
() => collectLocalTreeEntries(
selected,
{},
() => {
// Cancel as soon as progress starts flowing.
cancel = true;
},
() => cancel,
),
/cancelled/i,
);
} finally {
await fs.promises.rm(root, { recursive: true, force: true });
}
});
test("collectLocalTreeEntries bounds and parallelizes metadata reads within a directory", async (t) => {
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-parallel-"));
const selected = path.join(root, "project");
@@ -215,6 +265,34 @@ test("collectLocalTreeEntries bounds and parallelizes metadata reads within a di
assert.ok(maxActive <= 32, `metadata concurrency must stay bounded, got ${maxActive}`);
});
test("collectLocalTreeEntries skips a child that disappears during inspection", async (t) => {
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-vanished-"));
const selected = path.join(root, "project");
await fs.promises.mkdir(selected, { recursive: true });
await fs.promises.writeFile(path.join(selected, "stable.txt"), "stable");
await fs.promises.writeFile(path.join(selected, "vanished.txt"), "vanished");
const originalLstat = fs.promises.lstat;
fs.promises.lstat = async (candidate, ...args) => {
if (candidate === path.join(selected, "vanished.txt")) {
const error = new Error("file disappeared");
error.code = "ENOENT";
throw error;
}
return originalLstat.call(fs.promises, candidate, ...args);
};
t.after(async () => {
fs.promises.lstat = originalLstat;
await fs.promises.rm(root, { recursive: true, force: true });
});
const entries = await collectLocalTreeEntries(selected);
assert.deepEqual(entries.map((entry) => entry.relativePath), [
"project",
"project/stable.txt",
]);
});
test("collectLocalTreeEntries does not follow a directory symlink cycle", async (t) => {
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-cycle-"));
const selected = path.join(root, "project");
+192 -280
View File
@@ -2241,76 +2241,23 @@ async function uploadFile(
transfer.uploadStrategy = "scp";
logTransferDiag(transfer, "strategy", { strategy: "scp" });
const backend = getScpBackendForClient(client);
let scpSourcePath = localPath;
let digestPath = null;
let snapshotPath = null;
let openReadStream = null;
let initialSource = null;
try {
if (!transfer.sourceIsOwnedTemp) {
initialSource = await fs.promises.stat(localPath);
const snapshotId = crypto.createHash("sha256")
.update(String(transfer.transferId || localPath))
.digest("hex")
.slice(0, 16);
digestPath = tempDirBridge.getTransferTempFilePath(
`upload-digest-${snapshotId}`,
"ranges.sha256",
);
snapshotPath = tempDirBridge.getTransferTempFilePath(
`upload-source-${snapshotId}`,
"snapshot.bin",
);
await createUploadDigestBaseline(localPath, digestPath, fileSize, transfer);
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
const sourceAfterBaseline = await fs.promises.stat(localPath);
assertSourceMetadataUnchanged(initialSource, sourceAfterBaseline, fileSize, {
contentVerifiedSeparately: true,
});
if (typeof onBytesCommitted === "function") {
await createVerifiedUploadSnapshot(
localPath,
snapshotPath,
digestPath,
fileSize,
transfer,
);
scpSourcePath = snapshotPath;
} else {
// Remote staging can safely discard a failed upload. Verify every
// source chunk as SCP reads it, then rescan before promotion, without
// requiring another full local copy of large files.
snapshotPath = null;
openReadStream = () => createVerifiedUploadReadStream(
localPath,
digestPath,
fileSize,
transfer,
);
}
}
await backend.uploadFile(scpSourcePath, remotePath, {
fileSize,
transfer,
encoding,
signal: transfer.signal,
openReadStream,
onProgress: (transferred, total) => sendProgress(transferred, total || fileSize),
});
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
if (digestPath && !snapshotPath) {
const latestSource = await fs.promises.stat(localPath);
assertSourceMetadataUnchanged(initialSource, latestSource, fileSize, {
contentVerifiedSeparately: true,
});
await verifyUploadDigestBaseline(localPath, digestPath, fileSize, transfer);
}
onBytesCommitted?.();
return;
} finally {
if (snapshotPath) await fs.promises.rm(snapshotPath, { force: true }).catch(() => {});
if (digestPath) await fs.promises.rm(digestPath, { force: true }).catch(() => {});
}
// Stream the live local path directly — no whole-file digest / snapshot.
// openReadStream is optional for scpBackend (falls back to createReadStream);
// provide a plain file stream so tests/backends that always call it still work.
await backend.uploadFile(localPath, remotePath, {
fileSize,
transfer,
encoding,
signal: transfer.signal,
openReadStream: () => {
const stream = fs.createReadStream(localPath, { highWaterMark: 256 * 1024 });
return { stream, completed: Promise.resolve() };
},
onProgress: (transferred, total) => sendProgress(transferred, total || fileSize),
});
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
onBytesCommitted?.();
return;
}
await requireSftpChannel(client);
@@ -2329,79 +2276,21 @@ async function uploadFile(
else lastPipelineError = new Error(String(err || "SFTP upload failed"));
};
if (!transfer.sourceIsOwnedTemp) {
const digestId = crypto.createHash("sha256")
.update(String(transfer.transferId || "upload"))
.digest("hex")
.slice(0, 16);
const digestPath = tempDirBridge.getTransferTempFilePath(
`upload-digest-${digestId}`,
"ranges.sha256",
);
transfer.sourceDigestPath = digestPath;
const previousPhase = transfer.phase;
transfer.phase = "verifying";
transfer.publishCurrentProgress?.();
try {
await createUploadDigestBaseline(
originalLocalPath,
digestPath,
fileSize,
transfer,
() => {
// Keep remote transferredBytes at the resume checkpoint while hashing
// so large uploads do not look stuck at 0 B with no status (#2712).
sendProgress(Math.max(0, Number(transfer.checkpointBytes) || 0), fileSize, {
force: true,
});
},
);
} finally {
transfer.phase = previousPhase || "transferring";
if (!transfer.cancelled && !transfer.signal?.aborted) {
transfer.publishCurrentProgress?.();
}
}
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
const sourceAfterBaseline = await fs.promises.stat(originalLocalPath);
// Digest was just built; only size/content matter from here.
assertSourceMetadataUnchanged(initialSource, sourceAfterBaseline, fileSize, {
contentVerifiedSeparately: true,
});
}
const cleanupSourceDigest = async () => {
if (!transfer.sourceDigestPath) return;
const digestPath = transfer.sourceDigestPath;
transfer.sourceDigestPath = null;
await fs.promises.rm(digestPath, { force: true }).catch(() => {});
};
// Industry-standard SFTP clients (FileZilla / WinSCP / OpenSSH) resume by
// size only — no whole-file content digest before or during the body transfer.
// Pause/resume durability lives in checkpointBytes + the remote .part stage.
const finishSuccessfulUpload = async () => {
try {
if (initialSource) {
const latestSource = await fs.promises.stat(originalLocalPath);
// Prefer digest re-scan for same-size rewrites. Hard-failing on ctime
// alone false-positives long pause/resume uploads on macOS.
assertSourceMetadataUnchanged(initialSource, latestSource, fileSize, {
contentVerifiedSeparately: Boolean(transfer.sourceDigestPath),
});
}
// Metadata alone cannot catch same-size rewrites with unchanged/coarse
// timestamps (e.g. all ranges already verified before the rewrite).
// Re-scan the source against the digest baseline before promotion.
if (transfer.sourceDigestPath) {
await verifyUploadDigestBaseline(
originalLocalPath,
transfer.sourceDigestPath,
fileSize,
transfer,
);
}
await assertRemoteUploadSize(client, remotePath, fileSize);
} finally {
await cleanupSourceDigest();
if (initialSource) {
const latestSource = await fs.promises.stat(originalLocalPath);
// Soft size + mtime/ino (no full-file re-hash). Do not claim separate
// content proof — same-size rewrites that bump mtime still fail closed.
// ignoreCtime: macOS xattr/Spotlight noise must not abort a true match.
assertSourceMetadataUnchanged(initialSource, latestSource, fileSize, {
ignoreCtime: true,
});
}
await assertRemoteUploadSize(client, remotePath, fileSize);
};
// Prefer an isolated SFTP channel so cancellation cannot kill the browse session.
@@ -2502,8 +2391,6 @@ async function uploadFile(
&& !transfer.resumable
) {
let fastPutOk = false;
let fastPutSourcePath = localPath;
let fastPutSnapshotPath = null;
try {
// Wait for any prior/in-flight write OPEN on this path (including our
// own concurrent attempt's still-pending OPEN) before fastPut truncates
@@ -2519,29 +2406,12 @@ async function uploadFile(
logTransferDiag(transfer, "strategy", { strategy: "fastPut-isolated" });
transfer.pauseSupported = false;
transfer.pauseUnavailableReason = "Pause is unavailable during fastPut upload";
if (!transfer.sourceIsOwnedTemp) {
const snapshotId = crypto.createHash("sha256")
.update(String(transfer.transferId || localPath))
.digest("hex")
.slice(0, 16);
fastPutSnapshotPath = tempDirBridge.getTransferTempFilePath(
`upload-source-${snapshotId}`,
"snapshot.bin",
);
await createVerifiedUploadSnapshot(
originalLocalPath,
fastPutSnapshotPath,
transfer.sourceDigestPath,
fileSize,
transfer,
);
fastPutSourcePath = fastPutSnapshotPath;
}
// Stream the live local path (no whole-file snapshot / digest sidecar).
sendProgress(Math.max(0, Number(transfer.checkpointBytes) || 0), fileSize, {
force: true,
});
await uploadViaFastPut(
fastPutSourcePath,
originalLocalPath,
remotePath,
isolated,
fileSize,
@@ -2551,10 +2421,10 @@ async function uploadFile(
);
fastPutOk = true;
} catch (err) {
// Gate-wait / snapshot / fastPut failure: end the reopened isolated
// channel before nulling. Rethrow paths skip the post-block else-if
// end, and fallthrough also clears isolated; either way we must not
// leak the SSH subsystem opened for this attempt (#2755 Bugbot).
// Gate-wait / fastPut failure: end the reopened isolated channel before
// nulling. Rethrow paths skip the post-block else-if end, and fallthrough
// also clears isolated; either way we must not leak the SSH subsystem
// opened for this attempt (#2755 Bugbot).
if (isolated && typeof isolated.end === "function") {
try { isolated.end(); } catch { /* ignore */ }
}
@@ -2576,10 +2446,6 @@ async function uploadFile(
"[transferBridge] isolated fastPut failed, trying next pipelined strategy:",
err?.message || String(err),
);
} finally {
if (fastPutSnapshotPath) {
await fs.promises.rm(fastPutSnapshotPath, { force: true }).catch(() => {});
}
}
if (fastPutOk) {
onBytesCommitted?.();
@@ -2648,7 +2514,10 @@ async function uploadFile(
const error = new Error(message, cause ? { cause } : undefined);
if (cause?.code !== undefined) error.code = cause.code;
if (cause?.noTransferFallback) error.noTransferFallback = true;
await cleanupSourceDigest();
if (transfer.sourceDigestPath) {
try { await fs.promises.rm(transfer.sourceDigestPath, { force: true }); } catch { /* ignore */ }
transfer.sourceDigestPath = null;
}
throw error;
}
@@ -3730,10 +3599,20 @@ function isTransferCancelled(transfer) {
}
const UPLOAD_DIGEST_SCAN_SIZE = TRANSFER_CHUNK_SIZE * 128;
const EMPTY_DIGEST_SLOT = Buffer.alloc(32);
function uploadDigestByteLength(fileSize) {
return Math.ceil(Math.max(0, Number(fileSize) || 0) / TRANSFER_CHUNK_SIZE) * 32;
}
function isUnsetDigestSlot(buffer, bytesRead) {
if (bytesRead !== 32) return true;
return buffer.equals(EMPTY_DIGEST_SLOT);
}
async function assertUploadDigestCapacity(digestPath, fileSize) {
if (typeof fs.promises.statfs !== "function") return;
const requiredBytes = BigInt(Math.ceil(fileSize / TRANSFER_CHUNK_SIZE)) * 32n;
const requiredBytes = BigInt(uploadDigestByteLength(fileSize));
let stats;
try {
stats = await fs.promises.statfs(path.dirname(digestPath), { bigint: true });
@@ -3750,12 +3629,31 @@ async function assertUploadDigestCapacity(digestPath, fileSize) {
}
}
/**
* Allocate an empty per-chunk digest sidecar without reading the source.
* Chunk digests are filled on first verified read during the upload pass so
* multi-GB files no longer wait on a full pre-hash before the first WRITE.
*/
async function prepareUploadDigestSidecar(digestPath, fileSize) {
await fs.promises.rm(digestPath, { force: true });
await assertUploadDigestCapacity(digestPath, fileSize);
const requiredBytes = uploadDigestByteLength(fileSize);
const handle = await fs.promises.open(digestPath, "w");
try {
if (requiredBytes > 0) await handle.truncate(requiredBytes);
} finally {
await handle.close().catch(() => {});
}
}
async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer, onProgress = null) {
let sourceHandle = null;
let digestHandle = null;
try {
sourceHandle = await fs.promises.open(sourcePath, "r");
digestHandle = await fs.promises.open(digestPath, "r");
// r+ so a lazy sidecar can finish filling any still-empty slots on the
// post-upload re-scan (e.g. resume after a crash with a partial digest).
digestHandle = await fs.promises.open(digestPath, "r+");
const buffer = Buffer.allocUnsafe(Math.min(UPLOAD_DIGEST_SCAN_SIZE, Math.max(1, fileSize)));
let position = 0;
let chunkIndex = 0;
@@ -3771,7 +3669,18 @@ async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, tran
const actual = crypto.createHash("sha256")
.update(buffer.subarray(offset, Math.min(offset + TRANSFER_CHUNK_SIZE, length)))
.digest();
if (!expected.subarray(index * 32, (index + 1) * 32).equals(actual)) {
const slot = expected.subarray(index * 32, (index + 1) * 32);
if (isUnsetDigestSlot(slot, 32)) {
const writeResult = await digestHandle.write(
actual,
0,
32,
(chunkIndex + index) * 32,
);
if (!writeResult || writeResult.bytesWritten !== 32) {
throw new Error("Upload digest sidecar stopped accepting data");
}
} else if (!slot.equals(actual)) {
throw createSourceContentChangedError();
}
}
@@ -3785,63 +3694,80 @@ async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, tran
}
}
async function createUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer, onProgress = null) {
// A crashed attempt may have left this transfer's old baseline behind. It is
// fully replaceable and its blocks must be reclaimable before capacity is
// evaluated for the new baseline.
await fs.promises.rm(digestPath, { force: true });
await assertUploadDigestCapacity(digestPath, fileSize);
async function createUploadDigestBaseline(
sourcePath,
digestPath,
fileSize,
transfer,
onProgress = null,
options = {},
) {
// merge:true — fill/verify an existing sidecar while uploads are in flight.
// Default replaces the file (SCP / snapshot callers need a clean baseline).
const merge = options.merge === true;
if (!merge) {
await fs.promises.rm(digestPath, { force: true });
await prepareUploadDigestSidecar(digestPath, fileSize);
} else {
await assertUploadDigestCapacity(digestPath, fileSize);
try {
await fs.promises.access(digestPath);
} catch {
await prepareUploadDigestSidecar(digestPath, fileSize);
}
}
let sourceHandle = null;
let digestHandle = null;
let completed = false;
try {
sourceHandle = await fs.promises.open(sourcePath, "r");
digestHandle = await fs.promises.open(digestPath, "w");
digestHandle = await fs.promises.open(digestPath, "r+");
const buffer = Buffer.allocUnsafe(Math.min(UPLOAD_DIGEST_SCAN_SIZE, Math.max(1, fileSize)));
let position = 0;
let digestPosition = 0;
let chunkIndex = 0;
while (position < fileSize) {
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
const length = Math.min(buffer.length, fileSize - position);
await readLocalRange(sourceHandle, buffer, position, length);
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
const digestCount = Math.ceil(length / TRANSFER_CHUNK_SIZE);
const digests = Buffer.allocUnsafe(digestCount * 32);
for (let offset = 0, index = 0; offset < length; offset += TRANSFER_CHUNK_SIZE, index += 1) {
crypto.createHash("sha256")
const actual = crypto.createHash("sha256")
.update(buffer.subarray(offset, Math.min(offset + TRANSFER_CHUNK_SIZE, length)))
.digest()
.copy(digests, index * 32);
}
let written = 0;
while (written < digests.length) {
const result = await digestHandle.write(
digests,
written,
digests.length - written,
digestPosition + written,
);
if (!result || result.bytesWritten <= 0) {
throw new Error("Upload digest baseline stopped accepting data");
}
written += result.bytesWritten;
.digest();
const slotIndex = chunkIndex + index;
const digestOffset = slotIndex * 32;
await withDigestSlotLock(digestHandle, slotIndex, async () => {
const expected = Buffer.allocUnsafe(32);
const digestResult = await digestHandle.read(expected, 0, 32, digestOffset);
if (isUnsetDigestSlot(expected, digestResult.bytesRead)) {
const writeResult = await digestHandle.write(actual, 0, 32, digestOffset);
if (!writeResult || writeResult.bytesWritten !== 32) {
throw new Error("Upload digest baseline stopped accepting data");
}
} else if (!expected.equals(actual)) {
throw createSourceContentChangedError();
}
});
}
position += length;
digestPosition += digests.length;
chunkIndex += digestCount;
// Report scan progress so large uploads do not look stuck at 0 B/s while
// hashing (#2712 / #2556). Bytes here are local read progress, not remote.
onProgress?.(position, fileSize);
}
if (typeof digestHandle.sync === "function") {
await digestHandle.sync().catch(() => {});
}
completed = true;
} finally {
await sourceHandle?.close().catch(() => {});
await digestHandle?.close().catch(() => {});
if (!completed) await fs.promises.rm(digestPath, { force: true }).catch(() => {});
// Background filler must not delete a sidecar still used by live uploads.
if (!completed && !merge) {
await fs.promises.rm(digestPath, { force: true }).catch(() => {});
}
}
// Confirm the sidecar matches the source before any remote WRITE. Progress
// callbacks keep the UI in "verifying" instead of looking stuck (#2712).
await verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer, onProgress);
}
async function createVerifiedUploadSnapshot(
@@ -3858,7 +3784,7 @@ async function createVerifiedUploadSnapshot(
let completed = false;
try {
sourceHandle = await fs.promises.open(sourcePath, "r");
digestHandle = await fs.promises.open(digestPath, "r");
digestHandle = await fs.promises.open(digestPath, "r+");
snapshotHandle = await fs.promises.open(snapshotPath, "w");
const sourceStats = await sourceHandle.stat();
let position = 0;
@@ -3910,7 +3836,7 @@ function createVerifiedUploadReadStream(
let digestHandle = null;
try {
sourceHandle = await fs.promises.open(sourcePath, "r");
digestHandle = await fs.promises.open(digestPath, "r");
digestHandle = await fs.promises.open(digestPath, "r+");
let position = 0;
while (position < fileSize) {
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
@@ -3935,13 +3861,39 @@ function createVerifiedUploadReadStream(
return { stream, completed };
}
async function withDigestSlotLock(digestHandle, chunkIndex, fn) {
// Per-chunk locks keep pipelined uploads concurrent across ranges while still
// serializing r+ ops on the same 32-byte digest slot.
if (!digestHandle.__netcattySlotLocks) {
digestHandle.__netcattySlotLocks = new Map();
}
const locks = digestHandle.__netcattySlotLocks;
const previous = locks.get(chunkIndex) || Promise.resolve();
let release;
const gate = new Promise((resolve) => {
release = resolve;
});
locks.set(chunkIndex, previous.then(() => gate, () => gate));
try {
await previous;
return await fn();
} finally {
release();
}
}
async function readVerifiedUploadRange(
localHandle,
digestHandle,
position,
length,
fileSize,
options = {},
) {
// allowCreate: first pass fills empty digest slots while uploading (no full
// pre-hash). Resume re-reads must require an existing non-empty slot so a
// rewritten source cannot silently mint a new identity for already-sent bytes.
const allowCreate = options.allowCreate !== false;
const output = Buffer.allocUnsafe(length);
let outputOffset = 0;
while (outputOffset < length) {
@@ -3949,13 +3901,25 @@ async function readVerifiedUploadRange(
const chunkStart = Math.floor(rangePosition / TRANSFER_CHUNK_SIZE) * TRANSFER_CHUNK_SIZE;
const chunkLength = Math.min(TRANSFER_CHUNK_SIZE, fileSize - chunkStart);
const chunk = Buffer.allocUnsafe(chunkLength);
// Local reads may run concurrently; only digest-slot mutations are locked.
await readLocalRange(localHandle, chunk, chunkStart, chunkLength);
const expected = Buffer.allocUnsafe(32);
const chunkIndex = Math.floor(chunkStart / TRANSFER_CHUNK_SIZE);
const digestResult = await digestHandle.read(expected, 0, 32, chunkIndex * 32);
if (digestResult.bytesRead !== 32) throw createSourceContentChangedError();
const actual = crypto.createHash("sha256").update(chunk).digest();
if (!expected.equals(actual)) throw createSourceContentChangedError();
const chunkIndex = Math.floor(chunkStart / TRANSFER_CHUNK_SIZE);
const digestOffset = chunkIndex * 32;
await withDigestSlotLock(digestHandle, chunkIndex, async () => {
const expected = Buffer.allocUnsafe(32);
const digestResult = await digestHandle.read(expected, 0, 32, digestOffset);
if (isUnsetDigestSlot(expected, digestResult.bytesRead)) {
if (!allowCreate) throw createSourceContentChangedError();
// Record identity for this chunk as we upload it (single local read).
const writeResult = await digestHandle.write(actual, 0, 32, digestOffset);
if (!writeResult || writeResult.bytesWritten !== 32) {
throw new Error("Upload digest sidecar stopped accepting data");
}
} else if (!expected.equals(actual)) {
throw createSourceContentChangedError();
}
});
const chunkOffset = rangePosition - chunkStart;
const copyLength = Math.min(length - outputOffset, chunkLength - chunkOffset);
chunk.copy(output, outputOffset, chunkOffset, chunkOffset + copyLength);
@@ -3983,7 +3947,7 @@ async function readVerifiedUploadRange(
* @param {object|null|undefined} initialSource
* @param {object|null|undefined} latestSource
* @param {number} expectedSize
* @param {{ contentVerifiedSeparately?: boolean, allowSourceGrowth?: boolean }} [options]
* @param {{ contentVerifiedSeparately?: boolean, allowSourceGrowth?: boolean, ignoreCtime?: boolean }} [options]
*/
function assertSourceMetadataUnchanged(initialSource, latestSource, expectedSize, options = {}) {
const latestSize = Number(latestSource?.size);
@@ -4010,8 +3974,11 @@ function assertSourceMetadataUnchanged(initialSource, latestSource, expectedSize
return;
}
// No digest / per-range content proof: timestamps + inode are the durable
// same-size rewrite signal (remote SFTP download path).
const versionFields = ["mtimeMs", "ctimeMs", "mtime", "ctime", "ino"];
// same-size rewrite signal. Upload finish may set ignoreCtime so macOS
// xattr/Spotlight ctime bumps do not abort after a true size match.
const versionFields = options.ignoreCtime
? ["mtimeMs", "mtime", "ino"]
: ["mtimeMs", "ctimeMs", "mtime", "ctime", "ino"];
const changed = versionFields.some((field) => {
const before = Number(initialSource?.[field]);
const after = Number(latestSource?.[field]);
@@ -4416,8 +4383,6 @@ async function uploadFileConcurrent(
transfer.abort = abortEarly;
let localHandle = null;
let digestHandle = null;
let ephemeralDigestPath = null;
let initialSource = null;
let remoteHandle = null;
let failed = false;
@@ -4434,29 +4399,8 @@ async function uploadFileConcurrent(
throw localOpenError;
}
if (transfer.cancelled) throw new Error("Transfer cancelled");
if (transfer.sourceDigestPath) {
digestHandle = await fs.promises.open(transfer.sourceDigestPath, "r");
} else if (!transfer.sourceIsOwnedTemp) {
// Non-resumable range uploads do not receive uploadFile's persistent
// digest sidecar. Build the same stable baseline here so every range is
// verified immediately before its remote WRITE instead of relying on a
// before/after whole-file fingerprint that temporary rewrites can evade.
if (!transfer.sourceIsOwnedTemp) {
initialSource = await localHandle.stat();
const digestId = crypto.createHash("sha256")
.update(String(transfer.transferId || localPath))
.digest("hex")
.slice(0, 16);
ephemeralDigestPath = tempDirBridge.getTransferTempFilePath(
`upload-digest-${digestId}`,
"ranges.sha256",
);
await createUploadDigestBaseline(localPath, ephemeralDigestPath, fileSize, transfer);
if (transfer.cancelled) throw new Error("Transfer cancelled");
const sourceAfterBaseline = await localHandle.stat();
assertSourceMetadataUnchanged(initialSource, sourceAfterBaseline, fileSize, {
contentVerifiedSeparately: true,
});
digestHandle = await fs.promises.open(ephemeralDigestPath, "r");
}
if (transfer.cancelled) throw new Error("Transfer cancelled");
remoteHandle = await openSftpHandleForTransfer(
@@ -4524,19 +4468,10 @@ async function uploadFileConcurrent(
checkpoint,
concurrency: UPLOAD_TRANSFER_CONCURRENCY,
copyRange: async (position, length) => {
const buffer = digestHandle
? await readVerifiedUploadRange(
localHandle,
digestHandle,
position,
length,
fileSize,
)
: await (async () => {
const directBuffer = Buffer.allocUnsafe(length);
await readLocalRange(localHandle, directBuffer, position, length);
return directBuffer;
})();
// Size-based resume: read the source range at `position` and WRITE it.
// No per-chunk content digest (FileZilla / WinSCP style).
const buffer = Buffer.allocUnsafe(length);
await readLocalRange(localHandle, buffer, position, length);
if (transfer.cancelled) throw new Error("Transfer cancelled");
await writeSftpRange(sftp, remoteHandle, buffer, position, length);
},
@@ -4550,16 +4485,13 @@ async function uploadFileConcurrent(
// published at this point, so stop accepting cancellation before source
// revalidation and handle cleanup; staged uploads pass no callback.
options.onBytesCommitted?.();
const contentVerifiedSeparately = Boolean(digestHandle || ephemeralDigestPath || transfer.sourceDigestPath);
if (initialSource) {
const latestSource = await localHandle.stat();
// Soft size + mtime/ino only — no digest, but not fail-open either.
assertSourceMetadataUnchanged(initialSource, latestSource, fileSize, {
contentVerifiedSeparately,
ignoreCtime: true,
});
}
if (ephemeralDigestPath) {
await verifyUploadDigestBaseline(localPath, ephemeralDigestPath, fileSize, transfer);
}
} catch (error) {
failed = true;
throw error;
@@ -4592,12 +4524,6 @@ async function uploadFileConcurrent(
if (localHandle) {
await localHandle.close().catch(() => {});
}
if (digestHandle) {
await digestHandle.close().catch(() => {});
}
if (ephemeralDigestPath) {
await fs.promises.rm(ephemeralDigestPath, { force: true }).catch(() => {});
}
let remoteCloseError = null;
// Close remote handles while the channel is still live. On disposeChannel
// cancel/failure the channel is about to be ended (or already dead); a
@@ -5903,24 +5829,10 @@ async function startTransferNow(event, payload, onProgress) {
))
: 0;
sendProgress(transfer.checkpointBytes, fileSize, { force: true });
if (usesStage && transfer.checkpointBytes > 0) {
const verifyBytes = resumeContentVerifyBytes(
transfer.checkpointBytes,
transfer.sourceFingerprint,
);
await verifyResumeContent(
verifyBytes,
(options) => hashLocalPrefix(sourcePath, verifyBytes, options),
(options) => hashRemotePrefix(
client,
targetSftpId,
uploadTargetPath,
targetEncoding,
verifyBytes,
options,
),
);
}
// Resume is size-based only (WinSCP/FileZilla): the durable .part size
// is the checkpoint. Do not re-hash [0, checkpoint) on both ends —
// that blocked large-file resume for seconds-to-minutes with no bytes
// moving.
await uploadFile(
sourcePath,
encodedUploadPath,
File diff suppressed because it is too large. Load diff
+91 -2
View File
@@ -876,8 +876,97 @@ function createPreloadApi(ctx) {
statLocal: async (path) => {
return ipcRenderer.invoke("netcatty:local:stat", { path });
},
listLocalTree: async (path) => {
return ipcRenderer.invoke("netcatty:local:tree", { path });
listLocalTree: async (path, options = {}) => {
const onProgress = typeof options?.onProgress === "function" ? options.onProgress : null;
const onEntries = typeof options?.onEntries === "function" ? options.onEntries : null;
const token = typeof options?.scanId === "string" && options.scanId
? options.scanId
: `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
const progressChannel = onProgress
? `netcatty:local:tree-progress:${token}`
: undefined;
const entriesChannel = onEntries
? `netcatty:local:tree-entries:${token}`
: undefined;
const cancelChannel = `netcatty:local:tree-cancel:${token}`;
const progressHandler = onProgress && progressChannel
? (_event, stats) => {
try {
onProgress(stats);
} catch {
// Ignore renderer progress handler errors so the scan can finish.
}
}
: null;
// Entry batches arrive on a different channel than the invoke reply. If we
// tear down the listener when invoke resolves, nested-folder batches that
// are still in the IPC queue are dropped (top-level files look fine).
let settleEntriesStream = null;
const entriesStreamDone = onEntries && entriesChannel
? new Promise((resolve) => {
settleEntriesStream = resolve;
})
: null;
const entriesHandler = onEntries && entriesChannel
? (_event, payload) => {
if (payload && typeof payload === "object" && !Array.isArray(payload) && payload.type === "tree-end") {
settleEntriesStream?.();
return;
}
try {
if (Array.isArray(payload)) onEntries(payload);
} catch {
// Ignore batch handler errors so the scan can finish.
}
}
: null;
if (progressHandler && progressChannel) {
ipcRenderer.on(progressChannel, progressHandler);
}
if (entriesHandler && entriesChannel) {
ipcRenderer.on(entriesChannel, entriesHandler);
}
try {
const result = await ipcRenderer.invoke("netcatty:local:tree", {
path,
progressChannel,
entriesChannel,
cancelChannel,
limits: options?.limits,
});
if (entriesStreamDone) {
// Wait for the main-process end marker so late nested batches are not
// lost. Fail the scan if the marker never arrives (do not pretend a
// partial tree is complete).
let timedOut = false;
await Promise.race([
entriesStreamDone,
new Promise((resolve) => {
setTimeout(() => {
timedOut = true;
resolve();
}, 5_000);
}),
]);
if (timedOut) {
throw new Error("Local tree scan ended without a completion marker");
}
}
return result;
} finally {
settleEntriesStream?.();
if (progressHandler && progressChannel) {
ipcRenderer.removeListener(progressChannel, progressHandler);
}
if (entriesHandler && entriesChannel) {
ipcRenderer.removeListener(entriesChannel, entriesHandler);
}
}
},
cancelLocalTreeScan: async (scanId) => {
if (typeof scanId === "string" && scanId) {
ipcRenderer.send(`netcatty:local:tree-cancel:${scanId}`);
}
},
getHomeDir: async () => {
return ipcRenderer.invoke("netcatty:local:homedir");
@@ -106,3 +106,88 @@ test("failed upload path allocation releases the staging controller", async () =
await assert.rejects(api.stageUploadFile(createFile("file.bin", [[1]]), "failed-path"), /path unavailable/);
assert.deepEqual(await api.cancelStagedUploadFile("failed-path"), { success: false });
});
test("native tree scans use a bridge-safe cancellation id", async () => {
const sent = [];
const invoked = [];
const api = createPreloadApi({
webUtils: {},
ipcRenderer: {
on() {},
removeListener() {},
send(...args) { sent.push(args); },
async invoke(channel, payload) {
invoked.push({ channel, payload });
return [];
},
},
});
await api.listLocalTree("/tmp/project", {
scanId: "scan-123",
onProgress: () => {},
});
await api.cancelLocalTreeScan("scan-123");
assert.deepEqual(invoked, [{
channel: "netcatty:local:tree",
payload: {
path: "/tmp/project",
progressChannel: "netcatty:local:tree-progress:scan-123",
entriesChannel: undefined,
cancelChannel: "netcatty:local:tree-cancel:scan-123",
limits: undefined,
},
}]);
assert.deepEqual(sent, [["netcatty:local:tree-cancel:scan-123"]]);
});
test("listLocalTree keeps the entries listener until the tree-end marker arrives", async () => {
const listeners = new Map();
const batches = [];
let removed = false;
const api = createPreloadApi({
webUtils: {},
ipcRenderer: {
on(channel, handler) {
listeners.set(channel, handler);
},
removeListener(channel) {
if (channel.startsWith("netcatty:local:tree-entries:")) removed = true;
listeners.delete(channel);
},
send() {},
async invoke(channel, payload) {
assert.equal(channel, "netcatty:local:tree");
const entriesChannel = payload.entriesChannel;
const handler = listeners.get(entriesChannel);
assert.equal(typeof handler, "function");
// Simulate the invoke reply racing ahead of a late nested batch.
queueMicrotask(() => {
handler({}, [
{
localPath: "/tmp/project/nested/deep.txt",
relativePath: "project/nested/deep.txt",
type: "file",
size: 1,
lastModified: 1,
},
]);
handler({}, { type: "tree-end" });
});
return [];
},
},
});
await api.listLocalTree("/tmp/project", {
scanId: "scan-nested",
onEntries: (batch) => {
batches.push(batch);
},
});
assert.equal(batches.length, 1);
assert.equal(batches[0][0].relativePath, "project/nested/deep.txt");
assert.equal(removed, true);
});
+4 -1
View File
@@ -204,7 +204,10 @@ export const STORAGE_KEY_AI_EXTERNAL_MCP_SILENT_SESSIONS = 'netcatty_ai_external
// SFTP Transfer Concurrency
export const STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY = 'netcatty_sftp_transfer_concurrency_v1';
/** Folder transfers: scan the full tree before moving bytes (stable file totals). */
/**
* Legacy key only. Folder full-tree pre-scan was removed; values are ignored
* so old localStorage / sync payloads do not resurrect a live setting.
*/
export const STORAGE_KEY_SFTP_FOLDER_PRESCAN = 'netcatty_sftp_folder_prescan_v1';
/** Skip files when target size + mtime already match the source (rsync-like). */
export const STORAGE_KEY_SFTP_SKIP_UNCHANGED = 'netcatty_sftp_skip_unchanged_v1';
+628
View File
@@ -0,0 +1,628 @@
import test from "node:test";
import assert from "node:assert/strict";
import { uploadLocalFoldersProgressively } from "./progressiveFolderUpload.ts";
import { UploadController } from "./uploadController.ts";
import type { LocalTreeListEntry } from "./sftpFileUtils.ts";
test("progressive folder upload starts file transfers before the tree walk finishes", async () => {
const events: string[] = [];
let releaseSecondBatch: (() => void) | null = null;
const secondBatchGate = new Promise<void>((resolve) => {
releaseSecondBatch = resolve;
});
const listLocalTree = async (
_path: string,
options: {
onEntries?: (entries: LocalTreeListEntry[]) => void;
},
) => {
options.onEntries?.([
{
localPath: "/tmp/docs",
relativePath: "docs",
type: "directory",
size: 0,
lastModified: 1,
},
{
localPath: "/tmp/docs/a.txt",
relativePath: "docs/a.txt",
type: "file",
size: 3,
lastModified: 1,
},
]);
events.push("batch1");
// First file should be uploadable while we still "discover" more.
await secondBatchGate;
options.onEntries?.([
{
localPath: "/tmp/docs/b.txt",
relativePath: "docs/b.txt",
type: "file",
size: 4,
lastModified: 2,
},
]);
events.push("batch2");
return [];
};
const transferred: string[] = [];
const controller = new UploadController();
const uploadPromise = uploadLocalFoldersProgressively(
[{ name: "docs", localPath: "/tmp/docs" }],
{
targetPath: "/remote",
sftpId: "sftp-1",
isLocal: false,
joinPath: (base, name) => `${base}/${name}`,
bridge: {
mkdirSftp: async () => {},
startStreamTransfer: async (payload) => {
transferred.push(payload.sourcePath);
events.push(`upload:${payload.sourcePath}`);
if (payload.sourcePath.endsWith("a.txt")) {
// Let the second discovery batch proceed only after the first upload started.
releaseSecondBatch?.();
}
return { transferId: payload.transferId };
},
},
listLocalTree,
callbacks: {
onTaskCreated: (task) => events.push(`created:${task.fileName}`),
onTaskCompleted: (taskId) => events.push(`completed:${taskId.slice(0, 8)}`),
onTaskProgress: (taskId, progress) => {
events.push(`progress:${progress.transferred}/${progress.total}:${progress.phase ?? ""}`);
},
},
},
controller,
);
const results = await uploadPromise;
assert.equal(results.filter((row) => row.success).length, 2);
assert.deepEqual(transferred, ["/tmp/docs/a.txt", "/tmp/docs/b.txt"]);
// First upload must happen before the second discovery batch finishes.
const uploadA = events.indexOf("upload:/tmp/docs/a.txt");
const batch2 = events.indexOf("batch2");
assert.ok(uploadA >= 0 && batch2 >= 0, "expected both upload and batch events");
assert.ok(uploadA < batch2, "first file must upload while scan still running");
});
test("progressive folder upload enqueues nested subdirectory files", async () => {
const transferred: string[] = [];
const mkdirs: string[] = [];
const listLocalTree = async (
_path: string,
options: {
onEntries?: (entries: LocalTreeListEntry[]) => void;
},
) => {
options.onEntries?.([
{
localPath: "/tmp/docs",
relativePath: "docs",
type: "directory",
size: 0,
lastModified: 1,
},
{
localPath: "/tmp/docs/a.txt",
relativePath: "docs/a.txt",
type: "file",
size: 1,
lastModified: 1,
},
]);
options.onEntries?.([
{
localPath: "/tmp/docs/nested",
relativePath: "docs/nested",
type: "directory",
size: 0,
lastModified: 1,
},
{
localPath: "/tmp/docs/nested/deep",
relativePath: "docs/nested/deep",
type: "directory",
size: 0,
lastModified: 1,
},
{
localPath: "/tmp/docs/nested/deep/b.txt",
relativePath: "docs/nested/deep/b.txt",
type: "file",
size: 2,
lastModified: 2,
},
]);
return [];
};
const results = await uploadLocalFoldersProgressively(
[{ name: "docs", localPath: "/tmp/docs" }],
{
targetPath: "/remote",
sftpId: "sftp-1",
isLocal: false,
joinPath: (base, name) => `${base}/${name}`,
bridge: {
mkdirSftp: async (_id, dirPath) => {
mkdirs.push(dirPath);
},
startStreamTransfer: async (payload) => {
transferred.push(payload.targetPath);
return { transferId: payload.transferId };
},
},
listLocalTree,
},
);
assert.equal(results.filter((row) => row.success).length, 2);
assert.deepEqual(transferred.sort(), [
"/remote/docs/a.txt",
"/remote/docs/nested/deep/b.txt",
].sort());
assert.ok(mkdirs.includes("/remote/docs/nested"));
assert.ok(mkdirs.includes("/remote/docs/nested/deep"));
});
test("progressive folder upload stops enqueueing children after cancel", async () => {
const controller = new UploadController();
const createdChildren: string[] = [];
let entriesCb: ((entries: LocalTreeListEntry[]) => void) | null = null;
const listLocalTree = async (
_path: string,
options: {
onEntries?: (entries: LocalTreeListEntry[]) => void;
abortSignal?: AbortSignal;
},
) => {
entriesCb = options.onEntries ?? null;
options.onEntries?.([
{
localPath: "/tmp/big/a.txt",
relativePath: "big/a.txt",
type: "file",
size: 1,
lastModified: 1,
},
]);
// Wait until cancelled.
await new Promise<void>((resolve) => {
if (options.abortSignal?.aborted) {
resolve();
return;
}
options.abortSignal?.addEventListener("abort", () => resolve(), { once: true });
});
return [];
};
const abort = new AbortController();
const uploadPromise = uploadLocalFoldersProgressively(
[{ name: "big", localPath: "/tmp/big" }],
{
targetPath: "/remote",
sftpId: "sftp-1",
isLocal: false,
joinPath: (base, name) => `${base}/${name}`,
abortSignal: abort.signal,
bridge: {
mkdirSftp: async () => {},
startStreamTransfer: async (payload) => {
// Cancel as soon as the first child begins.
await controller.cancel();
abort.abort();
// Push more discovered files after cancel — workers must not create more children.
entriesCb?.([
{
localPath: "/tmp/big/b.txt",
relativePath: "big/b.txt",
type: "file",
size: 1,
lastModified: 1,
},
{
localPath: "/tmp/big/c.txt",
relativePath: "big/c.txt",
type: "file",
size: 1,
lastModified: 1,
},
]);
return { transferId: payload.transferId, cancelled: true };
},
},
listLocalTree,
callbacks: {
onTaskCreated: (task) => {
if (!task.isDirectory) createdChildren.push(task.fileName);
},
},
},
controller,
);
await uploadPromise;
// At most the in-flight child should have been created.
assert.ok(createdChildren.length <= 1, `expected no post-cancel flood, got ${createdChildren.join(",")}`);
});
test("progressive folder upload stops enqueueing children while soft-paused", async () => {
const createdChildren: string[] = [];
const transferred: string[] = [];
let paused = true;
let releasePause!: () => void;
const pauseGate = new Promise<void>((resolve) => {
releasePause = resolve;
});
let releaseMoreFiles!: () => void;
const moreFilesGate = new Promise<void>((resolve) => {
releaseMoreFiles = resolve;
});
const waitWhilePaused = async () => {
while (paused) {
await pauseGate;
}
};
const listLocalTree = async (
_path: string,
options: {
onEntries?: (entries: LocalTreeListEntry[]) => void;
},
) => {
options.onEntries?.([
{
localPath: "/tmp/docs/a.txt",
relativePath: "docs/a.txt",
type: "file",
size: 1,
lastModified: 1,
},
]);
// Stay in the walk until the test unblocks the second batch (after pause assert).
await moreFilesGate;
options.onEntries?.([
{
localPath: "/tmp/docs/b.txt",
relativePath: "docs/b.txt",
type: "file",
size: 1,
lastModified: 2,
},
{
localPath: "/tmp/docs/c.txt",
relativePath: "docs/c.txt",
type: "file",
size: 1,
lastModified: 3,
},
]);
return [];
};
const uploadPromise = uploadLocalFoldersProgressively(
[{ name: "docs", localPath: "/tmp/docs" }],
{
targetPath: "/remote",
sftpId: "sftp-1",
isLocal: false,
joinPath: (base, name) => `${base}/${name}`,
waitWhilePaused,
bridge: {
mkdirSftp: async () => {},
startStreamTransfer: async (payload) => {
transferred.push(payload.sourcePath);
return { transferId: payload.transferId };
},
},
listLocalTree,
callbacks: {
onTaskCreated: (task) => {
if (!task.isDirectory) createdChildren.push(task.fileName);
},
},
},
);
// Give workers a turn; soft-pause must block child creation entirely.
await new Promise((resolve) => setTimeout(resolve, 40));
assert.deepEqual(createdChildren, [], "no children while paused");
assert.deepEqual(transferred, [], "no transfers while paused");
paused = false;
releasePause();
// Let first file create/upload, then release the rest of the tree.
await new Promise((resolve) => setTimeout(resolve, 40));
releaseMoreFiles();
const results = await uploadPromise;
assert.equal(results.filter((row) => row.success).length, 3);
assert.deepEqual(
createdChildren.sort(),
["docs/a.txt", "docs/b.txt", "docs/c.txt"].sort(),
);
assert.equal(transferred.length, 3);
});
test("progressive multi-root pause does not HOL-block an unpaused sibling root", async () => {
const transferred: string[] = [];
const pausedParents = new Set<string>(["parent-a"]);
const parentIds = new Map([
["folderA", "parent-a"],
["folderB", "parent-b"],
]);
const waitWhilePaused = async (parentId: string) => {
while (pausedParents.has(parentId)) {
await new Promise((resolve) => setTimeout(resolve, 10));
}
};
const listLocalTree = async (
localPath: string,
options: {
onEntries?: (entries: LocalTreeListEntry[]) => void;
},
) => {
if (localPath.endsWith("folderA")) {
options.onEntries?.([
{
localPath: "/tmp/folderA/a.txt",
relativePath: "folderA/a.txt",
type: "file",
size: 1,
lastModified: 1,
},
]);
} else {
options.onEntries?.([
{
localPath: "/tmp/folderB/b.txt",
relativePath: "folderB/b.txt",
type: "file",
size: 1,
lastModified: 1,
},
]);
}
return [];
};
const uploadPromise = uploadLocalFoldersProgressively(
[
{ name: "folderA", localPath: "/tmp/folderA" },
{ name: "folderB", localPath: "/tmp/folderB" },
],
{
targetPath: "/remote",
sftpId: "sftp-1",
isLocal: false,
joinPath: (base, name) => `${base}/${name}`,
parentTaskIds: parentIds,
waitWhilePaused,
isPaused: (parentId) => pausedParents.has(parentId),
bridge: {
mkdirSftp: async () => {},
startStreamTransfer: async (payload) => {
transferred.push(payload.sourcePath);
// Unpause A only after B has uploaded — proves HOL skip works.
if (payload.sourcePath.includes("folderB")) {
pausedParents.delete("parent-a");
}
return { transferId: payload.transferId };
},
},
listLocalTree,
},
);
const results = await uploadPromise;
assert.equal(results.filter((row) => row.success).length, 2);
assert.ok(transferred.some((p) => p.includes("folderB")), "unpaused root must transfer");
assert.ok(transferred.some((p) => p.includes("folderA")), "paused root resumes after unlatch");
// folderB must not wait behind folderA indefinitely — B completes first.
assert.ok(
transferred.indexOf(transferred.find((p) => p.includes("folderB"))!)
< transferred.indexOf(transferred.find((p) => p.includes("folderA"))!),
`expected B before A, got ${transferred.join(",")}`,
);
});
test("progressive backpressure wakes every parked enqueue waiter", async () => {
// Discovery floods the queue while workers hold transfers open so the queue
// stays above the high-water mark. Multiple enqueueBatch waiters must all
// be released when workers drain past the low-water mark.
const transferred: string[] = [];
let releaseTransfers!: () => void;
const transferGate = new Promise<void>((resolve) => {
releaseTransfers = resolve;
});
let started = 0;
const listLocalTree = async (
_path: string,
options: {
onEntries?: (entries: LocalTreeListEntry[]) => void;
},
) => {
const batches = Array.from({ length: 8 }, (_, batch) => (
Array.from({ length: 300 }, (__, i) => ({
localPath: `/tmp/docs/f-${batch}-${i}.txt`,
relativePath: `docs/f-${batch}-${i}.txt`,
type: "file" as const,
size: 1,
lastModified: 1,
}))
));
// Overlapping handlers so several park on waitIfQueueHigh together.
await Promise.all(batches.map((batch) => Promise.resolve().then(() => options.onEntries?.(batch))));
return [];
};
const uploadPromise = uploadLocalFoldersProgressively(
[{ name: "docs", localPath: "/tmp/docs" }],
{
targetPath: "/remote",
sftpId: "sftp-1",
isLocal: false,
joinPath: (base, name) => `${base}/${name}`,
bridge: {
mkdirSftp: async () => {},
startStreamTransfer: async (payload) => {
started += 1;
transferred.push(payload.sourcePath);
await transferGate;
return { transferId: payload.transferId };
},
},
listLocalTree,
},
);
// Let discovery fill past high-water (2000) while 2 workers hold the gate.
await new Promise((resolve) => setTimeout(resolve, 80));
assert.ok(started >= 1, "workers should have started");
releaseTransfers();
const results = await uploadPromise;
assert.equal(results.filter((row) => row.success).length, 8 * 300);
assert.equal(transferred.length, 8 * 300);
});
test("progressive root conflict skip does not overwrite an existing remote folder", async () => {
const transferred: string[] = [];
const result = await uploadLocalFoldersProgressively(
[{ name: "docs", localPath: "/tmp/docs" }],
{
targetPath: "/remote",
sftpId: "sftp-1",
isLocal: false,
joinPath: (base, name) => `${base}/${name}`,
resolveConflict: async () => "skip",
bridge: {
mkdirSftp: async () => {},
statSftp: async () => ({ type: "directory", size: 0, lastModified: 1 }),
startStreamTransfer: async (payload) => {
transferred.push(payload.sourcePath);
return { transferId: payload.transferId };
},
},
listLocalTree: async () => {
throw new Error("scan must not run after skip");
},
},
);
assert.equal(transferred.length, 0);
assert.equal(result[0]?.cancelled, true);
});
test("progressive multi-root resume of non-head parent unblocks while head stays paused", async () => {
const transferred: string[] = [];
// Both latched initially; head is always parent-a in FIFO order.
const pausedParents = new Set<string>(["parent-a", "parent-b"]);
const parentIds = new Map([
["folderA", "parent-a"],
["folderB", "parent-b"],
]);
const waiters = new Map<string, Array<() => void>>();
const waitWhilePaused = async (parentId: string) => {
while (pausedParents.has(parentId)) {
await new Promise<void>((resolve) => {
const list = waiters.get(parentId) ?? [];
list.push(resolve);
waiters.set(parentId, list);
});
}
};
const release = (parentId: string) => {
pausedParents.delete(parentId);
for (const resolve of waiters.get(parentId) ?? []) resolve();
waiters.delete(parentId);
};
const listLocalTree = async (
localPath: string,
options: {
onEntries?: (entries: LocalTreeListEntry[]) => void;
},
) => {
if (localPath.endsWith("folderA")) {
options.onEntries?.([
{
localPath: "/tmp/folderA/a.txt",
relativePath: "folderA/a.txt",
type: "file",
size: 1,
lastModified: 1,
},
]);
} else {
options.onEntries?.([
{
localPath: "/tmp/folderB/b.txt",
relativePath: "folderB/b.txt",
type: "file",
size: 1,
lastModified: 1,
},
]);
}
return [];
};
const uploadPromise = uploadLocalFoldersProgressively(
[
{ name: "folderA", localPath: "/tmp/folderA" },
{ name: "folderB", localPath: "/tmp/folderB" },
],
{
targetPath: "/remote",
sftpId: "sftp-1",
isLocal: false,
joinPath: (base, name) => `${base}/${name}`,
parentTaskIds: parentIds,
waitWhilePaused,
isPaused: (parentId) => pausedParents.has(parentId),
bridge: {
mkdirSftp: async () => {},
startStreamTransfer: async (payload) => {
transferred.push(payload.sourcePath);
return { transferId: payload.transferId };
},
},
listLocalTree,
},
);
// Let both jobs queue and workers park on all-paused race.
await new Promise((resolve) => setTimeout(resolve, 40));
assert.deepEqual(transferred, []);
// Resume only non-head parent B while A stays paused.
release("parent-b");
await new Promise((resolve) => setTimeout(resolve, 40));
assert.ok(
transferred.some((p) => p.includes("folderB")),
`B must transfer while A paused, got ${transferred.join(",")}`,
);
assert.equal(
transferred.some((p) => p.includes("folderA")),
false,
"A must stay blocked",
);
release("parent-a");
const results = await uploadPromise;
assert.equal(results.filter((row) => row.success).length, 2);
});
+669
View File
@@ -0,0 +1,669 @@
/**
* Progressive local-folder upload: stream discovery batches from listLocalTree
* and upload files as they arrive (edge-scan / edge-transfer).
*/
import type { DropEntry } from "./sftpFileUtils";
import { localTreeToDropEntries, type LocalTreeListEntry } from "./sftpFileUtils";
import type { UploadBridge, UploadCallbacks, UploadResult } from "./uploadService.types";
import type { UploadController } from "./uploadController";
import {
canReplaceSftpConflict,
describeSftpExistingKind,
describeSftpIncomingKind,
getSftpConflictTypeKey,
} from "../domain/sftpConflict";
const formatUploadError = (error: unknown): string =>
error instanceof Error ? error.message : String(error);
const UPLOAD_CONCURRENCY = 2;
/** Pause discovery when the upload queue grows this large (memory backpressure). */
const QUEUE_HIGH_WATER = 2_000;
/** Resume discovery once the queue drains to this size. */
const QUEUE_LOW_WATER = 500;
export type ProgressiveLocalRoot = {
name: string;
localPath: string;
};
export type ListLocalTreeStreaming = (
path: string,
options: {
onProgress?: (progress: { fileCount: number; directoryCount: number; entryCount: number }) => void;
onEntries?: (entries: LocalTreeListEntry[]) => void;
abortSignal?: AbortSignal;
},
) => Promise<LocalTreeListEntry[]>;
export type ProgressiveConflictAction = "stop" | "skip" | "replace" | "duplicate" | "merge";
export type ProgressiveFolderUploadConfig = {
targetPath: string;
sftpId: string | null;
targetHostId?: string;
isLocal: boolean;
bridge: UploadBridge;
joinPath: (base: string, name: string) => string;
callbacks?: UploadCallbacks;
listLocalTree: ListLocalTreeStreaming;
/** Optional pre-created parent task ids (e.g. scanning task id for single root). */
parentTaskIds?: Map<string, string>;
abortSignal?: AbortSignal;
/**
* Soft-pause gate (process-global latch). Must resolve only after the parent
* folder is resumed. Used before discovery enqueue and before creating child
* UI rows so Pause does not keep filling the transfer queue.
*/
waitWhilePaused?: (parentTaskId: string) => Promise<void>;
/** Sync probe so multi-root queues can skip latched parents (no HOL block). */
isPaused?: (parentTaskId: string) => boolean;
/**
* Root-level conflict dialog (same contract as uploadEntries). Progressive
* path must not overwrite existing remote folders without confirmation.
*/
resolveConflict?: (conflict: {
fileName: string;
targetPath: string;
isDirectory: boolean;
existingType?: "file" | "directory" | "symlink";
existingSize: number;
newSize: number;
existingModified: number;
newModified: number;
applyToAllCount: number;
}) => Promise<ProgressiveConflictAction>;
};
function entryToDrop(entry: LocalTreeListEntry): DropEntry {
return localTreeToDropEntries([entry])[0];
}
/**
* Walk each local root with streaming batches and upload files concurrently
* while discovery continues.
*/
export async function uploadLocalFoldersProgressively(
roots: ProgressiveLocalRoot[],
config: ProgressiveFolderUploadConfig,
controller?: UploadController,
): Promise<UploadResult[]> {
const {
targetPath,
sftpId,
targetHostId,
isLocal,
bridge,
joinPath,
callbacks,
listLocalTree,
parentTaskIds,
abortSignal,
waitWhilePaused,
isPaused,
resolveConflict,
} = config;
if (roots.length === 0) return [];
if (!isLocal && !sftpId) {
throw new Error("No SFTP session for progressive folder upload");
}
const isStopped = () => !!(controller?.isCancelled() || abortSignal?.aborted);
/** Block while the folder soft-pause latch is held; re-check cancel after wake. */
const awaitUnpaused = async (parentTaskId: string): Promise<boolean> => {
if (!waitWhilePaused) return !isStopped();
while (!isStopped()) {
await waitWhilePaused(parentTaskId);
// waitWhilePaused may resolve spuriously if the latch was already free;
// always re-check stop intent after any await.
if (isStopped()) return false;
return true;
}
return false;
};
const results: UploadResult[] = [];
const createdDirs = new Set<string>();
const failedDirs = new Map<string, string>();
const parentIds = new Map<string, string>();
const parentStats = new Map<string, {
discovered: number;
completed: number;
failed: number;
dirFailures: number;
}>();
/** Source root name -> destination root segment after conflict resolution. */
const destRootNameBySource = new Map<string, string>();
const statTarget = async (path: string) => {
try {
if (isLocal) return await bridge.statLocal?.(path) ?? null;
if (sftpId) return await bridge.statSftp?.(sftpId, path) ?? null;
} catch {
return null;
}
return null;
};
const deleteTarget = async (path: string) => {
if (isLocal) await bridge.deleteLocalFile?.(path);
else if (sftpId) await bridge.deleteSftp?.(sftpId, path);
};
const getDuplicateName = async (name: string) => {
for (let index = 1; index < 1000; index++) {
const suffix = index === 1 ? " (copy)" : ` (copy ${index})`;
const candidate = `${name}${suffix}`;
const existing = await statTarget(joinPath(targetPath, candidate));
if (!existing) return candidate;
}
return `${name} (copy ${Date.now()})`;
};
// Root-level conflict preflight — match uploadEntries so progressive drops
// cannot overwrite remote content without Skip/Replace/Duplicate/Merge.
let activeRoots = [...roots];
if (resolveConflict) {
const existingByRoot = await Promise.all(activeRoots.map(async (root) => ({
root,
existing: await statTarget(joinPath(targetPath, root.name)),
})));
const conflictCounts = new Map<string, number>();
for (const { existing } of existingByRoot) {
if (!existing) continue;
const key = getSftpConflictTypeKey(true, existing.type);
conflictCounts.set(key, (conflictCounts.get(key) ?? 0) + 1);
}
const kept: ProgressiveLocalRoot[] = [];
for (const { root, existing } of existingByRoot) {
if (isStopped()) break;
if (!existing) {
destRootNameBySource.set(root.name, root.name);
kept.push(root);
continue;
}
const conflictKey = getSftpConflictTypeKey(true, existing.type);
const action = await resolveConflict({
fileName: root.name,
targetPath: joinPath(targetPath, root.name),
isDirectory: true,
existingType: existing.type,
existingSize: existing.size,
// Progressive has not walked yet; byte total is unknown.
newSize: 0,
existingModified: existing.lastModified,
newModified: Date.now(),
applyToAllCount: conflictCounts.get(conflictKey) ?? 1,
});
if (action === "stop") {
await controller?.cancel();
return [{ fileName: root.name, success: false, cancelled: true }, ...results];
}
if (action === "skip") {
results.push({ fileName: root.name, success: false, cancelled: true });
const scanningId = parentTaskIds?.get(root.name);
if (scanningId) callbacks?.onTaskCancelled?.(scanningId);
continue;
}
if (action === "replace") {
if (!canReplaceSftpConflict(true, existing.type)) {
results.push({
fileName: root.name,
success: false,
error: `Cannot replace existing ${describeSftpExistingKind(existing.type)} with ${describeSftpIncomingKind(true)}: ${joinPath(targetPath, root.name)}`,
});
const scanningId = parentTaskIds?.get(root.name);
if (scanningId) {
callbacks?.onTaskFailed?.(
scanningId,
`Cannot replace existing ${describeSftpExistingKind(existing.type)}`,
);
}
continue;
}
await deleteTarget(joinPath(targetPath, root.name));
destRootNameBySource.set(root.name, root.name);
kept.push(root);
continue;
}
if (action === "duplicate") {
const duplicateName = await getDuplicateName(root.name);
destRootNameBySource.set(root.name, duplicateName);
kept.push(root);
continue;
}
if (action === "merge" && !(existing.type === "directory")) {
results.push({
fileName: root.name,
success: false,
error: `Cannot merge existing ${describeSftpExistingKind(existing.type)} with ${describeSftpIncomingKind(true)}: ${joinPath(targetPath, root.name)}`,
});
const scanningId = parentTaskIds?.get(root.name);
if (scanningId) {
callbacks?.onTaskFailed?.(scanningId, `Cannot merge existing ${describeSftpExistingKind(existing.type)}`);
}
continue;
}
destRootNameBySource.set(root.name, root.name);
kept.push(root);
}
activeRoots = kept;
} else {
for (const root of activeRoots) destRootNameBySource.set(root.name, root.name);
}
if (activeRoots.length === 0) return results;
for (const root of activeRoots) {
const id = parentTaskIds?.get(root.name) ?? crypto.randomUUID();
parentIds.set(root.name, id);
parentStats.set(id, { discovered: 0, completed: 0, failed: 0, dirFailures: 0 });
const destName = destRootNameBySource.get(root.name) ?? root.name;
// Skip create if caller already opened a scanning row with this id.
if (!parentTaskIds?.has(root.name)) {
callbacks?.onTaskCreated?.({
id,
fileName: destName,
displayName: destName,
isDirectory: true,
progressMode: "files",
totalBytes: 0,
transferredBytes: 0,
speed: 0,
fileCount: 0,
completedCount: 0,
sourcePath: root.localPath,
});
} else {
// Promote scanning row into live file-count progress.
callbacks?.onTaskProgress?.(id, {
transferred: 0,
total: 0,
speed: 0,
percent: 0,
phase: "scanning",
});
}
}
const ensureDirectory = async (dirPath: string): Promise<void> => {
if (createdDirs.has(dirPath)) return;
if (failedDirs.has(dirPath)) {
throw new Error(failedDirs.get(dirPath) || "Directory creation failed");
}
try {
if (isLocal) {
await bridge.mkdirLocal?.(dirPath);
} else if (sftpId) {
await bridge.mkdirSftp(sftpId, dirPath);
}
createdDirs.add(dirPath);
} catch (error) {
const message = formatUploadError(error);
// Concurrent workers / merge into existing remote dirs race here.
if (/exist|EEXIST|file exists|already/i.test(message)) {
createdDirs.add(dirPath);
return;
}
failedDirs.set(dirPath, message);
throw error;
}
};
const remapRelativePath = (sourceRootName: string, relativePath: string): string => {
const dest = destRootNameBySource.get(sourceRootName) ?? sourceRootName;
if (dest === sourceRootName) return relativePath;
if (relativePath === sourceRootName) return dest;
if (relativePath.startsWith(`${sourceRootName}/`)) {
return `${dest}/${relativePath.slice(sourceRootName.length + 1)}`;
}
return relativePath;
};
const ensureParentsForFile = async (relativePath: string): Promise<void> => {
const parts = relativePath.replace(/\\/g, "/").split("/").filter(Boolean);
if (parts.length <= 1) return;
let cursor = targetPath;
for (let i = 0; i < parts.length - 1; i++) {
cursor = joinPath(cursor, parts[i]);
await ensureDirectory(cursor);
}
};
type FileJob = { entry: DropEntry; parentId: string; rootName: string };
const fileQueue: FileJob[] = [];
let scanDone = false;
/** In-flight onEntries handlers (may await soft-pause before queueing). */
let pendingEnqueues = 0;
let scanError: unknown;
let wakeWaiters: Array<() => void> = [];
const wake = () => {
const waiters = wakeWaiters;
wakeWaiters = [];
for (const resolve of waiters) resolve();
};
const waitForWork = () => new Promise<void>((resolve) => {
wakeWaiters.push(resolve);
});
const discoverySettled = () => scanDone && pendingEnqueues === 0;
// Multiple enqueueBatch handlers can park on backpressure at once. A single
// resolver would orphan older waiters and leave pendingEnqueues stuck forever.
let pauseScanWaiters: Array<() => void> = [];
const waitIfQueueHigh = async () => {
while (fileQueue.length >= QUEUE_HIGH_WATER && !isStopped()) {
await new Promise<void>((resolve) => {
pauseScanWaiters.push(resolve);
});
}
};
const maybeResumeScan = () => {
if (fileQueue.length > QUEUE_LOW_WATER || pauseScanWaiters.length === 0) return;
const waiters = pauseScanWaiters;
pauseScanWaiters = [];
for (const resolve of waiters) resolve();
};
const publishParentProgress = (parentId: string, phase?: "scanning" | "transferring") => {
const stats = parentStats.get(parentId);
if (!stats) return;
const total = Math.max(stats.discovered, stats.completed);
callbacks?.onTaskProgress?.(parentId, {
transferred: stats.completed,
total,
speed: 0,
percent: total > 0 ? (stats.completed / total) * 100 : 0,
phase: phase ?? (stats.completed > 0 ? "transferring" : "scanning"),
});
};
const enqueueBatch = async (rootName: string, batch: LocalTreeListEntry[]) => {
pendingEnqueues += 1;
try {
if (isStopped()) {
fileQueue.length = 0;
return;
}
const parentId = parentIds.get(rootName);
if (!parentId) return;
const stats = parentStats.get(parentId);
if (!stats) return;
// Soft-pause: do not grow the work queue or create remote dirs until resume.
// Otherwise Pause still looks "alive" as hundreds of pending children appear.
if (!(await awaitUnpaused(parentId))) {
fileQueue.length = 0;
return;
}
for (const row of batch) {
if (isStopped()) {
fileQueue.length = 0;
return;
}
// Re-check pause between entries so a mid-batch Pause freezes the rest.
if (waitWhilePaused && !(await awaitUnpaused(parentId))) {
fileQueue.length = 0;
return;
}
const drop = entryToDrop(row);
const remappedRelative = remapRelativePath(rootName, drop.relativePath);
const remappedDrop = remappedRelative === drop.relativePath
? drop
: { ...drop, relativePath: remappedRelative };
if (remappedDrop.isDirectory) {
// Create remote dirs early when we see them. Empty-directory failures
// must not be silently ignored (no later file will retry the mkdir).
try {
await ensureDirectory(joinPath(targetPath, remappedDrop.relativePath));
} catch (error) {
stats.dirFailures += 1;
results.push({
fileName: remappedDrop.relativePath,
success: false,
error: formatUploadError(error),
});
}
continue;
}
stats.discovered += 1;
fileQueue.push({ entry: remappedDrop, parentId, rootName });
}
publishParentProgress(parentId);
wake();
maybeResumeScan();
await waitIfQueueHigh();
} finally {
pendingEnqueues = Math.max(0, pendingEnqueues - 1);
// Workers may have seen an empty queue while we were still paused inside
// this handler — wake them once discovery work is actually settled.
wake();
}
};
const scanPromise = (async () => {
try {
for (const root of activeRoots) {
if (isStopped()) break;
await listLocalTree(root.localPath, {
abortSignal,
onProgress: () => {
// Counts are derived from entry batches so UI stays consistent.
},
onEntries: (batch) => {
void enqueueBatch(root.name, batch);
},
});
}
} catch (error) {
scanError = error;
} finally {
scanDone = true;
wake();
if (pauseScanWaiters.length > 0) {
const waiters = pauseScanWaiters;
pauseScanWaiters = [];
for (const resolve of waiters) resolve();
}
}
})();
const uploadSingle = async (job: FileJob): Promise<void> => {
if (isStopped()) return;
const { entry, parentId } = job;
const stats = parentStats.get(parentId);
if (!stats) return;
// Soft-pause and cancel both block before a child row is created. Creating
// the UI task first made Pause look broken: new "pending" children kept
// flooding the panel even though streams were soft-drained.
if (!(await awaitUnpaused(parentId))) return;
const entryTargetPath = joinPath(targetPath, entry.relativePath);
const childId = crypto.randomUUID();
const fileTotalBytes = entry.size ?? 0;
callbacks?.onTaskCreated?.({
id: childId,
fileName: entry.relativePath,
displayName: entry.relativePath,
isDirectory: false,
progressMode: "bytes",
parentTaskId: parentId,
totalBytes: fileTotalBytes,
transferredBytes: 0,
speed: 0,
fileCount: 1,
completedCount: 0,
sourcePath: entry.localPath,
});
if (isStopped()) {
callbacks?.onTaskCancelled?.(childId);
return;
}
// Pause may have been hit between create and stream open — wait again so we
// never start a new write under a paused parent.
if (!(await awaitUnpaused(parentId))) {
callbacks?.onTaskCancelled?.(childId);
return;
}
try {
await ensureParentsForFile(entry.relativePath);
const localFilePath = entry.localPath;
if (!localFilePath || !bridge.startStreamTransfer) {
throw new Error("A local file path is required for streaming SFTP upload");
}
controller?.addActiveTransfer(childId);
let streamResult: { error?: string; cancelled?: boolean } | undefined;
try {
streamResult = await bridge.startStreamTransfer({
transferId: childId,
sourcePath: localFilePath,
targetPath: entryTargetPath,
sourceType: "local",
targetType: isLocal ? "local" : "sftp",
targetSftpId: isLocal ? undefined : sftpId ?? undefined,
targetHostId: isLocal ? undefined : targetHostId,
totalBytes: fileTotalBytes,
resumable: true,
checkpointBytes: 0,
});
} finally {
controller?.removeActiveTransfer(childId);
}
if (streamResult?.cancelled || streamResult?.error?.includes("cancelled")) {
callbacks?.onTaskCancelled?.(childId);
return;
}
if (streamResult?.error) {
throw new Error(streamResult.error);
}
results.push({ fileName: entry.relativePath, success: true });
stats.completed += 1;
callbacks?.onTaskCompleted?.(childId, fileTotalBytes);
publishParentProgress(parentId, "transferring");
} catch (error) {
if (controller?.isCancelled()) {
callbacks?.onTaskCancelled?.(childId);
return;
}
const message = formatUploadError(error);
results.push({ fileName: entry.relativePath, success: false, error: message });
stats.failed += 1;
stats.completed += 1;
callbacks?.onTaskFailed?.(childId, message);
publishParentProgress(parentId, "transferring");
} finally {
maybeResumeScan();
wake();
}
};
const workers = Array.from({ length: UPLOAD_CONCURRENCY }, async () => {
while (true) {
if (isStopped()) {
fileQueue.length = 0;
return;
}
if (fileQueue.length === 0) {
// onEntries handlers may still be awaiting soft-pause before pushing
// jobs — do not treat scanDone alone as terminal.
if (discoverySettled()) return;
await waitForWork();
continue;
}
// Soft-pause is per-parent. Prefer an unpaused job so pausing parent A
// does not head-of-line block parent B on the shared FIFO.
if (fileQueue.length === 0) continue;
let pickIndex = 0;
if (isPaused) {
const freeIndex = fileQueue.findIndex((j) => !isPaused(j.parentId));
if (freeIndex < 0) {
// Every pending parent is latched. Race-wait on *all* of them so
// resuming a non-head parent does not stall behind a still-paused head.
const pausedParents = [...new Set(fileQueue.map((j) => j.parentId))];
let anyReleased = false;
await Promise.race(
pausedParents.map(async (parentId) => {
if (await awaitUnpaused(parentId)) anyReleased = true;
}),
);
if (!anyReleased || isStopped()) {
fileQueue.length = 0;
return;
}
continue;
}
pickIndex = freeIndex;
} else if (waitWhilePaused) {
if (!(await awaitUnpaused(fileQueue[0].parentId))) {
fileQueue.length = 0;
return;
}
}
if (isStopped()) {
fileQueue.length = 0;
return;
}
if (fileQueue.length === 0) continue;
const [job] = fileQueue.splice(pickIndex, 1);
if (!job) continue;
maybeResumeScan();
if (isStopped()) {
fileQueue.length = 0;
return;
}
await uploadSingle(job);
}
});
await Promise.all([scanPromise, ...workers]);
if (controller?.isCancelled()) {
for (const parentId of parentIds.values()) {
callbacks?.onTaskCancelled?.(parentId);
}
return [{ fileName: "", success: false, cancelled: true }, ...results];
}
if (scanError) {
for (const parentId of parentIds.values()) {
callbacks?.onTaskFailed?.(parentId, formatUploadError(scanError));
}
throw scanError;
}
for (const [parentId, stats] of parentStats) {
if (stats.failed > 0 || stats.dirFailures > 0) {
const parts: string[] = [];
if (stats.failed > 0) {
parts.push(
stats.failed === stats.discovered && stats.discovered > 0
? `All ${stats.failed} files failed`
: `${stats.failed} of ${stats.discovered} files failed`,
);
}
if (stats.dirFailures > 0) {
parts.push(
stats.dirFailures === 1
? "1 directory could not be created"
: `${stats.dirFailures} directories could not be created`,
);
}
callbacks?.onTaskFailed?.(parentId, parts.join("; "));
} else {
callbacks?.onTaskCompleted?.(parentId, stats.discovered);
}
}
return results;
}
+131 -1
View File
@@ -1,7 +1,15 @@
import test from "node:test";
import assert from "node:assert/strict";
import { getFileExtension, hasFileExtension } from "./sftpFileUtils.ts";
import {
captureDropPayload,
formatDropScanLabel,
getFileExtension,
hasFileExtension,
localTreeToDropEntries,
materializeDropEntries,
type LocalTreeListEntry,
} from "./sftpFileUtils.ts";
test("hasFileExtension identifies extensionless and dotted filenames", () => {
assert.equal(hasFileExtension("nginx"), false);
@@ -13,3 +21,125 @@ test("hasFileExtension identifies extensionless and dotted filenames", () => {
assert.equal(hasFileExtension(".bashrc"), false);
assert.equal(hasFileExtension("archive.tar.gz"), true);
});
test("formatDropScanLabel summarizes dropped root names", () => {
assert.equal(formatDropScanLabel([]), "Scanning files...");
assert.equal(formatDropScanLabel([{ name: "docs", isDirectory: true }]), "docs");
assert.equal(
formatDropScanLabel([
{ name: "a", isDirectory: true },
{ name: "b", isDirectory: true },
]),
"a, b",
);
assert.equal(
formatDropScanLabel([
{ name: "a", isDirectory: true },
{ name: "b", isDirectory: true },
{ name: "c", isDirectory: true },
]),
"a, b +1",
);
});
test("captureDropPayload reads webkit entries synchronously", () => {
const file = new File(["x"], "note.txt");
Object.defineProperty(file, "path", { value: "/tmp/note.txt" });
const dataTransfer = {
items: [{
kind: "file",
getAsFile: () => file,
webkitGetAsEntry: () => ({
name: "note.txt",
isFile: true,
isDirectory: false,
}),
}],
files: [file],
} as unknown as DataTransfer;
const payload = captureDropPayload(dataTransfer);
assert.equal(payload.roots.length, 1);
assert.equal(payload.roots[0].name, "note.txt");
assert.equal(payload.roots[0].isDirectory, false);
assert.equal(payload.roots[0].localPath, "/tmp/note.txt");
});
test("materializeDropEntries prefers listLocalTree for directory roots with paths", async () => {
const progress: Array<{ fileCount: number; directoryCount: number }> = [];
const tree: LocalTreeListEntry[] = [
{
localPath: "/tmp/project",
relativePath: "project",
type: "directory",
size: 0,
lastModified: 1,
},
{
localPath: "/tmp/project/src",
relativePath: "project/src",
type: "directory",
size: 0,
lastModified: 1,
},
{
localPath: "/tmp/project/src/main.ts",
relativePath: "project/src/main.ts",
type: "file",
size: 12,
lastModified: 2,
},
];
const entries = await materializeDropEntries(
{
roots: [{
name: "project",
isDirectory: true,
localPath: "/tmp/project",
}],
filesFallback: [],
},
{
listLocalTree: async (path, options) => {
assert.equal(path, "/tmp/project");
options?.onProgress?.({ fileCount: 1, directoryCount: 2, entryCount: 3 });
return tree;
},
onProgress: (p) => progress.push({ fileCount: p.fileCount, directoryCount: p.directoryCount }),
},
);
assert.deepEqual(
entries.map((entry) => ({
relativePath: entry.relativePath,
isDirectory: entry.isDirectory,
localPath: entry.localPath,
size: entry.size,
})),
[
{ relativePath: "project", isDirectory: true, localPath: "/tmp/project", size: undefined },
{ relativePath: "project/src", isDirectory: true, localPath: "/tmp/project/src", size: undefined },
{
relativePath: "project/src/main.ts",
isDirectory: false,
localPath: "/tmp/project/src/main.ts",
size: 12,
},
],
);
assert.ok(progress.some((p) => p.fileCount === 1 && p.directoryCount === 2));
});
test("localTreeToDropEntries preserves file sizes without File handles", () => {
const entries = localTreeToDropEntries([{
localPath: "/tmp/a.txt",
relativePath: "a.txt",
type: "file",
size: 42,
lastModified: 9,
}]);
assert.equal(entries[0].file, null);
assert.equal(entries[0].localPath, "/tmp/a.txt");
assert.equal(entries[0].size, 42);
});
+446 -98
View File
@@ -242,6 +242,59 @@ export interface DropEntry {
isDirectory: boolean;
}
/** Local tree row from main-process `listLocalTree`. */
export interface LocalTreeListEntry {
localPath: string;
relativePath: string;
type: "file" | "directory";
size: number;
lastModified: number;
}
/** Live scan counters while expanding a dropped folder. */
export interface DropScanProgress {
fileCount: number;
directoryCount: number;
entryCount: number;
/** Optional UI label (e.g. root folder names). */
label?: string;
}
/**
* Sync snapshot of a drop. DataTransfer must be read before any await —
* after that, items/files may be empty.
*/
export interface CapturedDropRoot {
name: string;
isDirectory: boolean;
localPath?: string;
file?: File | null;
size?: number;
/** Folder-picker relative path when the File carries webkitRelativePath */
relativePath?: string;
/** webkit entry for Chromium walk fallback when no local path is available */
fsEntry?: FileSystemEntry;
}
export interface CapturedDropPayload {
roots: CapturedDropRoot[];
filesFallback: File[];
}
export interface MaterializeDropOptions {
listLocalTree?: (
path: string,
options?: {
onProgress?: (progress: DropScanProgress) => void;
abortSignal?: AbortSignal;
},
) => Promise<LocalTreeListEntry[]>;
onProgress?: (progress: DropScanProgress) => void;
/** Cooperative cancel for webkit walks and native listLocalTree. */
abortSignal?: AbortSignal;
isCancelled?: () => boolean;
}
export const getDropEntryLocalPath = (entry: DropEntry): string | undefined =>
entry.localPath ?? (entry.file ? getPathForFile(entry.file) : undefined);
@@ -291,74 +344,133 @@ async function readAllDirectoryEntries(
return allEntries;
}
function joinLocalRelativePath(rootPath: string, relativePath: string): string {
const normalizedRelative = relativePath.replace(/\\/g, "/");
const parts = normalizedRelative.split("/");
// relativePath is rooted at the drop root name; local path already points at that root.
const nested = parts.length > 1 ? parts.slice(1).join("/") : "";
if (!nested) return rootPath;
const separator = rootPath.includes("\\") ? "\\" : "/";
return rootPath + separator + nested.replace(/\//g, separator);
}
/**
* Process file system entries iteratively (non-recursive) to handle large folders
* Uses a queue-based approach to avoid stack overflow
* @param rootEntries - The root entries to process
* @returns Array of DropEntry objects with files and their relative paths
* Process file system entries iteratively (non-recursive) to handle large folders.
* Prefer reconstructing local paths from the drop root so we can skip per-file
* `entry.file()` when Electron already exposed the folder path.
*/
export function isDropScanCancelledError(error: unknown): boolean {
if (!error || typeof error !== "object") return false;
const code = (error as { code?: string }).code;
return code === "ERR_DROP_SCAN_CANCELLED" || code === "ERR_LOCAL_TREE_CANCELLED";
}
function throwIfDropScanCancelled(options: {
abortSignal?: AbortSignal;
isCancelled?: () => boolean;
}): void {
if (options.abortSignal?.aborted || options.isCancelled?.()) {
const error = new Error("Drop scan cancelled");
(error as Error & { code?: string }).code = "ERR_DROP_SCAN_CANCELLED";
throw error;
}
}
async function processEntriesIteratively(
rootEntries: FileSystemEntry[]
rootEntries: FileSystemEntry[],
options: {
rootPathByName?: Map<string, string>;
onProgress?: (progress: DropScanProgress) => void;
abortSignal?: AbortSignal;
isCancelled?: () => boolean;
} = {},
): Promise<DropEntry[]> {
const results: DropEntry[] = [];
const rootPathByName = options.rootPathByName ?? new Map<string, string>();
// Queue of entries to process: [entry, basePath]
// Index-based queue avoids O(n²) Array.shift on huge trees.
const queue: Array<{ entry: FileSystemEntry; basePath: string }> = [];
// Initialize queue with root entries
for (const entry of rootEntries) {
queue.push({ entry, basePath: "" });
}
let queueIndex = 0;
let processedCount = 0;
const YIELD_INTERVAL = 100; // Yield to main thread every N items
let fileCount = 0;
let directoryCount = 0;
const YIELD_INTERVAL = 100;
const PROGRESS_INTERVAL = 32;
while (queue.length > 0) {
const { entry, basePath } = queue.shift()!;
const reportProgress = (force = false) => {
if (!options.onProgress) return;
if (!force && processedCount % PROGRESS_INTERVAL !== 0) return;
options.onProgress({
fileCount,
directoryCount,
entryCount: fileCount + directoryCount,
});
};
while (queueIndex < queue.length) {
throwIfDropScanCancelled(options);
const { entry, basePath } = queue[queueIndex++];
const relativePath = basePath ? `${basePath}/${entry.name}` : entry.name;
const rootName = relativePath.split("/")[0] ?? entry.name;
const rootLocalPath = rootPathByName.get(rootName);
if (entry.isFile) {
const fileEntry = entry as FileSystemFileEntry;
try {
const file = await entryToFile(fileEntry);
if (rootLocalPath) {
// Native path is enough for stream upload; avoid Chromium File materialization.
results.push({
file,
relativePath: basePath ? `${basePath}/${entry.name}` : entry.name,
file: null,
localPath: joinLocalRelativePath(rootLocalPath, relativePath),
relativePath,
isDirectory: false,
});
} catch (error) {
console.warn(`Failed to read file entry: ${entry.name}`, error);
fileCount++;
} else {
try {
const file = await entryToFile(fileEntry);
results.push({
file,
relativePath,
isDirectory: false,
});
fileCount++;
} catch (error) {
console.warn(`Failed to read file entry: ${entry.name}`, error);
}
}
} else if (entry.isDirectory) {
const dirEntry = entry as FileSystemDirectoryEntry;
const currentPath = basePath ? `${basePath}/${entry.name}` : entry.name;
// Add directory entry
results.push({
file: null,
relativePath: currentPath,
localPath: rootLocalPath ? joinLocalRelativePath(rootLocalPath, relativePath) : undefined,
relativePath,
isDirectory: true,
});
directoryCount++;
try {
const reader = dirEntry.createReader();
const childEntries = await readAllDirectoryEntries(reader);
// Add child entries to the queue (not recursive!)
for (const childEntry of childEntries) {
queue.push({ entry: childEntry, basePath: currentPath });
queue.push({ entry: childEntry, basePath: relativePath });
}
} catch (error) {
console.warn(`Failed to read directory: ${entry.name}`, error);
}
}
// Yield to main thread periodically to keep UI responsive
processedCount++;
reportProgress();
if (processedCount % YIELD_INTERVAL === 0) {
await new Promise<void>(resolve => setTimeout(resolve, 0));
await new Promise<void>((resolve) => setTimeout(resolve, 0));
}
}
reportProgress(true);
return results;
}
@@ -378,87 +490,323 @@ export function getPathForFile(file: File): string | undefined {
}
}
/** Build a short label for the scanning task (folder names visible immediately). */
export function formatDropScanLabel(roots: readonly CapturedDropRoot[]): string {
const names = roots.map((root) => root.name).filter(Boolean);
if (names.length === 0) return "Scanning files...";
if (names.length === 1) return names[0];
if (names.length === 2) return `${names[0]}, ${names[1]}`;
return `${names[0]}, ${names[1]} +${names.length - 2}`;
}
/**
* Extract all files and directories from a DataTransfer object
* Supports both regular files and folders dropped from the OS
*
* Uses the webkitGetAsEntry() API for folder access, with fallback
* to regular FileList for browsers that don't support it.
*
* @param dataTransfer - The DataTransfer object from a drop event
* @returns Array of DropEntry objects with files and relative paths
* Synchronously capture drop roots. Must run during the drop/paste event —
* before any await — or DataTransfer becomes empty.
*/
export async function extractDropEntries(
dataTransfer: DataTransfer
): Promise<DropEntry[]> {
export function captureDropPayload(dataTransfer: DataTransfer): CapturedDropPayload {
const filesFallback: File[] = [];
const files = dataTransfer.files;
for (let i = 0; i < files.length; i++) {
filesFallback.push(files[i]);
}
const roots: CapturedDropRoot[] = [];
const items = dataTransfer.items;
// Build a map of file/folder name to path from the original files in DataTransfer.files
const filePathMap = new Map<string, string>();
const filesWithPath = dataTransfer.files;
for (let i = 0; i < filesWithPath.length; i++) {
const f = filesWithPath[i];
const path = getPathForFile(f);
if (path) {
filePathMap.set(f.name, path);
}
}
const relativePathForFile = (file: File): string | undefined => {
const relative = (file as File & { webkitRelativePath?: string }).webkitRelativePath;
return relative && relative.length > 0 ? relative : undefined;
};
// Check if webkitGetAsEntry is supported (for folder access)
if (items && items.length > 0 && typeof items[0].webkitGetAsEntry === 'function') {
// Collect all entries first (getAsEntry must be called synchronously)
const entries: FileSystemEntry[] = [];
if (items && items.length > 0 && typeof items[0].webkitGetAsEntry === "function") {
for (let i = 0; i < items.length; i++) {
const item = items[i];
if (item.kind === 'file') {
const entry = item.webkitGetAsEntry();
if (entry) {
entries.push(entry);
}
if (item.kind !== "file") continue;
const entry = item.webkitGetAsEntry();
const file = typeof item.getAsFile === "function" ? item.getAsFile() : (filesFallback[i] ?? null);
const localPath = file ? getPathForFile(file) : undefined;
const relativePath = file ? relativePathForFile(file) : undefined;
if (entry) {
roots.push({
name: entry.name,
isDirectory: entry.isDirectory,
localPath,
file: entry.isFile ? file : null,
size: file?.size,
relativePath: entry.isFile ? relativePath : undefined,
fsEntry: entry,
});
continue;
}
if (file) {
roots.push({
name: file.name,
isDirectory: false,
localPath: getPathForFile(file),
file,
size: file.size,
relativePath,
});
}
}
// Process entries iteratively (non-recursive) to avoid stack overflow
const results = await processEntriesIteratively(entries);
if (results.length === 0) {
return createDropEntriesFromFiles(dataTransfer.files);
}
// Preserve local paths on DropEntry instead of mutating File.path. Modern
// runtimes expose File.path as read-only when it exists.
for (const result of results) {
if (result.file) {
// First try to get path directly from the file
const directPath = getPathForFile(result.file);
if (directPath) {
result.localPath = directPath;
} else {
// Fallback: try to reconstruct from root folder path
const pathParts = result.relativePath.split('/');
const rootName = pathParts[0];
const rootPath = filePathMap.get(rootName);
if (rootPath) {
if (pathParts.length === 1) {
// Root-level file: use the path directly
result.localPath = rootPath;
} else {
// Nested file in a folder: construct full path
// rootPath is the path to the root folder, we need to append the rest
const restOfPath = pathParts.slice(1).join('/');
const separator = rootPath.includes('\\') ? '\\' : '/';
const fullPath = rootPath + separator + restOfPath.replace(/\//g, separator);
result.localPath = fullPath;
}
}
}
}
}
return results;
} else {
// Fallback: use regular FileList (no folder support)
// Files from FileList in Electron already have the 'path' property
return createDropEntriesFromFiles(dataTransfer.files);
for (const file of filesFallback) {
roots.push({
name: file.name,
isDirectory: false,
localPath: getPathForFile(file),
file,
size: file.size,
relativePath: relativePathForFile(file),
});
}
}
return { roots, filesFallback };
}
/** Map main-process local tree rows into upload DropEntry records. */
export function localTreeToDropEntries(tree: readonly LocalTreeListEntry[]): DropEntry[] {
return tree.map((entry) => {
if (entry.type === "directory") {
return {
file: null,
localPath: entry.localPath,
relativePath: entry.relativePath,
isDirectory: true,
// Do not use directory metadata size in conflict / compressed totals.
};
}
return {
file: null,
localPath: entry.localPath,
relativePath: entry.relativePath,
isDirectory: false,
size: entry.size,
};
});
}
function countDropEntries(entries: readonly DropEntry[]): DropScanProgress {
let fileCount = 0;
let directoryCount = 0;
for (const entry of entries) {
if (entry.isDirectory) directoryCount += 1;
else fileCount += 1;
}
return {
fileCount,
directoryCount,
entryCount: fileCount + directoryCount,
};
}
/**
* Expand a captured drop into full DropEntry rows.
* Prefer Electron `listLocalTree` (native fs) for directory roots with a local
* path; fall back to Chromium FileSystemEntry walk only when necessary.
*/
export async function materializeDropEntries(
payload: CapturedDropPayload,
options: MaterializeDropOptions = {},
): Promise<DropEntry[]> {
const { listLocalTree, onProgress, abortSignal, isCancelled } = options;
const results: DropEntry[] = [];
let fileCount = 0;
let directoryCount = 0;
const report = (label?: string, forcePartial?: DropScanProgress) => {
if (!onProgress) return;
if (forcePartial) {
onProgress({
...forcePartial,
label,
});
return;
}
onProgress({
fileCount,
directoryCount,
entryCount: fileCount + directoryCount,
label,
});
};
const nativeDirectoryRoots: CapturedDropRoot[] = [];
const webkitDirectoryRoots: CapturedDropRoot[] = [];
const fileRoots: CapturedDropRoot[] = [];
for (const root of payload.roots) {
if (root.isDirectory) {
if (root.localPath && listLocalTree) {
nativeDirectoryRoots.push(root);
} else if (root.fsEntry) {
webkitDirectoryRoots.push(root);
} else {
console.warn(`[SFTP] Skipping directory drop root without path or entry: ${root.name}`);
}
continue;
}
fileRoots.push(root);
}
throwIfDropScanCancelled({ abortSignal, isCancelled });
// Parallel native walks — one IPC per root folder, much faster than webkit.
if (nativeDirectoryRoots.length > 0 && listLocalTree) {
// Cumulative progress across roots: each walk reports its own counts.
const partialByRoot = new Map<string, DropScanProgress>();
const emitCombined = (label?: string) => {
let files = fileCount;
let dirs = directoryCount;
for (const partial of partialByRoot.values()) {
files += partial.fileCount;
dirs += partial.directoryCount;
}
report(label, { fileCount: files, directoryCount: dirs, entryCount: files + dirs });
};
// Local controller so a single root failure aborts sibling walks too.
const siblingAbort = new AbortController();
const stopSiblingScans = () => {
try {
siblingAbort.abort();
} catch {
// ignore
}
};
if (abortSignal) {
if (abortSignal.aborted) stopSiblingScans();
else abortSignal.addEventListener("abort", stopSiblingScans, { once: true });
}
const walkSignal = siblingAbort.signal;
const walkCancelled = () => (
walkSignal.aborted || abortSignal?.aborted === true || isCancelled?.() === true
);
const walkPromises = nativeDirectoryRoots.map(async (root) => {
throwIfDropScanCancelled({ abortSignal: walkSignal, isCancelled: walkCancelled });
const tree = await listLocalTree(root.localPath!, {
abortSignal: walkSignal,
onProgress: (partial) => {
partialByRoot.set(root.localPath!, partial);
emitCombined(root.name);
},
});
return { root, tree };
});
let trees: Array<{ root: CapturedDropRoot; tree: LocalTreeListEntry[] }>;
try {
trees = await Promise.all(walkPromises);
} catch (error) {
stopSiblingScans();
// Drain remaining native walks so retry does not pile more I/O on top.
await Promise.allSettled(walkPromises);
throw error;
} finally {
abortSignal?.removeEventListener("abort", stopSiblingScans);
}
for (const { root, tree } of trees) {
const entries = localTreeToDropEntries(tree);
for (const entry of entries) {
results.push(entry);
if (entry.isDirectory) directoryCount += 1;
else fileCount += 1;
}
partialByRoot.delete(root.localPath!);
report(root.name);
}
}
for (const root of fileRoots) {
results.push({
file: root.file ?? null,
localPath: root.localPath,
relativePath: root.relativePath || root.name,
isDirectory: false,
size: root.size ?? root.file?.size,
});
fileCount += 1;
}
if (fileRoots.length > 0) {
report();
}
if (webkitDirectoryRoots.length > 0) {
const rootPathByName = new Map<string, string>();
for (const root of webkitDirectoryRoots) {
if (root.localPath) rootPathByName.set(root.name, root.localPath);
}
const walked = await processEntriesIteratively(
webkitDirectoryRoots.map((root) => root.fsEntry!).filter(Boolean),
{
rootPathByName,
abortSignal,
isCancelled,
onProgress: (partial) => {
report(undefined, {
fileCount: fileCount + partial.fileCount,
directoryCount: directoryCount + partial.directoryCount,
entryCount: fileCount + directoryCount + partial.entryCount,
});
},
},
);
// Attach reconstructed paths when we only know the root path.
for (const entry of walked) {
if (!entry.localPath) {
const rootName = entry.relativePath.split("/")[0];
const rootPath = rootPathByName.get(rootName);
if (rootPath) {
entry.localPath = joinLocalRelativePath(rootPath, entry.relativePath);
} else if (entry.file) {
entry.localPath = getPathForFile(entry.file);
}
}
results.push(entry);
if (entry.isDirectory) directoryCount += 1;
else fileCount += 1;
}
report();
}
if (results.length === 0 && payload.filesFallback.length > 0) {
const fallback = createDropEntriesFromFiles(payload.filesFallback);
const counts = countDropEntries(fallback);
report(undefined, counts);
return fallback;
}
report(undefined, { fileCount, directoryCount, entryCount: fileCount + directoryCount });
return results;
}
/**
* Extract all files and directories from a DataTransfer object.
* Supports both regular files and folders dropped from the OS.
*
* Prefer Electron native tree walk when local paths are available; otherwise
* use webkitGetAsEntry with a path-reconstruction fast path.
*/
export async function extractDropEntries(
dataTransfer: DataTransfer,
options: MaterializeDropOptions = {},
): Promise<DropEntry[]> {
const payload = captureDropPayload(dataTransfer);
const bridge = netcattyBridge.get();
return materializeDropEntries(payload, {
listLocalTree: options.listLocalTree
?? (bridge?.listLocalTree
? (path, treeOptions) => bridge.listLocalTree!(path, treeOptions)
: undefined),
onProgress: options.onProgress,
abortSignal: options.abortSignal,
isCancelled: options.isCancelled,
});
}
+45
View File
@@ -48,3 +48,48 @@ test("compressed folder uploads use reconstructed drop-entry paths", async () =>
}
}
});
test("compressed folder totals exclude directory entries", async () => {
const previousWindow = globalThis.window;
const started: Array<{ totalBytes: number; folderPath: string }> = [];
Object.defineProperty(globalThis, "window", {
configurable: true,
value: {
netcatty: {
checkCompressedUploadSupport: async () => ({
supported: true,
localTar: true,
remoteTar: true,
}),
startCompressedUpload: async (options: { totalBytes: number; folderPath: string; compressionId: string }) => {
started.push({ totalBytes: options.totalBytes, folderPath: options.folderPath });
return { compressionId: options.compressionId, success: true };
},
},
},
});
try {
const results = await uploadFoldersCompressed(
[["docs", [
{ file: null, localPath: "/tmp/docs", relativePath: "docs", isDirectory: true, size: 4096 },
{ file: null, localPath: "/tmp/docs/sub", relativePath: "docs/sub", isDirectory: true, size: 4096 },
{ file: null, localPath: "/tmp/docs/a.txt", relativePath: "docs/a.txt", isDirectory: false, size: 10 },
]]],
"/remote",
"sftp-1",
);
assert.deepEqual(results, [{ fileName: "docs", success: true }]);
assert.deepEqual(started, [{ totalBytes: 10, folderPath: "/tmp/docs" }]);
} finally {
if (previousWindow === undefined) {
Reflect.deleteProperty(globalThis, "window");
} else {
Object.defineProperty(globalThis, "window", {
configurable: true,
value: previousWindow,
});
}
}
});
+17 -7
View File
@@ -1,5 +1,7 @@
import type { DropEntry } from "./sftpFileUtils";
import { getDropEntryLocalPath } from "./sftpFileUtils";
const getDropEntrySize = (entry: DropEntry): number => entry.file?.size ?? entry.size ?? 0;
import type { UploadCallbacks, UploadResult } from "./uploadService.types";
import type { UploadController } from "./uploadController";
@@ -23,14 +25,17 @@ export async function uploadFoldersCompressed(
break;
}
// Get the local folder path from the first file in the folder
const firstFile = entries.find(e => e.file);
if (!firstFile?.file) {
// Prefer any file-like entry with a resolvable local path (native tree scans
// set localPath without a browser File handle).
const firstFile = entries.find((entry) => (
!entry.isDirectory && (!!entry.file || !!getDropEntryLocalPath(entry))
));
if (!firstFile) {
// Empty folder - mark for fallback to regular upload which will create the directory
results.push({ fileName: folderName, success: false, error: "Compressed upload not supported - fallback needed" });
continue;
}
const localFilePath = getDropEntryLocalPath(firstFile);
if (!localFilePath) {
results.push({ fileName: folderName, success: false, error: "Could not get local file path" });
@@ -41,7 +46,10 @@ export async function uploadFoldersCompressed(
// Use DropEntry.relativePath which works for both file input and drag-drop scenarios
// For file input: webkitRelativePath is set (e.g., "folder/subdir/file.txt")
// For drag-drop: DropEntry.relativePath contains the correct path from extractDropEntries
const relativePath = firstFile.relativePath || (firstFile.file as File & { webkitRelativePath?: string }).webkitRelativePath || firstFile.file.name;
const relativePath = firstFile.relativePath
|| (firstFile.file as (File & { webkitRelativePath?: string }) | null)?.webkitRelativePath
|| firstFile.file?.name
|| folderName;
// Normalize path separators for cross-platform compatibility
const normalizePathSeparators = (path: string) => path.replace(/\\/g, '/');
@@ -114,7 +122,9 @@ export async function uploadFoldersCompressed(
controller?.addActiveCompression(compressionId);
// Create a task for this folder compression
const totalBytes = entries.reduce((sum, entry) => sum + (entry.file?.size || 0), 0);
// Path-only drop entries (listLocalTree) carry size without a File handle.
const fileEntries = entries.filter((entry) => !entry.isDirectory);
const totalBytes = fileEntries.reduce((sum, entry) => sum + getDropEntrySize(entry), 0);
taskId = compressionId;
if (callbacks?.onTaskCreated) {
@@ -127,7 +137,7 @@ export async function uploadFoldersCompressed(
totalBytes,
transferredBytes: 0,
speed: 0,
fileCount: entries.length,
fileCount: fileEntries.length,
completedCount: 0,
sourcePath: folderPath,
controlKind: 'compressed-upload',
+34 -1
View File
@@ -6,12 +6,31 @@ export class UploadController {
private activeCompressionIds = new Set<string>();
private currentTransferId = "";
private bridge: UploadBridge | null = null;
private cancelListeners = new Set<() => void>();
/**
* Register a listener fired as soon as cancel() is requested (before async
* cleanup). Used to abort in-flight local tree scans.
*/
addCancelListener(listener: () => void): () => void {
this.cancelListeners.add(listener);
return () => {
this.cancelListeners.delete(listener);
};
}
/**
* Cancel all active uploads
*/
async cancel(): Promise<void> {
this.cancelled = true;
for (const listener of Array.from(this.cancelListeners)) {
try {
listener();
} catch {
// Ignore listener errors so cancel still drains transfers.
}
}
// Cancel all active compressed uploads
const activeCompressionIds = Array.from(this.activeCompressionIds);
@@ -73,10 +92,24 @@ export class UploadController {
}
/**
* Reset controller state for new upload
* Reset controller state for a brand-new upload session.
* Prefer prepareForEntries when the controller already owns an external drop
* (scan cancel listeners must stay attached until the drop settles).
*/
reset(): void {
this.cancelled = false;
this.cancelListeners.clear();
this.activeFileTransferIds.clear();
this.activeCompressionIds.clear();
this.currentTransferId = "";
}
/**
* Soft prepare for entry upload without clearing cancel latches/listeners.
* External drop flows attach scan/conflict cancel listeners before entries
* are ready; a full reset would leave the scanning row uncancelable.
*/
prepareForEntries(): void {
this.activeFileTransferIds.clear();
this.activeCompressionIds.clear();
this.currentTransferId = "";
+28 -10
View File
@@ -37,7 +37,9 @@ import type { UploadBridge, UploadCallbacks, UploadConfig, UploadResult } from "
const formatUploadError = (error: unknown): string =>
error instanceof Error ? error.message : String(error);
const getDropEntrySize = (entry: DropEntry): number => entry.file?.size ?? entry.size ?? 0;
const getDropEntrySize = (entry: DropEntry): number => (
entry.isDirectory ? 0 : entry.file?.size ?? entry.size ?? 0
);
const getRootDropLocalPath = (rootName: string, entries: DropEntry[]): string | undefined => {
const entry = entries.find((candidate) => getDropEntryLocalPath(candidate));
const localPath = entry ? getDropEntryLocalPath(entry) : undefined;
@@ -61,9 +63,10 @@ export interface UploadScanningTask {
export function startUploadScanningTask(
callbacks?: UploadCallbacks,
taskId = crypto.randomUUID(),
info?: { label?: string },
): UploadScanningTask {
let open = true;
callbacks?.onScanningStart?.(taskId);
callbacks?.onScanningStart?.(taskId, info);
const close = (settle: () => void) => {
if (!open) return;
@@ -160,11 +163,24 @@ export async function uploadFromDataTransfer(
const scanningTask = startUploadScanningTask(callbacks);
let entries: DropEntry[];
try {
entries = await extractDropEntries(dataTransfer);
entries = await extractDropEntries(dataTransfer, {
onProgress: (progress) => {
callbacks?.onScanningProgress?.(scanningTask.taskId, progress);
},
isCancelled: () => controller?.isCancelled() === true,
});
} catch (error) {
scanningTask.complete();
if (controller?.isCancelled() || /cancel/i.test(error instanceof Error ? error.message : String(error))) {
scanningTask.cancel();
} else {
scanningTask.fail(error);
}
throw error;
}
if (controller?.isCancelled()) {
scanningTask.cancel();
return [{ fileName: "", success: false, cancelled: true }];
}
scanningTask.complete();
logger.debug(`[SFTP:perf] extractDropEntries — ${entries.length} entries — ${(performance.now() - scanT0).toFixed(0)}ms`);
@@ -668,14 +684,14 @@ async function uploadEntries(
const isStandaloneFile = rootName.startsWith("__file__");
if (isStandaloneFile) continue;
// Calculate total bytes for this folder
// Calculate total bytes for this folder (path-only entries from listLocalTree
// carry size without a browser File handle).
let totalBytes = 0;
let fileCount = 0;
for (const entry of rootEntries) {
if (!entry.isDirectory && entry.file) {
totalBytes += entry.file.size;
fileCount++;
}
if (!isUploadableFileEntry(entry)) continue;
totalBytes += getDropEntrySize(entry);
fileCount++;
}
if (fileCount === 0) continue;
@@ -1036,7 +1052,9 @@ export async function uploadEntriesDirect(
}
if (controller) {
controller.reset();
// Keep cancel latches/listeners from the external drop scan so the user
// can still cancel while pre-task work (stat / conflict / mkdir) runs.
controller.prepareForEntries();
controller.setBridge(config.bridge);
}
+8 -1
View File
@@ -49,7 +49,14 @@ export interface UploadCallbacks {
/** Called when a task is cancelled */
onTaskCancelled?: (taskId: string) => void;
/** Called when scanning starts (for showing placeholder) */
onScanningStart?: (taskId: string) => void;
onScanningStart?: (taskId: string, info?: { label?: string }) => void;
/** Live scan counters (file/dir totals) while enumerating a drop */
onScanningProgress?: (taskId: string, progress: {
fileCount: number;
directoryCount: number;
entryCount: number;
label?: string;
}) => void;
/** Called when scanning ends */
onScanningEnd?: (taskId: string) => void;
/** Called when task name needs to be updated (for phase changes) */
+15
View File
@@ -28,3 +28,18 @@ test("ZMODEM drops use reconstructed paths without buffering the file", async ()
}]);
assert.equal(buffered, false);
});
test("ZMODEM drops accept path-only files from native folder scans", async () => {
const files = await buildZmodemDragDropFiles([{
file: null,
localPath: "/tmp/project/src/main.ts",
relativePath: "project/src/main.ts",
isDirectory: false,
}]);
assert.deepEqual(files, [{
path: "/tmp/project/src/main.ts",
name: "main.ts",
remoteName: "main.ts",
}]);
});
+8 -4
View File
@@ -60,23 +60,27 @@ export async function buildZmodemDragDropFiles(
const files: ZmodemDragDropFile[] = [];
for (const entry of dropEntries) {
if (entry.isDirectory || !entry.file) continue;
if (entry.isDirectory) continue;
const remoteName = getZmodemRemoteName(entry.relativePath, entry.file.name);
const fileName = entry.file?.name
|| entry.relativePath.replace(/\\/g, "/").split("/").pop()
|| entry.relativePath;
const remoteName = getZmodemRemoteName(entry.relativePath, fileName);
const localPath = getDropEntryLocalPath(entry);
if (localPath) {
files.push({
path: localPath,
name: entry.file.name,
name: fileName,
remoteName,
});
continue;
}
if (!entry.file) continue;
const data = await entry.file.arrayBuffer();
files.push({
name: entry.file.name,
name: fileName,
remoteName,
data,
});
+26 -2
View File
@@ -143,13 +143,37 @@ declare global {
renameLocalFile?(oldPath: string, newPath: string): Promise<void>;
mkdirLocal?(path: string): Promise<void>;
statLocal?(path: string): Promise<SftpStatResult>;
listLocalTree?(path: string): Promise<Array<{
listLocalTree?(
path: string,
options?: {
onProgress?: (progress: {
fileCount: number;
directoryCount: number;
entryCount: number;
}) => void;
/** Stream discovered rows while the walk continues (edge-scan/upload). */
onEntries?: (entries: Array<{
localPath: string;
relativePath: string;
type: 'file' | 'directory';
size: number;
lastModified: number;
}>) => void;
/** Renderer-generated ID used by cancelLocalTreeScan. */
scanId?: string;
limits?: {
maxDirectories?: number;
maxEntries?: number;
};
},
): Promise<Array<{
localPath: string;
relativePath: string;
type: 'file' | 'directory';
size: number;
lastModified: number;
}>>;
}>>;
cancelLocalTreeScan?(scanId: string): Promise<void>;
getHomeDir?(): Promise<string>;
listDrives?(): Promise<string[]>;
getSystemInfo?(): Promise<{ username: string; hostname: string }>;