mirror of
https://github.com/binaricat/Netcatty.git
synced 2026-11-12 09:24:24 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9fe24fa461 | ||
|
|
96378f6fca | ||
|
|
a35945ae8e | ||
|
|
fee1953267 | ||
|
|
12471c1723 | ||
|
|
d43960902c | ||
|
|
8ed5777a63 | ||
|
|
b0e198f836 | ||
|
|
908d9a544b | ||
|
|
14d5c4301d | ||
|
|
f87e4fddef | ||
|
|
96eaf837f4 | ||
|
|
375c14be68 | ||
|
|
bfc821d7da | ||
|
|
bb725765d3 | ||
|
|
163af7176f | ||
|
|
938c4e4675 | ||
|
|
b8ecf1adfd |
No files matched your search
@@ -268,6 +268,8 @@ export const enVaultMessages: Messages = {
|
||||
'sftp.transfers.calculatingTotal': 'Calculating total size...',
|
||||
'sftp.transfers.filesCount': '{count} files',
|
||||
'sftp.transfers.filesProgress': '{current}/{total} files',
|
||||
// Progressive folder walk: total is "found so far", not a fixed grand total.
|
||||
'sftp.transfers.filesDiscoveredProgress': '{completed} done · {discovered} found',
|
||||
'sftp.transfers.expandChildren': 'Show files',
|
||||
'sftp.transfers.collapseChildren': 'Hide files',
|
||||
'sftp.transfers.expandChildList': 'Show detail',
|
||||
@@ -560,8 +562,6 @@ export const enVaultMessages: Messages = {
|
||||
'settings.sftp.compressedUpload.enable': 'Enable folder compression',
|
||||
'settings.sftp.compressedUpload.enableDesc': 'Automatically compress folders using tar before transfer. Requires tar support on the server. Falls back to regular transfer if not available.',
|
||||
|
||||
'settings.sftp.folderPrescan.enable': 'Pre-scan folders before transfer',
|
||||
'settings.sftp.folderPrescan.enableDesc': 'Walk the full folder tree first so the transfer center shows a stable file total before bytes move. Slightly delays the first transfer on very large trees.',
|
||||
'settings.sftp.skipUnchanged.enable': 'Skip unchanged files',
|
||||
'settings.sftp.skipUnchanged.enableDesc': 'Skip files whose size and modification time already match the destination (rsync-style). Disabled when replacing a folder into a staging path.',
|
||||
|
||||
|
||||
@@ -305,6 +305,7 @@ export const ruVaultMessages: Messages = {
|
||||
'sftp.transfers.calculatingTotal': 'Вычисление общего размера...',
|
||||
'sftp.transfers.filesCount': '{count} файл(ов)',
|
||||
'sftp.transfers.filesProgress': '{current}/{total} файл(ов)',
|
||||
'sftp.transfers.filesDiscoveredProgress': '{completed} готово · {discovered} найдено',
|
||||
'sftp.transfers.expandChildren': 'Показать файлы',
|
||||
'sftp.transfers.collapseChildren': 'Скрыть файлы',
|
||||
'sftp.transfers.expandChildList': 'Показать детали',
|
||||
@@ -597,8 +598,6 @@ export const ruVaultMessages: Messages = {
|
||||
'settings.sftp.compressedUpload.enable': 'Включить сжатие папок',
|
||||
'settings.sftp.compressedUpload.enableDesc': 'Автоматически сжимать папки с помощью tar перед передачей. Требует поддержки tar на сервере. Если она недоступна, будет использована обычная передача.',
|
||||
|
||||
'settings.sftp.folderPrescan.enable': 'Предварительное сканирование папок',
|
||||
'settings.sftp.folderPrescan.enableDesc': 'Сначала обойти всё дерево папок, чтобы центр передачи показал стабильное число файлов до начала передачи байтов. На очень больших деревьях немного откладывает первую передачу.',
|
||||
'settings.sftp.skipUnchanged.enable': 'Пропускать неизменённые файлы',
|
||||
'settings.sftp.skipUnchanged.enableDesc': 'Пропускать файлы, у которых размер и время изменения уже совпадают с назначением (в стиле rsync). Отключается при замене папки во временный путь.',
|
||||
|
||||
|
||||
@@ -863,6 +863,7 @@ export const zhCNCoreMessages: Messages = {
|
||||
'sftp.transfers.calculatingTotal': '正在统计总大小...',
|
||||
'sftp.transfers.filesCount': '{count} 个文件',
|
||||
'sftp.transfers.filesProgress': '{current}/{total} 个文件',
|
||||
'sftp.transfers.filesDiscoveredProgress': '已完成 {completed} · 已发现 {discovered}',
|
||||
'sftp.transfers.expandChildren': '展开文件',
|
||||
'sftp.transfers.collapseChildren': '收起文件',
|
||||
'sftp.transfers.expandChildList': '展开详情',
|
||||
|
||||
@@ -206,8 +206,6 @@ export const zhCNTerminalMessages: Messages = {
|
||||
'settings.sftp.compressedUpload.enable': '启用文件夹压缩',
|
||||
'settings.sftp.compressedUpload.enableDesc': '自动使用 tar 压缩文件夹后再传输。需要服务器支持 tar 命令,不支持时自动回退到普通传输。',
|
||||
|
||||
'settings.sftp.folderPrescan.enable': '传输前预扫描文件夹',
|
||||
'settings.sftp.folderPrescan.enableDesc': '先完整遍历文件夹树,传输中心在开始传字节前就能显示稳定的文件总数。超大目录会略微推迟首个文件开传。',
|
||||
'settings.sftp.skipUnchanged.enable': '跳过未改文件',
|
||||
'settings.sftp.skipUnchanged.enableDesc': '当目标文件的大小与修改时间已与源一致时跳过传输(类似 rsync)。替换到暂存路径时自动禁用。',
|
||||
|
||||
|
||||
@@ -863,6 +863,7 @@ export const zhTWCoreMessages: Messages = {
|
||||
'sftp.transfers.calculatingTotal': '正在統計總大小...',
|
||||
'sftp.transfers.filesCount': '{count} 個檔案',
|
||||
'sftp.transfers.filesProgress': '{current}/{total} 個檔案',
|
||||
'sftp.transfers.filesDiscoveredProgress': '已完成 {completed} · 已發現 {discovered}',
|
||||
'sftp.transfers.expandChildren': '展開檔案',
|
||||
'sftp.transfers.collapseChildren': '收起檔案',
|
||||
'sftp.transfers.expandChildList': '展開詳情',
|
||||
|
||||
@@ -206,8 +206,6 @@ export const zhTWTerminalMessages: Messages = {
|
||||
'settings.sftp.compressedUpload.enable': '啟用資料夾壓縮',
|
||||
'settings.sftp.compressedUpload.enableDesc': '自動使用 tar 壓縮資料夾後再傳輸。需要伺服器支援 tar 指令,不支援時自動回退到普通傳輸。',
|
||||
|
||||
'settings.sftp.folderPrescan.enable': '傳輸前預掃描資料夾',
|
||||
'settings.sftp.folderPrescan.enableDesc': '先完整遍歷資料夾樹,傳輸中心在開始傳位元組前就能顯示穩定的檔案總數。超大目錄會略微推遲首個檔案開傳。',
|
||||
'settings.sftp.skipUnchanged.enable': '跳過未改檔案',
|
||||
'settings.sftp.skipUnchanged.enableDesc': '當目標檔案的大小與修改時間已與來源一致時跳過傳輸(類似 rsync)。替換到暫存路徑時自動停用。',
|
||||
|
||||
|
||||
@@ -28,7 +28,6 @@ import {
|
||||
STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY,
|
||||
STORAGE_KEY_SSH_TRANSPORT_IDLE_TTL_MS,
|
||||
STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD,
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SHOW_ONLY_UNGROUPED_HOSTS_IN_ROOT,
|
||||
STORAGE_KEY_SHOW_RECENT_HOSTS,
|
||||
@@ -81,7 +80,6 @@ interface UseSettingsStorageSyncParams {
|
||||
sftpAutoSync: boolean;
|
||||
sftpShowHiddenFiles: boolean;
|
||||
sftpUseCompressedUpload: boolean;
|
||||
sftpFolderPrescan: boolean;
|
||||
sftpSkipUnchanged: boolean;
|
||||
sftpAutoOpenSidebar: boolean;
|
||||
sftpFollowTerminalCwd: boolean;
|
||||
@@ -129,7 +127,6 @@ interface UseSettingsStorageSyncParams {
|
||||
setSftpAutoSync: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpShowHiddenFiles: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpUseCompressedUpload: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpFolderPrescan: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpSkipUnchanged: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpAutoOpenSidebar: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpFollowTerminalCwd: Dispatch<SetStateAction<boolean>>;
|
||||
@@ -171,7 +168,7 @@ export function useSettingsStorageSync({
|
||||
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
|
||||
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
|
||||
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
|
||||
@@ -180,7 +177,7 @@ export function useSettingsStorageSync({
|
||||
setTerminalThemeId, setTerminalThemeDarkId, setTerminalThemeLightId,
|
||||
setFollowAppTerminalThemeState, setTerminalFontFamilyId, setTerminalFontSize,
|
||||
setSftpDoubleClickBehavior, setSftpAutoSync, setSftpShowHiddenFiles,
|
||||
setSftpUseCompressedUpload, setSftpFolderPrescan, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
|
||||
setSftpUseCompressedUpload, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
|
||||
setShowRecentHostsState, setHostClickBehaviorState, setShowOnlyUngroupedHostsInRootState, setShowSftpTabState, setShowHostTreeSidebarState, setTerminalSidePanelAutoOpenState, setTerminalSidePanelAutoOpenTabState, setShellOnlyTabNumberShortcutsState, setDisableTerminalFontZoomState, setRestorePreviousSessionState, setRestoreTerminalCwdState,
|
||||
setEditorWordWrapState, setSessionLogsEnabled, setSessionLogsDir, setSessionLogsFormat, setSessionLogsTimestampsEnabled, setSshDebugLogsEnabled, setSshDeepLinkEnabledState, setJmsDeepLinkEnabledState, setExplorerContextMenuEnabledState,
|
||||
setGlobalHotkeyEnabled, setWindowOpacity, setAppIconVariant, setAutoUpdateEnabled, setWorkspaceFocusStyleState,
|
||||
@@ -195,7 +192,7 @@ export function useSettingsStorageSync({
|
||||
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
|
||||
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
|
||||
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
|
||||
@@ -205,7 +202,7 @@ export function useSettingsStorageSync({
|
||||
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
|
||||
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
|
||||
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
|
||||
@@ -385,12 +382,6 @@ export function useSettingsStorageSync({
|
||||
setSftpUseCompressedUpload(newValue);
|
||||
}
|
||||
}
|
||||
if (e.key === STORAGE_KEY_SFTP_FOLDER_PRESCAN && e.newValue !== null) {
|
||||
const newValue = e.newValue === 'true';
|
||||
if (newValue !== s.sftpFolderPrescan) {
|
||||
setSftpFolderPrescan(newValue);
|
||||
}
|
||||
}
|
||||
if (e.key === STORAGE_KEY_SFTP_SKIP_UNCHANGED && e.newValue !== null) {
|
||||
const newValue = e.newValue === 'true';
|
||||
if (newValue !== s.sftpSkipUnchanged) {
|
||||
@@ -560,7 +551,6 @@ export function useSettingsStorageSync({
|
||||
setSftpTransferConcurrencyState,
|
||||
setSshTransportIdleTtlMsState,
|
||||
setSftpUseCompressedUpload,
|
||||
setSftpFolderPrescan,
|
||||
setSftpSkipUnchanged,
|
||||
setShowOnlyUngroupedHostsInRootState,
|
||||
setShowHostTreeSidebarState,
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import type { TransferTask } from "../../../domain/models";
|
||||
import {
|
||||
isExternalDragDropFileUpload,
|
||||
retryExternalDragDropFileUpload,
|
||||
} from "./externalDragDropRetry";
|
||||
|
||||
const baseTask = (overrides: Partial<TransferTask> = {}): TransferTask => ({
|
||||
id: "child-1",
|
||||
fileName: "a.txt",
|
||||
sourcePath: "/tmp/docs/a.txt",
|
||||
targetPath: "/remote/docs/a.txt",
|
||||
sourceConnectionId: "external",
|
||||
targetConnectionId: "conn-1",
|
||||
targetHostId: "host-1",
|
||||
direction: "upload",
|
||||
status: "failed",
|
||||
totalBytes: 12,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
startTime: 1,
|
||||
isDirectory: false,
|
||||
origin: "drag-drop",
|
||||
parentTaskId: "folder-1",
|
||||
error: "Network error",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
test("detects progressive drag-drop file children as retryable", () => {
|
||||
assert.equal(isExternalDragDropFileUpload(baseTask()), true);
|
||||
assert.equal(isExternalDragDropFileUpload(baseTask({ isDirectory: true })), false);
|
||||
assert.equal(isExternalDragDropFileUpload(baseTask({ origin: "manual" })), false);
|
||||
assert.equal(isExternalDragDropFileUpload(baseTask({ sourceConnectionId: "local" })), false);
|
||||
assert.equal(isExternalDragDropFileUpload(baseTask({ status: "completed" })), false);
|
||||
assert.equal(isExternalDragDropFileUpload(baseTask({ retryable: false })), false);
|
||||
});
|
||||
|
||||
test("retry reuses the same transfer id and starts a stream upload", async () => {
|
||||
const patches: Array<{ id: string; updates: Partial<TransferTask> }> = [];
|
||||
const streams: Array<Record<string, unknown>> = [];
|
||||
|
||||
// No pool acquire → falls back to browse sftp id.
|
||||
const result = await retryExternalDragDropFileUpload(baseTask(), {
|
||||
getBrowseSftpId: () => "sftp-live",
|
||||
startStreamTransfer: async (options) => {
|
||||
streams.push(options as unknown as Record<string, unknown>);
|
||||
return { transferId: options.transferId };
|
||||
},
|
||||
onPatch: (taskId, updates) => patches.push({ id: taskId, updates }),
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(streams.length, 1);
|
||||
assert.equal(streams[0].transferId, "child-1");
|
||||
assert.equal(streams[0].sourcePath, "/tmp/docs/a.txt");
|
||||
assert.equal(streams[0].targetPath, "/remote/docs/a.txt");
|
||||
assert.equal(streams[0].targetSftpId, "sftp-live");
|
||||
assert.equal(patches[0].updates.status, "transferring");
|
||||
assert.equal(patches.at(-1)?.updates.status, "completed");
|
||||
});
|
||||
|
||||
test("retry prefers a dedicated pool session over the browse sftp id", async () => {
|
||||
let released = false;
|
||||
let acquired = false;
|
||||
const result = await retryExternalDragDropFileUpload(baseTask(), {
|
||||
getBrowseSftpId: () => "sftp-live",
|
||||
acquireTransferSession: async () => {
|
||||
acquired = true;
|
||||
return {
|
||||
sftpId: "sftp-pool",
|
||||
poolKey: "host-1",
|
||||
release: () => { released = true; },
|
||||
discard: () => {},
|
||||
};
|
||||
},
|
||||
startStreamTransfer: async (options) => {
|
||||
assert.equal(options.targetSftpId, "sftp-pool");
|
||||
return {};
|
||||
},
|
||||
onPatch: () => {},
|
||||
});
|
||||
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(acquired, true);
|
||||
assert.equal(released, true);
|
||||
});
|
||||
|
||||
test("retry falls back to browse when pool acquire is unavailable", async () => {
|
||||
const result = await retryExternalDragDropFileUpload(baseTask(), {
|
||||
getBrowseSftpId: () => "sftp-live",
|
||||
startStreamTransfer: async (options) => {
|
||||
assert.equal(options.targetSftpId, "sftp-live");
|
||||
return {};
|
||||
},
|
||||
onPatch: () => {},
|
||||
});
|
||||
assert.equal(result.success, true);
|
||||
});
|
||||
|
||||
test("retry rolls parent to completed when all siblings succeed", async () => {
|
||||
const patches: Array<{ id: string; updates: Partial<TransferTask> }> = [];
|
||||
const result = await retryExternalDragDropFileUpload(baseTask(), {
|
||||
getBrowseSftpId: () => "sftp-live",
|
||||
startStreamTransfer: async () => ({}),
|
||||
getTask: (id) => (
|
||||
id === "folder-1"
|
||||
? baseTask({
|
||||
id: "folder-1",
|
||||
isDirectory: true,
|
||||
status: "failed",
|
||||
sourcePath: "/tmp/docs",
|
||||
targetPath: "/remote/docs",
|
||||
error: "1 of 2 files failed",
|
||||
totalBytes: 2,
|
||||
transferredBytes: 1,
|
||||
parentTaskId: undefined,
|
||||
})
|
||||
: undefined
|
||||
),
|
||||
getChildTasks: () => [
|
||||
baseTask({ id: "child-1", status: "transferring" }),
|
||||
baseTask({ id: "child-2", status: "completed", sourcePath: "/tmp/docs/b.txt" }),
|
||||
],
|
||||
onPatch: (taskId, updates) => patches.push({ id: taskId, updates }),
|
||||
});
|
||||
assert.equal(result.success, true);
|
||||
const parentPatch = patches.find((p) => p.id === "folder-1");
|
||||
assert.ok(parentPatch);
|
||||
assert.equal(parentPatch?.updates.status, "completed");
|
||||
});
|
||||
|
||||
test("retry does not rollup a still-transferring progressive parent", async () => {
|
||||
const patches: Array<{ id: string; updates: Partial<TransferTask> }> = [];
|
||||
await retryExternalDragDropFileUpload(baseTask(), {
|
||||
getBrowseSftpId: () => "sftp-live",
|
||||
startStreamTransfer: async () => ({}),
|
||||
getTask: (id) => (
|
||||
id === "folder-1"
|
||||
? baseTask({
|
||||
id: "folder-1",
|
||||
isDirectory: true,
|
||||
status: "transferring",
|
||||
sourcePath: "/tmp/docs",
|
||||
parentTaskId: undefined,
|
||||
})
|
||||
: undefined
|
||||
),
|
||||
getChildTasks: () => [
|
||||
baseTask({ id: "child-1", status: "transferring" }),
|
||||
baseTask({ id: "child-2", status: "completed", sourcePath: "/tmp/docs/b.txt" }),
|
||||
],
|
||||
onPatch: (taskId, updates) => patches.push({ id: taskId, updates }),
|
||||
});
|
||||
assert.equal(patches.some((p) => p.id === "folder-1"), false);
|
||||
});
|
||||
|
||||
test("retry fails clearly when no sftp session can be opened", async () => {
|
||||
const patches: Array<Partial<TransferTask>> = [];
|
||||
const result = await retryExternalDragDropFileUpload(baseTask(), {
|
||||
getBrowseSftpId: () => undefined,
|
||||
startStreamTransfer: async () => {
|
||||
throw new Error("should not start");
|
||||
},
|
||||
onPatch: (_id, updates) => patches.push(updates),
|
||||
});
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.match(result.error || "", /No SFTP session/);
|
||||
assert.equal(patches.at(-1)?.status, "failed");
|
||||
});
|
||||
|
||||
test("stream error keeps the row failed with the backend message", async () => {
|
||||
const patches: Array<Partial<TransferTask>> = [];
|
||||
const result = await retryExternalDragDropFileUpload(baseTask(), {
|
||||
getBrowseSftpId: () => "sftp-live",
|
||||
startStreamTransfer: async () => ({ error: "Permission denied" }),
|
||||
onPatch: (_id, updates) => patches.push(updates),
|
||||
});
|
||||
|
||||
assert.equal(result.success, false);
|
||||
assert.equal(result.error, "Permission denied");
|
||||
assert.equal(patches.at(-1)?.status, "failed");
|
||||
assert.equal(patches.at(-1)?.error, "Permission denied");
|
||||
});
|
||||
@@ -0,0 +1,267 @@
|
||||
/**
|
||||
* Low-cost retry for progressive / external drag-drop file uploads.
|
||||
*
|
||||
* These rows use sourceConnectionId "external" and never have dual-pane
|
||||
* endpoints, so the generic processTransfer retry path silently no-ops.
|
||||
* Retry re-opens a single startStreamTransfer with the stored local path.
|
||||
*/
|
||||
|
||||
import type { TransferTask } from "../../../domain/models";
|
||||
import type { TransferConnectionLease } from "./transferConnectionPool";
|
||||
import { isTransferCancelledFlag } from "./transferCancelLatch";
|
||||
|
||||
export function isExternalDragDropFileUpload(
|
||||
task: Pick<
|
||||
TransferTask,
|
||||
| "origin"
|
||||
| "direction"
|
||||
| "sourceConnectionId"
|
||||
| "isDirectory"
|
||||
| "sourcePath"
|
||||
| "targetPath"
|
||||
| "retryable"
|
||||
| "status"
|
||||
>,
|
||||
): boolean {
|
||||
if (task.retryable === false) return false;
|
||||
if (task.origin !== "drag-drop") return false;
|
||||
if (task.direction !== "upload") return false;
|
||||
if (task.sourceConnectionId !== "external") return false;
|
||||
if (task.isDirectory) return false;
|
||||
if (!task.sourcePath || task.sourcePath === "local") return false;
|
||||
if (!task.targetPath) return false;
|
||||
return task.status === "failed" || task.status === "cancelled" || task.status === "attention";
|
||||
}
|
||||
|
||||
export type ExternalDragDropRetryDeps = {
|
||||
getBrowseSftpId: (connectionId: string) => string | undefined;
|
||||
acquireTransferSession?: (
|
||||
hostId: string,
|
||||
transferId: string,
|
||||
) => Promise<TransferConnectionLease>;
|
||||
startStreamTransfer: (options: {
|
||||
transferId: string;
|
||||
sourcePath: string;
|
||||
targetPath: string;
|
||||
sourceType: "local";
|
||||
targetType: "local" | "sftp";
|
||||
targetSftpId?: string;
|
||||
targetHostId?: string;
|
||||
totalBytes?: number;
|
||||
resumable?: boolean;
|
||||
checkpointBytes?: number;
|
||||
}) => Promise<{ error?: string; cancelled?: boolean } | undefined>;
|
||||
clearPendingCancel?: (transferId: string) => Promise<unknown>;
|
||||
cleanupArtifacts?: (task: TransferTask) => Promise<void>;
|
||||
onPatch: (taskId: string, updates: Partial<TransferTask>) => void;
|
||||
/** Live store lookup for terminal cancel races and completion bytes. */
|
||||
getTask?: (taskId: string) => TransferTask | undefined;
|
||||
/** Children of a progressive parent (for rollup after child success). */
|
||||
getChildTasks?: (parentTaskId: string) => TransferTask[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Restart a failed/cancelled external drag-drop file upload in place (same id).
|
||||
* Returns true when the stream completed successfully.
|
||||
*/
|
||||
export async function retryExternalDragDropFileUpload(
|
||||
task: TransferTask,
|
||||
deps: ExternalDragDropRetryDeps,
|
||||
): Promise<{ success: boolean; error?: string }> {
|
||||
if (!isExternalDragDropFileUpload(task)) {
|
||||
return { success: false, error: "Not an external drag-drop file upload" };
|
||||
}
|
||||
|
||||
try {
|
||||
await deps.clearPendingCancel?.(task.id);
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
try {
|
||||
await deps.cleanupArtifacts?.(task);
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
|
||||
const targetIsLocal = task.targetConnectionId === "local" || !task.targetHostId;
|
||||
let lease: TransferConnectionLease | null = null;
|
||||
let targetSftpId: string | undefined;
|
||||
|
||||
try {
|
||||
if (!targetIsLocal) {
|
||||
// Prefer a dedicated pool session when hostId is known — same policy as
|
||||
// progressive upload (do not pin the browse/tab session).
|
||||
if (task.targetHostId && deps.acquireTransferSession) {
|
||||
lease = await deps.acquireTransferSession(task.targetHostId, `${task.id}:retry`);
|
||||
targetSftpId = lease.sftpId;
|
||||
}
|
||||
if (!targetSftpId) {
|
||||
targetSftpId = deps.getBrowseSftpId(task.targetConnectionId);
|
||||
}
|
||||
if (!targetSftpId) {
|
||||
const error = "No SFTP session available to retry this upload. Reconnect and try again.";
|
||||
deps.onPatch(task.id, {
|
||||
status: "failed",
|
||||
error,
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
});
|
||||
return { success: false, error };
|
||||
}
|
||||
}
|
||||
|
||||
deps.onPatch(task.id, {
|
||||
status: "transferring",
|
||||
error: undefined,
|
||||
transferredBytes: 0,
|
||||
checkpointBytes: 0,
|
||||
speed: 0,
|
||||
endTime: undefined,
|
||||
phase: "transferring",
|
||||
reconnectRequired: false,
|
||||
pauseUnavailableReason: undefined,
|
||||
startTime: Date.now(),
|
||||
});
|
||||
|
||||
const result = await deps.startStreamTransfer({
|
||||
transferId: task.id,
|
||||
sourcePath: task.sourcePath,
|
||||
targetPath: task.targetPath,
|
||||
sourceType: "local",
|
||||
targetType: targetIsLocal ? "local" : "sftp",
|
||||
targetSftpId: targetIsLocal ? undefined : targetSftpId,
|
||||
targetHostId: targetIsLocal ? undefined : task.targetHostId,
|
||||
totalBytes: task.totalBytes > 0 ? task.totalBytes : undefined,
|
||||
resumable: true,
|
||||
checkpointBytes: 0,
|
||||
});
|
||||
|
||||
// Late cancel can settle the row before this return; never resurrect it.
|
||||
if (isTransferCancelledFlag(task.id) || result?.cancelled) {
|
||||
deps.onPatch(task.id, {
|
||||
status: "cancelled",
|
||||
error: undefined,
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
phase: undefined,
|
||||
});
|
||||
return { success: false, error: "Transfer cancelled" };
|
||||
}
|
||||
if (result?.error) {
|
||||
deps.onPatch(task.id, {
|
||||
status: "failed",
|
||||
error: result.error,
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
phase: undefined,
|
||||
});
|
||||
return { success: false, error: result.error };
|
||||
}
|
||||
|
||||
const live = deps.getTask?.(task.id);
|
||||
const completedBytes = Math.max(
|
||||
live?.totalBytes ?? 0,
|
||||
live?.transferredBytes ?? 0,
|
||||
task.totalBytes,
|
||||
0,
|
||||
);
|
||||
deps.onPatch(task.id, {
|
||||
status: "completed",
|
||||
error: undefined,
|
||||
transferredBytes: completedBytes,
|
||||
totalBytes: Math.max(live?.totalBytes ?? 0, task.totalBytes, completedBytes),
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
phase: undefined,
|
||||
});
|
||||
|
||||
// Progressive parents finalize once with "N of M failed". After a child
|
||||
// retry succeeds, re-roll the parent when no failed children remain.
|
||||
if (task.parentTaskId && deps.getChildTasks) {
|
||||
rollupParentAfterChildSuccess(task.parentTaskId, task.id, deps);
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (isTransferCancelledFlag(task.id) || /cancel/i.test(message)) {
|
||||
deps.onPatch(task.id, {
|
||||
status: "cancelled",
|
||||
error: undefined,
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
phase: undefined,
|
||||
});
|
||||
return { success: false, error: "Transfer cancelled" };
|
||||
}
|
||||
deps.onPatch(task.id, {
|
||||
status: "failed",
|
||||
error: message,
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
phase: undefined,
|
||||
});
|
||||
if (lease && /session|sftp|disconnect|not found/i.test(message)) {
|
||||
try { lease.discard(); } catch { /* best-effort */ }
|
||||
lease = null;
|
||||
}
|
||||
return { success: false, error: message };
|
||||
} finally {
|
||||
try { lease?.release(); } catch { /* best-effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
function rollupParentAfterChildSuccess(
|
||||
parentTaskId: string,
|
||||
completedChildId: string,
|
||||
deps: ExternalDragDropRetryDeps,
|
||||
): void {
|
||||
// Only re-roll parents that already finished the progressive walk as failed.
|
||||
// Never promote a still-scanning/transferring parent mid-walk.
|
||||
const parent = deps.getTask?.(parentTaskId);
|
||||
if (
|
||||
parent
|
||||
&& parent.status !== "failed"
|
||||
&& parent.status !== "attention"
|
||||
&& parent.status !== "cancelled"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const children = deps.getChildTasks?.(parentTaskId) ?? [];
|
||||
if (children.length === 0) return;
|
||||
const stillFailed = children.some(
|
||||
(child) =>
|
||||
child.id !== completedChildId
|
||||
&& (child.status === "failed" || child.status === "attention"),
|
||||
);
|
||||
if (stillFailed) return;
|
||||
const active = children.some(
|
||||
(child) =>
|
||||
child.id !== completedChildId
|
||||
&& (
|
||||
child.status === "transferring"
|
||||
|| child.status === "pending"
|
||||
|| child.status === "queued"
|
||||
|| child.status === "pausing"
|
||||
|| child.status === "paused"
|
||||
),
|
||||
);
|
||||
if (active) return;
|
||||
const completedCount = children.filter(
|
||||
(child) => child.id === completedChildId || child.status === "completed",
|
||||
).length;
|
||||
const total = Math.max(
|
||||
Number(parent?.totalBytes) || 0,
|
||||
children.length,
|
||||
completedCount,
|
||||
);
|
||||
deps.onPatch(parentTaskId, {
|
||||
status: "completed",
|
||||
error: undefined,
|
||||
transferredBytes: completedCount,
|
||||
totalBytes: total,
|
||||
speed: 0,
|
||||
endTime: Date.now(),
|
||||
phase: undefined,
|
||||
});
|
||||
}
|
||||
@@ -4,10 +4,8 @@ import assert from "node:assert/strict";
|
||||
import {
|
||||
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
|
||||
DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY,
|
||||
DEFAULT_SFTP_FOLDER_PRESCAN,
|
||||
DEFAULT_SFTP_SKIP_UNCHANGED,
|
||||
resolveSftpDirectoryListingConcurrency,
|
||||
resolveSftpFolderPrescanEnabled,
|
||||
resolveSftpSkipUnchangedEnabled,
|
||||
resolveSftpTransferConcurrency,
|
||||
runBoundedConcurrency,
|
||||
@@ -27,12 +25,9 @@ test("defaults directory listing fanout to four concurrent readdirs", () => {
|
||||
assert.equal(DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY, 4);
|
||||
});
|
||||
|
||||
test("defaults folder pre-scan and skip-unchanged to enabled", () => {
|
||||
assert.equal(resolveSftpFolderPrescanEnabled(() => null), DEFAULT_SFTP_FOLDER_PRESCAN);
|
||||
test("defaults skip-unchanged to enabled", () => {
|
||||
assert.equal(resolveSftpSkipUnchangedEnabled(() => null), DEFAULT_SFTP_SKIP_UNCHANGED);
|
||||
assert.equal(DEFAULT_SFTP_FOLDER_PRESCAN, true);
|
||||
assert.equal(DEFAULT_SFTP_SKIP_UNCHANGED, true);
|
||||
assert.equal(resolveSftpFolderPrescanEnabled(() => false), false);
|
||||
assert.equal(resolveSftpSkipUnchangedEnabled(() => false), false);
|
||||
});
|
||||
|
||||
|
||||
@@ -13,8 +13,6 @@ export const DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY = 4;
|
||||
export const MIN_SFTP_DIRECTORY_LISTING_CONCURRENCY = 1;
|
||||
export const MAX_SFTP_DIRECTORY_LISTING_CONCURRENCY = 8;
|
||||
|
||||
/** Default on: accurate folder totals before first byte (rsync --no-i-r style). */
|
||||
export const DEFAULT_SFTP_FOLDER_PRESCAN = true;
|
||||
/** Default on: skip size+mtime matches like rsync's generator. */
|
||||
export const DEFAULT_SFTP_SKIP_UNCHANGED = true;
|
||||
|
||||
@@ -38,13 +36,6 @@ export function resolveSftpDirectoryListingConcurrency(
|
||||
: DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY;
|
||||
}
|
||||
|
||||
export function resolveSftpFolderPrescanEnabled(
|
||||
readStoredValue: () => boolean | null | undefined,
|
||||
): boolean {
|
||||
const stored = readStoredValue();
|
||||
return stored == null ? DEFAULT_SFTP_FOLDER_PRESCAN : stored;
|
||||
}
|
||||
|
||||
export function resolveSftpSkipUnchangedEnabled(
|
||||
readStoredValue: () => boolean | null | undefined,
|
||||
): boolean {
|
||||
|
||||
@@ -1,311 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import type { SftpFileEntry } from "../../../domain/models";
|
||||
import { compareDirectoryTraversalPaths } from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { createSftpDirectoryTraversalBudget } from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { discoverTransferTree } from "./transferDirectoryDiscovery";
|
||||
|
||||
const directoryEntry = (name: string): SftpFileEntry => ({
|
||||
name,
|
||||
type: "directory",
|
||||
size: 0,
|
||||
sizeFormatted: "0 B",
|
||||
lastModified: 0,
|
||||
lastModifiedFormatted: "",
|
||||
});
|
||||
|
||||
const fileEntry = (name: string, size = 1, lastModified = 1_700_000_000): SftpFileEntry => ({
|
||||
name,
|
||||
type: "file",
|
||||
size,
|
||||
sizeFormatted: `${size} B`,
|
||||
lastModified,
|
||||
lastModifiedFormatted: "",
|
||||
});
|
||||
|
||||
test("discoverTransferTree returns a stable flat file list with bounded listings", async () => {
|
||||
const previousWindow = (globalThis as { window?: unknown }).window;
|
||||
(globalThis as { window?: unknown }).window = {
|
||||
netcatty: {
|
||||
realpathSftp: async (_sftpId: string, remotePath: string) => remotePath,
|
||||
},
|
||||
};
|
||||
|
||||
let active = 0;
|
||||
let maxActive = 0;
|
||||
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
|
||||
active += 1;
|
||||
maxActive = Math.max(maxActive, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
active -= 1;
|
||||
if (path === "/source") {
|
||||
return [directoryEntry("a"), directoryEntry("b"), fileEntry("root.txt")];
|
||||
}
|
||||
if (path === "/source/a") return [fileEntry("a1.txt"), fileEntry("a2.txt")];
|
||||
if (path === "/source/b") return [fileEntry("b1.txt")];
|
||||
return [];
|
||||
};
|
||||
|
||||
try {
|
||||
const counts: number[] = [];
|
||||
const result = await discoverTransferTree({
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: false,
|
||||
sourceSftpId: "sftp-1",
|
||||
sourceEncoding: "auto",
|
||||
listLocalFiles: async () => [],
|
||||
listRemoteFiles,
|
||||
listingConcurrency: 2,
|
||||
onDiscoveredFiles: (total) => counts.push(total),
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
result.files.map((file) => file.sourcePath),
|
||||
[...result.files.map((file) => file.sourcePath)].sort((left, right) =>
|
||||
compareDirectoryTraversalPaths(left, right)
|
||||
),
|
||||
"pre-scan file order must match resume comparator",
|
||||
);
|
||||
assert.deepEqual(
|
||||
result.files.map((file) => file.sourcePath).sort(),
|
||||
["/source/a/a1.txt", "/source/a/a2.txt", "/source/b/b1.txt", "/source/root.txt"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
result.directories.map((dir) => dir.sourcePath).sort(),
|
||||
["/source", "/source/a", "/source/b"],
|
||||
);
|
||||
assert.equal(counts.at(-1), 4);
|
||||
assert.equal(result.omittedSymlinkDirectoryErrors, 0);
|
||||
assert.ok(maxActive >= 2, `expected parallel listings, got ${maxActive}`);
|
||||
assert.ok(maxActive <= 2, `listing concurrency exceeded: ${maxActive}`);
|
||||
} finally {
|
||||
(globalThis as { window?: unknown }).window = previousWindow;
|
||||
}
|
||||
});
|
||||
|
||||
test("discoverTransferTree skips symlink directory cycles across BFS waves", async () => {
|
||||
const previousWindow = (globalThis as { window?: unknown }).window;
|
||||
const listCalls = new Map<string, number>();
|
||||
(globalThis as { window?: unknown }).window = {
|
||||
netcatty: {
|
||||
realpathSftp: async (_sftpId: string, remotePath: string) => (
|
||||
remotePath === "/source/loop" ? "/source" : remotePath
|
||||
),
|
||||
},
|
||||
};
|
||||
|
||||
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
|
||||
listCalls.set(path, (listCalls.get(path) ?? 0) + 1);
|
||||
if (path === "/source") {
|
||||
return [
|
||||
directoryEntry("child"),
|
||||
{ ...directoryEntry("loop"), type: "symlink", linkTarget: "directory" },
|
||||
fileEntry("root.txt"),
|
||||
];
|
||||
}
|
||||
if (path === "/source/child") return [fileEntry("nested.txt")];
|
||||
throw new Error(`unexpected list of ${path}`);
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await discoverTransferTree({
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: false,
|
||||
sourceSftpId: "sftp-1",
|
||||
sourceEncoding: "auto",
|
||||
followSymlinks: true,
|
||||
listLocalFiles: async () => [],
|
||||
listRemoteFiles,
|
||||
listingConcurrency: 4,
|
||||
});
|
||||
|
||||
assert.equal(listCalls.has("/source/loop"), false);
|
||||
assert.equal(listCalls.get("/source"), 1);
|
||||
assert.equal(listCalls.get("/source/child"), 1);
|
||||
assert.deepEqual(
|
||||
result.files.map((file) => file.sourcePath).sort(),
|
||||
["/source/child/nested.txt", "/source/root.txt"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
result.directories.map((dir) => dir.sourcePath).sort(),
|
||||
["/source", "/source/child"],
|
||||
"cyclic symlink dirs must not appear in the creation plan",
|
||||
);
|
||||
assert.equal(result.omittedSymlinkDirectoryErrors, 0);
|
||||
} finally {
|
||||
(globalThis as { window?: unknown }).window = previousWindow;
|
||||
}
|
||||
});
|
||||
|
||||
test("discoverTransferTree copies sibling symlink aliases to distinct targets", async () => {
|
||||
const previousWindow = (globalThis as { window?: unknown }).window;
|
||||
const listCalls = new Map<string, number>();
|
||||
(globalThis as { window?: unknown }).window = {
|
||||
netcatty: {
|
||||
realpathSftp: async (_sftpId: string, remotePath: string) => {
|
||||
if (remotePath === "/source/alias-a" || remotePath === "/source/alias-b") {
|
||||
return "/shared";
|
||||
}
|
||||
return remotePath;
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
|
||||
listCalls.set(path, (listCalls.get(path) ?? 0) + 1);
|
||||
if (path === "/source") {
|
||||
return [
|
||||
{ ...directoryEntry("alias-a"), type: "symlink", linkTarget: "directory" },
|
||||
{ ...directoryEntry("alias-b"), type: "symlink", linkTarget: "directory" },
|
||||
];
|
||||
}
|
||||
if (path === "/source/alias-a" || path === "/source/alias-b") {
|
||||
return [fileEntry("shared.txt")];
|
||||
}
|
||||
throw new Error(`unexpected list of ${path}`);
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await discoverTransferTree({
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: false,
|
||||
sourceSftpId: "sftp-1",
|
||||
sourceEncoding: "auto",
|
||||
followSymlinks: true,
|
||||
listLocalFiles: async () => [],
|
||||
listRemoteFiles,
|
||||
listingConcurrency: 4,
|
||||
});
|
||||
|
||||
assert.equal(listCalls.get("/source/alias-a"), 1);
|
||||
assert.equal(listCalls.get("/source/alias-b"), 1);
|
||||
assert.deepEqual(
|
||||
result.files.map((file) => file.targetPath).sort(),
|
||||
["/target/alias-a/shared.txt", "/target/alias-b/shared.txt"],
|
||||
);
|
||||
} finally {
|
||||
(globalThis as { window?: unknown }).window = previousWindow;
|
||||
}
|
||||
});
|
||||
|
||||
test("discoverTransferTree counts max-depth symlink directory omissions as errors", async () => {
|
||||
const previousWindow = (globalThis as { window?: unknown }).window;
|
||||
(globalThis as { window?: unknown }).window = {
|
||||
netcatty: {
|
||||
realpathSftp: async (_sftpId: string, remotePath: string) => remotePath,
|
||||
},
|
||||
};
|
||||
|
||||
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
|
||||
const depth = path === "/source"
|
||||
? 0
|
||||
: path.startsWith("/source/")
|
||||
? path.slice("/source/".length).split("/").filter(Boolean).length
|
||||
: -1;
|
||||
if (depth < 0 || depth > 32) throw new Error(`unexpected list of ${path}`);
|
||||
const nextLink = { ...directoryEntry("deep"), type: "symlink" as const, linkTarget: "directory" as const };
|
||||
if (depth === 0) {
|
||||
return [
|
||||
nextLink,
|
||||
{
|
||||
name: "link.txt",
|
||||
type: "symlink",
|
||||
linkTarget: "file",
|
||||
size: 3,
|
||||
sizeFormatted: "3 B",
|
||||
lastModified: 1_700_000_000,
|
||||
lastModifiedFormatted: "",
|
||||
},
|
||||
];
|
||||
}
|
||||
return [nextLink];
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await discoverTransferTree({
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: false,
|
||||
sourceSftpId: "sftp-1",
|
||||
sourceEncoding: "auto",
|
||||
followSymlinks: true,
|
||||
listLocalFiles: async () => [],
|
||||
listRemoteFiles,
|
||||
listingConcurrency: 2,
|
||||
});
|
||||
|
||||
assert.equal(result.omittedSymlinkDirectoryErrors, 1);
|
||||
assert.equal(result.files.length, 1);
|
||||
assert.equal(result.files[0]?.isSymlink, true);
|
||||
assert.equal(result.files[0]?.name, "link.txt");
|
||||
} finally {
|
||||
(globalThis as { window?: unknown }).window = previousWindow;
|
||||
}
|
||||
});
|
||||
|
||||
test("discoverTransferTree bounds local folder pre-scan memory", async () => {
|
||||
const listLocalFiles = async (path: string): Promise<SftpFileEntry[]> => {
|
||||
if (path === "/local") return [directoryEntry("a"), fileEntry("root.txt")];
|
||||
if (path === "/local/a") return [fileEntry("nested.txt")];
|
||||
return [];
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() => discoverTransferTree({
|
||||
sourcePath: "/local",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: true,
|
||||
sourceSftpId: null,
|
||||
sourceEncoding: "auto",
|
||||
listLocalFiles,
|
||||
listRemoteFiles: async () => [],
|
||||
listingConcurrency: 2,
|
||||
traversalBudget: createSftpDirectoryTraversalBudget({ maxDirectories: 1, maxEntries: 200_000 }),
|
||||
}),
|
||||
/Directory traversal directory limit exceeded/,
|
||||
);
|
||||
});
|
||||
|
||||
test("discoverTransferTree waits while paused between BFS waves", async () => {
|
||||
let paused = true;
|
||||
let listCalls = 0;
|
||||
let pauseWaits = 0;
|
||||
const listLocalFiles = async (path: string): Promise<SftpFileEntry[]> => {
|
||||
listCalls += 1;
|
||||
if (path === "/local") return [directoryEntry("a"), fileEntry("root.txt")];
|
||||
if (path === "/local/a") return [fileEntry("nested.txt")];
|
||||
return [];
|
||||
};
|
||||
|
||||
const discovery = discoverTransferTree({
|
||||
sourcePath: "/local",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: true,
|
||||
sourceSftpId: null,
|
||||
sourceEncoding: "auto",
|
||||
listLocalFiles,
|
||||
listRemoteFiles: async () => [],
|
||||
listingConcurrency: 1,
|
||||
waitWhilePaused: async () => {
|
||||
pauseWaits += 1;
|
||||
while (paused) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// First waitWhilePaused (wave start) should block before any listing.
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
assert.equal(listCalls, 0, "pause must block pre-scan listings");
|
||||
assert.ok(pauseWaits >= 1, "expected waitWhilePaused to be invoked");
|
||||
|
||||
paused = false;
|
||||
const result = await discovery;
|
||||
assert.equal(result.files.length, 2);
|
||||
assert.ok(listCalls >= 2);
|
||||
assert.ok(pauseWaits >= 2, "expected pause checks across waves/listings");
|
||||
});
|
||||
@@ -1,279 +0,0 @@
|
||||
import type { SftpFileEntry, SftpFilenameEncoding } from "../../../domain/models";
|
||||
import {
|
||||
accountSftpDirectoryEntries,
|
||||
claimSftpDirectoryVisit,
|
||||
compareDirectoryTraversalPaths,
|
||||
createSftpDirectoryBranchAncestors,
|
||||
createSftpDirectoryTraversalBudget,
|
||||
releaseSftpDirectoryVisit,
|
||||
shouldFollowSftpSymlinkDirectory,
|
||||
type SftpDirectoryTraversalBudget,
|
||||
} from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { netcattyBridge } from "../../../infrastructure/services/netcattyBridge";
|
||||
import { logger } from "../../../lib/logger";
|
||||
import {
|
||||
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
|
||||
resolveSftpDirectoryListingConcurrency,
|
||||
runBoundedConcurrency,
|
||||
} from "./transferConcurrency";
|
||||
import { joinPath, joinTransferTargetPath } from "./utils";
|
||||
|
||||
export type DiscoveredTransferFile = {
|
||||
name: string;
|
||||
sourcePath: string;
|
||||
targetPath: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
/**
|
||||
* True when the listing entry was a symlink. Link-node size/mtime must not
|
||||
* drive skip-unchanged; transfer follows the target bytes.
|
||||
*/
|
||||
isSymlink?: boolean;
|
||||
};
|
||||
|
||||
export type DiscoveredTransferDirectory = {
|
||||
sourcePath: string;
|
||||
targetPath: string;
|
||||
};
|
||||
|
||||
export type DiscoverTransferTreeResult = {
|
||||
files: DiscoveredTransferFile[];
|
||||
directories: DiscoveredTransferDirectory[];
|
||||
/** Symlink directories omitted at max follow depth (incomplete tree). */
|
||||
omittedSymlinkDirectoryErrors: number;
|
||||
};
|
||||
|
||||
type ListingGate = {
|
||||
run: <T>(fn: () => Promise<T>) => Promise<T>;
|
||||
};
|
||||
|
||||
function createListingGate(concurrency = DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY): ListingGate {
|
||||
const limit = Math.max(1, Math.floor(concurrency) || 1);
|
||||
let active = 0;
|
||||
const waiters: Array<() => void> = [];
|
||||
const acquire = async () => {
|
||||
if (active < limit) {
|
||||
active += 1;
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
waiters.push(() => {
|
||||
active += 1;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
const release = () => {
|
||||
active = Math.max(0, active - 1);
|
||||
const next = waiters.shift();
|
||||
if (next) next();
|
||||
};
|
||||
return {
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
await acquire();
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function getEntrySize(entry: SftpFileEntry): number {
|
||||
if (typeof entry.size === "string") {
|
||||
const parsed = parseInt(entry.size, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
|
||||
}
|
||||
return typeof entry.size === "number" && entry.size > 0 ? entry.size : 0;
|
||||
}
|
||||
|
||||
export type DiscoverTransferTreeOptions = {
|
||||
sourcePath: string;
|
||||
targetPath: string;
|
||||
sourceIsLocal: boolean;
|
||||
sourceSftpId: string | null;
|
||||
sourceEncoding: SftpFilenameEncoding;
|
||||
followSymlinks?: boolean;
|
||||
listLocalFiles: (path: string) => Promise<SftpFileEntry[]>;
|
||||
listRemoteFiles: (sftpId: string, path: string, encoding?: SftpFilenameEncoding) => Promise<SftpFileEntry[]>;
|
||||
shouldAbort?: () => boolean;
|
||||
/** Honor transfer-center pause between BFS waves / listings. */
|
||||
waitWhilePaused?: () => Promise<void>;
|
||||
onDiscoveredFiles?: (totalFiles: number) => void;
|
||||
listingConcurrency?: number;
|
||||
traversalBudget?: SftpDirectoryTraversalBudget;
|
||||
};
|
||||
|
||||
/**
|
||||
* Full-tree discovery (rsync --no-inc-recursive style): walk every directory
|
||||
* before transfer starts so the UI can show a stable file total.
|
||||
* Listings are bounded-parallel; nested walks share one semaphore.
|
||||
*/
|
||||
export async function discoverTransferTree(
|
||||
options: DiscoverTransferTreeOptions,
|
||||
): Promise<DiscoverTransferTreeResult> {
|
||||
const followSymlinks = options.followSymlinks === true;
|
||||
const listingConcurrency = options.listingConcurrency
|
||||
?? resolveSftpDirectoryListingConcurrency();
|
||||
const listingGate = createListingGate(listingConcurrency);
|
||||
const traversal = options.traversalBudget ?? createSftpDirectoryTraversalBudget();
|
||||
const files: DiscoveredTransferFile[] = [];
|
||||
const directories: DiscoveredTransferDirectory[] = [
|
||||
{ sourcePath: options.sourcePath, targetPath: options.targetPath },
|
||||
];
|
||||
let omittedSymlinkDirectoryErrors = 0;
|
||||
|
||||
type DirJob = {
|
||||
sourcePath: string;
|
||||
targetPath: string;
|
||||
symlinkDepth: number;
|
||||
/** Ancestors on this BFS branch (copied for parallel sibling aliases). */
|
||||
branchAncestors: Set<string>;
|
||||
/** False for the root (already in `directories`); true for accepted children. */
|
||||
includeInCreationPlan: boolean;
|
||||
};
|
||||
|
||||
const queue: DirJob[] = [{
|
||||
sourcePath: options.sourcePath,
|
||||
targetPath: options.targetPath,
|
||||
symlinkDepth: 0,
|
||||
branchAncestors: createSftpDirectoryBranchAncestors(),
|
||||
includeInCreationPlan: false,
|
||||
}];
|
||||
|
||||
const publishCount = () => {
|
||||
options.onDiscoveredFiles?.(files.length);
|
||||
};
|
||||
|
||||
const processDir = async (job: DirJob): Promise<DirJob[]> => {
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
await options.waitWhilePaused?.();
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
let claimedCanonicalPath: string | null = null;
|
||||
try {
|
||||
let canonicalPath = job.sourcePath;
|
||||
if (!options.sourceIsLocal && options.sourceSftpId) {
|
||||
const bridge = netcattyBridge.get();
|
||||
canonicalPath = await bridge?.realpathSftp?.(
|
||||
options.sourceSftpId,
|
||||
job.sourcePath,
|
||||
options.sourceEncoding,
|
||||
).catch(() => job.sourcePath) ?? job.sourcePath;
|
||||
}
|
||||
// Bound both remote and local pre-scan trees (Codex P1 on 86c64f48).
|
||||
claimedCanonicalPath = claimSftpDirectoryVisit(
|
||||
traversal,
|
||||
canonicalPath,
|
||||
job.branchAncestors,
|
||||
);
|
||||
// Cycle / already visited: do not create an empty target dir for this path.
|
||||
if (!claimedCanonicalPath) return [];
|
||||
|
||||
// Only record directories after the visit is accepted so cyclic symlink
|
||||
// children (e.g. /source/loop -> /source) are omitted from mkdir plans.
|
||||
if (job.includeInCreationPlan) {
|
||||
directories.push({ sourcePath: job.sourcePath, targetPath: job.targetPath });
|
||||
}
|
||||
|
||||
const listed = await listingGate.run(async () => {
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
await options.waitWhilePaused?.();
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
if (options.sourceIsLocal) {
|
||||
return options.listLocalFiles(job.sourcePath);
|
||||
}
|
||||
if (!options.sourceSftpId) throw new Error("No source connection");
|
||||
return options.listRemoteFiles(
|
||||
options.sourceSftpId,
|
||||
job.sourcePath,
|
||||
options.sourceEncoding,
|
||||
);
|
||||
});
|
||||
|
||||
const filtered = listed.filter((entry) => entry.name !== "." && entry.name !== "..");
|
||||
accountSftpDirectoryEntries(traversal, filtered.length);
|
||||
|
||||
const childDirs: DirJob[] = [];
|
||||
const regularFiles: SftpFileEntry[] = [];
|
||||
for (const entry of filtered) {
|
||||
if (entry.type === "directory") {
|
||||
childDirs.push({
|
||||
sourcePath: joinPath(job.sourcePath, entry.name),
|
||||
targetPath: joinTransferTargetPath(job.targetPath, entry.name),
|
||||
symlinkDepth: job.symlinkDepth,
|
||||
branchAncestors: createSftpDirectoryBranchAncestors(job.branchAncestors),
|
||||
includeInCreationPlan: true,
|
||||
});
|
||||
} else if (
|
||||
followSymlinks
|
||||
&& entry.type === "symlink"
|
||||
&& entry.linkTarget === "directory"
|
||||
) {
|
||||
if (shouldFollowSftpSymlinkDirectory(job.symlinkDepth)) {
|
||||
childDirs.push({
|
||||
sourcePath: joinPath(job.sourcePath, entry.name),
|
||||
targetPath: joinTransferTargetPath(job.targetPath, entry.name),
|
||||
symlinkDepth: job.symlinkDepth + 1,
|
||||
branchAncestors: createSftpDirectoryBranchAncestors(job.branchAncestors),
|
||||
includeInCreationPlan: true,
|
||||
});
|
||||
} else {
|
||||
omittedSymlinkDirectoryErrors += 1;
|
||||
logger.warn(
|
||||
`[SFTP] Skipping symlink directory at max depth during pre-scan: ${joinPath(job.sourcePath, entry.name)}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
regularFiles.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
childDirs.sort((left, right) => left.sourcePath.localeCompare(right.sourcePath));
|
||||
regularFiles.sort((left, right) => left.name.localeCompare(right.name));
|
||||
|
||||
for (const entry of regularFiles) {
|
||||
files.push({
|
||||
name: entry.name,
|
||||
sourcePath: joinPath(job.sourcePath, entry.name),
|
||||
targetPath: joinTransferTargetPath(job.targetPath, entry.name),
|
||||
size: getEntrySize(entry),
|
||||
lastModified: typeof entry.lastModified === "number" ? entry.lastModified : 0,
|
||||
isSymlink: entry.type === "symlink",
|
||||
});
|
||||
}
|
||||
publishCount();
|
||||
return childDirs;
|
||||
} finally {
|
||||
if (claimedCanonicalPath) {
|
||||
releaseSftpDirectoryVisit(traversal, claimedCanonicalPath, job.branchAncestors);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// BFS waves: each wave lists up to listingConcurrency directories in parallel.
|
||||
while (queue.length > 0) {
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
await options.waitWhilePaused?.();
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
const wave = queue.splice(0, queue.length);
|
||||
const nested: DirJob[][] = new Array(wave.length);
|
||||
await runBoundedConcurrency(
|
||||
wave,
|
||||
listingConcurrency,
|
||||
async (job, index) => {
|
||||
nested[index] = await processDir(job);
|
||||
},
|
||||
);
|
||||
for (const children of nested) {
|
||||
if (!children) continue;
|
||||
for (const child of children) queue.push(child);
|
||||
}
|
||||
}
|
||||
|
||||
// Match dedicated-resume / transfer-center ordering so directoryEntryIndex and
|
||||
// manifest checkpoints stay valid across interrupt + rebuild.
|
||||
files.sort((left, right) => compareDirectoryTraversalPaths(left.sourcePath, right.sourcePath));
|
||||
|
||||
return { files, directories, omittedSymlinkDirectoryErrors };
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import type { MutableRefObject } from "react";
|
||||
|
||||
import type { TransferTask } from "../../../domain/models";
|
||||
import { STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY } from "../../../infrastructure/config/storageKeys";
|
||||
import { transferDiscoveredFiles } from "./transferDirectoryFileBatch";
|
||||
|
||||
const memoryStore = new Map<string, string>();
|
||||
(globalThis as { localStorage?: Storage }).localStorage = {
|
||||
getItem: (key: string) => memoryStore.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => {
|
||||
memoryStore.set(key, String(value));
|
||||
},
|
||||
removeItem: (key: string) => {
|
||||
memoryStore.delete(key);
|
||||
},
|
||||
clear: () => memoryStore.clear(),
|
||||
key: () => null,
|
||||
length: 0,
|
||||
} as Storage;
|
||||
memoryStore.set(STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY, "1");
|
||||
|
||||
const rootTask = (): TransferTask => ({
|
||||
id: "root",
|
||||
fileName: "source",
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceConnectionId: "local",
|
||||
targetConnectionId: "local",
|
||||
direction: "upload",
|
||||
status: "transferring",
|
||||
totalBytes: 1,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
startTime: 0,
|
||||
isDirectory: true,
|
||||
});
|
||||
|
||||
test("transferDiscoveredFiles re-stats source before skip-unchanged", async () => {
|
||||
const transfersRef = { current: [rootTask()] } as MutableRefObject<TransferTask[]>;
|
||||
const cancelledTasksRef = { current: new Set<string>() } as MutableRefObject<Set<string>>;
|
||||
const activeChildIdsRef = {
|
||||
current: new Map<string, Set<string>>(),
|
||||
} as MutableRefObject<Map<string, Set<string>>>;
|
||||
let transfers = transfersRef.current;
|
||||
let transferCalls = 0;
|
||||
let sourceStatCalls = 0;
|
||||
|
||||
const staleMtime = 1_700_000_000_000;
|
||||
const freshMtime = 1_700_000_100_000;
|
||||
|
||||
await transferDiscoveredFiles({
|
||||
rootTask: rootTask(),
|
||||
files: [{
|
||||
name: "a.txt",
|
||||
sourcePath: "/source/a.txt",
|
||||
targetPath: "/target/a.txt",
|
||||
size: 10,
|
||||
lastModified: staleMtime,
|
||||
}],
|
||||
sourceSftpId: null,
|
||||
targetSftpId: null,
|
||||
sourceIsLocal: true,
|
||||
targetIsLocal: true,
|
||||
sourceEncoding: "auto",
|
||||
targetEncoding: "auto",
|
||||
rootTaskId: "root",
|
||||
skipUnchanged: true,
|
||||
startEntryIndex: 0,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers: (updater) => {
|
||||
transfers = typeof updater === "function" ? updater(transfers) : updater;
|
||||
transfersRef.current = transfers;
|
||||
},
|
||||
waitWhileTransferPaused: async () => {},
|
||||
isPauseLatched: () => false,
|
||||
transferFile: async () => {
|
||||
transferCalls += 1;
|
||||
},
|
||||
tryStatTarget: async () => ({
|
||||
size: 10,
|
||||
lastModified: staleMtime,
|
||||
type: "file",
|
||||
}),
|
||||
tryStatSource: async () => {
|
||||
sourceStatCalls += 1;
|
||||
return {
|
||||
size: 10,
|
||||
lastModified: freshMtime,
|
||||
type: "file",
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(sourceStatCalls, 1, "must re-stat source before skip check");
|
||||
assert.equal(transferCalls, 1, "stale listing must not skip a changed source");
|
||||
const child = transfers.find((row) => row.parentTaskId === "root");
|
||||
assert.ok(child);
|
||||
assert.equal(child?.status, "completed");
|
||||
assert.equal(child?.sourceLastModified, freshMtime);
|
||||
});
|
||||
|
||||
test("transferDiscoveredFiles does not skip when source re-stat fails", async () => {
|
||||
const transfersRef = { current: [rootTask()] } as MutableRefObject<TransferTask[]>;
|
||||
const cancelledTasksRef = { current: new Set<string>() } as MutableRefObject<Set<string>>;
|
||||
const activeChildIdsRef = {
|
||||
current: new Map<string, Set<string>>(),
|
||||
} as MutableRefObject<Map<string, Set<string>>>;
|
||||
let transfers = transfersRef.current;
|
||||
let transferCalls = 0;
|
||||
let transferredTotalBytes: number | undefined;
|
||||
let transferredSourceLastModified: number | undefined;
|
||||
const mtime = 1_700_000_000_000;
|
||||
|
||||
await transferDiscoveredFiles({
|
||||
rootTask: rootTask(),
|
||||
files: [{
|
||||
name: "a.txt",
|
||||
sourcePath: "/source/a.txt",
|
||||
targetPath: "/target/a.txt",
|
||||
size: 10,
|
||||
lastModified: mtime,
|
||||
}],
|
||||
sourceSftpId: null,
|
||||
targetSftpId: null,
|
||||
sourceIsLocal: true,
|
||||
targetIsLocal: true,
|
||||
sourceEncoding: "auto",
|
||||
targetEncoding: "auto",
|
||||
rootTaskId: "root",
|
||||
skipUnchanged: true,
|
||||
startEntryIndex: 0,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers: (updater) => {
|
||||
transfers = typeof updater === "function" ? updater(transfers) : updater;
|
||||
transfersRef.current = transfers;
|
||||
},
|
||||
waitWhileTransferPaused: async () => {},
|
||||
isPauseLatched: () => false,
|
||||
transferFile: async (task) => {
|
||||
transferCalls += 1;
|
||||
transferredTotalBytes = task.totalBytes;
|
||||
transferredSourceLastModified = task.sourceLastModified;
|
||||
},
|
||||
tryStatTarget: async () => ({
|
||||
size: 10,
|
||||
lastModified: mtime,
|
||||
type: "file",
|
||||
}),
|
||||
tryStatSource: async () => null,
|
||||
});
|
||||
|
||||
assert.equal(transferCalls, 1, "missing fresh source meta must not skip");
|
||||
assert.equal(transferredTotalBytes, 0, "must not pass stale listing size after re-stat failure");
|
||||
assert.equal(
|
||||
transferredSourceLastModified,
|
||||
undefined,
|
||||
"must clear stale listing mtime after re-stat failure",
|
||||
);
|
||||
});
|
||||
|
||||
test("transferDiscoveredFiles skips only when fresh source still matches", async () => {
|
||||
const transfersRef = { current: [rootTask()] } as MutableRefObject<TransferTask[]>;
|
||||
const cancelledTasksRef = { current: new Set<string>() } as MutableRefObject<Set<string>>;
|
||||
const activeChildIdsRef = {
|
||||
current: new Map<string, Set<string>>(),
|
||||
} as MutableRefObject<Map<string, Set<string>>>;
|
||||
let transfers = transfersRef.current;
|
||||
let transferCalls = 0;
|
||||
const mtime = 1_700_000_000_000;
|
||||
|
||||
await transferDiscoveredFiles({
|
||||
rootTask: rootTask(),
|
||||
files: [{
|
||||
name: "a.txt",
|
||||
sourcePath: "/source/a.txt",
|
||||
targetPath: "/target/a.txt",
|
||||
size: 10,
|
||||
lastModified: mtime - 60_000,
|
||||
}],
|
||||
sourceSftpId: null,
|
||||
targetSftpId: null,
|
||||
sourceIsLocal: true,
|
||||
targetIsLocal: true,
|
||||
sourceEncoding: "auto",
|
||||
targetEncoding: "auto",
|
||||
rootTaskId: "root",
|
||||
skipUnchanged: true,
|
||||
startEntryIndex: 0,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers: (updater) => {
|
||||
transfers = typeof updater === "function" ? updater(transfers) : updater;
|
||||
transfersRef.current = transfers;
|
||||
},
|
||||
waitWhileTransferPaused: async () => {},
|
||||
isPauseLatched: () => false,
|
||||
transferFile: async () => {
|
||||
transferCalls += 1;
|
||||
},
|
||||
tryStatTarget: async () => ({
|
||||
size: 10,
|
||||
lastModified: mtime,
|
||||
type: "file",
|
||||
}),
|
||||
tryStatSource: async () => ({
|
||||
size: 10,
|
||||
lastModified: mtime,
|
||||
type: "file",
|
||||
}),
|
||||
});
|
||||
|
||||
assert.equal(transferCalls, 0, "matching fresh identity must skip transfer");
|
||||
const child = transfers.find((row) => row.parentTaskId === "root");
|
||||
assert.equal(child?.status, "completed");
|
||||
assert.equal(child?.transferredBytes, 10);
|
||||
});
|
||||
@@ -1,354 +0,0 @@
|
||||
import type { Dispatch, MutableRefObject, SetStateAction } from "react";
|
||||
import type {
|
||||
SftpFilenameEncoding,
|
||||
TransferStatus,
|
||||
TransferTask,
|
||||
} from "../../../domain/models";
|
||||
import { createDirectoryEntryIdentity } from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { isUnchangedTransferCandidate } from "../../../domain/sftpTransferSkip";
|
||||
import { STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY } from "../../../infrastructure/config/storageKeys";
|
||||
import { localStorageAdapter } from "../../../infrastructure/persistence/localStorageAdapter";
|
||||
import { logger } from "../../../lib/logger";
|
||||
import { isSessionError } from "./errors";
|
||||
import type { DiscoveredTransferFile } from "./transferDirectoryDiscovery";
|
||||
import { runSftpTransferWorkers } from "./transferConcurrency";
|
||||
import { isTransferCancelledError } from "./transferRetry";
|
||||
|
||||
type TransferOneFile = (
|
||||
task: TransferTask,
|
||||
sourceSftpId: string | null,
|
||||
targetSftpId: string | null,
|
||||
sourceIsLocal: boolean,
|
||||
targetIsLocal: boolean,
|
||||
sourceEncoding: SftpFilenameEncoding,
|
||||
targetEncoding: SftpFilenameEncoding,
|
||||
rootTaskId: string,
|
||||
sameHost?: boolean,
|
||||
) => Promise<void>;
|
||||
|
||||
export type TransferDiscoveredFilesParams = {
|
||||
rootTask: TransferTask;
|
||||
files: DiscoveredTransferFile[];
|
||||
sourceSftpId: string | null;
|
||||
targetSftpId: string | null;
|
||||
sourceIsLocal: boolean;
|
||||
targetIsLocal: boolean;
|
||||
sourceEncoding: SftpFilenameEncoding;
|
||||
targetEncoding: SftpFilenameEncoding;
|
||||
rootTaskId: string;
|
||||
sameHost?: boolean;
|
||||
skipUnchanged: boolean;
|
||||
startEntryIndex: number;
|
||||
cancelledTasksRef: MutableRefObject<Set<string>>;
|
||||
activeChildIdsRef: MutableRefObject<Map<string, Set<string>>>;
|
||||
transfersRef: MutableRefObject<TransferTask[]>;
|
||||
setTransfers: Dispatch<SetStateAction<TransferTask[]>>;
|
||||
waitWhileTransferPaused: (rootTaskId: string, taskId?: string) => Promise<void>;
|
||||
isPauseLatched: (rootTaskId: string, taskId?: string) => boolean;
|
||||
transferFile: TransferOneFile;
|
||||
tryStatTarget: (
|
||||
targetPath: string,
|
||||
) => Promise<{ size: number; lastModified: number; type: string } | null>;
|
||||
/** Fresh source identity immediately before skip-unchanged (Codex P1). */
|
||||
tryStatSource: (
|
||||
sourcePath: string,
|
||||
) => Promise<{ size: number; lastModified: number; type: string } | null>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Transfer a flat pre-scanned file list with the same child-task bookkeeping as
|
||||
* the interleaved directory walk.
|
||||
*/
|
||||
export async function transferDiscoveredFiles(
|
||||
params: TransferDiscoveredFilesParams,
|
||||
): Promise<number> {
|
||||
const {
|
||||
rootTask,
|
||||
files,
|
||||
sourceSftpId,
|
||||
targetSftpId,
|
||||
sourceIsLocal,
|
||||
targetIsLocal,
|
||||
sourceEncoding,
|
||||
targetEncoding,
|
||||
rootTaskId,
|
||||
sameHost,
|
||||
skipUnchanged,
|
||||
startEntryIndex,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers,
|
||||
waitWhileTransferPaused,
|
||||
isPauseLatched,
|
||||
transferFile,
|
||||
tryStatTarget,
|
||||
tryStatSource,
|
||||
} = params;
|
||||
|
||||
if (files.length === 0) return 0;
|
||||
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? { ...candidate, phase: "transferring" }
|
||||
: candidate));
|
||||
|
||||
const errors: Error[] = [];
|
||||
let sessionLostError: Error | null = null;
|
||||
|
||||
await runSftpTransferWorkers(
|
||||
files,
|
||||
() => localStorageAdapter.readNumber(STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY),
|
||||
async (file, fileIndex) => {
|
||||
if (sessionLostError) throw sessionLostError;
|
||||
if (cancelledTasksRef.current.has(rootTask.id) || cancelledTasksRef.current.has(rootTaskId)) {
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
|
||||
const fileSize = file.size;
|
||||
const sourcePath = file.sourcePath;
|
||||
const targetPath = file.targetPath;
|
||||
let skipSourceSize = fileSize;
|
||||
let skipSourceLastModified = file.lastModified;
|
||||
const directoryEntryIndex = startEntryIndex + fileIndex;
|
||||
let directoryEntryIdentity = createDirectoryEntryIdentity({
|
||||
sourcePath,
|
||||
targetPath,
|
||||
size: fileSize,
|
||||
lastModified: file.lastModified,
|
||||
});
|
||||
const persistedChild = transfersRef.current.find((candidate) => (
|
||||
candidate.parentTaskId === rootTaskId
|
||||
&& candidate.sourcePath === sourcePath
|
||||
&& candidate.targetPath === targetPath
|
||||
));
|
||||
if (persistedChild?.status === "completed") return;
|
||||
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
if (cancelledTasksRef.current.has(rootTask.id) || cancelledTasksRef.current.has(rootTaskId)) {
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
if (isPauseLatched(rootTaskId)) {
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
}
|
||||
|
||||
// Symlink listing attrs are the link node; transfer follows target bytes.
|
||||
// Re-stat regular files so skip and transfer both use current size/mtime
|
||||
// (Codex P1: pre-scan listing attrs can go stale before transfer starts).
|
||||
let freshSourceOk = false;
|
||||
if (!file.isSymlink) {
|
||||
const freshSource = await tryStatSource(sourcePath);
|
||||
if (freshSource && freshSource.type !== "directory") {
|
||||
skipSourceSize = freshSource.size;
|
||||
skipSourceLastModified = freshSource.lastModified;
|
||||
directoryEntryIdentity = createDirectoryEntryIdentity({
|
||||
sourcePath,
|
||||
targetPath,
|
||||
size: skipSourceSize,
|
||||
lastModified: skipSourceLastModified,
|
||||
});
|
||||
freshSourceOk = true;
|
||||
} else {
|
||||
// Do not pass stale pre-scan size as an explicit snapshot (Codex P2).
|
||||
// Child totalBytes 0 is omitted at startStreamTransfer via `|| undefined`,
|
||||
// so the dedicated transfer session re-stats the source itself.
|
||||
skipSourceSize = 0;
|
||||
skipSourceLastModified = 0;
|
||||
}
|
||||
}
|
||||
if (skipUnchanged && !file.isSymlink && freshSourceOk) {
|
||||
const existing = await tryStatTarget(targetPath);
|
||||
if (
|
||||
existing
|
||||
&& existing.type !== "directory"
|
||||
&& isUnchangedTransferCandidate(
|
||||
{ size: skipSourceSize, lastModified: skipSourceLastModified, mtimeUnit: "ms" },
|
||||
{ size: existing.size, lastModified: existing.lastModified, mtimeUnit: "ms" },
|
||||
)
|
||||
) {
|
||||
const skippedId = persistedChild?.id ?? crypto.randomUUID();
|
||||
setTransfers((prev) => {
|
||||
const hasChild = prev.some((row) => row.id === skippedId);
|
||||
const next = hasChild
|
||||
? prev.map((row) => row.id === skippedId
|
||||
? {
|
||||
...row,
|
||||
status: "completed" as TransferStatus,
|
||||
transferredBytes: skipSourceSize,
|
||||
totalBytes: skipSourceSize,
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
error: undefined,
|
||||
}
|
||||
: row)
|
||||
: [...prev, {
|
||||
...rootTask,
|
||||
id: skippedId,
|
||||
fileName: file.name,
|
||||
originalFileName: file.name,
|
||||
sourcePath,
|
||||
targetPath,
|
||||
isDirectory: false,
|
||||
progressMode: "bytes" as const,
|
||||
parentTaskId: rootTaskId,
|
||||
totalBytes: skipSourceSize,
|
||||
transferredBytes: skipSourceSize,
|
||||
sourceLastModified: skipSourceLastModified,
|
||||
directoryEntryIndex,
|
||||
directoryEntryIdentity,
|
||||
status: "completed" as TransferStatus,
|
||||
speed: 0,
|
||||
startTime: Date.now(),
|
||||
endTime: Date.now(),
|
||||
}];
|
||||
return next.map((row) => {
|
||||
if (row.id !== rootTaskId) return row;
|
||||
if (row.status === "paused" || row.status === "pausing" || isPauseLatched(rootTaskId)) {
|
||||
return { ...row, speed: 0 };
|
||||
}
|
||||
return { ...row, transferredBytes: row.transferredBytes + 1 };
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const fileId = persistedChild?.id ?? crypto.randomUUID();
|
||||
if (!activeChildIdsRef.current.has(rootTaskId)) {
|
||||
activeChildIdsRef.current.set(rootTaskId, new Set());
|
||||
}
|
||||
activeChildIdsRef.current.get(rootTaskId)!.add(fileId);
|
||||
|
||||
const childTask: TransferTask = {
|
||||
...rootTask,
|
||||
...persistedChild,
|
||||
id: fileId,
|
||||
fileName: file.name,
|
||||
originalFileName: file.name,
|
||||
sourcePath,
|
||||
targetPath,
|
||||
isDirectory: false,
|
||||
progressMode: "bytes",
|
||||
parentTaskId: rootTaskId,
|
||||
totalBytes: skipSourceSize,
|
||||
sourceLastModified: skipSourceLastModified || undefined,
|
||||
directoryEntryIndex,
|
||||
directoryEntryIdentity,
|
||||
retryable: rootTask.retryable,
|
||||
lifecycleEpoch: undefined,
|
||||
phase: undefined,
|
||||
pauseUnavailableReason: undefined,
|
||||
};
|
||||
|
||||
setTransfers((prev) => persistedChild
|
||||
? prev.map((candidate) => candidate.id === fileId ? {
|
||||
...childTask,
|
||||
status: "queued" as TransferStatus,
|
||||
speed: 0,
|
||||
error: undefined,
|
||||
endTime: undefined,
|
||||
lifecycleEpoch: undefined,
|
||||
} : candidate)
|
||||
: [...prev, {
|
||||
...childTask,
|
||||
status: "queued" as TransferStatus,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
startTime: Date.now(),
|
||||
lifecycleEpoch: undefined,
|
||||
}]);
|
||||
|
||||
try {
|
||||
await transferFile(
|
||||
childTask,
|
||||
sourceSftpId,
|
||||
targetSftpId,
|
||||
sourceIsLocal,
|
||||
targetIsLocal,
|
||||
sourceEncoding,
|
||||
targetEncoding,
|
||||
rootTaskId,
|
||||
sameHost,
|
||||
);
|
||||
|
||||
activeChildIdsRef.current.get(rootTaskId)?.delete(fileId);
|
||||
setTransfers((prev) => {
|
||||
const parentRow = prev.find((row) => row.id === rootTaskId);
|
||||
const parentFrozen = !!parentRow && (
|
||||
parentRow.status === "paused"
|
||||
|| parentRow.status === "pausing"
|
||||
|| isPauseLatched(rootTaskId)
|
||||
);
|
||||
return prev.map((t) => {
|
||||
if (t.id === fileId) {
|
||||
return {
|
||||
...t,
|
||||
status: "completed" as TransferStatus,
|
||||
endTime: Date.now(),
|
||||
transferredBytes: t.totalBytes,
|
||||
};
|
||||
}
|
||||
if (t.id === rootTaskId) {
|
||||
if (parentFrozen) return { ...t, speed: 0 };
|
||||
return { ...t, transferredBytes: t.transferredBytes + 1, speed: t.speed };
|
||||
}
|
||||
return t;
|
||||
});
|
||||
});
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
} catch (err) {
|
||||
activeChildIdsRef.current.get(rootTaskId)?.delete(fileId);
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
if (isTransferCancelledError(err)) {
|
||||
setTransfers((prev) => prev.map((t) => (
|
||||
t.id === fileId
|
||||
? { ...t, status: "cancelled" as TransferStatus, error: undefined, endTime: Date.now() }
|
||||
: t
|
||||
)));
|
||||
errors.push(err instanceof Error ? err : new Error(message));
|
||||
return;
|
||||
}
|
||||
setTransfers((prev) => prev.map((t) => (
|
||||
t.id === fileId
|
||||
? { ...t, status: "failed" as TransferStatus, error: message }
|
||||
: t
|
||||
)));
|
||||
if (isSessionError(err) && !sessionLostError) {
|
||||
sessionLostError = err instanceof Error ? err : new Error(message);
|
||||
setTransfers((prev) => prev.map((t) => (
|
||||
t.parentTaskId === rootTaskId
|
||||
&& (t.status === "queued" || t.status === "pending")
|
||||
? {
|
||||
...t,
|
||||
status: "failed" as TransferStatus,
|
||||
error: "SFTP session lost - reconnect and resume remaining files",
|
||||
speed: 0,
|
||||
endTime: Date.now(),
|
||||
}
|
||||
: t
|
||||
)));
|
||||
}
|
||||
errors.push(err instanceof Error ? err : new Error(message));
|
||||
if (sessionLostError) throw sessionLostError;
|
||||
if (isPauseLatched(rootTaskId)) {
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
beforeClaim: async () => {
|
||||
if (sessionLostError) return;
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
},
|
||||
},
|
||||
).catch((err) => {
|
||||
if (sessionLostError || isTransferCancelledError(err)) return;
|
||||
throw err;
|
||||
});
|
||||
|
||||
if (sessionLostError) {
|
||||
logger.warn("[SFTP] Directory transfer stopped early: session lost", sessionLostError.message);
|
||||
} else if (errors.length > 0) {
|
||||
logger.debug?.("[SFTP] Some files in directory transfer failed", errors);
|
||||
}
|
||||
return errors.length;
|
||||
}
|
||||
@@ -146,9 +146,9 @@ test("directory transfer discovers each directory once with bounded listing conc
|
||||
assert.equal(listCalls.size, directoryCount + 1);
|
||||
assert.ok(Array.from(listCalls.values()).every((count) => count === 1));
|
||||
assert.equal(listCalls.has("/source/loop"), false, "canonical symlink cycles must not be listed");
|
||||
// Bounded parallel discovery (default 4) - never unbounded fan-out.
|
||||
assert.ok(maxActiveListings >= 2, `expected parallel listings, got ${maxActiveListings}`);
|
||||
assert.ok(maxActiveListings <= 4, `listing concurrency exceeded budget: ${maxActiveListings}`);
|
||||
// Interleaved walk processes sibling subdirectories sequentially so resume
|
||||
// manifests stay deterministic (no full-tree pre-scan fan-out).
|
||||
assert.equal(maxActiveListings, 1, `expected sequential listings, got ${maxActiveListings}`);
|
||||
assert.equal(tasks.find((task) => task.id === root.id)?.totalBytes, directoryCount);
|
||||
} finally {
|
||||
await act(async () => {
|
||||
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
} from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { isUnchangedTransferCandidate } from "../../../domain/sftpTransferSkip";
|
||||
import {
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY,
|
||||
} from "../../../infrastructure/config/storageKeys";
|
||||
@@ -21,14 +20,9 @@ import { logger } from "../../../lib/logger";
|
||||
import {
|
||||
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
|
||||
resolveSftpDirectoryListingConcurrency,
|
||||
resolveSftpFolderPrescanEnabled,
|
||||
resolveSftpSkipUnchangedEnabled,
|
||||
runSftpTransferWorkers,
|
||||
} from "./transferConcurrency";
|
||||
import {
|
||||
discoverTransferTree,
|
||||
} from "./transferDirectoryDiscovery";
|
||||
import { transferDiscoveredFiles } from "./transferDirectoryFileBatch";
|
||||
import { getSftpTransferResourceKeys, globalSftpTransferScheduler } from "./globalTransferScheduler";
|
||||
import { resolveDedicatedStreamEndpointIds } from "../../../domain/sftpDedicatedStreamPolicy";
|
||||
import { isSessionError } from "./errors";
|
||||
@@ -103,38 +97,10 @@ function isCancelledLocalOrGlobal(
|
||||
return false;
|
||||
}
|
||||
|
||||
function readFolderPrescanEnabled(): boolean {
|
||||
return resolveSftpFolderPrescanEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN));
|
||||
}
|
||||
|
||||
function readSkipUnchangedEnabled(): boolean {
|
||||
return resolveSftpSkipUnchangedEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED));
|
||||
}
|
||||
|
||||
async function ensureTransferDirectory(
|
||||
targetPath: string,
|
||||
targetIsLocal: boolean,
|
||||
targetSftpId: string | null,
|
||||
targetEncoding: SftpFilenameEncoding,
|
||||
): Promise<void> {
|
||||
if (targetIsLocal) {
|
||||
try {
|
||||
await netcattyBridge.get()?.mkdirLocal?.(targetPath);
|
||||
} catch (mkdirErr: unknown) {
|
||||
const isEEXIST = mkdirErr instanceof Error && mkdirErr.message.includes("EEXIST");
|
||||
if (!isEEXIST) throw mkdirErr;
|
||||
const stat = await netcattyBridge.get()?.statLocal?.(targetPath);
|
||||
if (stat && stat.type !== "directory") {
|
||||
throw new Error(`Target path exists as a file: ${targetPath}`);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (targetSftpId) {
|
||||
await netcattyBridge.get()?.mkdirSftp(targetSftpId, targetPath, targetEncoding);
|
||||
}
|
||||
}
|
||||
|
||||
async function tryStatTransferPath(
|
||||
filePath: string,
|
||||
isLocal: boolean,
|
||||
@@ -577,102 +543,10 @@ export function useSftpDirectoryTransferOps({
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
|
||||
// Root-only pre-scan (rsync --no-inc-recursive): discover the full tree first
|
||||
// so the transfer center can show a stable file total before bytes move.
|
||||
if (!discoveryProgress && readFolderPrescanEnabled()) {
|
||||
const skipUnchanged = readSkipUnchangedEnabled()
|
||||
&& !task.replaceExistingTarget
|
||||
&& !String(task.targetPath).includes(".netcatty-");
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? {
|
||||
...candidate,
|
||||
phase: "scanning",
|
||||
totalBytes: Math.max(candidate.transferredBytes, candidate.totalBytes || 0),
|
||||
}
|
||||
: candidate));
|
||||
|
||||
const traversal = traversalBudget ?? createSftpDirectoryTraversalBudget();
|
||||
const discovered = await discoverTransferTree({
|
||||
sourcePath: task.sourcePath,
|
||||
targetPath: task.targetPath,
|
||||
sourceIsLocal,
|
||||
sourceSftpId,
|
||||
sourceEncoding,
|
||||
followSymlinks,
|
||||
listLocalFiles,
|
||||
listRemoteFiles,
|
||||
traversalBudget: traversal,
|
||||
shouldAbort: () => (
|
||||
cancelledTasksRef.current.has(task.id)
|
||||
|| cancelledTasksRef.current.has(rootTaskId)
|
||||
),
|
||||
waitWhilePaused: async () => {
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
},
|
||||
onDiscoveredFiles: (totalFiles) => {
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? {
|
||||
...candidate,
|
||||
phase: "scanning",
|
||||
totalBytes: Math.max(totalFiles, candidate.transferredBytes),
|
||||
}
|
||||
: candidate));
|
||||
},
|
||||
});
|
||||
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? {
|
||||
...candidate,
|
||||
totalBytes: Math.max(discovered.files.length, candidate.transferredBytes),
|
||||
}
|
||||
: candidate));
|
||||
|
||||
// Create directories breadth-first so parents exist before children.
|
||||
for (const dir of discovered.directories) {
|
||||
if (cancelledTasksRef.current.has(task.id) || cancelledTasksRef.current.has(rootTaskId)) {
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
await ensureTransferDirectory(dir.targetPath, targetIsLocal, targetSftpId, targetEncoding);
|
||||
}
|
||||
|
||||
const fileErrors = await transferDiscoveredFiles({
|
||||
rootTask: task,
|
||||
files: discovered.files,
|
||||
sourceSftpId,
|
||||
targetSftpId,
|
||||
sourceIsLocal,
|
||||
targetIsLocal,
|
||||
sourceEncoding,
|
||||
targetEncoding,
|
||||
rootTaskId,
|
||||
sameHost,
|
||||
skipUnchanged,
|
||||
startEntryIndex: 0,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers,
|
||||
waitWhileTransferPaused,
|
||||
isPauseLatched,
|
||||
transferFile,
|
||||
tryStatTarget: (targetPath) => tryStatTransferTarget(
|
||||
targetPath,
|
||||
targetIsLocal,
|
||||
targetSftpId,
|
||||
targetEncoding,
|
||||
),
|
||||
tryStatSource: (sourcePath) => tryStatTransferPath(
|
||||
sourcePath,
|
||||
sourceIsLocal,
|
||||
sourceSftpId,
|
||||
sourceEncoding,
|
||||
),
|
||||
});
|
||||
// Match interleaved walk: max-depth symlink omissions count as failures.
|
||||
return fileErrors + discovered.omittedSymlinkDirectoryErrors;
|
||||
}
|
||||
|
||||
// Always interleave discovery with transfer (rsync --inc-recursive style).
|
||||
// Full-tree pre-scan was removed: non-compressed folder uploads start bytes
|
||||
// as soon as each directory is listed; compressed upload is a separate path.
|
||||
// UI should treat totalBytes as "discovered so far", not a fixed grand total.
|
||||
let totalErrors = 0;
|
||||
const progress = discoveryProgress ?? {
|
||||
discoveredFiles: 0,
|
||||
@@ -772,7 +646,9 @@ export function useSftpDirectoryTransferOps({
|
||||
? {
|
||||
...candidate,
|
||||
totalBytes: Math.max(progress.discoveredFiles, candidate.transferredBytes),
|
||||
phase: "scanning",
|
||||
// Keep scanning only until the first file completes; later lists must
|
||||
// not flip an in-progress folder bar back to indeterminate thrash.
|
||||
...(candidate.transferredBytes <= 0 ? { phase: "scanning" as const } : null),
|
||||
}
|
||||
: candidate));
|
||||
|
||||
|
||||
@@ -59,3 +59,96 @@ test("upload task callbacks write through the transfer store without page callba
|
||||
assert.equal(patches[1].updates.transferredBytes, 100);
|
||||
assert.deepEqual(dismissed, []);
|
||||
});
|
||||
|
||||
test("progress promotes pending scanning folder rows into transferring", () => {
|
||||
const patches: Array<{ taskId: string; updates: Partial<TransferTask> }> = [];
|
||||
const liveTask: TransferTask = {
|
||||
id: "folder-1",
|
||||
fileName: "docs",
|
||||
sourcePath: "local",
|
||||
targetPath: "/remote/docs",
|
||||
sourceConnectionId: "external",
|
||||
targetConnectionId: "connection-1",
|
||||
direction: "upload",
|
||||
status: "pending",
|
||||
totalBytes: 0,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
startTime: 1,
|
||||
isDirectory: true,
|
||||
progressMode: "files",
|
||||
phase: "scanning",
|
||||
};
|
||||
const store = {
|
||||
upsertTasks: () => {},
|
||||
patchTask: (taskId: string, updates: Partial<TransferTask>) => patches.push({ taskId, updates }),
|
||||
dismiss: () => {},
|
||||
getTask: (taskId: string) => (taskId === "folder-1" ? liveTask : undefined),
|
||||
};
|
||||
const callbacks = createUploadTaskCallbacks({
|
||||
ownerId: "owner-1",
|
||||
connectionId: "connection-1",
|
||||
targetPath: "/remote",
|
||||
store,
|
||||
});
|
||||
|
||||
callbacks.onTaskProgress?.("folder-1", {
|
||||
transferred: 12,
|
||||
total: 400,
|
||||
speed: 0,
|
||||
percent: 3,
|
||||
phase: "transferring",
|
||||
});
|
||||
|
||||
assert.equal(patches[0].taskId, "folder-1");
|
||||
assert.equal(patches[0].updates.status, "transferring");
|
||||
assert.equal(patches[0].updates.transferredBytes, 12);
|
||||
assert.equal(patches[0].updates.totalBytes, 400);
|
||||
assert.equal(patches[0].updates.phase, "transferring");
|
||||
});
|
||||
|
||||
test("scanning callbacks expose live file counts in files progress mode", () => {
|
||||
const upserts: TransferTask[][] = [];
|
||||
const patches: Array<{ taskId: string; updates: Partial<TransferTask> }> = [];
|
||||
const store = {
|
||||
upsertTasks: (tasks: readonly TransferTask[]) => upserts.push([...tasks]),
|
||||
patchTask: (taskId: string, updates: Partial<TransferTask>) => patches.push({ taskId, updates }),
|
||||
dismiss: () => {},
|
||||
};
|
||||
const callbacks = createUploadTaskCallbacks({
|
||||
ownerId: "owner-1",
|
||||
connectionId: "connection-1",
|
||||
targetPath: "/remote",
|
||||
store,
|
||||
});
|
||||
|
||||
callbacks.onScanningStart?.("scan-1", { label: "docs" });
|
||||
callbacks.onScanningProgress?.("scan-1", {
|
||||
fileCount: 1284,
|
||||
directoryCount: 40,
|
||||
entryCount: 1324,
|
||||
label: "docs",
|
||||
});
|
||||
|
||||
assert.equal(upserts[0][0].fileName, "docs");
|
||||
assert.equal(upserts[0][0].phase, "scanning");
|
||||
assert.equal(upserts[0][0].progressMode, "files");
|
||||
assert.equal(upserts[0][0].status, "pending");
|
||||
assert.deepEqual(patches[0], {
|
||||
taskId: "scan-1",
|
||||
updates: {
|
||||
// Found files live in totalBytes; completed stays 0 during scan.
|
||||
transferredBytes: 0,
|
||||
totalBytes: 1284,
|
||||
progressMode: "files",
|
||||
phase: "scanning",
|
||||
fileName: "docs",
|
||||
},
|
||||
});
|
||||
|
||||
// Display contract: 0 done · N found (not N done · N found).
|
||||
const discovered = Math.max(patches[0].updates.totalBytes ?? 0, patches[0].updates.transferredBytes ?? 0);
|
||||
const completed = patches[0].updates.transferredBytes ?? 0;
|
||||
assert.equal(completed, 0);
|
||||
assert.equal(discovered, 1284);
|
||||
});
|
||||
@@ -6,7 +6,7 @@ import { joinPath } from "./utils";
|
||||
type UploadTransferStore = Pick<
|
||||
typeof sftpTransferCenterStore,
|
||||
"upsertTasks" | "patchTask" | "dismiss"
|
||||
>;
|
||||
> & Partial<Pick<typeof sftpTransferCenterStore, "getTask">>;
|
||||
|
||||
interface UploadTaskCallbacksParams {
|
||||
ownerId: string;
|
||||
@@ -27,11 +27,11 @@ export const createUploadTaskCallbacks = ({
|
||||
targetConnectionKey,
|
||||
store = sftpTransferCenterStore,
|
||||
}: UploadTaskCallbacksParams): UploadCallbacks => ({
|
||||
onScanningStart: (taskId: string) => {
|
||||
onScanningStart: (taskId: string, info?: { label?: string }) => {
|
||||
store.upsertTasks([{
|
||||
id: taskId,
|
||||
ownerId,
|
||||
fileName: "Scanning files...",
|
||||
fileName: info?.label?.trim() || "Scanning files...",
|
||||
sourcePath: "local",
|
||||
targetPath,
|
||||
sourceConnectionId: "external",
|
||||
@@ -42,18 +42,30 @@ export const createUploadTaskCallbacks = ({
|
||||
targetConnectionKey,
|
||||
direction: "upload",
|
||||
status: "pending" as TransferStatus,
|
||||
// Progressive discovery: totalBytes = found so far, completed stays 0
|
||||
// until real transfers start (UI: "0 done · N found").
|
||||
totalBytes: 0,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
startTime: Date.now(),
|
||||
isDirectory: true,
|
||||
progressMode: "bytes",
|
||||
progressMode: "files",
|
||||
origin: "drag-drop",
|
||||
background: false,
|
||||
resumable: true,
|
||||
phase: "scanning",
|
||||
}]);
|
||||
},
|
||||
onScanningProgress: (taskId: string, progress) => {
|
||||
store.patchTask(taskId, {
|
||||
// Found count is the growing total; nothing is completed during scan.
|
||||
totalBytes: Math.max(0, progress.fileCount),
|
||||
transferredBytes: 0,
|
||||
progressMode: "files",
|
||||
phase: "scanning",
|
||||
...(progress.label?.trim() ? { fileName: progress.label.trim() } : null),
|
||||
});
|
||||
},
|
||||
onScanningEnd: (taskId: string) => {
|
||||
store.dismiss(taskId);
|
||||
},
|
||||
@@ -90,8 +102,36 @@ export const createUploadTaskCallbacks = ({
|
||||
const durableCheckpoint = Number.isFinite(Number(progress.checkpointBytes))
|
||||
? Math.max(0, Math.trunc(Number(progress.checkpointBytes)))
|
||||
: progress.transferred;
|
||||
// Progressive folder walks keep the scanning row as pending until real
|
||||
// progress arrives. Promote pending/queued → transferring so the panel
|
||||
// leaves "Waiting..." and matches the transfer-center live state.
|
||||
const current = "getTask" in store && typeof store.getTask === "function"
|
||||
? store.getTask(taskId)
|
||||
: undefined;
|
||||
const shouldPromote =
|
||||
!!current
|
||||
&& (current.status === "pending" || current.status === "queued")
|
||||
&& current.reconnectRequired !== true
|
||||
&& (
|
||||
progress.phase === "scanning"
|
||||
|| progress.phase === "transferring"
|
||||
|| progress.transferred > 0
|
||||
|| progress.total > 0
|
||||
);
|
||||
// Only patch fingerprint/checkpoint while paused — do not keep animating
|
||||
// high-water transferred after the user hit Pause.
|
||||
const isPausedLike = current?.status === "paused" || current?.status === "pausing";
|
||||
if (isPausedLike) {
|
||||
store.patchTask(taskId, {
|
||||
checkpointBytes: durableCheckpoint,
|
||||
resumable: progress.resumable,
|
||||
pauseUnavailableReason: progress.pauseUnavailableReason,
|
||||
...("sourceFingerprint" in progress && progress.sourceFingerprint
|
||||
? { sourceFingerprint: progress.sourceFingerprint as string }
|
||||
: null),
|
||||
});
|
||||
return;
|
||||
}
|
||||
store.patchTask(taskId, {
|
||||
transferredBytes: progress.transferred,
|
||||
totalBytes: progress.total,
|
||||
@@ -101,6 +141,7 @@ export const createUploadTaskCallbacks = ({
|
||||
phase: progress.phase,
|
||||
resumable: progress.resumable,
|
||||
pauseUnavailableReason: progress.pauseUnavailableReason,
|
||||
...(shouldPromote ? { status: "transferring" as TransferStatus } : null),
|
||||
// Durable pause identity may arrive on a forced progress event while
|
||||
// status is already paused — keep it for restart/resume safety.
|
||||
...("sourceFingerprint" in progress && progress.sourceFingerprint
|
||||
@@ -121,6 +162,7 @@ export const createUploadTaskCallbacks = ({
|
||||
endTime: Date.now(),
|
||||
transferredBytes: totalBytes,
|
||||
speed: 0,
|
||||
phase: undefined,
|
||||
});
|
||||
},
|
||||
onTaskFailed: (taskId: string, error: string) => {
|
||||
@@ -129,6 +171,7 @@ export const createUploadTaskCallbacks = ({
|
||||
endTime: Date.now(),
|
||||
error,
|
||||
speed: 0,
|
||||
phase: undefined,
|
||||
});
|
||||
},
|
||||
onTaskCancelled: (taskId: string) => {
|
||||
@@ -136,6 +179,7 @@ export const createUploadTaskCallbacks = ({
|
||||
status: "cancelled" as TransferStatus,
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
phase: undefined,
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -15,11 +15,63 @@ import {
|
||||
UploadResult,
|
||||
startUploadScanningTask,
|
||||
} from "../../../lib/uploadService";
|
||||
import { extractDropEntries, type DropEntry } from "../../../lib/sftpFileUtils";
|
||||
import { uploadLocalFoldersProgressively } from "../../../lib/progressiveFolderUpload";
|
||||
import {
|
||||
captureDropPayload,
|
||||
formatDropScanLabel,
|
||||
isDropScanCancelledError,
|
||||
localTreeToDropEntries,
|
||||
materializeDropEntries,
|
||||
type DropEntry,
|
||||
type LocalTreeListEntry,
|
||||
} from "../../../lib/sftpFileUtils";
|
||||
|
||||
function isUploadScanCancelled(controller: UploadController, error?: unknown): boolean {
|
||||
return controller.isCancelled() || isDropScanCancelledError(error);
|
||||
}
|
||||
|
||||
// Re-export UploadResult for external usage
|
||||
export type { UploadResult };
|
||||
|
||||
type LocalTreeScanOptions = {
|
||||
onProgress?: (progress: { fileCount: number; directoryCount: number; entryCount: number }) => void;
|
||||
onEntries?: (entries: LocalTreeListEntry[]) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
};
|
||||
|
||||
function createDropScanCancelledError(): Error {
|
||||
const error = new Error("Drop scan cancelled");
|
||||
(error as Error & { code?: string }).code = "ERR_DROP_SCAN_CANCELLED";
|
||||
return error;
|
||||
}
|
||||
|
||||
async function listLocalTreeWithAbort(
|
||||
bridge: NetcattyBridge,
|
||||
localPath: string,
|
||||
options: LocalTreeScanOptions = {},
|
||||
): Promise<LocalTreeListEntry[]> {
|
||||
const scanId = `drop-scan-${crypto.randomUUID()}`;
|
||||
const abortSignal = options.abortSignal;
|
||||
const requestCancel = () => {
|
||||
void bridge.cancelLocalTreeScan?.(scanId);
|
||||
};
|
||||
if (abortSignal?.aborted) {
|
||||
throw createDropScanCancelledError();
|
||||
}
|
||||
abortSignal?.addEventListener("abort", requestCancel, { once: true });
|
||||
const { abortSignal: _abortSignal, ...bridgeOptions } = options;
|
||||
try {
|
||||
return await bridge.listLocalTree!(localPath, { ...bridgeOptions, scanId });
|
||||
} catch (error) {
|
||||
if (abortSignal?.aborted || isDropScanCancelledError(error)) {
|
||||
throw createDropScanCancelledError();
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
abortSignal?.removeEventListener("abort", requestCancel);
|
||||
}
|
||||
}
|
||||
|
||||
import type { UseSftpExternalOperationsParams, SftpExternalOperationsResult } from "./useSftpExternalOperations.types";
|
||||
import { getSftpTransferResourceKeys, globalSftpTransferScheduler } from "./globalTransferScheduler";
|
||||
import { localStorageAdapter } from "../../../infrastructure/persistence/localStorageAdapter";
|
||||
@@ -48,6 +100,10 @@ import {
|
||||
registerExternalUploadController,
|
||||
unregisterExternalUploadController,
|
||||
} from "./externalUploadRuntime";
|
||||
import {
|
||||
isTransferPauseLatched,
|
||||
waitWhileTransferOrRootPaused,
|
||||
} from "./transferPauseLatch";
|
||||
|
||||
type UploadConflictResolver = {
|
||||
resolve: (action: FileConflictAction) => void;
|
||||
@@ -151,9 +207,12 @@ export const useSftpExternalOperations = (
|
||||
callbacks: UploadCallbacks,
|
||||
): UploadCallbacks => ({
|
||||
...callbacks,
|
||||
onScanningStart: (taskId) => {
|
||||
onScanningStart: (taskId, info) => {
|
||||
registerUploadController(taskId, controller);
|
||||
callbacks.onScanningStart?.(taskId);
|
||||
callbacks.onScanningStart?.(taskId, info);
|
||||
},
|
||||
onScanningProgress: (taskId, progress) => {
|
||||
callbacks.onScanningProgress?.(taskId, progress);
|
||||
},
|
||||
onTaskCreated: (task) => {
|
||||
registerUploadController(task.id, controller);
|
||||
@@ -764,9 +823,12 @@ export const useSftpExternalOperations = (
|
||||
conflictDefaults.set(conflictType, action);
|
||||
},
|
||||
});
|
||||
controller.addCancelListener(() => {
|
||||
cancelPendingUploadConflicts(controller);
|
||||
});
|
||||
});
|
||||
};
|
||||
}, []);
|
||||
}, [cancelPendingUploadConflicts]);
|
||||
|
||||
// Create upload bridge that wraps netcattyBridge.
|
||||
// Pass connect-time Host so pooled stream uploads open the pinned endpoint
|
||||
@@ -888,52 +950,271 @@ export const useSftpExternalOperations = (
|
||||
|
||||
const uploadExternalFiles = useCallback(
|
||||
async (side: "left" | "right", dataTransfer: DataTransfer, targetPath?: string): Promise<UploadResult[]> => {
|
||||
// DataTransfer is only valid during the drop event. Capture entries BEFORE
|
||||
// DataTransfer is only valid during the drop event. Capture roots BEFORE
|
||||
// any await (session reconnect can take seconds while a transfer is busy).
|
||||
// Otherwise extractDropEntries returns [] and the UI toasts "Uploaded files: 0".
|
||||
let capturedEntries: DropEntry[];
|
||||
try {
|
||||
capturedEntries = await extractDropEntries(dataTransfer);
|
||||
} catch (error) {
|
||||
logger.error("[SFTP] Failed to read dropped files:", error);
|
||||
throw error;
|
||||
}
|
||||
if (capturedEntries.length === 0) {
|
||||
// Native tree expansion (listLocalTree) happens after the scanning UI is up.
|
||||
const dropPayload = captureDropPayload(dataTransfer);
|
||||
if (dropPayload.roots.length === 0 && dropPayload.filesFallback.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const pane = getActivePane(side);
|
||||
if (!pane?.connection) {
|
||||
throw new Error("No active connection");
|
||||
}
|
||||
const bridge = netcattyBridge.get();
|
||||
if (!bridge) {
|
||||
throw new Error("Bridge not available");
|
||||
}
|
||||
|
||||
const uploadTargetPath = targetPath || pane.connection.currentPath;
|
||||
const controller = new UploadController();
|
||||
const callbacks = bindUploadControllerCallbacks(
|
||||
controller,
|
||||
createUploadCallbacks(
|
||||
pane.connection.id,
|
||||
uploadTargetPath,
|
||||
pane.connection.isLocal ? undefined : pane.connection.hostId,
|
||||
pane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(pane.connection.id),
|
||||
pane.connection.isLocal ? undefined : pane.connection.hostLabel,
|
||||
),
|
||||
);
|
||||
|
||||
const pathBackedFolderRoots = dropPayload.roots
|
||||
.filter((root) => root.isDirectory && !!root.localPath)
|
||||
.map((root) => ({ name: root.name, localPath: root.localPath! }));
|
||||
// Progressive only when *every* root is a path-backed folder. Mixed drops
|
||||
// (folder + files, path-less dirs) must fall through to materialize so
|
||||
// sibling items are not silently dropped.
|
||||
const canProgressiveUpload = pathBackedFolderRoots.length > 0
|
||||
&& pathBackedFolderRoots.length === dropPayload.roots.length
|
||||
&& !!bridge.listLocalTree
|
||||
&& !useCompressedUpload;
|
||||
const needsDeepScan = dropPayload.roots.some((root) => root.isDirectory);
|
||||
const scanLabel = formatDropScanLabel(dropPayload.roots);
|
||||
// Instant feedback: scanning / parent row appears before work begins.
|
||||
const scanningTask = needsDeepScan
|
||||
? startUploadScanningTask(callbacks, crypto.randomUUID(), { label: scanLabel })
|
||||
: null;
|
||||
if (scanningTask && typeof window !== "undefined") {
|
||||
window.dispatchEvent(new CustomEvent("netcatty:open-sftp-transfer-center"));
|
||||
}
|
||||
|
||||
const scanAbort = new AbortController();
|
||||
const detachScanCancel = controller.addCancelListener(() => {
|
||||
scanAbort.abort();
|
||||
if (scanningTask?.isOpen()) scanningTask.cancel();
|
||||
});
|
||||
|
||||
// Path-backed folders (no compressed upload): edge-scan + edge-transfer.
|
||||
// Open the SFTP session first, then stream discovery batches into upload
|
||||
// workers so the first files start while the rest of the tree is still walking.
|
||||
if (canProgressiveUpload) {
|
||||
const runProgressive = async (forceReconnect = false): Promise<UploadResult[]> => {
|
||||
const { sftpId, release } = await resolveRemoteSftpId(side, { forceReconnect });
|
||||
const livePane = getActivePane(side) ?? pane;
|
||||
if (!livePane.connection) {
|
||||
release();
|
||||
throw new Error("No active connection");
|
||||
}
|
||||
const uploadPaneId = livePane.id;
|
||||
const liveTargetPath = targetPath || livePane.connection.currentPath;
|
||||
const connectHost = resolveUploadConnectHost(uploadPaneId, livePane.connection.isLocal);
|
||||
const uploadBridge = createUploadBridge(connectHost);
|
||||
const liveCallbacks = livePane.connection.id === pane.connection.id
|
||||
&& liveTargetPath === uploadTargetPath
|
||||
? callbacks
|
||||
: bindUploadControllerCallbacks(
|
||||
controller,
|
||||
createUploadCallbacks(
|
||||
livePane.connection.id,
|
||||
liveTargetPath,
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
|
||||
livePane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(livePane.connection.id),
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
|
||||
),
|
||||
);
|
||||
|
||||
const parentTaskIds = new Map<string, string>();
|
||||
if (scanningTask && pathBackedFolderRoots.length === 1) {
|
||||
parentTaskIds.set(pathBackedFolderRoots[0].name, scanningTask.taskId);
|
||||
// Keep the scanning row as the folder parent (do not dismiss it).
|
||||
scanningTask.complete = () => {
|
||||
/* no-op: progressive path settles via onTaskCompleted/Cancelled */
|
||||
};
|
||||
} else if (scanningTask?.isOpen()) {
|
||||
scanningTask.complete();
|
||||
}
|
||||
|
||||
try {
|
||||
if (controller.isCancelled()) {
|
||||
scanningTask?.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
const results = await uploadLocalFoldersProgressively(
|
||||
pathBackedFolderRoots,
|
||||
{
|
||||
targetPath: liveTargetPath,
|
||||
sftpId,
|
||||
targetHostId: livePane.connection.isLocal ? undefined : livePane.connection.hostId,
|
||||
isLocal: livePane.connection.isLocal,
|
||||
bridge: uploadBridge,
|
||||
joinPath,
|
||||
callbacks: liveCallbacks,
|
||||
parentTaskIds,
|
||||
abortSignal: scanAbort.signal,
|
||||
// Soft-pause must freeze discovery enqueue + child UI rows, not
|
||||
// only the open streams (otherwise Pause still floods the queue).
|
||||
waitWhilePaused: (parentTaskId) => waitWhileTransferOrRootPaused(parentTaskId),
|
||||
isPaused: (parentTaskId) => isTransferPauseLatched(parentTaskId),
|
||||
resolveConflict: createUploadConflictResolver(controller),
|
||||
listLocalTree: (localPath, treeOptions) => listLocalTreeWithAbort(bridge, localPath, {
|
||||
...treeOptions,
|
||||
abortSignal: scanAbort.signal,
|
||||
}),
|
||||
},
|
||||
controller,
|
||||
);
|
||||
if (clearDirCacheEntry && targetPath) {
|
||||
clearDirCacheEntry(livePane.connection.id, liveTargetPath);
|
||||
}
|
||||
if (liveTargetPath === livePane.connection.currentPath) {
|
||||
await refresh(side, { tabId: uploadPaneId });
|
||||
}
|
||||
return results;
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
return await runProgressive(false);
|
||||
} catch (error) {
|
||||
if (isSessionError(error) && ensureRemoteSftpId) {
|
||||
logger.warn("[SFTP] Progressive upload session lost; reconnecting once", error);
|
||||
try {
|
||||
return await runProgressive(true);
|
||||
} catch (retryError) {
|
||||
if (isUploadScanCancelled(controller, retryError)) {
|
||||
scanningTask?.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
if (scanningTask?.isOpen()) scanningTask.fail(retryError);
|
||||
throw retryError;
|
||||
}
|
||||
}
|
||||
if (isUploadScanCancelled(controller, error)) {
|
||||
scanningTask?.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
if (scanningTask?.isOpen()) scanningTask.fail(error);
|
||||
logger.error("[SFTP] Progressive folder upload failed:", error);
|
||||
throw error;
|
||||
} finally {
|
||||
detachScanCancel();
|
||||
unregisterUploadController(controller);
|
||||
}
|
||||
}
|
||||
|
||||
let capturedEntries: DropEntry[] = [];
|
||||
try {
|
||||
const scanT0 = performance.now();
|
||||
capturedEntries = await materializeDropEntries(dropPayload, {
|
||||
abortSignal: scanAbort.signal,
|
||||
isCancelled: () => controller.isCancelled(),
|
||||
listLocalTree: bridge.listLocalTree
|
||||
? (localPath, treeOptions) => listLocalTreeWithAbort(bridge, localPath, {
|
||||
...treeOptions,
|
||||
abortSignal: scanAbort.signal,
|
||||
})
|
||||
: undefined,
|
||||
onProgress: scanningTask
|
||||
? (progress) => {
|
||||
callbacks.onScanningProgress?.(scanningTask.taskId, {
|
||||
...progress,
|
||||
label: progress.label || scanLabel,
|
||||
});
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
logger.debug(
|
||||
`[SFTP:perf] materializeDropEntries — ${capturedEntries.length} entries — ${(performance.now() - scanT0).toFixed(0)}ms`,
|
||||
);
|
||||
} catch (error) {
|
||||
if (scanningTask?.isOpen()) {
|
||||
if (isUploadScanCancelled(controller, error)) {
|
||||
scanningTask.cancel();
|
||||
} else {
|
||||
scanningTask.fail(error);
|
||||
}
|
||||
}
|
||||
detachScanCancel();
|
||||
unregisterUploadController(controller);
|
||||
if (isUploadScanCancelled(controller, error)) {
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
logger.error("[SFTP] Failed to read dropped files:", error);
|
||||
throw error;
|
||||
}
|
||||
|
||||
if (controller.isCancelled()) {
|
||||
scanningTask?.cancel();
|
||||
detachScanCancel();
|
||||
unregisterUploadController(controller);
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
if (capturedEntries.length === 0) {
|
||||
scanningTask?.complete();
|
||||
detachScanCancel();
|
||||
unregisterUploadController(controller);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Keep scanning visible through session open so the UI never goes blank
|
||||
// between "N files found" and the real transfer rows.
|
||||
const run = async (forceReconnect = false): Promise<UploadResult[]> => {
|
||||
const pane = getActivePane(side);
|
||||
if (!pane?.connection) {
|
||||
const { sftpId, release } = await resolveRemoteSftpId(side, { forceReconnect });
|
||||
const livePane = getActivePane(side) ?? pane;
|
||||
if (!livePane.connection) {
|
||||
release();
|
||||
throw new Error("No active connection");
|
||||
}
|
||||
|
||||
const bridge = netcattyBridge.get();
|
||||
if (!bridge) {
|
||||
throw new Error("Bridge not available");
|
||||
}
|
||||
|
||||
const { sftpId, release } = await resolveRemoteSftpId(side, { forceReconnect });
|
||||
const livePane = getActivePane(side) ?? pane;
|
||||
if (!livePane.connection) throw new Error("No active connection");
|
||||
|
||||
const uploadPaneId = livePane.id;
|
||||
const uploadTargetPath = targetPath || livePane.connection.currentPath;
|
||||
const controller = new UploadController();
|
||||
const callbacks = bindUploadControllerCallbacks(
|
||||
controller,
|
||||
createUploadCallbacks(
|
||||
livePane.connection.id,
|
||||
uploadTargetPath,
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
|
||||
livePane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(livePane.connection.id),
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
|
||||
),
|
||||
);
|
||||
const liveTargetPath = targetPath || livePane.connection.currentPath;
|
||||
const connectHost = resolveUploadConnectHost(uploadPaneId, livePane.connection.isLocal);
|
||||
const uploadBridge = createUploadBridge(connectHost);
|
||||
|
||||
// Retarget callbacks if the pane path/id changed after reconnect.
|
||||
const liveCallbacks = livePane.connection.id === pane.connection.id
|
||||
&& liveTargetPath === uploadTargetPath
|
||||
? callbacks
|
||||
: bindUploadControllerCallbacks(
|
||||
controller,
|
||||
createUploadCallbacks(
|
||||
livePane.connection.id,
|
||||
liveTargetPath,
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
|
||||
livePane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(livePane.connection.id),
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
|
||||
),
|
||||
);
|
||||
const transferCallbacks = scanningTask
|
||||
? {
|
||||
...liveCallbacks,
|
||||
onTaskCreated: (task: Parameters<NonNullable<UploadCallbacks["onTaskCreated"]>>[0]) => {
|
||||
scanningTask.complete();
|
||||
liveCallbacks.onTaskCreated?.(task);
|
||||
},
|
||||
}
|
||||
: liveCallbacks;
|
||||
|
||||
let transferSucceeded = false;
|
||||
try {
|
||||
if (controller.isCancelled()) {
|
||||
scanningTask?.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
const hasDirectory = capturedEntries.some((entry) => (
|
||||
entry.isDirectory || entry.relativePath.replace(/\\/g, "/").includes("/")
|
||||
));
|
||||
@@ -951,13 +1232,13 @@ export const useSftpExternalOperations = (
|
||||
run: async (uploadSftpId) => uploadEntriesDirect(
|
||||
capturedEntries,
|
||||
{
|
||||
targetPath: uploadTargetPath,
|
||||
targetPath: liveTargetPath,
|
||||
sftpId: uploadSftpId,
|
||||
targetHostId: livePane.connection!.isLocal ? undefined : livePane.connection!.hostId,
|
||||
isLocal: livePane.connection!.isLocal,
|
||||
bridge: uploadBridge,
|
||||
joinPath,
|
||||
callbacks,
|
||||
callbacks: transferCallbacks,
|
||||
useCompressedUpload,
|
||||
resolveConflict: createUploadConflictResolver(controller),
|
||||
},
|
||||
@@ -966,15 +1247,19 @@ export const useSftpExternalOperations = (
|
||||
});
|
||||
|
||||
if (clearDirCacheEntry && targetPath) {
|
||||
clearDirCacheEntry(livePane.connection.id, uploadTargetPath);
|
||||
clearDirCacheEntry(livePane.connection.id, liveTargetPath);
|
||||
}
|
||||
if (uploadTargetPath === livePane.connection.currentPath) {
|
||||
if (liveTargetPath === livePane.connection.currentPath) {
|
||||
await refresh(side, { tabId: uploadPaneId });
|
||||
}
|
||||
transferSucceeded = true;
|
||||
return results;
|
||||
} finally {
|
||||
if (scanningTask?.isOpen()) {
|
||||
if (controller.isCancelled()) scanningTask.cancel();
|
||||
else if (transferSucceeded) scanningTask.complete();
|
||||
}
|
||||
release();
|
||||
unregisterUploadController(controller);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -983,10 +1268,26 @@ export const useSftpExternalOperations = (
|
||||
} catch (error) {
|
||||
if (isSessionError(error) && ensureRemoteSftpId) {
|
||||
logger.warn("[SFTP] Upload session lost; reconnecting and retrying once", error);
|
||||
return await run(true);
|
||||
try {
|
||||
return await run(true);
|
||||
} catch (retryError) {
|
||||
if (isUploadScanCancelled(controller, retryError)) {
|
||||
scanningTask?.cancel();
|
||||
} else if (scanningTask?.isOpen()) {
|
||||
scanningTask.fail(retryError);
|
||||
}
|
||||
throw retryError;
|
||||
}
|
||||
}
|
||||
if (scanningTask?.isOpen()) {
|
||||
if (controller.isCancelled()) scanningTask.cancel();
|
||||
else scanningTask.fail(error);
|
||||
}
|
||||
logger.error("[SFTP] Upload failed:", error);
|
||||
throw error;
|
||||
} finally {
|
||||
detachScanCancel();
|
||||
unregisterUploadController(controller);
|
||||
}
|
||||
},
|
||||
[
|
||||
@@ -1146,105 +1447,125 @@ export const useSftpExternalOperations = (
|
||||
connectionCacheKeyMapRef.current,
|
||||
);
|
||||
|
||||
const initialUploadTargetPath = targetPath || originatingPane.connection.currentPath;
|
||||
const controller = new UploadController();
|
||||
const callbacks = bindUploadControllerCallbacks(
|
||||
controller,
|
||||
createUploadCallbacks(
|
||||
originatingPane.connection.id,
|
||||
initialUploadTargetPath,
|
||||
originatingPane.connection.isLocal ? undefined : originatingPane.connection.hostId,
|
||||
originatingPane.connection.isLocal
|
||||
? undefined
|
||||
: connectionCacheKeyMapRef.current.get(originatingPane.connection.id),
|
||||
originatingPane.connection.isLocal ? undefined : originatingPane.connection.hostLabel,
|
||||
),
|
||||
);
|
||||
const folderName = folderPath.replace(/\\/g, "/").split("/").filter(Boolean).pop()
|
||||
|| folderPath;
|
||||
const scanningTask = startUploadScanningTask(callbacks, crypto.randomUUID(), {
|
||||
label: folderName,
|
||||
});
|
||||
if (typeof window !== "undefined") {
|
||||
window.dispatchEvent(new CustomEvent("netcatty:open-sftp-transfer-center"));
|
||||
}
|
||||
const scanAbort = new AbortController();
|
||||
const detachScanCancel = controller.addCancelListener(() => {
|
||||
scanAbort.abort();
|
||||
if (scanningTask.isOpen()) scanningTask.cancel();
|
||||
});
|
||||
let capturedEntries: DropEntry[] | null = null;
|
||||
|
||||
const run = async (forceReconnect = false): Promise<UploadResult[]> => {
|
||||
const pane = resolveUploadTargetPane({
|
||||
side,
|
||||
tabId: originatingTabId,
|
||||
getActivePane,
|
||||
getPaneByTabId,
|
||||
getPaneByConnectionId,
|
||||
});
|
||||
assertUploadEndpointUnchanged(
|
||||
pane.connection,
|
||||
originatingEndpoint,
|
||||
connectionCacheKeyMapRef.current,
|
||||
);
|
||||
const bridge = netcattyBridge.get();
|
||||
if (!bridge) throw new Error("Bridge not available");
|
||||
if (!bridge.listLocalTree) throw new Error("Folder upload not supported");
|
||||
|
||||
const { sftpId, release } = await resolveRemoteSftpId(side, {
|
||||
forceReconnect,
|
||||
connectionId: pane.connection.id,
|
||||
tabId: originatingTabId,
|
||||
});
|
||||
// Never re-resolve via getActivePane after awaits — focus may have moved.
|
||||
const livePane = resolveUploadTargetPane({
|
||||
side,
|
||||
tabId: originatingTabId,
|
||||
getActivePane,
|
||||
getPaneByTabId,
|
||||
getPaneByConnectionId,
|
||||
});
|
||||
assertUploadEndpointUnchanged(
|
||||
livePane.connection,
|
||||
originatingEndpoint,
|
||||
connectionCacheKeyMapRef.current,
|
||||
);
|
||||
|
||||
const uploadPaneId = livePane.id;
|
||||
const uploadTargetPath = targetPath || livePane.connection.currentPath;
|
||||
const controller = new UploadController();
|
||||
|
||||
const callbacks = bindUploadControllerCallbacks(
|
||||
controller,
|
||||
createUploadCallbacks(
|
||||
livePane.connection.id,
|
||||
uploadTargetPath,
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
|
||||
livePane.connection.isLocal ? undefined : connectionCacheKeyMapRef.current.get(livePane.connection.id),
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
|
||||
),
|
||||
);
|
||||
// Pin connect-time Host before listLocalTree: a slow folder scan can
|
||||
// outlive a same-hostId tab rebind, and resolveUploadConnectHost would
|
||||
// otherwise open the pooled stream bridge on the newly selected endpoint.
|
||||
const uploadConnectHost = resolveUploadConnectHost(
|
||||
uploadPaneId,
|
||||
livePane.connection.isLocal,
|
||||
);
|
||||
const uploadBridge = createUploadBridge(uploadConnectHost);
|
||||
|
||||
const scanningTask = startUploadScanningTask(callbacks);
|
||||
|
||||
if (controller.isCancelled()) {
|
||||
scanningTask.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
let release = () => {};
|
||||
try {
|
||||
const localEntries = await bridge.listLocalTree(folderPath);
|
||||
const pane = resolveUploadTargetPane({
|
||||
side,
|
||||
tabId: originatingTabId,
|
||||
getActivePane,
|
||||
getPaneByTabId,
|
||||
getPaneByConnectionId,
|
||||
});
|
||||
assertUploadEndpointUnchanged(
|
||||
pane.connection,
|
||||
originatingEndpoint,
|
||||
connectionCacheKeyMapRef.current,
|
||||
);
|
||||
const bridge = netcattyBridge.get();
|
||||
if (!bridge) throw new Error("Bridge not available");
|
||||
if (!bridge.listLocalTree) throw new Error("Folder upload not supported");
|
||||
|
||||
const resolved = await resolveRemoteSftpId(side, {
|
||||
forceReconnect,
|
||||
connectionId: pane.connection.id,
|
||||
tabId: originatingTabId,
|
||||
});
|
||||
const sftpId = resolved.sftpId;
|
||||
release = resolved.release;
|
||||
// Never re-resolve via getActivePane after awaits — focus may have moved.
|
||||
const livePane = resolveUploadTargetPane({
|
||||
side,
|
||||
tabId: originatingTabId,
|
||||
getActivePane,
|
||||
getPaneByTabId,
|
||||
getPaneByConnectionId,
|
||||
});
|
||||
assertUploadEndpointUnchanged(
|
||||
livePane.connection,
|
||||
originatingEndpoint,
|
||||
connectionCacheKeyMapRef.current,
|
||||
);
|
||||
|
||||
const uploadPaneId = livePane.id;
|
||||
const uploadTargetPath = targetPath || livePane.connection.currentPath;
|
||||
// Pin connect-time Host before listLocalTree: a slow folder scan can
|
||||
// outlive a same-hostId tab rebind, and resolveUploadConnectHost would
|
||||
// otherwise open the pooled stream bridge on the newly selected endpoint.
|
||||
const uploadConnectHost = resolveUploadConnectHost(
|
||||
uploadPaneId,
|
||||
livePane.connection.isLocal,
|
||||
);
|
||||
const uploadBridge = createUploadBridge(uploadConnectHost);
|
||||
const liveCallbacks = bindUploadControllerCallbacks(
|
||||
controller,
|
||||
createUploadCallbacks(
|
||||
livePane.connection.id,
|
||||
uploadTargetPath,
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostId,
|
||||
livePane.connection.isLocal
|
||||
? undefined
|
||||
: connectionCacheKeyMapRef.current.get(livePane.connection.id),
|
||||
livePane.connection.isLocal ? undefined : livePane.connection.hostLabel,
|
||||
),
|
||||
);
|
||||
const transferCallbacks = {
|
||||
...liveCallbacks,
|
||||
onTaskCreated: (task: Parameters<NonNullable<UploadCallbacks["onTaskCreated"]>>[0]) => {
|
||||
scanningTask.complete();
|
||||
liveCallbacks.onTaskCreated?.(task);
|
||||
},
|
||||
};
|
||||
|
||||
if (capturedEntries === null) {
|
||||
const localEntries = await listLocalTreeWithAbort(bridge, folderPath, {
|
||||
abortSignal: scanAbort.signal,
|
||||
onProgress: (progress) => {
|
||||
callbacks.onScanningProgress?.(scanningTask.taskId, {
|
||||
...progress,
|
||||
label: folderName,
|
||||
});
|
||||
},
|
||||
});
|
||||
capturedEntries = localTreeToDropEntries(localEntries);
|
||||
}
|
||||
if (controller.isCancelled()) {
|
||||
scanningTask.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
scanningTask.complete();
|
||||
|
||||
const entries: DropEntry[] = localEntries.map((entry) => {
|
||||
if (entry.type === "directory") {
|
||||
return {
|
||||
file: null,
|
||||
relativePath: entry.relativePath,
|
||||
isDirectory: true,
|
||||
};
|
||||
}
|
||||
|
||||
const file = {
|
||||
name: entry.relativePath.split("/").pop() || entry.relativePath,
|
||||
size: entry.size,
|
||||
lastModified: entry.lastModified,
|
||||
type: "",
|
||||
path: entry.localPath,
|
||||
arrayBuffer: async () => {
|
||||
const currentBridge = netcattyBridge.get();
|
||||
if (!currentBridge?.readLocalFile) {
|
||||
throw new Error("Local file reading not supported");
|
||||
}
|
||||
return currentBridge.readLocalFile(entry.localPath);
|
||||
},
|
||||
} as File & { path?: string };
|
||||
|
||||
return {
|
||||
file,
|
||||
relativePath: entry.relativePath,
|
||||
isDirectory: false,
|
||||
};
|
||||
});
|
||||
|
||||
const results = await runWithCompressedUploadSession({
|
||||
enabled: useCompressedUpload,
|
||||
@@ -1258,7 +1579,7 @@ export const useSftpExternalOperations = (
|
||||
: undefined,
|
||||
shouldDiscard: isSessionError,
|
||||
run: async (uploadSftpId) => uploadEntriesDirect(
|
||||
entries,
|
||||
capturedEntries,
|
||||
{
|
||||
targetPath: uploadTargetPath,
|
||||
sftpId: uploadSftpId,
|
||||
@@ -1266,7 +1587,7 @@ export const useSftpExternalOperations = (
|
||||
isLocal: livePane.connection!.isLocal,
|
||||
bridge: uploadBridge,
|
||||
joinPath,
|
||||
callbacks,
|
||||
callbacks: transferCallbacks,
|
||||
useCompressedUpload,
|
||||
resolveConflict: createUploadConflictResolver(controller),
|
||||
},
|
||||
@@ -1282,18 +1603,8 @@ export const useSftpExternalOperations = (
|
||||
await refresh(refreshSide, { tabId: uploadPaneId });
|
||||
}
|
||||
return results;
|
||||
} catch (error) {
|
||||
if (controller.isCancelled()) {
|
||||
scanningTask.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
if (scanningTask.isOpen()) {
|
||||
scanningTask.fail(error);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
release();
|
||||
unregisterUploadController(controller);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1302,11 +1613,33 @@ export const useSftpExternalOperations = (
|
||||
} catch (error) {
|
||||
if (isSessionError(error) && ensureRemoteSftpId) {
|
||||
logger.warn("[SFTP] Folder upload session lost; reconnecting and retrying once", error);
|
||||
return await run(true);
|
||||
try {
|
||||
return await run(true);
|
||||
} catch (retryError) {
|
||||
if (isUploadScanCancelled(controller, retryError)) {
|
||||
scanningTask.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
if (scanningTask.isOpen()) scanningTask.fail(retryError);
|
||||
throw retryError;
|
||||
}
|
||||
}
|
||||
if (isUploadScanCancelled(controller, error)) {
|
||||
scanningTask.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
if (scanningTask.isOpen()) scanningTask.fail(error);
|
||||
logger.error("[SFTP] Folder picker upload failed:", error);
|
||||
throw error;
|
||||
}
|
||||
finally {
|
||||
detachScanCancel();
|
||||
unregisterUploadController(controller);
|
||||
if (scanningTask.isOpen()) {
|
||||
if (controller.isCancelled()) scanningTask.cancel();
|
||||
else scanningTask.complete();
|
||||
}
|
||||
}
|
||||
},
|
||||
[
|
||||
acquireTransferSession,
|
||||
|
||||
@@ -60,6 +60,10 @@ import {
|
||||
} from "./transferConflictLifecycle";
|
||||
import { getParentPath, joinPath } from "./utils";
|
||||
import { promoteDirectoryReplaceStage as promoteDirectoryReplacePaths } from "./directoryReplacePromotion";
|
||||
import {
|
||||
isExternalDragDropFileUpload,
|
||||
retryExternalDragDropFileUpload,
|
||||
} from "./externalDragDropRetry";
|
||||
|
||||
/** Keep the MutableRefObject mirror in sync with the process-global latch set. */
|
||||
function syncPausedTasksRef(ref: { current: Set<string> }, taskId: string, latched: boolean) {
|
||||
@@ -1294,8 +1298,62 @@ export const useSftpTransfers = ({
|
||||
|
||||
const retryTransfer = useCallback(
|
||||
async (transferId: string) => {
|
||||
const task = transfersRef.current.find((t) => t.id === transferId);
|
||||
// Prefer the live owner list; fall back to the center store (drag-drop
|
||||
// children are upserted there and mirrored via subscribe).
|
||||
const task = transfersRef.current.find((t) => t.id === transferId)
|
||||
?? sftpTransferCenterStore.getTask(transferId);
|
||||
if (!task || task.retryable === false) return;
|
||||
|
||||
// Progressive / external drag-drop children never have dual-pane endpoints
|
||||
// (sourceConnectionId is "external"). Re-run startStreamTransfer in place.
|
||||
if (isExternalDragDropFileUpload(task)) {
|
||||
if (inFlightTransferIdsRef.current.has(transferId)) return;
|
||||
inFlightTransferIdsRef.current.add(transferId);
|
||||
clearTransferCancelled(transferId);
|
||||
cancelledTasksRef.current.delete(transferId);
|
||||
try {
|
||||
const result = await retryExternalDragDropFileUpload(task, {
|
||||
getBrowseSftpId: (connectionId) => sftpSessionsRef.current.get(connectionId),
|
||||
acquireTransferSession: acquireTransferSession
|
||||
? (hostId, id) => acquireTransferSession(hostId, id)
|
||||
: undefined,
|
||||
startStreamTransfer: async (options) => {
|
||||
const bridge = netcattyBridge.get();
|
||||
if (!bridge?.startStreamTransfer) {
|
||||
return { error: "Stream transfer is unavailable" };
|
||||
}
|
||||
return bridge.startStreamTransfer(options);
|
||||
},
|
||||
clearPendingCancel: (id) => netcattyBridge.get()?.clearPendingTransferCancel?.(id),
|
||||
cleanupArtifacts: cleanupTaskArtifacts,
|
||||
getTask: (id) => sftpTransferCenterStore.getTask(id)
|
||||
?? transfersRef.current.find((row) => row.id === id),
|
||||
getChildTasks: (parentId) => {
|
||||
const fromStore = sftpTransferCenterStore.getOwnerTasks(ownerId)
|
||||
.filter((row) => row.parentTaskId === parentId);
|
||||
if (fromStore.length > 0) return fromStore;
|
||||
return transfersRef.current.filter((row) => row.parentTaskId === parentId);
|
||||
},
|
||||
onPatch: (taskId, updates) => {
|
||||
transferRuntime.patchTask(taskId, updates);
|
||||
setTransfers((prev) => {
|
||||
if (!prev.some((row) => row.id === taskId)) {
|
||||
// Store-only child: mirror the patch by re-pulling owner rows.
|
||||
return sftpTransferCenterStore.getOwnerTasks(ownerId);
|
||||
}
|
||||
return prev.map((row) => (row.id === taskId ? { ...row, ...updates } : row));
|
||||
});
|
||||
},
|
||||
});
|
||||
if (!result.success && result.error && !/cancelled/i.test(result.error)) {
|
||||
notify.warning(result.error, "SFTP");
|
||||
}
|
||||
} finally {
|
||||
inFlightTransferIdsRef.current.delete(transferId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
await cleanupTaskArtifacts(task);
|
||||
|
||||
const retriedTask: TransferTask = {
|
||||
@@ -1316,7 +1374,13 @@ export const useSftpTransfers = ({
|
||||
};
|
||||
|
||||
const endpoints = resolveTaskEndpoints(task);
|
||||
if (!endpoints) return;
|
||||
if (!endpoints) {
|
||||
notify.warning(
|
||||
"Could not resolve transfer endpoints for retry. Reconnect the target and try again.",
|
||||
"SFTP",
|
||||
);
|
||||
return;
|
||||
}
|
||||
const { targetSide, sourcePane, targetPane } = endpoints;
|
||||
|
||||
const completionHandler = completionHandlersRef.current.get(transferId);
|
||||
@@ -1335,7 +1399,7 @@ export const useSftpTransfers = ({
|
||||
await processTransfer(retriedTask, sourcePane, targetPane, targetSide);
|
||||
},
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps -- processTransfer is defined inline
|
||||
[cleanupTaskArtifacts, resolveTaskEndpoints, setTransfers],
|
||||
[acquireTransferSession, cleanupTaskArtifacts, ownerId, resolveTaskEndpoints, setTransfers, sftpSessionsRef],
|
||||
);
|
||||
|
||||
const clearCompletedTransfers = useCallback(() => {
|
||||
|
||||
@@ -35,7 +35,6 @@ import {
|
||||
STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR,
|
||||
STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD,
|
||||
STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY,
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SSH_TRANSPORT_IDLE_TTL_MS,
|
||||
STORAGE_KEY_SFTP_TRANSFER_POOL_IDLE_TTL_MS,
|
||||
@@ -88,7 +87,6 @@ import { localStorageAdapter } from '../../infrastructure/persistence/localStora
|
||||
import { netcattyBridge } from '../../infrastructure/services/netcattyBridge';
|
||||
import {
|
||||
resolveSftpTransferConcurrency,
|
||||
resolveSftpFolderPrescanEnabled,
|
||||
resolveSftpSkipUnchangedEnabled,
|
||||
} from './sftp/transferConcurrency';
|
||||
import { resolveSshTransportIdleTtlMs } from '../../infrastructure/config/sshTransportIdleTtl';
|
||||
@@ -426,9 +424,6 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
// Folder transfer concurrency is renderer-only (runSftpTransferWorkers).
|
||||
// Do not push it into main-process host admission - that made multi-select
|
||||
// top-level files queue against each other and against folder children.
|
||||
const [sftpFolderPrescan, setSftpFolderPrescan] = useState<boolean>(() =>
|
||||
resolveSftpFolderPrescanEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN)),
|
||||
);
|
||||
const [sftpSkipUnchanged, setSftpSkipUnchanged] = useState<boolean>(() =>
|
||||
resolveSftpSkipUnchangedEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED)),
|
||||
);
|
||||
@@ -902,8 +897,6 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
if (storedHidden === 'true' || storedHidden === 'false') setSftpShowHiddenFiles(storedHidden === 'true');
|
||||
const storedCompress = readStoredString(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD);
|
||||
if (storedCompress === 'true' || storedCompress === 'false') setSftpUseCompressedUpload(storedCompress === 'true');
|
||||
const storedFolderPrescan = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN);
|
||||
if (storedFolderPrescan != null) setSftpFolderPrescan(storedFolderPrescan);
|
||||
const storedSkipUnchanged = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED);
|
||||
if (storedSkipUnchanged != null) setSftpSkipUnchanged(storedSkipUnchanged);
|
||||
const storedAutoOpenSidebar = readStoredString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR);
|
||||
@@ -1137,7 +1130,7 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
|
||||
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
|
||||
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
|
||||
@@ -1147,7 +1140,7 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
setTerminalThemeId, setTerminalThemeDarkId, setTerminalThemeLightId,
|
||||
setFollowAppTerminalThemeState, setTerminalFontFamilyId, setTerminalFontSize: applyIncomingTerminalFontSize,
|
||||
setSftpDoubleClickBehavior, setSftpAutoSync, setSftpShowHiddenFiles,
|
||||
setSftpUseCompressedUpload, setSftpFolderPrescan, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
|
||||
setSftpUseCompressedUpload, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
|
||||
setShowRecentHostsState, setHostClickBehaviorState, setShowOnlyUngroupedHostsInRootState, setShowSftpTabState, setShowHostTreeSidebarState, setTerminalSidePanelAutoOpenState, setTerminalSidePanelAutoOpenTabState, setShellOnlyTabNumberShortcutsState, setDisableTerminalFontZoomState, setRestorePreviousSessionState, setRestoreTerminalCwdState,
|
||||
setEditorWordWrapState, setSessionLogsEnabled, setSessionLogsDir, setSessionLogsFormat, setSessionLogsTimestampsEnabled, setSshDebugLogsEnabled, setSshDeepLinkEnabledState: applyIncomingSshDeepLinkEnabled, setJmsDeepLinkEnabledState: applyIncomingJmsDeepLinkEnabled, setExplorerContextMenuEnabledState: applyIncomingExplorerContextMenuEnabled,
|
||||
setGlobalHotkeyEnabled, setWindowOpacity: applyIncomingWindowOpacity, setAppIconVariant, setAutoUpdateEnabled, setWorkspaceFocusStyleState,
|
||||
@@ -1375,12 +1368,6 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
notifySettingsChanged(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD, sftpUseCompressedUpload);
|
||||
}, [sftpUseCompressedUpload, notifySettingsChanged]);
|
||||
|
||||
useEffect(() => {
|
||||
localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN, sftpFolderPrescan);
|
||||
if (!persistMountedRef.current) return;
|
||||
notifySettingsChanged(STORAGE_KEY_SFTP_FOLDER_PRESCAN, sftpFolderPrescan);
|
||||
}, [sftpFolderPrescan, notifySettingsChanged]);
|
||||
|
||||
useEffect(() => {
|
||||
localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED, sftpSkipUnchanged);
|
||||
if (!persistMountedRef.current) return;
|
||||
@@ -1772,8 +1759,6 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
setSftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload,
|
||||
setSftpUseCompressedUpload,
|
||||
sftpFolderPrescan,
|
||||
setSftpFolderPrescan,
|
||||
sftpSkipUnchanged,
|
||||
setSftpSkipUnchanged,
|
||||
sftpAutoOpenSidebar,
|
||||
@@ -1860,7 +1845,7 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
uiFontFamilyId, uiLanguage, customCSS,
|
||||
terminalThemeId, terminalFontFamilyId, terminalFontSize, terminalSettings,
|
||||
customKeyBindings, editorWordWrap,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles, sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles, sftpUseCompressedUpload, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom,
|
||||
customThemes, workspaceFocusStyle, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
]),
|
||||
|
||||
@@ -74,7 +74,6 @@ import {
|
||||
STORAGE_KEY_SFTP_AUTO_SYNC,
|
||||
STORAGE_KEY_SFTP_SHOW_HIDDEN_FILES,
|
||||
STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD,
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR,
|
||||
STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD,
|
||||
@@ -288,7 +287,6 @@ export const SYNCABLE_SETTING_STORAGE_KEYS = [
|
||||
STORAGE_KEY_SFTP_AUTO_SYNC,
|
||||
STORAGE_KEY_SFTP_SHOW_HIDDEN_FILES,
|
||||
STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD,
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR,
|
||||
STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD,
|
||||
@@ -494,8 +492,6 @@ export function collectSyncableSettings(): SyncPayload['settings'] {
|
||||
if (hidden === 'true' || hidden === 'false') settings.sftpShowHiddenFiles = hidden === 'true';
|
||||
const compress = localStorageAdapter.readString(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD);
|
||||
if (compress === 'true' || compress === 'false') settings.sftpUseCompressedUpload = compress === 'true';
|
||||
const folderPrescan = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN);
|
||||
if (folderPrescan != null) settings.sftpFolderPrescan = folderPrescan;
|
||||
const skipUnchanged = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED);
|
||||
if (skipUnchanged != null) settings.sftpSkipUnchanged = skipUnchanged;
|
||||
const autoOpenSidebar = localStorageAdapter.readString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR);
|
||||
@@ -734,7 +730,6 @@ async function applySyncableSettings(settings: NonNullable<SyncPayload['settings
|
||||
if (settings.sftpAutoSync != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_AUTO_SYNC, String(settings.sftpAutoSync));
|
||||
if (settings.sftpShowHiddenFiles != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_SHOW_HIDDEN_FILES, String(settings.sftpShowHiddenFiles));
|
||||
if (settings.sftpUseCompressedUpload != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD, String(settings.sftpUseCompressedUpload));
|
||||
if (settings.sftpFolderPrescan != null) localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN, settings.sftpFolderPrescan);
|
||||
if (settings.sftpSkipUnchanged != null) localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED, settings.sftpSkipUnchanged);
|
||||
if (settings.sftpAutoOpenSidebar != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR, String(settings.sftpAutoOpenSidebar));
|
||||
if (settings.sftpFollowTerminalCwd != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD, String(settings.sftpFollowTerminalCwd));
|
||||
|
||||
@@ -129,8 +129,11 @@ test("directory parent detection matches side-queue file-count semantics", () =>
|
||||
}), false);
|
||||
});
|
||||
|
||||
test("folder parent progress uses file counts not byte formatting", () => {
|
||||
test("folder parent progress uses progressive discovery labels not byte formatting", () => {
|
||||
const t = (key: string, params?: Record<string, string | number>) => {
|
||||
if (key === "sftp.transfers.filesDiscoveredProgress") {
|
||||
return `${params?.completed} done · ${params?.discovered} found`;
|
||||
}
|
||||
if (key === "sftp.transfers.filesProgress") {
|
||||
return `${params?.current}/${params?.total} files`;
|
||||
}
|
||||
@@ -146,14 +149,35 @@ test("folder parent progress uses file counts not byte formatting", () => {
|
||||
totalBytes: 12,
|
||||
transferredBytes: 1,
|
||||
speed: 0,
|
||||
phase: "transferring",
|
||||
}, t);
|
||||
assert.equal(display.percent, (1 / 12) * 100);
|
||||
assert.equal(display.detail, "1/12 files");
|
||||
assert.equal(display.detail, "1 done · 12 found");
|
||||
assert.equal(display.indeterminate, false);
|
||||
// Must not look like "1 Bytes / 12 Bytes"
|
||||
assert.doesNotMatch(display.detail, /Bytes/i);
|
||||
});
|
||||
|
||||
test("folder parent scanning stays indeterminate while nothing has completed", () => {
|
||||
const t = (key: string, params?: Record<string, string | number>) => {
|
||||
if (key === "sftp.transfers.filesDiscoveredProgress") {
|
||||
return `${params?.completed} done · ${params?.discovered} found`;
|
||||
}
|
||||
return key;
|
||||
};
|
||||
const display = buildGlobalTransferProgressDisplay({
|
||||
status: "transferring",
|
||||
isDirectory: true,
|
||||
progressMode: "files",
|
||||
totalBytes: 40,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
phase: "scanning",
|
||||
}, t);
|
||||
assert.equal(display.detail, "0 done · 40 found");
|
||||
assert.equal(display.indeterminate, true);
|
||||
});
|
||||
|
||||
test("single-file progress still shows byte totals", () => {
|
||||
const t = (key: string) => key;
|
||||
const display = buildGlobalTransferProgressDisplay({
|
||||
|
||||
@@ -181,28 +181,56 @@ export type GlobalTransferProgressDisplay = {
|
||||
/**
|
||||
* Build progress labels for a top-level row. Directory parents use file counts
|
||||
* so we never show "1 Bytes / 12 Bytes" for n/m files.
|
||||
*
|
||||
* Folder walks interleave discovery with transfer: totalBytes is "found so far",
|
||||
* not a fixed grand total. Active rows use progressive copy so the UI does not
|
||||
* look like a lying overall percentage.
|
||||
*/
|
||||
export function buildGlobalTransferProgressDisplay(
|
||||
task: Pick<TransferTask, "status" | "isDirectory" | "parentTaskId" | "progressMode" | "totalBytes" | "transferredBytes" | "speed">,
|
||||
task: Pick<TransferTask, "status" | "isDirectory" | "parentTaskId" | "progressMode" | "totalBytes" | "transferredBytes" | "speed" | "phase">,
|
||||
t: (key: string, params?: Record<string, string | number>) => string,
|
||||
): GlobalTransferProgressDisplay {
|
||||
const isDirParent = isDirectoryParentTask(task);
|
||||
const percent = getGlobalTransferProgressPercent(task);
|
||||
const hasTotal = task.totalBytes > 0;
|
||||
const discovered = Math.max(task.totalBytes, task.transferredBytes);
|
||||
|
||||
if (isDirParent) {
|
||||
const indeterminate = task.status === "transferring" && !hasTotal;
|
||||
const isActive = task.status === "transferring"
|
||||
|| task.status === "pausing"
|
||||
|| task.status === "queued"
|
||||
|| task.status === "pending";
|
||||
// Still discovering, or no completed files yet — pulse rather than freeze a fake %.
|
||||
const indeterminate = isActive && (
|
||||
task.phase === "scanning"
|
||||
|| !hasTotal
|
||||
|| task.transferredBytes <= 0
|
||||
);
|
||||
let detail = "";
|
||||
if (task.status === "transferring" || task.status === "pausing" || task.status === "queued" || task.status === "pending") {
|
||||
detail = hasTotal
|
||||
? t("sftp.transfers.filesProgress", { current: task.transferredBytes, total: task.totalBytes })
|
||||
: t("sftp.transfers.filesCount", { count: task.transferredBytes });
|
||||
if (isActive) {
|
||||
if (discovered > 0) {
|
||||
detail = t("sftp.transfers.filesDiscoveredProgress", {
|
||||
completed: task.transferredBytes,
|
||||
discovered,
|
||||
});
|
||||
}
|
||||
} else if (task.status === "completed" && hasTotal) {
|
||||
detail = t("sftp.transfers.filesCount", { count: task.totalBytes });
|
||||
} else if (hasTotal) {
|
||||
detail = t("sftp.transfers.filesProgress", { current: task.transferredBytes, total: task.totalBytes });
|
||||
} else if (discovered > 0) {
|
||||
detail = t("sftp.transfers.filesDiscoveredProgress", {
|
||||
completed: task.transferredBytes,
|
||||
discovered,
|
||||
});
|
||||
}
|
||||
return { percent, detail, indeterminate };
|
||||
// Soft percent of files found so far (discovered may still grow).
|
||||
const softPercent = discovered > 0
|
||||
? Math.min(100, (task.transferredBytes / discovered) * 100)
|
||||
: 0;
|
||||
return {
|
||||
percent: task.status === "completed" ? 100 : softPercent,
|
||||
detail,
|
||||
indeterminate,
|
||||
};
|
||||
}
|
||||
|
||||
const detailParts: string[] = [];
|
||||
@@ -394,6 +422,14 @@ function TransferRow({
|
||||
if (task.status === "pausing" || task.status === "paused" || task.status === "interrupted") {
|
||||
return t(statusLabelKey(task.status));
|
||||
}
|
||||
// Scanning runs as pending/transferring; never override terminal statuses
|
||||
// (cancelled/failed/completed) that still carry a stale phase value.
|
||||
if (
|
||||
task.phase === "scanning"
|
||||
&& (task.status === "pending" || task.status === "queued" || task.status === "transferring")
|
||||
) {
|
||||
return t("sftp.transferCenter.phase.scanning");
|
||||
}
|
||||
if (task.phase && task.status === "transferring") {
|
||||
return t(`sftp.transferCenter.phase.${task.phase}`);
|
||||
}
|
||||
@@ -687,6 +723,13 @@ export function GlobalSftpTransferCenter() {
|
||||
// progress ticks used to re-render this TopTabs child on every byte.
|
||||
const badge = useSftpTransferCenterBadge();
|
||||
const [open, setOpen] = useState(false);
|
||||
// Folder drops fire this so the scanning row is visible immediately without
|
||||
// requiring the user to notice the badge first.
|
||||
useEffect(() => {
|
||||
const openCenter = () => setOpen(true);
|
||||
window.addEventListener("netcatty:open-sftp-transfer-center", openCenter);
|
||||
return () => window.removeEventListener("netcatty:open-sftp-transfer-center", openCenter);
|
||||
}, []);
|
||||
const snapshot = useSftpTransferCenterWhenOpen(open);
|
||||
const [bucket, setBucket] = useState<GlobalTransferBucket>("all");
|
||||
const [showBackground, setShowBackground] = useState(false);
|
||||
|
||||
@@ -64,6 +64,70 @@ test("renders active transfer cancel action with an item-specific label", () =>
|
||||
assert.match(markup, /aria-label="Pause: archive\.tar\.gz"/);
|
||||
});
|
||||
|
||||
test("directory parents show transfer-center done/found progress instead of Waiting", () => {
|
||||
const markup = renderTransferItem({
|
||||
...baseTask,
|
||||
id: "folder-1",
|
||||
fileName: "big-tree",
|
||||
isDirectory: true,
|
||||
progressMode: "files",
|
||||
status: "pending",
|
||||
phase: "transferring",
|
||||
transferredBytes: 65,
|
||||
totalBytes: 108202,
|
||||
error: undefined,
|
||||
speed: 0,
|
||||
});
|
||||
|
||||
assert.match(markup, /65 done · 108202 found/);
|
||||
assert.doesNotMatch(markup, />Waiting\.\.\.</);
|
||||
});
|
||||
|
||||
test("scanning directory parents keep scanning label with live discovery counts", () => {
|
||||
const markup = renderTransferItem({
|
||||
...baseTask,
|
||||
id: "folder-scan",
|
||||
fileName: "docs",
|
||||
isDirectory: true,
|
||||
progressMode: "files",
|
||||
status: "pending",
|
||||
phase: "scanning",
|
||||
transferredBytes: 0,
|
||||
totalBytes: 1284,
|
||||
error: undefined,
|
||||
speed: 0,
|
||||
});
|
||||
|
||||
assert.match(markup, /Scanning/);
|
||||
assert.match(markup, /0 done · 1284 found/);
|
||||
});
|
||||
|
||||
test("child rows reserve an action column wide enough for pause and cancel", () => {
|
||||
const markup = renderTransferItem(
|
||||
{
|
||||
...baseTask,
|
||||
id: "child-transfer-actions",
|
||||
parentTaskId: "transfer-1",
|
||||
status: "transferring",
|
||||
error: undefined,
|
||||
transferredBytes: 256,
|
||||
totalBytes: 1024,
|
||||
speed: 128,
|
||||
resumable: true,
|
||||
},
|
||||
{
|
||||
isChild: true,
|
||||
childNameColumnWidth: 260,
|
||||
onResizeNameColumn: () => {},
|
||||
onSetNameColumnWidth: () => {},
|
||||
},
|
||||
);
|
||||
|
||||
assert.match(markup, /grid-template-columns:24px 260px 10px minmax\(0, 1fr\) 56px/);
|
||||
assert.match(markup, /aria-label="Pause: archive\.tar\.gz"/);
|
||||
assert.match(markup, /aria-label="Cancel: archive\.tar\.gz"/);
|
||||
});
|
||||
|
||||
test("renders paused transfer resume action", () => {
|
||||
const markup = renderTransferItem({
|
||||
...baseTask,
|
||||
|
||||
@@ -40,7 +40,7 @@ export default function SettingsFileAssociationsTab() {
|
||||
sftpAutoSync, setSftpAutoSync,
|
||||
sftpShowHiddenFiles, setSftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, setSftpUseCompressedUpload,
|
||||
sftpFolderPrescan, setSftpFolderPrescan,
|
||||
|
||||
sftpSkipUnchanged, setSftpSkipUnchanged,
|
||||
sftpAutoOpenSidebar, setSftpAutoOpenSidebar,
|
||||
sftpFollowTerminalCwd, setSftpFollowTerminalCwd,
|
||||
@@ -211,12 +211,6 @@ export default function SettingsFileAssociationsTab() {
|
||||
</span>
|
||||
</div>
|
||||
</SettingRow>
|
||||
<SettingRow
|
||||
label={t('settings.sftp.folderPrescan.enable')}
|
||||
description={t('settings.sftp.folderPrescan.enableDesc')}
|
||||
>
|
||||
<Toggle checked={sftpFolderPrescan} onChange={setSftpFolderPrescan} />
|
||||
</SettingRow>
|
||||
<SettingRow
|
||||
label={t('settings.sftp.skipUnchanged.enable')}
|
||||
description={t('settings.sftp.skipUnchanged.enableDesc')}
|
||||
|
||||
@@ -26,10 +26,17 @@ import { getParentPath } from '../../application/state/sftp/utils';
|
||||
import { useSftpTransferTask } from '../../application/state/sftpTransferCenterStore';
|
||||
import { cn } from '../../lib/utils';
|
||||
import { TransferTask } from '../../types';
|
||||
import {
|
||||
buildGlobalTransferProgressDisplay,
|
||||
isDirectoryParentTask,
|
||||
} from '../GlobalSftpTransferCenter';
|
||||
import { Button } from '../ui/button';
|
||||
import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '../ui/tooltip';
|
||||
import { formatSpeed, formatTransferBytes } from './utils';
|
||||
|
||||
/** Child rows need room for Pause + Cancel (2×24px icons + gap). */
|
||||
const CHILD_ACTIONS_COLUMN_PX = 56;
|
||||
|
||||
interface SftpTransferItemProps {
|
||||
task: TransferTask;
|
||||
isChild?: boolean;
|
||||
@@ -132,16 +139,20 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
|
||||
// Optimistic spinner from click until store status moves off paused/interrupted.
|
||||
const [resumeClicked, setResumeClicked] = useState(false);
|
||||
|
||||
// Align with global transfer center: directory parents default to file-count
|
||||
// progress unless explicitly compressed/byte mode.
|
||||
const progressMode = task.progressMode
|
||||
?? (task.isDirectory && !task.parentTaskId ? 'files' : 'bytes');
|
||||
const isDirParent = task.isDirectory && !task.parentTaskId && progressMode === 'files';
|
||||
const hasKnownTotal = task.totalBytes > 0 || (!isDirParent && !!task.sourceLastModified);
|
||||
const progress = hasKnownTotal
|
||||
? Math.min((task.transferredBytes / task.totalBytes) * 100, 100)
|
||||
: 0;
|
||||
const isIndeterminate = task.status === 'transferring' && !hasKnownTotal;
|
||||
// Same progress model as the global transfer center (done · found for folders).
|
||||
const isDirParent = isDirectoryParentTask(task);
|
||||
const centerProgress = buildGlobalTransferProgressDisplay(task, t);
|
||||
const hasKnownTotal = isDirParent
|
||||
? task.totalBytes > 0 && task.transferredBytes > 0 && task.phase !== 'scanning'
|
||||
: task.totalBytes > 0 || !!task.sourceLastModified;
|
||||
const progress = isDirParent
|
||||
? centerProgress.percent
|
||||
: hasKnownTotal
|
||||
? Math.min((task.transferredBytes / task.totalBytes) * 100, 100)
|
||||
: 0;
|
||||
const isIndeterminate = isDirParent
|
||||
? centerProgress.indeterminate && (task.status === 'transferring' || task.status === 'pending' || task.status === 'queued' || task.status === 'pausing')
|
||||
: task.status === 'transferring' && !hasKnownTotal;
|
||||
const isActiveTransfer = task.status === 'transferring' || task.status === 'pausing';
|
||||
// Reconnect / dedicated resume window — keep the action slot as a spinner
|
||||
// until the first real progress clears reconnectRequired.
|
||||
@@ -178,13 +189,9 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
|
||||
? formatTransferBytes(task.totalBytes)
|
||||
: '';
|
||||
|
||||
const fileCountDisplay = isDirParent && (isActiveTransfer || isPausedLike)
|
||||
? (task.totalBytes > 0
|
||||
? t('sftp.transfers.filesProgress', { current: task.transferredBytes, total: task.totalBytes })
|
||||
: t('sftp.transfers.filesCount', { count: task.transferredBytes }))
|
||||
: isDirParent && task.status === 'completed' && task.totalBytes > 0
|
||||
? t('sftp.transfers.filesCount', { count: task.totalBytes })
|
||||
: '';
|
||||
// Prefer the transfer-center detail string so the panel never lags behind
|
||||
// "N done · M found" while status is still pending during progressive walks.
|
||||
const fileCountDisplay = isDirParent ? centerProgress.detail : '';
|
||||
|
||||
const speedFormatted = effectiveSpeed > 0 ? formatSpeed(effectiveSpeed) : '';
|
||||
const targetDirectoryPath = task.isDirectory ? task.targetPath : getParentPath(task.targetPath);
|
||||
@@ -193,29 +200,42 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
|
||||
// the backend drained in-flight chunks ("finish current step").
|
||||
const pausingLabel = t('sftp.transferCenter.status.pausing');
|
||||
const resumingLabel = t('sftp.transferCenter.status.resuming');
|
||||
const isLiveScanning = task.phase === 'scanning'
|
||||
&& (task.status === 'pending' || task.status === 'queued' || task.status === 'transferring');
|
||||
const progressOverlayText = isResuming
|
||||
? resumingLabel
|
||||
: task.status === 'pending'
|
||||
? t('sftp.task.waiting')
|
||||
: isLiveScanning
|
||||
? (fileCountDisplay
|
||||
? `${t('sftp.transferCenter.phase.scanning')} · ${fileCountDisplay}`
|
||||
: t('sftp.transferCenter.phase.scanning'))
|
||||
: task.status === 'pausing'
|
||||
? pausingLabel
|
||||
: isIndeterminate
|
||||
? t('sftp.transfer.preparing')
|
||||
: isDirParent
|
||||
? (fileCountDisplay
|
||||
? `${fileCountDisplay}${hasKnownTotal ? ` • ${Math.round(progress)}%` : ''}`
|
||||
: hasKnownTotal
|
||||
? `${Math.round(progress)}%`
|
||||
: '...')
|
||||
: bytesDisplay
|
||||
? `${bytesDisplay}${hasKnownTotal ? ` • ${Math.round(progress)}%` : ''}`
|
||||
: hasKnownTotal
|
||||
? `${Math.round(progress)}%`
|
||||
: '...';
|
||||
: isDirParent
|
||||
? (fileCountDisplay
|
||||
|| (task.status === 'pending' || task.status === 'queued'
|
||||
? t('sftp.task.waiting')
|
||||
: isIndeterminate
|
||||
? '...'
|
||||
: `${Math.round(progress)}%`))
|
||||
: task.status === 'pending'
|
||||
? t('sftp.task.waiting')
|
||||
: isIndeterminate
|
||||
? t('sftp.transfer.preparing')
|
||||
: bytesDisplay
|
||||
? `${bytesDisplay}${hasKnownTotal ? ` • ${Math.round(progress)}%` : ''}`
|
||||
: hasKnownTotal
|
||||
? `${Math.round(progress)}%`
|
||||
: '...';
|
||||
|
||||
const progressBarWidth = task.status === 'pending' || (task.status === 'transferring' && !hasKnownTotal) || isIndeterminate
|
||||
? (task.status === 'pending' || !hasKnownTotal ? '100%' : `${progress}%`)
|
||||
: `${progress}%`;
|
||||
const progressBarWidth = isDirParent
|
||||
? (centerProgress.indeterminate || isLiveScanning
|
||||
? '100%'
|
||||
: `${progress}%`)
|
||||
: task.status === 'pending'
|
||||
|| (task.status === 'transferring' && !hasKnownTotal)
|
||||
|| isIndeterminate
|
||||
? (task.status === 'pending' || !hasKnownTotal ? '100%' : `${progress}%`)
|
||||
: `${progress}%`;
|
||||
|
||||
const statusIcon = isResuming
|
||||
? <Loader2 size={12} className="animate-spin text-primary" />
|
||||
@@ -269,7 +289,12 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
|
||||
</div>
|
||||
);
|
||||
|
||||
const progressSummaryText = isResuming || isActiveTransfer || isPausedLike || task.status === 'pending'
|
||||
const progressSummaryText = isResuming
|
||||
|| isActiveTransfer
|
||||
|| isPausedLike
|
||||
|| task.status === 'pending'
|
||||
|| task.status === 'queued'
|
||||
|| (isDirParent && !!fileCountDisplay)
|
||||
? [speedFormatted, progressOverlayText].filter(Boolean).join(' • ')
|
||||
: '';
|
||||
const showTransferSizeCalculation = task.status === 'transferring' && !hasKnownTotal && !isDirParent;
|
||||
@@ -437,7 +462,9 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
|
||||
data-transfer-status={task.status}
|
||||
data-transfer-direction={task.direction}
|
||||
style={{
|
||||
gridTemplateColumns: `24px ${childNameColumnWidth}px 10px minmax(0, 1fr) 24px`,
|
||||
// Last column reserves space for Pause + Cancel so the
|
||||
// progress bar never paints under the action buttons.
|
||||
gridTemplateColumns: `24px ${childNameColumnWidth}px 10px minmax(0, 1fr) ${CHILD_ACTIONS_COLUMN_PX}px`,
|
||||
}}
|
||||
>
|
||||
<div className="flex h-full items-center justify-center text-muted-foreground">
|
||||
@@ -468,10 +495,10 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top">{resizeNameColumnLabel}</TooltipContent>
|
||||
</Tooltip>
|
||||
<div className="min-w-0">
|
||||
<div className="min-w-0 overflow-hidden">
|
||||
{childProgressBar}
|
||||
</div>
|
||||
<div className="flex h-full items-center justify-center">
|
||||
<div className="flex h-full min-w-0 items-center justify-end gap-0.5 pl-1">
|
||||
{actionButtons}
|
||||
</div>
|
||||
</div>
|
||||
@@ -551,12 +578,16 @@ const SftpTransferItemInner: React.FC<SftpTransferItemProps> = ({
|
||||
{toggleChildrenButton}
|
||||
|
||||
{progressSummaryText && (
|
||||
<span className="ml-auto shrink-0 whitespace-nowrap text-[10px] text-muted-foreground font-mono">
|
||||
<span className="ml-auto min-w-0 max-w-[50%] truncate whitespace-nowrap text-right text-[10px] text-muted-foreground font-mono">
|
||||
{progressSummaryText}
|
||||
</span>
|
||||
)}
|
||||
|
||||
{actionButtons}
|
||||
{/* Keep pause/cancel outside the progress summary so long
|
||||
"N done · M found" labels never crowd the action buttons. */}
|
||||
<div className="ml-1 shrink-0">
|
||||
{actionButtons}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{showBelowParentProgress && (
|
||||
@@ -622,6 +653,9 @@ const arePropsEqual = (
|
||||
if (prev.fileName !== next.fileName) return false;
|
||||
if (prev.targetPath !== next.targetPath) return false;
|
||||
if (prev.totalBytes !== next.totalBytes) return false;
|
||||
if (prev.transferredBytes !== next.transferredBytes) return false;
|
||||
if (prev.phase !== next.phase) return false;
|
||||
if (prev.progressMode !== next.progressMode) return false;
|
||||
if ((prevProps.canRevealTarget ?? false) !== (nextProps.canRevealTarget ?? false)) return false;
|
||||
if ((prevProps.canCopyTargetPath ?? false) !== (nextProps.canCopyTargetPath ?? false)) return false;
|
||||
if ((prevProps.isChild ?? false) !== (nextProps.isChild ?? false)) return false;
|
||||
@@ -634,20 +668,10 @@ const arePropsEqual = (
|
||||
if ((prevProps.childListId ?? '') !== (nextProps.childListId ?? '')) return false;
|
||||
if ((prevProps.resizeHandleTabIndex ?? 0) !== (nextProps.resizeHandleTabIndex ?? 0)) return false;
|
||||
|
||||
if (next.status === 'transferring' || next.status === 'pausing') {
|
||||
if (next.totalBytes <= 0 && prev.transferredBytes !== next.transferredBytes) return false;
|
||||
|
||||
const prevProgress = prev.totalBytes > 0 ? (prev.transferredBytes / prev.totalBytes) * 100 : 0;
|
||||
const nextProgress = next.totalBytes > 0 ? (next.transferredBytes / next.totalBytes) * 100 : 0;
|
||||
if (Math.abs(nextProgress - prevProgress) >= 0.1) return false;
|
||||
|
||||
if (next.status === 'transferring' || next.status === 'pausing' || next.status === 'pending' || next.status === 'queued') {
|
||||
if (next.speed !== prev.speed) return false;
|
||||
}
|
||||
|
||||
if (next.status === 'pending') {
|
||||
return true;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
|
||||
@@ -463,7 +463,7 @@ export const useSftpKeyboardShortcuts = ({
|
||||
const hasClipboardItems = (dataTransfer?.items?.length ?? 0) > 0;
|
||||
// webkitGetAsEntry must be invoked synchronously during the paste event.
|
||||
const dropEntriesPromise = dataTransfer?.items?.length
|
||||
? extractDropEntries(dataTransfer)
|
||||
? extractDropEntries(dataTransfer).catch(() => [])
|
||||
: null;
|
||||
const pastedFileSnapshot = dataTransfer?.files?.length
|
||||
? Array.from(dataTransfer.files).filter((file) => file.name)
|
||||
|
||||
@@ -20,8 +20,8 @@ import { assertSftpFileFitsBuiltinEditor } from "../sftpEditorFileLimits";
|
||||
/** Local multi-select blob downloads read whole files into ArrayBuffers. */
|
||||
const LOCAL_BLOB_DOWNLOAD_CONCURRENCY = 1;
|
||||
/**
|
||||
* Multi-select roots each run folder pre-scan before the transfer scheduler.
|
||||
* Bound them so listing fan-out cannot multiply by selection size.
|
||||
* Multi-select roots each start their own interleaved folder walk / session
|
||||
* work. Bound them so many selected directories cannot stampede the scheduler.
|
||||
*/
|
||||
const MULTI_SELECT_ROOT_DOWNLOAD_CONCURRENCY = DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY;
|
||||
|
||||
@@ -576,8 +576,8 @@ export const useSftpViewFileOps = ({
|
||||
const selectedDirectory = await selectDirectory(t("sftp.context.download"));
|
||||
if (!selectedDirectory) return;
|
||||
|
||||
// Bound root jobs: directory roots run full pre-scan before the scheduler,
|
||||
// so unbounded Promise.allSettled would multiply LIST concurrency.
|
||||
// Bound root jobs: each directory root walks and transfers independently,
|
||||
// so unbounded Promise.allSettled would multiply session / LIST pressure.
|
||||
const results: Array<PromiseSettledResult<{ file: SftpFileEntry; status: TransferStatus }>> = [];
|
||||
await runBoundedConcurrency(
|
||||
files,
|
||||
|
||||
@@ -111,4 +111,16 @@ describe('extractRootPathsFromDropEntries', () => {
|
||||
['/home/user/folder'],
|
||||
);
|
||||
});
|
||||
|
||||
it('uses native path-only entries from a folder scan', () => {
|
||||
assert.deepEqual(
|
||||
extractRootPathsFromDropEntries([{
|
||||
file: null,
|
||||
localPath: '/home/user/folder/src/main.ts',
|
||||
relativePath: 'folder/src/main.ts',
|
||||
isDirectory: false,
|
||||
}]),
|
||||
['/home/user/folder'],
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -44,8 +44,6 @@ export function extractRootPathsFromDropEntries(dropEntries: DropEntry[]): strin
|
||||
const seenPaths = new Set<string>();
|
||||
|
||||
for (const entry of dropEntries) {
|
||||
if (!entry.file) continue;
|
||||
|
||||
const fullPath = getDropEntryLocalPath(entry);
|
||||
if (!fullPath) continue;
|
||||
|
||||
|
||||
@@ -324,7 +324,6 @@ export interface SyncPayload {
|
||||
sftpAutoSync?: boolean;
|
||||
sftpShowHiddenFiles?: boolean;
|
||||
sftpUseCompressedUpload?: boolean;
|
||||
sftpFolderPrescan?: boolean;
|
||||
sftpSkipUnchanged?: boolean;
|
||||
sftpAutoOpenSidebar?: boolean;
|
||||
sftpFollowTerminalCwd?: boolean;
|
||||
|
||||
@@ -307,31 +307,69 @@ async function statLocal(event, payload) {
|
||||
};
|
||||
}
|
||||
|
||||
async function collectLocalTreeEntries(rootPath, limits = {}) {
|
||||
function throwIfLocalTreeCancelled(isCancelled) {
|
||||
if (typeof isCancelled === "function" && isCancelled()) {
|
||||
const error = new Error("Local directory traversal cancelled");
|
||||
error.code = "ERR_LOCAL_TREE_CANCELLED";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const LOCAL_TREE_ENTRY_BATCH_SIZE = 64;
|
||||
|
||||
async function collectLocalTreeEntries(rootPath, limits = {}, onProgress, isCancelled, onEntries) {
|
||||
const rootStat = await fs.promises.stat(rootPath);
|
||||
if (!rootStat.isDirectory()) {
|
||||
throw new Error("Selected path is not a directory");
|
||||
}
|
||||
throwIfLocalTreeCancelled(isCancelled);
|
||||
|
||||
const traversalBudget = createLocalTreeTraversalBudget(limits);
|
||||
claimLocalTreeDirectory(traversalBudget);
|
||||
const rootName = path.basename(rootPath);
|
||||
const rootRealPath = await fs.promises.realpath(rootPath);
|
||||
const entries = [{
|
||||
const rootEntry = {
|
||||
localPath: rootPath,
|
||||
relativePath: rootName,
|
||||
type: "directory",
|
||||
size: rootStat.size,
|
||||
lastModified: rootStat.mtime.getTime(),
|
||||
}];
|
||||
};
|
||||
// When streaming batches, avoid retaining the full tree in memory for 100k+
|
||||
// drops. Callers that only need the complete array still get it below.
|
||||
const retainAll = typeof onEntries !== "function";
|
||||
const entries = retainAll ? [rootEntry] : null;
|
||||
let pendingBatch = [rootEntry];
|
||||
const flushBatch = (force = false) => {
|
||||
if (typeof onEntries !== "function") return;
|
||||
if (!force && pendingBatch.length < LOCAL_TREE_ENTRY_BATCH_SIZE) return;
|
||||
if (pendingBatch.length === 0) return;
|
||||
const batch = pendingBatch;
|
||||
pendingBatch = [];
|
||||
onEntries(batch);
|
||||
};
|
||||
flushBatch(true);
|
||||
|
||||
const queue = [{
|
||||
localPath: rootPath,
|
||||
relativePath: rootName,
|
||||
ancestorRealPaths: new Set([rootRealPath]),
|
||||
}];
|
||||
let queueIndex = 0;
|
||||
let fileCount = 0;
|
||||
let directoryCount = 1;
|
||||
let lastReportedTotal = 0;
|
||||
const reportProgress = (force = false) => {
|
||||
if (typeof onProgress !== "function") return;
|
||||
const entryCount = fileCount + directoryCount;
|
||||
if (!force && entryCount - lastReportedTotal < 32) return;
|
||||
lastReportedTotal = entryCount;
|
||||
onProgress({ fileCount, directoryCount, entryCount });
|
||||
};
|
||||
reportProgress(true);
|
||||
|
||||
while (queueIndex < queue.length) {
|
||||
throwIfLocalTreeCancelled(isCancelled);
|
||||
const current = queue[queueIndex++];
|
||||
const children = await fs.promises.readdir(current.localPath, { withFileTypes: true });
|
||||
accountLocalTreeEntries(traversalBudget, children.length);
|
||||
@@ -339,35 +377,44 @@ async function collectLocalTreeEntries(rootPath, limits = {}) {
|
||||
|
||||
const metadataConcurrency = 32;
|
||||
for (let start = 0; start < children.length; start += metadataConcurrency) {
|
||||
const inspected = await Promise.all(
|
||||
throwIfLocalTreeCancelled(isCancelled);
|
||||
const inspected = (await Promise.all(
|
||||
children.slice(start, start + metadataConcurrency).map(async (child) => {
|
||||
const childPath = path.join(current.localPath, child.name);
|
||||
const childRelativePath = `${current.relativePath}/${child.name}`;
|
||||
// Use lstat to distinguish links, then stat the target. Directory
|
||||
// links retain the established folder-upload behavior, while the
|
||||
// real-path ancestor chain prevents junction/symlink cycles.
|
||||
const linkStat = await fs.promises.lstat(childPath);
|
||||
const stat = linkStat.isSymbolicLink()
|
||||
? await fs.promises.stat(childPath).catch(() => linkStat)
|
||||
: linkStat;
|
||||
const isDirectory = stat.isDirectory();
|
||||
const realPath = isDirectory
|
||||
? await fs.promises.realpath(childPath)
|
||||
: null;
|
||||
const isCycle = !!realPath && current.ancestorRealPaths.has(realPath);
|
||||
const ancestorRealPaths = realPath
|
||||
? new Set([...current.ancestorRealPaths, realPath])
|
||||
: current.ancestorRealPaths;
|
||||
return {
|
||||
childPath,
|
||||
childRelativePath,
|
||||
stat,
|
||||
isDirectory,
|
||||
isCycle,
|
||||
ancestorRealPaths,
|
||||
};
|
||||
try {
|
||||
// Use lstat to distinguish links, then stat the target. Directory
|
||||
// links retain the established folder-upload behavior, while the
|
||||
// real-path ancestor chain prevents junction/symlink cycles.
|
||||
const linkStat = await fs.promises.lstat(childPath);
|
||||
const stat = linkStat.isSymbolicLink()
|
||||
? await fs.promises.stat(childPath).catch(() => linkStat)
|
||||
: linkStat;
|
||||
const isDirectory = stat.isDirectory();
|
||||
const realPath = isDirectory
|
||||
? await fs.promises.realpath(childPath)
|
||||
: null;
|
||||
const isCycle = !!realPath && current.ancestorRealPaths.has(realPath);
|
||||
const ancestorRealPaths = realPath
|
||||
? new Set([...current.ancestorRealPaths, realPath])
|
||||
: current.ancestorRealPaths;
|
||||
return {
|
||||
childPath,
|
||||
childRelativePath,
|
||||
stat,
|
||||
isDirectory,
|
||||
isCycle,
|
||||
ancestorRealPaths,
|
||||
};
|
||||
} catch (error) {
|
||||
// A folder can change while it is being scanned. Match the
|
||||
// tolerant browser traversal and skip entries that disappear or
|
||||
// become inaccessible instead of aborting the whole drop.
|
||||
console.warn(`Could not inspect ${childPath}:`, error.message);
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
);
|
||||
)).filter(Boolean);
|
||||
|
||||
// Promise.all preserves the sorted child order, so restart manifests stay
|
||||
// deterministic while metadata I/O is parallelized.
|
||||
@@ -379,30 +426,106 @@ async function collectLocalTreeEntries(rootPath, limits = {}) {
|
||||
// Cyclic links cannot be represented by a finite copied tree. Skip the
|
||||
// loop itself instead of misreporting it as a file that later fails.
|
||||
if (child.isCycle) continue;
|
||||
entries.push({
|
||||
const row = {
|
||||
localPath: child.childPath,
|
||||
relativePath: child.childRelativePath,
|
||||
type: child.isDirectory ? "directory" : "file",
|
||||
size: child.stat.size,
|
||||
lastModified: child.stat.mtime.getTime(),
|
||||
});
|
||||
|
||||
};
|
||||
if (retainAll) entries.push(row);
|
||||
pendingBatch.push(row);
|
||||
if (child.isDirectory) {
|
||||
directoryCount += 1;
|
||||
queue.push({
|
||||
localPath: child.childPath,
|
||||
relativePath: child.childRelativePath,
|
||||
ancestorRealPaths: child.ancestorRealPaths,
|
||||
});
|
||||
} else {
|
||||
fileCount += 1;
|
||||
}
|
||||
}
|
||||
flushBatch();
|
||||
reportProgress();
|
||||
}
|
||||
}
|
||||
|
||||
return entries;
|
||||
throwIfLocalTreeCancelled(isCancelled);
|
||||
flushBatch(true);
|
||||
reportProgress(true);
|
||||
return retainAll ? entries : [];
|
||||
}
|
||||
|
||||
async function listLocalTree(event, payload) {
|
||||
return collectLocalTreeEntries(payload.path);
|
||||
const progressChannel = typeof payload?.progressChannel === "string" && payload.progressChannel
|
||||
? payload.progressChannel
|
||||
: null;
|
||||
const cancelChannel = typeof payload?.cancelChannel === "string" && payload.cancelChannel
|
||||
? payload.cancelChannel
|
||||
: null;
|
||||
let cancelled = false;
|
||||
const onCancel = () => {
|
||||
cancelled = true;
|
||||
};
|
||||
// Lazy-require so unit tests can import collectLocalTreeEntries without a
|
||||
// full Electron binary (top-level require("electron") breaks node --test).
|
||||
let electronIpcMain = null;
|
||||
if (cancelChannel) {
|
||||
try {
|
||||
electronIpcMain = require("electron").ipcMain;
|
||||
electronIpcMain.on(cancelChannel, onCancel);
|
||||
} catch {
|
||||
electronIpcMain = null;
|
||||
}
|
||||
}
|
||||
const onProgress = progressChannel
|
||||
? (stats) => {
|
||||
try {
|
||||
event.sender.send(progressChannel, stats);
|
||||
} catch {
|
||||
// Renderer may have gone away mid-scan.
|
||||
}
|
||||
}
|
||||
: undefined;
|
||||
const entriesChannel = typeof payload?.entriesChannel === "string" && payload.entriesChannel
|
||||
? payload.entriesChannel
|
||||
: null;
|
||||
const onEntries = entriesChannel
|
||||
? (batch) => {
|
||||
try {
|
||||
// Always send plain arrays for entry batches. The stream end marker is
|
||||
// a separate object so the preload can keep its listener until every
|
||||
// nested batch has been delivered (invoke reply races with send).
|
||||
event.sender.send(entriesChannel, batch);
|
||||
} catch {
|
||||
// Renderer may have gone away mid-scan.
|
||||
}
|
||||
}
|
||||
: undefined;
|
||||
try {
|
||||
return await collectLocalTreeEntries(
|
||||
payload.path,
|
||||
payload.limits || {},
|
||||
onProgress,
|
||||
() => cancelled,
|
||||
onEntries,
|
||||
);
|
||||
} finally {
|
||||
// Must be sent after the last entry batch and before the invoke resolves
|
||||
// is not enough alone — preload must wait for this marker before removing
|
||||
// its listener, otherwise deep nested files (discovered late) are dropped.
|
||||
if (entriesChannel) {
|
||||
try {
|
||||
event.sender.send(entriesChannel, { type: "tree-end" });
|
||||
} catch {
|
||||
// Renderer may have gone away mid-scan.
|
||||
}
|
||||
}
|
||||
if (cancelChannel && electronIpcMain) {
|
||||
electronIpcMain.removeListener(cancelChannel, onCancel);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -183,6 +183,56 @@ test("collectLocalTreeEntries preserves empty directories in selected folders",
|
||||
}
|
||||
});
|
||||
|
||||
test("collectLocalTreeEntries reports progressive file counts during scan", async () => {
|
||||
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-progress-"));
|
||||
const selected = path.join(root, "project");
|
||||
await fs.promises.mkdir(path.join(selected, "src"), { recursive: true });
|
||||
await fs.promises.writeFile(path.join(selected, "readme.txt"), "hi");
|
||||
await fs.promises.writeFile(path.join(selected, "src", "main.txt"), "hello");
|
||||
|
||||
const progress = [];
|
||||
try {
|
||||
const entries = await collectLocalTreeEntries(selected, {}, (stats) => {
|
||||
progress.push({ ...stats });
|
||||
});
|
||||
assert.equal(entries.filter((entry) => entry.type === "file").length, 2);
|
||||
assert.ok(progress.length >= 1, "expected at least one progress event");
|
||||
const final = progress[progress.length - 1];
|
||||
assert.equal(final.fileCount, 2);
|
||||
assert.equal(final.directoryCount, 2);
|
||||
assert.equal(final.entryCount, 4);
|
||||
} finally {
|
||||
await fs.promises.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("collectLocalTreeEntries aborts when cancelled mid-scan", async () => {
|
||||
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-cancel-"));
|
||||
const selected = path.join(root, "project");
|
||||
await fs.promises.mkdir(path.join(selected, "a"), { recursive: true });
|
||||
await fs.promises.mkdir(path.join(selected, "b"), { recursive: true });
|
||||
await fs.promises.writeFile(path.join(selected, "a", "1.txt"), "1");
|
||||
await fs.promises.writeFile(path.join(selected, "b", "2.txt"), "2");
|
||||
|
||||
let cancel = false;
|
||||
try {
|
||||
await assert.rejects(
|
||||
() => collectLocalTreeEntries(
|
||||
selected,
|
||||
{},
|
||||
() => {
|
||||
// Cancel as soon as progress starts flowing.
|
||||
cancel = true;
|
||||
},
|
||||
() => cancel,
|
||||
),
|
||||
/cancelled/i,
|
||||
);
|
||||
} finally {
|
||||
await fs.promises.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("collectLocalTreeEntries bounds and parallelizes metadata reads within a directory", async (t) => {
|
||||
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-parallel-"));
|
||||
const selected = path.join(root, "project");
|
||||
@@ -215,6 +265,34 @@ test("collectLocalTreeEntries bounds and parallelizes metadata reads within a di
|
||||
assert.ok(maxActive <= 32, `metadata concurrency must stay bounded, got ${maxActive}`);
|
||||
});
|
||||
|
||||
test("collectLocalTreeEntries skips a child that disappears during inspection", async (t) => {
|
||||
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-vanished-"));
|
||||
const selected = path.join(root, "project");
|
||||
await fs.promises.mkdir(selected, { recursive: true });
|
||||
await fs.promises.writeFile(path.join(selected, "stable.txt"), "stable");
|
||||
await fs.promises.writeFile(path.join(selected, "vanished.txt"), "vanished");
|
||||
|
||||
const originalLstat = fs.promises.lstat;
|
||||
fs.promises.lstat = async (candidate, ...args) => {
|
||||
if (candidate === path.join(selected, "vanished.txt")) {
|
||||
const error = new Error("file disappeared");
|
||||
error.code = "ENOENT";
|
||||
throw error;
|
||||
}
|
||||
return originalLstat.call(fs.promises, candidate, ...args);
|
||||
};
|
||||
t.after(async () => {
|
||||
fs.promises.lstat = originalLstat;
|
||||
await fs.promises.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const entries = await collectLocalTreeEntries(selected);
|
||||
assert.deepEqual(entries.map((entry) => entry.relativePath), [
|
||||
"project",
|
||||
"project/stable.txt",
|
||||
]);
|
||||
});
|
||||
|
||||
test("collectLocalTreeEntries does not follow a directory symlink cycle", async (t) => {
|
||||
const root = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-upload-tree-cycle-"));
|
||||
const selected = path.join(root, "project");
|
||||
|
||||
+192
-280
@@ -2241,76 +2241,23 @@ async function uploadFile(
|
||||
transfer.uploadStrategy = "scp";
|
||||
logTransferDiag(transfer, "strategy", { strategy: "scp" });
|
||||
const backend = getScpBackendForClient(client);
|
||||
let scpSourcePath = localPath;
|
||||
let digestPath = null;
|
||||
let snapshotPath = null;
|
||||
let openReadStream = null;
|
||||
let initialSource = null;
|
||||
try {
|
||||
if (!transfer.sourceIsOwnedTemp) {
|
||||
initialSource = await fs.promises.stat(localPath);
|
||||
const snapshotId = crypto.createHash("sha256")
|
||||
.update(String(transfer.transferId || localPath))
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
digestPath = tempDirBridge.getTransferTempFilePath(
|
||||
`upload-digest-${snapshotId}`,
|
||||
"ranges.sha256",
|
||||
);
|
||||
snapshotPath = tempDirBridge.getTransferTempFilePath(
|
||||
`upload-source-${snapshotId}`,
|
||||
"snapshot.bin",
|
||||
);
|
||||
await createUploadDigestBaseline(localPath, digestPath, fileSize, transfer);
|
||||
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
|
||||
const sourceAfterBaseline = await fs.promises.stat(localPath);
|
||||
assertSourceMetadataUnchanged(initialSource, sourceAfterBaseline, fileSize, {
|
||||
contentVerifiedSeparately: true,
|
||||
});
|
||||
if (typeof onBytesCommitted === "function") {
|
||||
await createVerifiedUploadSnapshot(
|
||||
localPath,
|
||||
snapshotPath,
|
||||
digestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
);
|
||||
scpSourcePath = snapshotPath;
|
||||
} else {
|
||||
// Remote staging can safely discard a failed upload. Verify every
|
||||
// source chunk as SCP reads it, then rescan before promotion, without
|
||||
// requiring another full local copy of large files.
|
||||
snapshotPath = null;
|
||||
openReadStream = () => createVerifiedUploadReadStream(
|
||||
localPath,
|
||||
digestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
);
|
||||
}
|
||||
}
|
||||
await backend.uploadFile(scpSourcePath, remotePath, {
|
||||
fileSize,
|
||||
transfer,
|
||||
encoding,
|
||||
signal: transfer.signal,
|
||||
openReadStream,
|
||||
onProgress: (transferred, total) => sendProgress(transferred, total || fileSize),
|
||||
});
|
||||
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
|
||||
if (digestPath && !snapshotPath) {
|
||||
const latestSource = await fs.promises.stat(localPath);
|
||||
assertSourceMetadataUnchanged(initialSource, latestSource, fileSize, {
|
||||
contentVerifiedSeparately: true,
|
||||
});
|
||||
await verifyUploadDigestBaseline(localPath, digestPath, fileSize, transfer);
|
||||
}
|
||||
onBytesCommitted?.();
|
||||
return;
|
||||
} finally {
|
||||
if (snapshotPath) await fs.promises.rm(snapshotPath, { force: true }).catch(() => {});
|
||||
if (digestPath) await fs.promises.rm(digestPath, { force: true }).catch(() => {});
|
||||
}
|
||||
// Stream the live local path directly — no whole-file digest / snapshot.
|
||||
// openReadStream is optional for scpBackend (falls back to createReadStream);
|
||||
// provide a plain file stream so tests/backends that always call it still work.
|
||||
await backend.uploadFile(localPath, remotePath, {
|
||||
fileSize,
|
||||
transfer,
|
||||
encoding,
|
||||
signal: transfer.signal,
|
||||
openReadStream: () => {
|
||||
const stream = fs.createReadStream(localPath, { highWaterMark: 256 * 1024 });
|
||||
return { stream, completed: Promise.resolve() };
|
||||
},
|
||||
onProgress: (transferred, total) => sendProgress(transferred, total || fileSize),
|
||||
});
|
||||
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
|
||||
onBytesCommitted?.();
|
||||
return;
|
||||
}
|
||||
|
||||
await requireSftpChannel(client);
|
||||
@@ -2329,79 +2276,21 @@ async function uploadFile(
|
||||
else lastPipelineError = new Error(String(err || "SFTP upload failed"));
|
||||
};
|
||||
|
||||
if (!transfer.sourceIsOwnedTemp) {
|
||||
const digestId = crypto.createHash("sha256")
|
||||
.update(String(transfer.transferId || "upload"))
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
const digestPath = tempDirBridge.getTransferTempFilePath(
|
||||
`upload-digest-${digestId}`,
|
||||
"ranges.sha256",
|
||||
);
|
||||
transfer.sourceDigestPath = digestPath;
|
||||
const previousPhase = transfer.phase;
|
||||
transfer.phase = "verifying";
|
||||
transfer.publishCurrentProgress?.();
|
||||
try {
|
||||
await createUploadDigestBaseline(
|
||||
originalLocalPath,
|
||||
digestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
() => {
|
||||
// Keep remote transferredBytes at the resume checkpoint while hashing
|
||||
// so large uploads do not look stuck at 0 B with no status (#2712).
|
||||
sendProgress(Math.max(0, Number(transfer.checkpointBytes) || 0), fileSize, {
|
||||
force: true,
|
||||
});
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
transfer.phase = previousPhase || "transferring";
|
||||
if (!transfer.cancelled && !transfer.signal?.aborted) {
|
||||
transfer.publishCurrentProgress?.();
|
||||
}
|
||||
}
|
||||
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
|
||||
const sourceAfterBaseline = await fs.promises.stat(originalLocalPath);
|
||||
// Digest was just built; only size/content matter from here.
|
||||
assertSourceMetadataUnchanged(initialSource, sourceAfterBaseline, fileSize, {
|
||||
contentVerifiedSeparately: true,
|
||||
});
|
||||
}
|
||||
|
||||
const cleanupSourceDigest = async () => {
|
||||
if (!transfer.sourceDigestPath) return;
|
||||
const digestPath = transfer.sourceDigestPath;
|
||||
transfer.sourceDigestPath = null;
|
||||
await fs.promises.rm(digestPath, { force: true }).catch(() => {});
|
||||
};
|
||||
// Industry-standard SFTP clients (FileZilla / WinSCP / OpenSSH) resume by
|
||||
// size only — no whole-file content digest before or during the body transfer.
|
||||
// Pause/resume durability lives in checkpointBytes + the remote .part stage.
|
||||
|
||||
const finishSuccessfulUpload = async () => {
|
||||
try {
|
||||
if (initialSource) {
|
||||
const latestSource = await fs.promises.stat(originalLocalPath);
|
||||
// Prefer digest re-scan for same-size rewrites. Hard-failing on ctime
|
||||
// alone false-positives long pause/resume uploads on macOS.
|
||||
assertSourceMetadataUnchanged(initialSource, latestSource, fileSize, {
|
||||
contentVerifiedSeparately: Boolean(transfer.sourceDigestPath),
|
||||
});
|
||||
}
|
||||
// Metadata alone cannot catch same-size rewrites with unchanged/coarse
|
||||
// timestamps (e.g. all ranges already verified before the rewrite).
|
||||
// Re-scan the source against the digest baseline before promotion.
|
||||
if (transfer.sourceDigestPath) {
|
||||
await verifyUploadDigestBaseline(
|
||||
originalLocalPath,
|
||||
transfer.sourceDigestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
);
|
||||
}
|
||||
await assertRemoteUploadSize(client, remotePath, fileSize);
|
||||
} finally {
|
||||
await cleanupSourceDigest();
|
||||
if (initialSource) {
|
||||
const latestSource = await fs.promises.stat(originalLocalPath);
|
||||
// Soft size + mtime/ino (no full-file re-hash). Do not claim separate
|
||||
// content proof — same-size rewrites that bump mtime still fail closed.
|
||||
// ignoreCtime: macOS xattr/Spotlight noise must not abort a true match.
|
||||
assertSourceMetadataUnchanged(initialSource, latestSource, fileSize, {
|
||||
ignoreCtime: true,
|
||||
});
|
||||
}
|
||||
await assertRemoteUploadSize(client, remotePath, fileSize);
|
||||
};
|
||||
|
||||
// Prefer an isolated SFTP channel so cancellation cannot kill the browse session.
|
||||
@@ -2502,8 +2391,6 @@ async function uploadFile(
|
||||
&& !transfer.resumable
|
||||
) {
|
||||
let fastPutOk = false;
|
||||
let fastPutSourcePath = localPath;
|
||||
let fastPutSnapshotPath = null;
|
||||
try {
|
||||
// Wait for any prior/in-flight write OPEN on this path (including our
|
||||
// own concurrent attempt's still-pending OPEN) before fastPut truncates
|
||||
@@ -2519,29 +2406,12 @@ async function uploadFile(
|
||||
logTransferDiag(transfer, "strategy", { strategy: "fastPut-isolated" });
|
||||
transfer.pauseSupported = false;
|
||||
transfer.pauseUnavailableReason = "Pause is unavailable during fastPut upload";
|
||||
if (!transfer.sourceIsOwnedTemp) {
|
||||
const snapshotId = crypto.createHash("sha256")
|
||||
.update(String(transfer.transferId || localPath))
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
fastPutSnapshotPath = tempDirBridge.getTransferTempFilePath(
|
||||
`upload-source-${snapshotId}`,
|
||||
"snapshot.bin",
|
||||
);
|
||||
await createVerifiedUploadSnapshot(
|
||||
originalLocalPath,
|
||||
fastPutSnapshotPath,
|
||||
transfer.sourceDigestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
);
|
||||
fastPutSourcePath = fastPutSnapshotPath;
|
||||
}
|
||||
// Stream the live local path (no whole-file snapshot / digest sidecar).
|
||||
sendProgress(Math.max(0, Number(transfer.checkpointBytes) || 0), fileSize, {
|
||||
force: true,
|
||||
});
|
||||
await uploadViaFastPut(
|
||||
fastPutSourcePath,
|
||||
originalLocalPath,
|
||||
remotePath,
|
||||
isolated,
|
||||
fileSize,
|
||||
@@ -2551,10 +2421,10 @@ async function uploadFile(
|
||||
);
|
||||
fastPutOk = true;
|
||||
} catch (err) {
|
||||
// Gate-wait / snapshot / fastPut failure: end the reopened isolated
|
||||
// channel before nulling. Rethrow paths skip the post-block else-if
|
||||
// end, and fallthrough also clears isolated; either way we must not
|
||||
// leak the SSH subsystem opened for this attempt (#2755 Bugbot).
|
||||
// Gate-wait / fastPut failure: end the reopened isolated channel before
|
||||
// nulling. Rethrow paths skip the post-block else-if end, and fallthrough
|
||||
// also clears isolated; either way we must not leak the SSH subsystem
|
||||
// opened for this attempt (#2755 Bugbot).
|
||||
if (isolated && typeof isolated.end === "function") {
|
||||
try { isolated.end(); } catch { /* ignore */ }
|
||||
}
|
||||
@@ -2576,10 +2446,6 @@ async function uploadFile(
|
||||
"[transferBridge] isolated fastPut failed, trying next pipelined strategy:",
|
||||
err?.message || String(err),
|
||||
);
|
||||
} finally {
|
||||
if (fastPutSnapshotPath) {
|
||||
await fs.promises.rm(fastPutSnapshotPath, { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
if (fastPutOk) {
|
||||
onBytesCommitted?.();
|
||||
@@ -2648,7 +2514,10 @@ async function uploadFile(
|
||||
const error = new Error(message, cause ? { cause } : undefined);
|
||||
if (cause?.code !== undefined) error.code = cause.code;
|
||||
if (cause?.noTransferFallback) error.noTransferFallback = true;
|
||||
await cleanupSourceDigest();
|
||||
if (transfer.sourceDigestPath) {
|
||||
try { await fs.promises.rm(transfer.sourceDigestPath, { force: true }); } catch { /* ignore */ }
|
||||
transfer.sourceDigestPath = null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
@@ -3730,10 +3599,20 @@ function isTransferCancelled(transfer) {
|
||||
}
|
||||
|
||||
const UPLOAD_DIGEST_SCAN_SIZE = TRANSFER_CHUNK_SIZE * 128;
|
||||
const EMPTY_DIGEST_SLOT = Buffer.alloc(32);
|
||||
|
||||
function uploadDigestByteLength(fileSize) {
|
||||
return Math.ceil(Math.max(0, Number(fileSize) || 0) / TRANSFER_CHUNK_SIZE) * 32;
|
||||
}
|
||||
|
||||
function isUnsetDigestSlot(buffer, bytesRead) {
|
||||
if (bytesRead !== 32) return true;
|
||||
return buffer.equals(EMPTY_DIGEST_SLOT);
|
||||
}
|
||||
|
||||
async function assertUploadDigestCapacity(digestPath, fileSize) {
|
||||
if (typeof fs.promises.statfs !== "function") return;
|
||||
const requiredBytes = BigInt(Math.ceil(fileSize / TRANSFER_CHUNK_SIZE)) * 32n;
|
||||
const requiredBytes = BigInt(uploadDigestByteLength(fileSize));
|
||||
let stats;
|
||||
try {
|
||||
stats = await fs.promises.statfs(path.dirname(digestPath), { bigint: true });
|
||||
@@ -3750,12 +3629,31 @@ async function assertUploadDigestCapacity(digestPath, fileSize) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Allocate an empty per-chunk digest sidecar without reading the source.
|
||||
* Chunk digests are filled on first verified read during the upload pass so
|
||||
* multi-GB files no longer wait on a full pre-hash before the first WRITE.
|
||||
*/
|
||||
async function prepareUploadDigestSidecar(digestPath, fileSize) {
|
||||
await fs.promises.rm(digestPath, { force: true });
|
||||
await assertUploadDigestCapacity(digestPath, fileSize);
|
||||
const requiredBytes = uploadDigestByteLength(fileSize);
|
||||
const handle = await fs.promises.open(digestPath, "w");
|
||||
try {
|
||||
if (requiredBytes > 0) await handle.truncate(requiredBytes);
|
||||
} finally {
|
||||
await handle.close().catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer, onProgress = null) {
|
||||
let sourceHandle = null;
|
||||
let digestHandle = null;
|
||||
try {
|
||||
sourceHandle = await fs.promises.open(sourcePath, "r");
|
||||
digestHandle = await fs.promises.open(digestPath, "r");
|
||||
// r+ so a lazy sidecar can finish filling any still-empty slots on the
|
||||
// post-upload re-scan (e.g. resume after a crash with a partial digest).
|
||||
digestHandle = await fs.promises.open(digestPath, "r+");
|
||||
const buffer = Buffer.allocUnsafe(Math.min(UPLOAD_DIGEST_SCAN_SIZE, Math.max(1, fileSize)));
|
||||
let position = 0;
|
||||
let chunkIndex = 0;
|
||||
@@ -3771,7 +3669,18 @@ async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, tran
|
||||
const actual = crypto.createHash("sha256")
|
||||
.update(buffer.subarray(offset, Math.min(offset + TRANSFER_CHUNK_SIZE, length)))
|
||||
.digest();
|
||||
if (!expected.subarray(index * 32, (index + 1) * 32).equals(actual)) {
|
||||
const slot = expected.subarray(index * 32, (index + 1) * 32);
|
||||
if (isUnsetDigestSlot(slot, 32)) {
|
||||
const writeResult = await digestHandle.write(
|
||||
actual,
|
||||
0,
|
||||
32,
|
||||
(chunkIndex + index) * 32,
|
||||
);
|
||||
if (!writeResult || writeResult.bytesWritten !== 32) {
|
||||
throw new Error("Upload digest sidecar stopped accepting data");
|
||||
}
|
||||
} else if (!slot.equals(actual)) {
|
||||
throw createSourceContentChangedError();
|
||||
}
|
||||
}
|
||||
@@ -3785,63 +3694,80 @@ async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, tran
|
||||
}
|
||||
}
|
||||
|
||||
async function createUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer, onProgress = null) {
|
||||
// A crashed attempt may have left this transfer's old baseline behind. It is
|
||||
// fully replaceable and its blocks must be reclaimable before capacity is
|
||||
// evaluated for the new baseline.
|
||||
await fs.promises.rm(digestPath, { force: true });
|
||||
await assertUploadDigestCapacity(digestPath, fileSize);
|
||||
async function createUploadDigestBaseline(
|
||||
sourcePath,
|
||||
digestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
onProgress = null,
|
||||
options = {},
|
||||
) {
|
||||
// merge:true — fill/verify an existing sidecar while uploads are in flight.
|
||||
// Default replaces the file (SCP / snapshot callers need a clean baseline).
|
||||
const merge = options.merge === true;
|
||||
if (!merge) {
|
||||
await fs.promises.rm(digestPath, { force: true });
|
||||
await prepareUploadDigestSidecar(digestPath, fileSize);
|
||||
} else {
|
||||
await assertUploadDigestCapacity(digestPath, fileSize);
|
||||
try {
|
||||
await fs.promises.access(digestPath);
|
||||
} catch {
|
||||
await prepareUploadDigestSidecar(digestPath, fileSize);
|
||||
}
|
||||
}
|
||||
let sourceHandle = null;
|
||||
let digestHandle = null;
|
||||
let completed = false;
|
||||
try {
|
||||
sourceHandle = await fs.promises.open(sourcePath, "r");
|
||||
digestHandle = await fs.promises.open(digestPath, "w");
|
||||
digestHandle = await fs.promises.open(digestPath, "r+");
|
||||
const buffer = Buffer.allocUnsafe(Math.min(UPLOAD_DIGEST_SCAN_SIZE, Math.max(1, fileSize)));
|
||||
let position = 0;
|
||||
let digestPosition = 0;
|
||||
let chunkIndex = 0;
|
||||
while (position < fileSize) {
|
||||
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
|
||||
const length = Math.min(buffer.length, fileSize - position);
|
||||
await readLocalRange(sourceHandle, buffer, position, length);
|
||||
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
|
||||
const digestCount = Math.ceil(length / TRANSFER_CHUNK_SIZE);
|
||||
const digests = Buffer.allocUnsafe(digestCount * 32);
|
||||
for (let offset = 0, index = 0; offset < length; offset += TRANSFER_CHUNK_SIZE, index += 1) {
|
||||
crypto.createHash("sha256")
|
||||
const actual = crypto.createHash("sha256")
|
||||
.update(buffer.subarray(offset, Math.min(offset + TRANSFER_CHUNK_SIZE, length)))
|
||||
.digest()
|
||||
.copy(digests, index * 32);
|
||||
}
|
||||
let written = 0;
|
||||
while (written < digests.length) {
|
||||
const result = await digestHandle.write(
|
||||
digests,
|
||||
written,
|
||||
digests.length - written,
|
||||
digestPosition + written,
|
||||
);
|
||||
if (!result || result.bytesWritten <= 0) {
|
||||
throw new Error("Upload digest baseline stopped accepting data");
|
||||
}
|
||||
written += result.bytesWritten;
|
||||
.digest();
|
||||
const slotIndex = chunkIndex + index;
|
||||
const digestOffset = slotIndex * 32;
|
||||
await withDigestSlotLock(digestHandle, slotIndex, async () => {
|
||||
const expected = Buffer.allocUnsafe(32);
|
||||
const digestResult = await digestHandle.read(expected, 0, 32, digestOffset);
|
||||
if (isUnsetDigestSlot(expected, digestResult.bytesRead)) {
|
||||
const writeResult = await digestHandle.write(actual, 0, 32, digestOffset);
|
||||
if (!writeResult || writeResult.bytesWritten !== 32) {
|
||||
throw new Error("Upload digest baseline stopped accepting data");
|
||||
}
|
||||
} else if (!expected.equals(actual)) {
|
||||
throw createSourceContentChangedError();
|
||||
}
|
||||
});
|
||||
}
|
||||
position += length;
|
||||
digestPosition += digests.length;
|
||||
chunkIndex += digestCount;
|
||||
// Report scan progress so large uploads do not look stuck at 0 B/s while
|
||||
// hashing (#2712 / #2556). Bytes here are local read progress, not remote.
|
||||
onProgress?.(position, fileSize);
|
||||
}
|
||||
if (typeof digestHandle.sync === "function") {
|
||||
await digestHandle.sync().catch(() => {});
|
||||
}
|
||||
completed = true;
|
||||
} finally {
|
||||
await sourceHandle?.close().catch(() => {});
|
||||
await digestHandle?.close().catch(() => {});
|
||||
if (!completed) await fs.promises.rm(digestPath, { force: true }).catch(() => {});
|
||||
// Background filler must not delete a sidecar still used by live uploads.
|
||||
if (!completed && !merge) {
|
||||
await fs.promises.rm(digestPath, { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
// Confirm the sidecar matches the source before any remote WRITE. Progress
|
||||
// callbacks keep the UI in "verifying" instead of looking stuck (#2712).
|
||||
await verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer, onProgress);
|
||||
}
|
||||
|
||||
async function createVerifiedUploadSnapshot(
|
||||
@@ -3858,7 +3784,7 @@ async function createVerifiedUploadSnapshot(
|
||||
let completed = false;
|
||||
try {
|
||||
sourceHandle = await fs.promises.open(sourcePath, "r");
|
||||
digestHandle = await fs.promises.open(digestPath, "r");
|
||||
digestHandle = await fs.promises.open(digestPath, "r+");
|
||||
snapshotHandle = await fs.promises.open(snapshotPath, "w");
|
||||
const sourceStats = await sourceHandle.stat();
|
||||
let position = 0;
|
||||
@@ -3910,7 +3836,7 @@ function createVerifiedUploadReadStream(
|
||||
let digestHandle = null;
|
||||
try {
|
||||
sourceHandle = await fs.promises.open(sourcePath, "r");
|
||||
digestHandle = await fs.promises.open(digestPath, "r");
|
||||
digestHandle = await fs.promises.open(digestPath, "r+");
|
||||
let position = 0;
|
||||
while (position < fileSize) {
|
||||
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
|
||||
@@ -3935,13 +3861,39 @@ function createVerifiedUploadReadStream(
|
||||
return { stream, completed };
|
||||
}
|
||||
|
||||
async function withDigestSlotLock(digestHandle, chunkIndex, fn) {
|
||||
// Per-chunk locks keep pipelined uploads concurrent across ranges while still
|
||||
// serializing r+ ops on the same 32-byte digest slot.
|
||||
if (!digestHandle.__netcattySlotLocks) {
|
||||
digestHandle.__netcattySlotLocks = new Map();
|
||||
}
|
||||
const locks = digestHandle.__netcattySlotLocks;
|
||||
const previous = locks.get(chunkIndex) || Promise.resolve();
|
||||
let release;
|
||||
const gate = new Promise((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
locks.set(chunkIndex, previous.then(() => gate, () => gate));
|
||||
try {
|
||||
await previous;
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
async function readVerifiedUploadRange(
|
||||
localHandle,
|
||||
digestHandle,
|
||||
position,
|
||||
length,
|
||||
fileSize,
|
||||
options = {},
|
||||
) {
|
||||
// allowCreate: first pass fills empty digest slots while uploading (no full
|
||||
// pre-hash). Resume re-reads must require an existing non-empty slot so a
|
||||
// rewritten source cannot silently mint a new identity for already-sent bytes.
|
||||
const allowCreate = options.allowCreate !== false;
|
||||
const output = Buffer.allocUnsafe(length);
|
||||
let outputOffset = 0;
|
||||
while (outputOffset < length) {
|
||||
@@ -3949,13 +3901,25 @@ async function readVerifiedUploadRange(
|
||||
const chunkStart = Math.floor(rangePosition / TRANSFER_CHUNK_SIZE) * TRANSFER_CHUNK_SIZE;
|
||||
const chunkLength = Math.min(TRANSFER_CHUNK_SIZE, fileSize - chunkStart);
|
||||
const chunk = Buffer.allocUnsafe(chunkLength);
|
||||
// Local reads may run concurrently; only digest-slot mutations are locked.
|
||||
await readLocalRange(localHandle, chunk, chunkStart, chunkLength);
|
||||
const expected = Buffer.allocUnsafe(32);
|
||||
const chunkIndex = Math.floor(chunkStart / TRANSFER_CHUNK_SIZE);
|
||||
const digestResult = await digestHandle.read(expected, 0, 32, chunkIndex * 32);
|
||||
if (digestResult.bytesRead !== 32) throw createSourceContentChangedError();
|
||||
const actual = crypto.createHash("sha256").update(chunk).digest();
|
||||
if (!expected.equals(actual)) throw createSourceContentChangedError();
|
||||
const chunkIndex = Math.floor(chunkStart / TRANSFER_CHUNK_SIZE);
|
||||
const digestOffset = chunkIndex * 32;
|
||||
await withDigestSlotLock(digestHandle, chunkIndex, async () => {
|
||||
const expected = Buffer.allocUnsafe(32);
|
||||
const digestResult = await digestHandle.read(expected, 0, 32, digestOffset);
|
||||
if (isUnsetDigestSlot(expected, digestResult.bytesRead)) {
|
||||
if (!allowCreate) throw createSourceContentChangedError();
|
||||
// Record identity for this chunk as we upload it (single local read).
|
||||
const writeResult = await digestHandle.write(actual, 0, 32, digestOffset);
|
||||
if (!writeResult || writeResult.bytesWritten !== 32) {
|
||||
throw new Error("Upload digest sidecar stopped accepting data");
|
||||
}
|
||||
} else if (!expected.equals(actual)) {
|
||||
throw createSourceContentChangedError();
|
||||
}
|
||||
});
|
||||
const chunkOffset = rangePosition - chunkStart;
|
||||
const copyLength = Math.min(length - outputOffset, chunkLength - chunkOffset);
|
||||
chunk.copy(output, outputOffset, chunkOffset, chunkOffset + copyLength);
|
||||
@@ -3983,7 +3947,7 @@ async function readVerifiedUploadRange(
|
||||
* @param {object|null|undefined} initialSource
|
||||
* @param {object|null|undefined} latestSource
|
||||
* @param {number} expectedSize
|
||||
* @param {{ contentVerifiedSeparately?: boolean, allowSourceGrowth?: boolean }} [options]
|
||||
* @param {{ contentVerifiedSeparately?: boolean, allowSourceGrowth?: boolean, ignoreCtime?: boolean }} [options]
|
||||
*/
|
||||
function assertSourceMetadataUnchanged(initialSource, latestSource, expectedSize, options = {}) {
|
||||
const latestSize = Number(latestSource?.size);
|
||||
@@ -4010,8 +3974,11 @@ function assertSourceMetadataUnchanged(initialSource, latestSource, expectedSize
|
||||
return;
|
||||
}
|
||||
// No digest / per-range content proof: timestamps + inode are the durable
|
||||
// same-size rewrite signal (remote SFTP download path).
|
||||
const versionFields = ["mtimeMs", "ctimeMs", "mtime", "ctime", "ino"];
|
||||
// same-size rewrite signal. Upload finish may set ignoreCtime so macOS
|
||||
// xattr/Spotlight ctime bumps do not abort after a true size match.
|
||||
const versionFields = options.ignoreCtime
|
||||
? ["mtimeMs", "mtime", "ino"]
|
||||
: ["mtimeMs", "ctimeMs", "mtime", "ctime", "ino"];
|
||||
const changed = versionFields.some((field) => {
|
||||
const before = Number(initialSource?.[field]);
|
||||
const after = Number(latestSource?.[field]);
|
||||
@@ -4416,8 +4383,6 @@ async function uploadFileConcurrent(
|
||||
transfer.abort = abortEarly;
|
||||
|
||||
let localHandle = null;
|
||||
let digestHandle = null;
|
||||
let ephemeralDigestPath = null;
|
||||
let initialSource = null;
|
||||
let remoteHandle = null;
|
||||
let failed = false;
|
||||
@@ -4434,29 +4399,8 @@ async function uploadFileConcurrent(
|
||||
throw localOpenError;
|
||||
}
|
||||
if (transfer.cancelled) throw new Error("Transfer cancelled");
|
||||
if (transfer.sourceDigestPath) {
|
||||
digestHandle = await fs.promises.open(transfer.sourceDigestPath, "r");
|
||||
} else if (!transfer.sourceIsOwnedTemp) {
|
||||
// Non-resumable range uploads do not receive uploadFile's persistent
|
||||
// digest sidecar. Build the same stable baseline here so every range is
|
||||
// verified immediately before its remote WRITE instead of relying on a
|
||||
// before/after whole-file fingerprint that temporary rewrites can evade.
|
||||
if (!transfer.sourceIsOwnedTemp) {
|
||||
initialSource = await localHandle.stat();
|
||||
const digestId = crypto.createHash("sha256")
|
||||
.update(String(transfer.transferId || localPath))
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
ephemeralDigestPath = tempDirBridge.getTransferTempFilePath(
|
||||
`upload-digest-${digestId}`,
|
||||
"ranges.sha256",
|
||||
);
|
||||
await createUploadDigestBaseline(localPath, ephemeralDigestPath, fileSize, transfer);
|
||||
if (transfer.cancelled) throw new Error("Transfer cancelled");
|
||||
const sourceAfterBaseline = await localHandle.stat();
|
||||
assertSourceMetadataUnchanged(initialSource, sourceAfterBaseline, fileSize, {
|
||||
contentVerifiedSeparately: true,
|
||||
});
|
||||
digestHandle = await fs.promises.open(ephemeralDigestPath, "r");
|
||||
}
|
||||
if (transfer.cancelled) throw new Error("Transfer cancelled");
|
||||
remoteHandle = await openSftpHandleForTransfer(
|
||||
@@ -4524,19 +4468,10 @@ async function uploadFileConcurrent(
|
||||
checkpoint,
|
||||
concurrency: UPLOAD_TRANSFER_CONCURRENCY,
|
||||
copyRange: async (position, length) => {
|
||||
const buffer = digestHandle
|
||||
? await readVerifiedUploadRange(
|
||||
localHandle,
|
||||
digestHandle,
|
||||
position,
|
||||
length,
|
||||
fileSize,
|
||||
)
|
||||
: await (async () => {
|
||||
const directBuffer = Buffer.allocUnsafe(length);
|
||||
await readLocalRange(localHandle, directBuffer, position, length);
|
||||
return directBuffer;
|
||||
})();
|
||||
// Size-based resume: read the source range at `position` and WRITE it.
|
||||
// No per-chunk content digest (FileZilla / WinSCP style).
|
||||
const buffer = Buffer.allocUnsafe(length);
|
||||
await readLocalRange(localHandle, buffer, position, length);
|
||||
if (transfer.cancelled) throw new Error("Transfer cancelled");
|
||||
await writeSftpRange(sftp, remoteHandle, buffer, position, length);
|
||||
},
|
||||
@@ -4550,16 +4485,13 @@ async function uploadFileConcurrent(
|
||||
// published at this point, so stop accepting cancellation before source
|
||||
// revalidation and handle cleanup; staged uploads pass no callback.
|
||||
options.onBytesCommitted?.();
|
||||
const contentVerifiedSeparately = Boolean(digestHandle || ephemeralDigestPath || transfer.sourceDigestPath);
|
||||
if (initialSource) {
|
||||
const latestSource = await localHandle.stat();
|
||||
// Soft size + mtime/ino only — no digest, but not fail-open either.
|
||||
assertSourceMetadataUnchanged(initialSource, latestSource, fileSize, {
|
||||
contentVerifiedSeparately,
|
||||
ignoreCtime: true,
|
||||
});
|
||||
}
|
||||
if (ephemeralDigestPath) {
|
||||
await verifyUploadDigestBaseline(localPath, ephemeralDigestPath, fileSize, transfer);
|
||||
}
|
||||
} catch (error) {
|
||||
failed = true;
|
||||
throw error;
|
||||
@@ -4592,12 +4524,6 @@ async function uploadFileConcurrent(
|
||||
if (localHandle) {
|
||||
await localHandle.close().catch(() => {});
|
||||
}
|
||||
if (digestHandle) {
|
||||
await digestHandle.close().catch(() => {});
|
||||
}
|
||||
if (ephemeralDigestPath) {
|
||||
await fs.promises.rm(ephemeralDigestPath, { force: true }).catch(() => {});
|
||||
}
|
||||
let remoteCloseError = null;
|
||||
// Close remote handles while the channel is still live. On disposeChannel
|
||||
// cancel/failure the channel is about to be ended (or already dead); a
|
||||
@@ -5903,24 +5829,10 @@ async function startTransferNow(event, payload, onProgress) {
|
||||
))
|
||||
: 0;
|
||||
sendProgress(transfer.checkpointBytes, fileSize, { force: true });
|
||||
if (usesStage && transfer.checkpointBytes > 0) {
|
||||
const verifyBytes = resumeContentVerifyBytes(
|
||||
transfer.checkpointBytes,
|
||||
transfer.sourceFingerprint,
|
||||
);
|
||||
await verifyResumeContent(
|
||||
verifyBytes,
|
||||
(options) => hashLocalPrefix(sourcePath, verifyBytes, options),
|
||||
(options) => hashRemotePrefix(
|
||||
client,
|
||||
targetSftpId,
|
||||
uploadTargetPath,
|
||||
targetEncoding,
|
||||
verifyBytes,
|
||||
options,
|
||||
),
|
||||
);
|
||||
}
|
||||
// Resume is size-based only (WinSCP/FileZilla): the durable .part size
|
||||
// is the checkpoint. Do not re-hash [0, checkpoint) on both ends —
|
||||
// that blocked large-file resume for seconds-to-minutes with no bytes
|
||||
// moving.
|
||||
await uploadFile(
|
||||
sourcePath,
|
||||
encodedUploadPath,
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -876,8 +876,97 @@ function createPreloadApi(ctx) {
|
||||
statLocal: async (path) => {
|
||||
return ipcRenderer.invoke("netcatty:local:stat", { path });
|
||||
},
|
||||
listLocalTree: async (path) => {
|
||||
return ipcRenderer.invoke("netcatty:local:tree", { path });
|
||||
listLocalTree: async (path, options = {}) => {
|
||||
const onProgress = typeof options?.onProgress === "function" ? options.onProgress : null;
|
||||
const onEntries = typeof options?.onEntries === "function" ? options.onEntries : null;
|
||||
const token = typeof options?.scanId === "string" && options.scanId
|
||||
? options.scanId
|
||||
: `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
const progressChannel = onProgress
|
||||
? `netcatty:local:tree-progress:${token}`
|
||||
: undefined;
|
||||
const entriesChannel = onEntries
|
||||
? `netcatty:local:tree-entries:${token}`
|
||||
: undefined;
|
||||
const cancelChannel = `netcatty:local:tree-cancel:${token}`;
|
||||
const progressHandler = onProgress && progressChannel
|
||||
? (_event, stats) => {
|
||||
try {
|
||||
onProgress(stats);
|
||||
} catch {
|
||||
// Ignore renderer progress handler errors so the scan can finish.
|
||||
}
|
||||
}
|
||||
: null;
|
||||
// Entry batches arrive on a different channel than the invoke reply. If we
|
||||
// tear down the listener when invoke resolves, nested-folder batches that
|
||||
// are still in the IPC queue are dropped (top-level files look fine).
|
||||
let settleEntriesStream = null;
|
||||
const entriesStreamDone = onEntries && entriesChannel
|
||||
? new Promise((resolve) => {
|
||||
settleEntriesStream = resolve;
|
||||
})
|
||||
: null;
|
||||
const entriesHandler = onEntries && entriesChannel
|
||||
? (_event, payload) => {
|
||||
if (payload && typeof payload === "object" && !Array.isArray(payload) && payload.type === "tree-end") {
|
||||
settleEntriesStream?.();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (Array.isArray(payload)) onEntries(payload);
|
||||
} catch {
|
||||
// Ignore batch handler errors so the scan can finish.
|
||||
}
|
||||
}
|
||||
: null;
|
||||
if (progressHandler && progressChannel) {
|
||||
ipcRenderer.on(progressChannel, progressHandler);
|
||||
}
|
||||
if (entriesHandler && entriesChannel) {
|
||||
ipcRenderer.on(entriesChannel, entriesHandler);
|
||||
}
|
||||
try {
|
||||
const result = await ipcRenderer.invoke("netcatty:local:tree", {
|
||||
path,
|
||||
progressChannel,
|
||||
entriesChannel,
|
||||
cancelChannel,
|
||||
limits: options?.limits,
|
||||
});
|
||||
if (entriesStreamDone) {
|
||||
// Wait for the main-process end marker so late nested batches are not
|
||||
// lost. Fail the scan if the marker never arrives (do not pretend a
|
||||
// partial tree is complete).
|
||||
let timedOut = false;
|
||||
await Promise.race([
|
||||
entriesStreamDone,
|
||||
new Promise((resolve) => {
|
||||
setTimeout(() => {
|
||||
timedOut = true;
|
||||
resolve();
|
||||
}, 5_000);
|
||||
}),
|
||||
]);
|
||||
if (timedOut) {
|
||||
throw new Error("Local tree scan ended without a completion marker");
|
||||
}
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
settleEntriesStream?.();
|
||||
if (progressHandler && progressChannel) {
|
||||
ipcRenderer.removeListener(progressChannel, progressHandler);
|
||||
}
|
||||
if (entriesHandler && entriesChannel) {
|
||||
ipcRenderer.removeListener(entriesChannel, entriesHandler);
|
||||
}
|
||||
}
|
||||
},
|
||||
cancelLocalTreeScan: async (scanId) => {
|
||||
if (typeof scanId === "string" && scanId) {
|
||||
ipcRenderer.send(`netcatty:local:tree-cancel:${scanId}`);
|
||||
}
|
||||
},
|
||||
getHomeDir: async () => {
|
||||
return ipcRenderer.invoke("netcatty:local:homedir");
|
||||
|
||||
@@ -106,3 +106,88 @@ test("failed upload path allocation releases the staging controller", async () =
|
||||
await assert.rejects(api.stageUploadFile(createFile("file.bin", [[1]]), "failed-path"), /path unavailable/);
|
||||
assert.deepEqual(await api.cancelStagedUploadFile("failed-path"), { success: false });
|
||||
});
|
||||
|
||||
test("native tree scans use a bridge-safe cancellation id", async () => {
|
||||
const sent = [];
|
||||
const invoked = [];
|
||||
const api = createPreloadApi({
|
||||
webUtils: {},
|
||||
ipcRenderer: {
|
||||
on() {},
|
||||
removeListener() {},
|
||||
send(...args) { sent.push(args); },
|
||||
async invoke(channel, payload) {
|
||||
invoked.push({ channel, payload });
|
||||
return [];
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await api.listLocalTree("/tmp/project", {
|
||||
scanId: "scan-123",
|
||||
onProgress: () => {},
|
||||
});
|
||||
await api.cancelLocalTreeScan("scan-123");
|
||||
|
||||
assert.deepEqual(invoked, [{
|
||||
channel: "netcatty:local:tree",
|
||||
payload: {
|
||||
path: "/tmp/project",
|
||||
progressChannel: "netcatty:local:tree-progress:scan-123",
|
||||
entriesChannel: undefined,
|
||||
cancelChannel: "netcatty:local:tree-cancel:scan-123",
|
||||
limits: undefined,
|
||||
},
|
||||
}]);
|
||||
assert.deepEqual(sent, [["netcatty:local:tree-cancel:scan-123"]]);
|
||||
});
|
||||
|
||||
test("listLocalTree keeps the entries listener until the tree-end marker arrives", async () => {
|
||||
const listeners = new Map();
|
||||
const batches = [];
|
||||
let removed = false;
|
||||
const api = createPreloadApi({
|
||||
webUtils: {},
|
||||
ipcRenderer: {
|
||||
on(channel, handler) {
|
||||
listeners.set(channel, handler);
|
||||
},
|
||||
removeListener(channel) {
|
||||
if (channel.startsWith("netcatty:local:tree-entries:")) removed = true;
|
||||
listeners.delete(channel);
|
||||
},
|
||||
send() {},
|
||||
async invoke(channel, payload) {
|
||||
assert.equal(channel, "netcatty:local:tree");
|
||||
const entriesChannel = payload.entriesChannel;
|
||||
const handler = listeners.get(entriesChannel);
|
||||
assert.equal(typeof handler, "function");
|
||||
// Simulate the invoke reply racing ahead of a late nested batch.
|
||||
queueMicrotask(() => {
|
||||
handler({}, [
|
||||
{
|
||||
localPath: "/tmp/project/nested/deep.txt",
|
||||
relativePath: "project/nested/deep.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
handler({}, { type: "tree-end" });
|
||||
});
|
||||
return [];
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await api.listLocalTree("/tmp/project", {
|
||||
scanId: "scan-nested",
|
||||
onEntries: (batch) => {
|
||||
batches.push(batch);
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(batches.length, 1);
|
||||
assert.equal(batches[0][0].relativePath, "project/nested/deep.txt");
|
||||
assert.equal(removed, true);
|
||||
});
|
||||
@@ -204,7 +204,10 @@ export const STORAGE_KEY_AI_EXTERNAL_MCP_SILENT_SESSIONS = 'netcatty_ai_external
|
||||
|
||||
// SFTP Transfer Concurrency
|
||||
export const STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY = 'netcatty_sftp_transfer_concurrency_v1';
|
||||
/** Folder transfers: scan the full tree before moving bytes (stable file totals). */
|
||||
/**
|
||||
* Legacy key only. Folder full-tree pre-scan was removed; values are ignored
|
||||
* so old localStorage / sync payloads do not resurrect a live setting.
|
||||
*/
|
||||
export const STORAGE_KEY_SFTP_FOLDER_PRESCAN = 'netcatty_sftp_folder_prescan_v1';
|
||||
/** Skip files when target size + mtime already match the source (rsync-like). */
|
||||
export const STORAGE_KEY_SFTP_SKIP_UNCHANGED = 'netcatty_sftp_skip_unchanged_v1';
|
||||
|
||||
@@ -0,0 +1,628 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { uploadLocalFoldersProgressively } from "./progressiveFolderUpload.ts";
|
||||
import { UploadController } from "./uploadController.ts";
|
||||
import type { LocalTreeListEntry } from "./sftpFileUtils.ts";
|
||||
|
||||
test("progressive folder upload starts file transfers before the tree walk finishes", async () => {
|
||||
const events: string[] = [];
|
||||
let releaseSecondBatch: (() => void) | null = null;
|
||||
const secondBatchGate = new Promise<void>((resolve) => {
|
||||
releaseSecondBatch = resolve;
|
||||
});
|
||||
|
||||
const listLocalTree = async (
|
||||
_path: string,
|
||||
options: {
|
||||
onEntries?: (entries: LocalTreeListEntry[]) => void;
|
||||
},
|
||||
) => {
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/docs",
|
||||
relativePath: "docs",
|
||||
type: "directory",
|
||||
size: 0,
|
||||
lastModified: 1,
|
||||
},
|
||||
{
|
||||
localPath: "/tmp/docs/a.txt",
|
||||
relativePath: "docs/a.txt",
|
||||
type: "file",
|
||||
size: 3,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
events.push("batch1");
|
||||
// First file should be uploadable while we still "discover" more.
|
||||
await secondBatchGate;
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/docs/b.txt",
|
||||
relativePath: "docs/b.txt",
|
||||
type: "file",
|
||||
size: 4,
|
||||
lastModified: 2,
|
||||
},
|
||||
]);
|
||||
events.push("batch2");
|
||||
return [];
|
||||
};
|
||||
|
||||
const transferred: string[] = [];
|
||||
const controller = new UploadController();
|
||||
const uploadPromise = uploadLocalFoldersProgressively(
|
||||
[{ name: "docs", localPath: "/tmp/docs" }],
|
||||
{
|
||||
targetPath: "/remote",
|
||||
sftpId: "sftp-1",
|
||||
isLocal: false,
|
||||
joinPath: (base, name) => `${base}/${name}`,
|
||||
bridge: {
|
||||
mkdirSftp: async () => {},
|
||||
startStreamTransfer: async (payload) => {
|
||||
transferred.push(payload.sourcePath);
|
||||
events.push(`upload:${payload.sourcePath}`);
|
||||
if (payload.sourcePath.endsWith("a.txt")) {
|
||||
// Let the second discovery batch proceed only after the first upload started.
|
||||
releaseSecondBatch?.();
|
||||
}
|
||||
return { transferId: payload.transferId };
|
||||
},
|
||||
},
|
||||
listLocalTree,
|
||||
callbacks: {
|
||||
onTaskCreated: (task) => events.push(`created:${task.fileName}`),
|
||||
onTaskCompleted: (taskId) => events.push(`completed:${taskId.slice(0, 8)}`),
|
||||
onTaskProgress: (taskId, progress) => {
|
||||
events.push(`progress:${progress.transferred}/${progress.total}:${progress.phase ?? ""}`);
|
||||
},
|
||||
},
|
||||
},
|
||||
controller,
|
||||
);
|
||||
|
||||
const results = await uploadPromise;
|
||||
assert.equal(results.filter((row) => row.success).length, 2);
|
||||
assert.deepEqual(transferred, ["/tmp/docs/a.txt", "/tmp/docs/b.txt"]);
|
||||
// First upload must happen before the second discovery batch finishes.
|
||||
const uploadA = events.indexOf("upload:/tmp/docs/a.txt");
|
||||
const batch2 = events.indexOf("batch2");
|
||||
assert.ok(uploadA >= 0 && batch2 >= 0, "expected both upload and batch events");
|
||||
assert.ok(uploadA < batch2, "first file must upload while scan still running");
|
||||
});
|
||||
|
||||
test("progressive folder upload enqueues nested subdirectory files", async () => {
|
||||
const transferred: string[] = [];
|
||||
const mkdirs: string[] = [];
|
||||
|
||||
const listLocalTree = async (
|
||||
_path: string,
|
||||
options: {
|
||||
onEntries?: (entries: LocalTreeListEntry[]) => void;
|
||||
},
|
||||
) => {
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/docs",
|
||||
relativePath: "docs",
|
||||
type: "directory",
|
||||
size: 0,
|
||||
lastModified: 1,
|
||||
},
|
||||
{
|
||||
localPath: "/tmp/docs/a.txt",
|
||||
relativePath: "docs/a.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/docs/nested",
|
||||
relativePath: "docs/nested",
|
||||
type: "directory",
|
||||
size: 0,
|
||||
lastModified: 1,
|
||||
},
|
||||
{
|
||||
localPath: "/tmp/docs/nested/deep",
|
||||
relativePath: "docs/nested/deep",
|
||||
type: "directory",
|
||||
size: 0,
|
||||
lastModified: 1,
|
||||
},
|
||||
{
|
||||
localPath: "/tmp/docs/nested/deep/b.txt",
|
||||
relativePath: "docs/nested/deep/b.txt",
|
||||
type: "file",
|
||||
size: 2,
|
||||
lastModified: 2,
|
||||
},
|
||||
]);
|
||||
return [];
|
||||
};
|
||||
|
||||
const results = await uploadLocalFoldersProgressively(
|
||||
[{ name: "docs", localPath: "/tmp/docs" }],
|
||||
{
|
||||
targetPath: "/remote",
|
||||
sftpId: "sftp-1",
|
||||
isLocal: false,
|
||||
joinPath: (base, name) => `${base}/${name}`,
|
||||
bridge: {
|
||||
mkdirSftp: async (_id, dirPath) => {
|
||||
mkdirs.push(dirPath);
|
||||
},
|
||||
startStreamTransfer: async (payload) => {
|
||||
transferred.push(payload.targetPath);
|
||||
return { transferId: payload.transferId };
|
||||
},
|
||||
},
|
||||
listLocalTree,
|
||||
},
|
||||
);
|
||||
|
||||
assert.equal(results.filter((row) => row.success).length, 2);
|
||||
assert.deepEqual(transferred.sort(), [
|
||||
"/remote/docs/a.txt",
|
||||
"/remote/docs/nested/deep/b.txt",
|
||||
].sort());
|
||||
assert.ok(mkdirs.includes("/remote/docs/nested"));
|
||||
assert.ok(mkdirs.includes("/remote/docs/nested/deep"));
|
||||
});
|
||||
|
||||
test("progressive folder upload stops enqueueing children after cancel", async () => {
|
||||
const controller = new UploadController();
|
||||
const createdChildren: string[] = [];
|
||||
let entriesCb: ((entries: LocalTreeListEntry[]) => void) | null = null;
|
||||
|
||||
const listLocalTree = async (
|
||||
_path: string,
|
||||
options: {
|
||||
onEntries?: (entries: LocalTreeListEntry[]) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
},
|
||||
) => {
|
||||
entriesCb = options.onEntries ?? null;
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/big/a.txt",
|
||||
relativePath: "big/a.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
// Wait until cancelled.
|
||||
await new Promise<void>((resolve) => {
|
||||
if (options.abortSignal?.aborted) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
options.abortSignal?.addEventListener("abort", () => resolve(), { once: true });
|
||||
});
|
||||
return [];
|
||||
};
|
||||
|
||||
const abort = new AbortController();
|
||||
const uploadPromise = uploadLocalFoldersProgressively(
|
||||
[{ name: "big", localPath: "/tmp/big" }],
|
||||
{
|
||||
targetPath: "/remote",
|
||||
sftpId: "sftp-1",
|
||||
isLocal: false,
|
||||
joinPath: (base, name) => `${base}/${name}`,
|
||||
abortSignal: abort.signal,
|
||||
bridge: {
|
||||
mkdirSftp: async () => {},
|
||||
startStreamTransfer: async (payload) => {
|
||||
// Cancel as soon as the first child begins.
|
||||
await controller.cancel();
|
||||
abort.abort();
|
||||
// Push more discovered files after cancel — workers must not create more children.
|
||||
entriesCb?.([
|
||||
{
|
||||
localPath: "/tmp/big/b.txt",
|
||||
relativePath: "big/b.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
{
|
||||
localPath: "/tmp/big/c.txt",
|
||||
relativePath: "big/c.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
return { transferId: payload.transferId, cancelled: true };
|
||||
},
|
||||
},
|
||||
listLocalTree,
|
||||
callbacks: {
|
||||
onTaskCreated: (task) => {
|
||||
if (!task.isDirectory) createdChildren.push(task.fileName);
|
||||
},
|
||||
},
|
||||
},
|
||||
controller,
|
||||
);
|
||||
|
||||
await uploadPromise;
|
||||
// At most the in-flight child should have been created.
|
||||
assert.ok(createdChildren.length <= 1, `expected no post-cancel flood, got ${createdChildren.join(",")}`);
|
||||
});
|
||||
|
||||
test("progressive folder upload stops enqueueing children while soft-paused", async () => {
|
||||
const createdChildren: string[] = [];
|
||||
const transferred: string[] = [];
|
||||
let paused = true;
|
||||
let releasePause!: () => void;
|
||||
const pauseGate = new Promise<void>((resolve) => {
|
||||
releasePause = resolve;
|
||||
});
|
||||
let releaseMoreFiles!: () => void;
|
||||
const moreFilesGate = new Promise<void>((resolve) => {
|
||||
releaseMoreFiles = resolve;
|
||||
});
|
||||
|
||||
const waitWhilePaused = async () => {
|
||||
while (paused) {
|
||||
await pauseGate;
|
||||
}
|
||||
};
|
||||
|
||||
const listLocalTree = async (
|
||||
_path: string,
|
||||
options: {
|
||||
onEntries?: (entries: LocalTreeListEntry[]) => void;
|
||||
},
|
||||
) => {
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/docs/a.txt",
|
||||
relativePath: "docs/a.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
// Stay in the walk until the test unblocks the second batch (after pause assert).
|
||||
await moreFilesGate;
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/docs/b.txt",
|
||||
relativePath: "docs/b.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 2,
|
||||
},
|
||||
{
|
||||
localPath: "/tmp/docs/c.txt",
|
||||
relativePath: "docs/c.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 3,
|
||||
},
|
||||
]);
|
||||
return [];
|
||||
};
|
||||
|
||||
const uploadPromise = uploadLocalFoldersProgressively(
|
||||
[{ name: "docs", localPath: "/tmp/docs" }],
|
||||
{
|
||||
targetPath: "/remote",
|
||||
sftpId: "sftp-1",
|
||||
isLocal: false,
|
||||
joinPath: (base, name) => `${base}/${name}`,
|
||||
waitWhilePaused,
|
||||
bridge: {
|
||||
mkdirSftp: async () => {},
|
||||
startStreamTransfer: async (payload) => {
|
||||
transferred.push(payload.sourcePath);
|
||||
return { transferId: payload.transferId };
|
||||
},
|
||||
},
|
||||
listLocalTree,
|
||||
callbacks: {
|
||||
onTaskCreated: (task) => {
|
||||
if (!task.isDirectory) createdChildren.push(task.fileName);
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Give workers a turn; soft-pause must block child creation entirely.
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
assert.deepEqual(createdChildren, [], "no children while paused");
|
||||
assert.deepEqual(transferred, [], "no transfers while paused");
|
||||
|
||||
paused = false;
|
||||
releasePause();
|
||||
// Let first file create/upload, then release the rest of the tree.
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
releaseMoreFiles();
|
||||
|
||||
const results = await uploadPromise;
|
||||
assert.equal(results.filter((row) => row.success).length, 3);
|
||||
assert.deepEqual(
|
||||
createdChildren.sort(),
|
||||
["docs/a.txt", "docs/b.txt", "docs/c.txt"].sort(),
|
||||
);
|
||||
assert.equal(transferred.length, 3);
|
||||
});
|
||||
|
||||
test("progressive multi-root pause does not HOL-block an unpaused sibling root", async () => {
|
||||
const transferred: string[] = [];
|
||||
const pausedParents = new Set<string>(["parent-a"]);
|
||||
const parentIds = new Map([
|
||||
["folderA", "parent-a"],
|
||||
["folderB", "parent-b"],
|
||||
]);
|
||||
|
||||
const waitWhilePaused = async (parentId: string) => {
|
||||
while (pausedParents.has(parentId)) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
};
|
||||
|
||||
const listLocalTree = async (
|
||||
localPath: string,
|
||||
options: {
|
||||
onEntries?: (entries: LocalTreeListEntry[]) => void;
|
||||
},
|
||||
) => {
|
||||
if (localPath.endsWith("folderA")) {
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/folderA/a.txt",
|
||||
relativePath: "folderA/a.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
} else {
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/folderB/b.txt",
|
||||
relativePath: "folderB/b.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const uploadPromise = uploadLocalFoldersProgressively(
|
||||
[
|
||||
{ name: "folderA", localPath: "/tmp/folderA" },
|
||||
{ name: "folderB", localPath: "/tmp/folderB" },
|
||||
],
|
||||
{
|
||||
targetPath: "/remote",
|
||||
sftpId: "sftp-1",
|
||||
isLocal: false,
|
||||
joinPath: (base, name) => `${base}/${name}`,
|
||||
parentTaskIds: parentIds,
|
||||
waitWhilePaused,
|
||||
isPaused: (parentId) => pausedParents.has(parentId),
|
||||
bridge: {
|
||||
mkdirSftp: async () => {},
|
||||
startStreamTransfer: async (payload) => {
|
||||
transferred.push(payload.sourcePath);
|
||||
// Unpause A only after B has uploaded — proves HOL skip works.
|
||||
if (payload.sourcePath.includes("folderB")) {
|
||||
pausedParents.delete("parent-a");
|
||||
}
|
||||
return { transferId: payload.transferId };
|
||||
},
|
||||
},
|
||||
listLocalTree,
|
||||
},
|
||||
);
|
||||
|
||||
const results = await uploadPromise;
|
||||
assert.equal(results.filter((row) => row.success).length, 2);
|
||||
assert.ok(transferred.some((p) => p.includes("folderB")), "unpaused root must transfer");
|
||||
assert.ok(transferred.some((p) => p.includes("folderA")), "paused root resumes after unlatch");
|
||||
// folderB must not wait behind folderA indefinitely — B completes first.
|
||||
assert.ok(
|
||||
transferred.indexOf(transferred.find((p) => p.includes("folderB"))!)
|
||||
< transferred.indexOf(transferred.find((p) => p.includes("folderA"))!),
|
||||
`expected B before A, got ${transferred.join(",")}`,
|
||||
);
|
||||
});
|
||||
|
||||
test("progressive backpressure wakes every parked enqueue waiter", async () => {
|
||||
// Discovery floods the queue while workers hold transfers open so the queue
|
||||
// stays above the high-water mark. Multiple enqueueBatch waiters must all
|
||||
// be released when workers drain past the low-water mark.
|
||||
const transferred: string[] = [];
|
||||
let releaseTransfers!: () => void;
|
||||
const transferGate = new Promise<void>((resolve) => {
|
||||
releaseTransfers = resolve;
|
||||
});
|
||||
let started = 0;
|
||||
|
||||
const listLocalTree = async (
|
||||
_path: string,
|
||||
options: {
|
||||
onEntries?: (entries: LocalTreeListEntry[]) => void;
|
||||
},
|
||||
) => {
|
||||
const batches = Array.from({ length: 8 }, (_, batch) => (
|
||||
Array.from({ length: 300 }, (__, i) => ({
|
||||
localPath: `/tmp/docs/f-${batch}-${i}.txt`,
|
||||
relativePath: `docs/f-${batch}-${i}.txt`,
|
||||
type: "file" as const,
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
}))
|
||||
));
|
||||
// Overlapping handlers so several park on waitIfQueueHigh together.
|
||||
await Promise.all(batches.map((batch) => Promise.resolve().then(() => options.onEntries?.(batch))));
|
||||
return [];
|
||||
};
|
||||
|
||||
const uploadPromise = uploadLocalFoldersProgressively(
|
||||
[{ name: "docs", localPath: "/tmp/docs" }],
|
||||
{
|
||||
targetPath: "/remote",
|
||||
sftpId: "sftp-1",
|
||||
isLocal: false,
|
||||
joinPath: (base, name) => `${base}/${name}`,
|
||||
bridge: {
|
||||
mkdirSftp: async () => {},
|
||||
startStreamTransfer: async (payload) => {
|
||||
started += 1;
|
||||
transferred.push(payload.sourcePath);
|
||||
await transferGate;
|
||||
return { transferId: payload.transferId };
|
||||
},
|
||||
},
|
||||
listLocalTree,
|
||||
},
|
||||
);
|
||||
|
||||
// Let discovery fill past high-water (2000) while 2 workers hold the gate.
|
||||
await new Promise((resolve) => setTimeout(resolve, 80));
|
||||
assert.ok(started >= 1, "workers should have started");
|
||||
releaseTransfers();
|
||||
const results = await uploadPromise;
|
||||
assert.equal(results.filter((row) => row.success).length, 8 * 300);
|
||||
assert.equal(transferred.length, 8 * 300);
|
||||
});
|
||||
|
||||
test("progressive root conflict skip does not overwrite an existing remote folder", async () => {
|
||||
const transferred: string[] = [];
|
||||
const result = await uploadLocalFoldersProgressively(
|
||||
[{ name: "docs", localPath: "/tmp/docs" }],
|
||||
{
|
||||
targetPath: "/remote",
|
||||
sftpId: "sftp-1",
|
||||
isLocal: false,
|
||||
joinPath: (base, name) => `${base}/${name}`,
|
||||
resolveConflict: async () => "skip",
|
||||
bridge: {
|
||||
mkdirSftp: async () => {},
|
||||
statSftp: async () => ({ type: "directory", size: 0, lastModified: 1 }),
|
||||
startStreamTransfer: async (payload) => {
|
||||
transferred.push(payload.sourcePath);
|
||||
return { transferId: payload.transferId };
|
||||
},
|
||||
},
|
||||
listLocalTree: async () => {
|
||||
throw new Error("scan must not run after skip");
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(transferred.length, 0);
|
||||
assert.equal(result[0]?.cancelled, true);
|
||||
});
|
||||
|
||||
test("progressive multi-root resume of non-head parent unblocks while head stays paused", async () => {
|
||||
const transferred: string[] = [];
|
||||
// Both latched initially; head is always parent-a in FIFO order.
|
||||
const pausedParents = new Set<string>(["parent-a", "parent-b"]);
|
||||
const parentIds = new Map([
|
||||
["folderA", "parent-a"],
|
||||
["folderB", "parent-b"],
|
||||
]);
|
||||
const waiters = new Map<string, Array<() => void>>();
|
||||
|
||||
const waitWhilePaused = async (parentId: string) => {
|
||||
while (pausedParents.has(parentId)) {
|
||||
await new Promise<void>((resolve) => {
|
||||
const list = waiters.get(parentId) ?? [];
|
||||
list.push(resolve);
|
||||
waiters.set(parentId, list);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const release = (parentId: string) => {
|
||||
pausedParents.delete(parentId);
|
||||
for (const resolve of waiters.get(parentId) ?? []) resolve();
|
||||
waiters.delete(parentId);
|
||||
};
|
||||
|
||||
const listLocalTree = async (
|
||||
localPath: string,
|
||||
options: {
|
||||
onEntries?: (entries: LocalTreeListEntry[]) => void;
|
||||
},
|
||||
) => {
|
||||
if (localPath.endsWith("folderA")) {
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/folderA/a.txt",
|
||||
relativePath: "folderA/a.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
} else {
|
||||
options.onEntries?.([
|
||||
{
|
||||
localPath: "/tmp/folderB/b.txt",
|
||||
relativePath: "folderB/b.txt",
|
||||
type: "file",
|
||||
size: 1,
|
||||
lastModified: 1,
|
||||
},
|
||||
]);
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const uploadPromise = uploadLocalFoldersProgressively(
|
||||
[
|
||||
{ name: "folderA", localPath: "/tmp/folderA" },
|
||||
{ name: "folderB", localPath: "/tmp/folderB" },
|
||||
],
|
||||
{
|
||||
targetPath: "/remote",
|
||||
sftpId: "sftp-1",
|
||||
isLocal: false,
|
||||
joinPath: (base, name) => `${base}/${name}`,
|
||||
parentTaskIds: parentIds,
|
||||
waitWhilePaused,
|
||||
isPaused: (parentId) => pausedParents.has(parentId),
|
||||
bridge: {
|
||||
mkdirSftp: async () => {},
|
||||
startStreamTransfer: async (payload) => {
|
||||
transferred.push(payload.sourcePath);
|
||||
return { transferId: payload.transferId };
|
||||
},
|
||||
},
|
||||
listLocalTree,
|
||||
},
|
||||
);
|
||||
|
||||
// Let both jobs queue and workers park on all-paused race.
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
assert.deepEqual(transferred, []);
|
||||
// Resume only non-head parent B while A stays paused.
|
||||
release("parent-b");
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
assert.ok(
|
||||
transferred.some((p) => p.includes("folderB")),
|
||||
`B must transfer while A paused, got ${transferred.join(",")}`,
|
||||
);
|
||||
assert.equal(
|
||||
transferred.some((p) => p.includes("folderA")),
|
||||
false,
|
||||
"A must stay blocked",
|
||||
);
|
||||
release("parent-a");
|
||||
const results = await uploadPromise;
|
||||
assert.equal(results.filter((row) => row.success).length, 2);
|
||||
});
|
||||
@@ -0,0 +1,669 @@
|
||||
/**
|
||||
* Progressive local-folder upload: stream discovery batches from listLocalTree
|
||||
* and upload files as they arrive (edge-scan / edge-transfer).
|
||||
*/
|
||||
|
||||
import type { DropEntry } from "./sftpFileUtils";
|
||||
import { localTreeToDropEntries, type LocalTreeListEntry } from "./sftpFileUtils";
|
||||
import type { UploadBridge, UploadCallbacks, UploadResult } from "./uploadService.types";
|
||||
import type { UploadController } from "./uploadController";
|
||||
import {
|
||||
canReplaceSftpConflict,
|
||||
describeSftpExistingKind,
|
||||
describeSftpIncomingKind,
|
||||
getSftpConflictTypeKey,
|
||||
} from "../domain/sftpConflict";
|
||||
|
||||
const formatUploadError = (error: unknown): string =>
|
||||
error instanceof Error ? error.message : String(error);
|
||||
|
||||
const UPLOAD_CONCURRENCY = 2;
|
||||
/** Pause discovery when the upload queue grows this large (memory backpressure). */
|
||||
const QUEUE_HIGH_WATER = 2_000;
|
||||
/** Resume discovery once the queue drains to this size. */
|
||||
const QUEUE_LOW_WATER = 500;
|
||||
|
||||
export type ProgressiveLocalRoot = {
|
||||
name: string;
|
||||
localPath: string;
|
||||
};
|
||||
|
||||
export type ListLocalTreeStreaming = (
|
||||
path: string,
|
||||
options: {
|
||||
onProgress?: (progress: { fileCount: number; directoryCount: number; entryCount: number }) => void;
|
||||
onEntries?: (entries: LocalTreeListEntry[]) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
},
|
||||
) => Promise<LocalTreeListEntry[]>;
|
||||
|
||||
export type ProgressiveConflictAction = "stop" | "skip" | "replace" | "duplicate" | "merge";
|
||||
|
||||
export type ProgressiveFolderUploadConfig = {
|
||||
targetPath: string;
|
||||
sftpId: string | null;
|
||||
targetHostId?: string;
|
||||
isLocal: boolean;
|
||||
bridge: UploadBridge;
|
||||
joinPath: (base: string, name: string) => string;
|
||||
callbacks?: UploadCallbacks;
|
||||
listLocalTree: ListLocalTreeStreaming;
|
||||
/** Optional pre-created parent task ids (e.g. scanning task id for single root). */
|
||||
parentTaskIds?: Map<string, string>;
|
||||
abortSignal?: AbortSignal;
|
||||
/**
|
||||
* Soft-pause gate (process-global latch). Must resolve only after the parent
|
||||
* folder is resumed. Used before discovery enqueue and before creating child
|
||||
* UI rows so Pause does not keep filling the transfer queue.
|
||||
*/
|
||||
waitWhilePaused?: (parentTaskId: string) => Promise<void>;
|
||||
/** Sync probe so multi-root queues can skip latched parents (no HOL block). */
|
||||
isPaused?: (parentTaskId: string) => boolean;
|
||||
/**
|
||||
* Root-level conflict dialog (same contract as uploadEntries). Progressive
|
||||
* path must not overwrite existing remote folders without confirmation.
|
||||
*/
|
||||
resolveConflict?: (conflict: {
|
||||
fileName: string;
|
||||
targetPath: string;
|
||||
isDirectory: boolean;
|
||||
existingType?: "file" | "directory" | "symlink";
|
||||
existingSize: number;
|
||||
newSize: number;
|
||||
existingModified: number;
|
||||
newModified: number;
|
||||
applyToAllCount: number;
|
||||
}) => Promise<ProgressiveConflictAction>;
|
||||
};
|
||||
|
||||
function entryToDrop(entry: LocalTreeListEntry): DropEntry {
|
||||
return localTreeToDropEntries([entry])[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk each local root with streaming batches and upload files concurrently
|
||||
* while discovery continues.
|
||||
*/
|
||||
export async function uploadLocalFoldersProgressively(
|
||||
roots: ProgressiveLocalRoot[],
|
||||
config: ProgressiveFolderUploadConfig,
|
||||
controller?: UploadController,
|
||||
): Promise<UploadResult[]> {
|
||||
const {
|
||||
targetPath,
|
||||
sftpId,
|
||||
targetHostId,
|
||||
isLocal,
|
||||
bridge,
|
||||
joinPath,
|
||||
callbacks,
|
||||
listLocalTree,
|
||||
parentTaskIds,
|
||||
abortSignal,
|
||||
waitWhilePaused,
|
||||
isPaused,
|
||||
resolveConflict,
|
||||
} = config;
|
||||
|
||||
if (roots.length === 0) return [];
|
||||
if (!isLocal && !sftpId) {
|
||||
throw new Error("No SFTP session for progressive folder upload");
|
||||
}
|
||||
|
||||
const isStopped = () => !!(controller?.isCancelled() || abortSignal?.aborted);
|
||||
|
||||
/** Block while the folder soft-pause latch is held; re-check cancel after wake. */
|
||||
const awaitUnpaused = async (parentTaskId: string): Promise<boolean> => {
|
||||
if (!waitWhilePaused) return !isStopped();
|
||||
while (!isStopped()) {
|
||||
await waitWhilePaused(parentTaskId);
|
||||
// waitWhilePaused may resolve spuriously if the latch was already free;
|
||||
// always re-check stop intent after any await.
|
||||
if (isStopped()) return false;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const results: UploadResult[] = [];
|
||||
const createdDirs = new Set<string>();
|
||||
const failedDirs = new Map<string, string>();
|
||||
const parentIds = new Map<string, string>();
|
||||
const parentStats = new Map<string, {
|
||||
discovered: number;
|
||||
completed: number;
|
||||
failed: number;
|
||||
dirFailures: number;
|
||||
}>();
|
||||
/** Source root name -> destination root segment after conflict resolution. */
|
||||
const destRootNameBySource = new Map<string, string>();
|
||||
|
||||
const statTarget = async (path: string) => {
|
||||
try {
|
||||
if (isLocal) return await bridge.statLocal?.(path) ?? null;
|
||||
if (sftpId) return await bridge.statSftp?.(sftpId, path) ?? null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
const deleteTarget = async (path: string) => {
|
||||
if (isLocal) await bridge.deleteLocalFile?.(path);
|
||||
else if (sftpId) await bridge.deleteSftp?.(sftpId, path);
|
||||
};
|
||||
|
||||
const getDuplicateName = async (name: string) => {
|
||||
for (let index = 1; index < 1000; index++) {
|
||||
const suffix = index === 1 ? " (copy)" : ` (copy ${index})`;
|
||||
const candidate = `${name}${suffix}`;
|
||||
const existing = await statTarget(joinPath(targetPath, candidate));
|
||||
if (!existing) return candidate;
|
||||
}
|
||||
return `${name} (copy ${Date.now()})`;
|
||||
};
|
||||
|
||||
// Root-level conflict preflight — match uploadEntries so progressive drops
|
||||
// cannot overwrite remote content without Skip/Replace/Duplicate/Merge.
|
||||
let activeRoots = [...roots];
|
||||
if (resolveConflict) {
|
||||
const existingByRoot = await Promise.all(activeRoots.map(async (root) => ({
|
||||
root,
|
||||
existing: await statTarget(joinPath(targetPath, root.name)),
|
||||
})));
|
||||
const conflictCounts = new Map<string, number>();
|
||||
for (const { existing } of existingByRoot) {
|
||||
if (!existing) continue;
|
||||
const key = getSftpConflictTypeKey(true, existing.type);
|
||||
conflictCounts.set(key, (conflictCounts.get(key) ?? 0) + 1);
|
||||
}
|
||||
|
||||
const kept: ProgressiveLocalRoot[] = [];
|
||||
for (const { root, existing } of existingByRoot) {
|
||||
if (isStopped()) break;
|
||||
if (!existing) {
|
||||
destRootNameBySource.set(root.name, root.name);
|
||||
kept.push(root);
|
||||
continue;
|
||||
}
|
||||
const conflictKey = getSftpConflictTypeKey(true, existing.type);
|
||||
const action = await resolveConflict({
|
||||
fileName: root.name,
|
||||
targetPath: joinPath(targetPath, root.name),
|
||||
isDirectory: true,
|
||||
existingType: existing.type,
|
||||
existingSize: existing.size,
|
||||
// Progressive has not walked yet; byte total is unknown.
|
||||
newSize: 0,
|
||||
existingModified: existing.lastModified,
|
||||
newModified: Date.now(),
|
||||
applyToAllCount: conflictCounts.get(conflictKey) ?? 1,
|
||||
});
|
||||
|
||||
if (action === "stop") {
|
||||
await controller?.cancel();
|
||||
return [{ fileName: root.name, success: false, cancelled: true }, ...results];
|
||||
}
|
||||
if (action === "skip") {
|
||||
results.push({ fileName: root.name, success: false, cancelled: true });
|
||||
const scanningId = parentTaskIds?.get(root.name);
|
||||
if (scanningId) callbacks?.onTaskCancelled?.(scanningId);
|
||||
continue;
|
||||
}
|
||||
if (action === "replace") {
|
||||
if (!canReplaceSftpConflict(true, existing.type)) {
|
||||
results.push({
|
||||
fileName: root.name,
|
||||
success: false,
|
||||
error: `Cannot replace existing ${describeSftpExistingKind(existing.type)} with ${describeSftpIncomingKind(true)}: ${joinPath(targetPath, root.name)}`,
|
||||
});
|
||||
const scanningId = parentTaskIds?.get(root.name);
|
||||
if (scanningId) {
|
||||
callbacks?.onTaskFailed?.(
|
||||
scanningId,
|
||||
`Cannot replace existing ${describeSftpExistingKind(existing.type)}`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
await deleteTarget(joinPath(targetPath, root.name));
|
||||
destRootNameBySource.set(root.name, root.name);
|
||||
kept.push(root);
|
||||
continue;
|
||||
}
|
||||
if (action === "duplicate") {
|
||||
const duplicateName = await getDuplicateName(root.name);
|
||||
destRootNameBySource.set(root.name, duplicateName);
|
||||
kept.push(root);
|
||||
continue;
|
||||
}
|
||||
if (action === "merge" && !(existing.type === "directory")) {
|
||||
results.push({
|
||||
fileName: root.name,
|
||||
success: false,
|
||||
error: `Cannot merge existing ${describeSftpExistingKind(existing.type)} with ${describeSftpIncomingKind(true)}: ${joinPath(targetPath, root.name)}`,
|
||||
});
|
||||
const scanningId = parentTaskIds?.get(root.name);
|
||||
if (scanningId) {
|
||||
callbacks?.onTaskFailed?.(scanningId, `Cannot merge existing ${describeSftpExistingKind(existing.type)}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
destRootNameBySource.set(root.name, root.name);
|
||||
kept.push(root);
|
||||
}
|
||||
activeRoots = kept;
|
||||
} else {
|
||||
for (const root of activeRoots) destRootNameBySource.set(root.name, root.name);
|
||||
}
|
||||
|
||||
if (activeRoots.length === 0) return results;
|
||||
|
||||
for (const root of activeRoots) {
|
||||
const id = parentTaskIds?.get(root.name) ?? crypto.randomUUID();
|
||||
parentIds.set(root.name, id);
|
||||
parentStats.set(id, { discovered: 0, completed: 0, failed: 0, dirFailures: 0 });
|
||||
const destName = destRootNameBySource.get(root.name) ?? root.name;
|
||||
// Skip create if caller already opened a scanning row with this id.
|
||||
if (!parentTaskIds?.has(root.name)) {
|
||||
callbacks?.onTaskCreated?.({
|
||||
id,
|
||||
fileName: destName,
|
||||
displayName: destName,
|
||||
isDirectory: true,
|
||||
progressMode: "files",
|
||||
totalBytes: 0,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
fileCount: 0,
|
||||
completedCount: 0,
|
||||
sourcePath: root.localPath,
|
||||
});
|
||||
} else {
|
||||
// Promote scanning row into live file-count progress.
|
||||
callbacks?.onTaskProgress?.(id, {
|
||||
transferred: 0,
|
||||
total: 0,
|
||||
speed: 0,
|
||||
percent: 0,
|
||||
phase: "scanning",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const ensureDirectory = async (dirPath: string): Promise<void> => {
|
||||
if (createdDirs.has(dirPath)) return;
|
||||
if (failedDirs.has(dirPath)) {
|
||||
throw new Error(failedDirs.get(dirPath) || "Directory creation failed");
|
||||
}
|
||||
try {
|
||||
if (isLocal) {
|
||||
await bridge.mkdirLocal?.(dirPath);
|
||||
} else if (sftpId) {
|
||||
await bridge.mkdirSftp(sftpId, dirPath);
|
||||
}
|
||||
createdDirs.add(dirPath);
|
||||
} catch (error) {
|
||||
const message = formatUploadError(error);
|
||||
// Concurrent workers / merge into existing remote dirs race here.
|
||||
if (/exist|EEXIST|file exists|already/i.test(message)) {
|
||||
createdDirs.add(dirPath);
|
||||
return;
|
||||
}
|
||||
failedDirs.set(dirPath, message);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const remapRelativePath = (sourceRootName: string, relativePath: string): string => {
|
||||
const dest = destRootNameBySource.get(sourceRootName) ?? sourceRootName;
|
||||
if (dest === sourceRootName) return relativePath;
|
||||
if (relativePath === sourceRootName) return dest;
|
||||
if (relativePath.startsWith(`${sourceRootName}/`)) {
|
||||
return `${dest}/${relativePath.slice(sourceRootName.length + 1)}`;
|
||||
}
|
||||
return relativePath;
|
||||
};
|
||||
|
||||
const ensureParentsForFile = async (relativePath: string): Promise<void> => {
|
||||
const parts = relativePath.replace(/\\/g, "/").split("/").filter(Boolean);
|
||||
if (parts.length <= 1) return;
|
||||
let cursor = targetPath;
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
cursor = joinPath(cursor, parts[i]);
|
||||
await ensureDirectory(cursor);
|
||||
}
|
||||
};
|
||||
|
||||
type FileJob = { entry: DropEntry; parentId: string; rootName: string };
|
||||
const fileQueue: FileJob[] = [];
|
||||
let scanDone = false;
|
||||
/** In-flight onEntries handlers (may await soft-pause before queueing). */
|
||||
let pendingEnqueues = 0;
|
||||
let scanError: unknown;
|
||||
let wakeWaiters: Array<() => void> = [];
|
||||
const wake = () => {
|
||||
const waiters = wakeWaiters;
|
||||
wakeWaiters = [];
|
||||
for (const resolve of waiters) resolve();
|
||||
};
|
||||
const waitForWork = () => new Promise<void>((resolve) => {
|
||||
wakeWaiters.push(resolve);
|
||||
});
|
||||
const discoverySettled = () => scanDone && pendingEnqueues === 0;
|
||||
|
||||
// Multiple enqueueBatch handlers can park on backpressure at once. A single
|
||||
// resolver would orphan older waiters and leave pendingEnqueues stuck forever.
|
||||
let pauseScanWaiters: Array<() => void> = [];
|
||||
const waitIfQueueHigh = async () => {
|
||||
while (fileQueue.length >= QUEUE_HIGH_WATER && !isStopped()) {
|
||||
await new Promise<void>((resolve) => {
|
||||
pauseScanWaiters.push(resolve);
|
||||
});
|
||||
}
|
||||
};
|
||||
const maybeResumeScan = () => {
|
||||
if (fileQueue.length > QUEUE_LOW_WATER || pauseScanWaiters.length === 0) return;
|
||||
const waiters = pauseScanWaiters;
|
||||
pauseScanWaiters = [];
|
||||
for (const resolve of waiters) resolve();
|
||||
};
|
||||
|
||||
const publishParentProgress = (parentId: string, phase?: "scanning" | "transferring") => {
|
||||
const stats = parentStats.get(parentId);
|
||||
if (!stats) return;
|
||||
const total = Math.max(stats.discovered, stats.completed);
|
||||
callbacks?.onTaskProgress?.(parentId, {
|
||||
transferred: stats.completed,
|
||||
total,
|
||||
speed: 0,
|
||||
percent: total > 0 ? (stats.completed / total) * 100 : 0,
|
||||
phase: phase ?? (stats.completed > 0 ? "transferring" : "scanning"),
|
||||
});
|
||||
};
|
||||
|
||||
const enqueueBatch = async (rootName: string, batch: LocalTreeListEntry[]) => {
|
||||
pendingEnqueues += 1;
|
||||
try {
|
||||
if (isStopped()) {
|
||||
fileQueue.length = 0;
|
||||
return;
|
||||
}
|
||||
const parentId = parentIds.get(rootName);
|
||||
if (!parentId) return;
|
||||
const stats = parentStats.get(parentId);
|
||||
if (!stats) return;
|
||||
|
||||
// Soft-pause: do not grow the work queue or create remote dirs until resume.
|
||||
// Otherwise Pause still looks "alive" as hundreds of pending children appear.
|
||||
if (!(await awaitUnpaused(parentId))) {
|
||||
fileQueue.length = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
for (const row of batch) {
|
||||
if (isStopped()) {
|
||||
fileQueue.length = 0;
|
||||
return;
|
||||
}
|
||||
// Re-check pause between entries so a mid-batch Pause freezes the rest.
|
||||
if (waitWhilePaused && !(await awaitUnpaused(parentId))) {
|
||||
fileQueue.length = 0;
|
||||
return;
|
||||
}
|
||||
const drop = entryToDrop(row);
|
||||
const remappedRelative = remapRelativePath(rootName, drop.relativePath);
|
||||
const remappedDrop = remappedRelative === drop.relativePath
|
||||
? drop
|
||||
: { ...drop, relativePath: remappedRelative };
|
||||
if (remappedDrop.isDirectory) {
|
||||
// Create remote dirs early when we see them. Empty-directory failures
|
||||
// must not be silently ignored (no later file will retry the mkdir).
|
||||
try {
|
||||
await ensureDirectory(joinPath(targetPath, remappedDrop.relativePath));
|
||||
} catch (error) {
|
||||
stats.dirFailures += 1;
|
||||
results.push({
|
||||
fileName: remappedDrop.relativePath,
|
||||
success: false,
|
||||
error: formatUploadError(error),
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
stats.discovered += 1;
|
||||
fileQueue.push({ entry: remappedDrop, parentId, rootName });
|
||||
}
|
||||
publishParentProgress(parentId);
|
||||
wake();
|
||||
maybeResumeScan();
|
||||
await waitIfQueueHigh();
|
||||
} finally {
|
||||
pendingEnqueues = Math.max(0, pendingEnqueues - 1);
|
||||
// Workers may have seen an empty queue while we were still paused inside
|
||||
// this handler — wake them once discovery work is actually settled.
|
||||
wake();
|
||||
}
|
||||
};
|
||||
|
||||
const scanPromise = (async () => {
|
||||
try {
|
||||
for (const root of activeRoots) {
|
||||
if (isStopped()) break;
|
||||
await listLocalTree(root.localPath, {
|
||||
abortSignal,
|
||||
onProgress: () => {
|
||||
// Counts are derived from entry batches so UI stays consistent.
|
||||
},
|
||||
onEntries: (batch) => {
|
||||
void enqueueBatch(root.name, batch);
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
scanError = error;
|
||||
} finally {
|
||||
scanDone = true;
|
||||
wake();
|
||||
if (pauseScanWaiters.length > 0) {
|
||||
const waiters = pauseScanWaiters;
|
||||
pauseScanWaiters = [];
|
||||
for (const resolve of waiters) resolve();
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
const uploadSingle = async (job: FileJob): Promise<void> => {
|
||||
if (isStopped()) return;
|
||||
const { entry, parentId } = job;
|
||||
const stats = parentStats.get(parentId);
|
||||
if (!stats) return;
|
||||
|
||||
// Soft-pause and cancel both block before a child row is created. Creating
|
||||
// the UI task first made Pause look broken: new "pending" children kept
|
||||
// flooding the panel even though streams were soft-drained.
|
||||
if (!(await awaitUnpaused(parentId))) return;
|
||||
|
||||
const entryTargetPath = joinPath(targetPath, entry.relativePath);
|
||||
const childId = crypto.randomUUID();
|
||||
const fileTotalBytes = entry.size ?? 0;
|
||||
|
||||
callbacks?.onTaskCreated?.({
|
||||
id: childId,
|
||||
fileName: entry.relativePath,
|
||||
displayName: entry.relativePath,
|
||||
isDirectory: false,
|
||||
progressMode: "bytes",
|
||||
parentTaskId: parentId,
|
||||
totalBytes: fileTotalBytes,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
fileCount: 1,
|
||||
completedCount: 0,
|
||||
sourcePath: entry.localPath,
|
||||
});
|
||||
|
||||
if (isStopped()) {
|
||||
callbacks?.onTaskCancelled?.(childId);
|
||||
return;
|
||||
}
|
||||
// Pause may have been hit between create and stream open — wait again so we
|
||||
// never start a new write under a paused parent.
|
||||
if (!(await awaitUnpaused(parentId))) {
|
||||
callbacks?.onTaskCancelled?.(childId);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureParentsForFile(entry.relativePath);
|
||||
const localFilePath = entry.localPath;
|
||||
if (!localFilePath || !bridge.startStreamTransfer) {
|
||||
throw new Error("A local file path is required for streaming SFTP upload");
|
||||
}
|
||||
controller?.addActiveTransfer(childId);
|
||||
let streamResult: { error?: string; cancelled?: boolean } | undefined;
|
||||
try {
|
||||
streamResult = await bridge.startStreamTransfer({
|
||||
transferId: childId,
|
||||
sourcePath: localFilePath,
|
||||
targetPath: entryTargetPath,
|
||||
sourceType: "local",
|
||||
targetType: isLocal ? "local" : "sftp",
|
||||
targetSftpId: isLocal ? undefined : sftpId ?? undefined,
|
||||
targetHostId: isLocal ? undefined : targetHostId,
|
||||
totalBytes: fileTotalBytes,
|
||||
resumable: true,
|
||||
checkpointBytes: 0,
|
||||
});
|
||||
} finally {
|
||||
controller?.removeActiveTransfer(childId);
|
||||
}
|
||||
|
||||
if (streamResult?.cancelled || streamResult?.error?.includes("cancelled")) {
|
||||
callbacks?.onTaskCancelled?.(childId);
|
||||
return;
|
||||
}
|
||||
if (streamResult?.error) {
|
||||
throw new Error(streamResult.error);
|
||||
}
|
||||
|
||||
results.push({ fileName: entry.relativePath, success: true });
|
||||
stats.completed += 1;
|
||||
callbacks?.onTaskCompleted?.(childId, fileTotalBytes);
|
||||
publishParentProgress(parentId, "transferring");
|
||||
} catch (error) {
|
||||
if (controller?.isCancelled()) {
|
||||
callbacks?.onTaskCancelled?.(childId);
|
||||
return;
|
||||
}
|
||||
const message = formatUploadError(error);
|
||||
results.push({ fileName: entry.relativePath, success: false, error: message });
|
||||
stats.failed += 1;
|
||||
stats.completed += 1;
|
||||
callbacks?.onTaskFailed?.(childId, message);
|
||||
publishParentProgress(parentId, "transferring");
|
||||
} finally {
|
||||
maybeResumeScan();
|
||||
wake();
|
||||
}
|
||||
};
|
||||
|
||||
const workers = Array.from({ length: UPLOAD_CONCURRENCY }, async () => {
|
||||
while (true) {
|
||||
if (isStopped()) {
|
||||
fileQueue.length = 0;
|
||||
return;
|
||||
}
|
||||
if (fileQueue.length === 0) {
|
||||
// onEntries handlers may still be awaiting soft-pause before pushing
|
||||
// jobs — do not treat scanDone alone as terminal.
|
||||
if (discoverySettled()) return;
|
||||
await waitForWork();
|
||||
continue;
|
||||
}
|
||||
// Soft-pause is per-parent. Prefer an unpaused job so pausing parent A
|
||||
// does not head-of-line block parent B on the shared FIFO.
|
||||
if (fileQueue.length === 0) continue;
|
||||
let pickIndex = 0;
|
||||
if (isPaused) {
|
||||
const freeIndex = fileQueue.findIndex((j) => !isPaused(j.parentId));
|
||||
if (freeIndex < 0) {
|
||||
// Every pending parent is latched. Race-wait on *all* of them so
|
||||
// resuming a non-head parent does not stall behind a still-paused head.
|
||||
const pausedParents = [...new Set(fileQueue.map((j) => j.parentId))];
|
||||
let anyReleased = false;
|
||||
await Promise.race(
|
||||
pausedParents.map(async (parentId) => {
|
||||
if (await awaitUnpaused(parentId)) anyReleased = true;
|
||||
}),
|
||||
);
|
||||
if (!anyReleased || isStopped()) {
|
||||
fileQueue.length = 0;
|
||||
return;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
pickIndex = freeIndex;
|
||||
} else if (waitWhilePaused) {
|
||||
if (!(await awaitUnpaused(fileQueue[0].parentId))) {
|
||||
fileQueue.length = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (isStopped()) {
|
||||
fileQueue.length = 0;
|
||||
return;
|
||||
}
|
||||
if (fileQueue.length === 0) continue;
|
||||
const [job] = fileQueue.splice(pickIndex, 1);
|
||||
if (!job) continue;
|
||||
maybeResumeScan();
|
||||
if (isStopped()) {
|
||||
fileQueue.length = 0;
|
||||
return;
|
||||
}
|
||||
await uploadSingle(job);
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all([scanPromise, ...workers]);
|
||||
|
||||
if (controller?.isCancelled()) {
|
||||
for (const parentId of parentIds.values()) {
|
||||
callbacks?.onTaskCancelled?.(parentId);
|
||||
}
|
||||
return [{ fileName: "", success: false, cancelled: true }, ...results];
|
||||
}
|
||||
|
||||
if (scanError) {
|
||||
for (const parentId of parentIds.values()) {
|
||||
callbacks?.onTaskFailed?.(parentId, formatUploadError(scanError));
|
||||
}
|
||||
throw scanError;
|
||||
}
|
||||
|
||||
for (const [parentId, stats] of parentStats) {
|
||||
if (stats.failed > 0 || stats.dirFailures > 0) {
|
||||
const parts: string[] = [];
|
||||
if (stats.failed > 0) {
|
||||
parts.push(
|
||||
stats.failed === stats.discovered && stats.discovered > 0
|
||||
? `All ${stats.failed} files failed`
|
||||
: `${stats.failed} of ${stats.discovered} files failed`,
|
||||
);
|
||||
}
|
||||
if (stats.dirFailures > 0) {
|
||||
parts.push(
|
||||
stats.dirFailures === 1
|
||||
? "1 directory could not be created"
|
||||
: `${stats.dirFailures} directories could not be created`,
|
||||
);
|
||||
}
|
||||
callbacks?.onTaskFailed?.(parentId, parts.join("; "));
|
||||
} else {
|
||||
callbacks?.onTaskCompleted?.(parentId, stats.discovered);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
+131
-1
@@ -1,7 +1,15 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { getFileExtension, hasFileExtension } from "./sftpFileUtils.ts";
|
||||
import {
|
||||
captureDropPayload,
|
||||
formatDropScanLabel,
|
||||
getFileExtension,
|
||||
hasFileExtension,
|
||||
localTreeToDropEntries,
|
||||
materializeDropEntries,
|
||||
type LocalTreeListEntry,
|
||||
} from "./sftpFileUtils.ts";
|
||||
|
||||
test("hasFileExtension identifies extensionless and dotted filenames", () => {
|
||||
assert.equal(hasFileExtension("nginx"), false);
|
||||
@@ -13,3 +21,125 @@ test("hasFileExtension identifies extensionless and dotted filenames", () => {
|
||||
assert.equal(hasFileExtension(".bashrc"), false);
|
||||
assert.equal(hasFileExtension("archive.tar.gz"), true);
|
||||
});
|
||||
|
||||
test("formatDropScanLabel summarizes dropped root names", () => {
|
||||
assert.equal(formatDropScanLabel([]), "Scanning files...");
|
||||
assert.equal(formatDropScanLabel([{ name: "docs", isDirectory: true }]), "docs");
|
||||
assert.equal(
|
||||
formatDropScanLabel([
|
||||
{ name: "a", isDirectory: true },
|
||||
{ name: "b", isDirectory: true },
|
||||
]),
|
||||
"a, b",
|
||||
);
|
||||
assert.equal(
|
||||
formatDropScanLabel([
|
||||
{ name: "a", isDirectory: true },
|
||||
{ name: "b", isDirectory: true },
|
||||
{ name: "c", isDirectory: true },
|
||||
]),
|
||||
"a, b +1",
|
||||
);
|
||||
});
|
||||
|
||||
test("captureDropPayload reads webkit entries synchronously", () => {
|
||||
const file = new File(["x"], "note.txt");
|
||||
Object.defineProperty(file, "path", { value: "/tmp/note.txt" });
|
||||
const dataTransfer = {
|
||||
items: [{
|
||||
kind: "file",
|
||||
getAsFile: () => file,
|
||||
webkitGetAsEntry: () => ({
|
||||
name: "note.txt",
|
||||
isFile: true,
|
||||
isDirectory: false,
|
||||
}),
|
||||
}],
|
||||
files: [file],
|
||||
} as unknown as DataTransfer;
|
||||
|
||||
const payload = captureDropPayload(dataTransfer);
|
||||
assert.equal(payload.roots.length, 1);
|
||||
assert.equal(payload.roots[0].name, "note.txt");
|
||||
assert.equal(payload.roots[0].isDirectory, false);
|
||||
assert.equal(payload.roots[0].localPath, "/tmp/note.txt");
|
||||
});
|
||||
|
||||
test("materializeDropEntries prefers listLocalTree for directory roots with paths", async () => {
|
||||
const progress: Array<{ fileCount: number; directoryCount: number }> = [];
|
||||
const tree: LocalTreeListEntry[] = [
|
||||
{
|
||||
localPath: "/tmp/project",
|
||||
relativePath: "project",
|
||||
type: "directory",
|
||||
size: 0,
|
||||
lastModified: 1,
|
||||
},
|
||||
{
|
||||
localPath: "/tmp/project/src",
|
||||
relativePath: "project/src",
|
||||
type: "directory",
|
||||
size: 0,
|
||||
lastModified: 1,
|
||||
},
|
||||
{
|
||||
localPath: "/tmp/project/src/main.ts",
|
||||
relativePath: "project/src/main.ts",
|
||||
type: "file",
|
||||
size: 12,
|
||||
lastModified: 2,
|
||||
},
|
||||
];
|
||||
|
||||
const entries = await materializeDropEntries(
|
||||
{
|
||||
roots: [{
|
||||
name: "project",
|
||||
isDirectory: true,
|
||||
localPath: "/tmp/project",
|
||||
}],
|
||||
filesFallback: [],
|
||||
},
|
||||
{
|
||||
listLocalTree: async (path, options) => {
|
||||
assert.equal(path, "/tmp/project");
|
||||
options?.onProgress?.({ fileCount: 1, directoryCount: 2, entryCount: 3 });
|
||||
return tree;
|
||||
},
|
||||
onProgress: (p) => progress.push({ fileCount: p.fileCount, directoryCount: p.directoryCount }),
|
||||
},
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
entries.map((entry) => ({
|
||||
relativePath: entry.relativePath,
|
||||
isDirectory: entry.isDirectory,
|
||||
localPath: entry.localPath,
|
||||
size: entry.size,
|
||||
})),
|
||||
[
|
||||
{ relativePath: "project", isDirectory: true, localPath: "/tmp/project", size: undefined },
|
||||
{ relativePath: "project/src", isDirectory: true, localPath: "/tmp/project/src", size: undefined },
|
||||
{
|
||||
relativePath: "project/src/main.ts",
|
||||
isDirectory: false,
|
||||
localPath: "/tmp/project/src/main.ts",
|
||||
size: 12,
|
||||
},
|
||||
],
|
||||
);
|
||||
assert.ok(progress.some((p) => p.fileCount === 1 && p.directoryCount === 2));
|
||||
});
|
||||
|
||||
test("localTreeToDropEntries preserves file sizes without File handles", () => {
|
||||
const entries = localTreeToDropEntries([{
|
||||
localPath: "/tmp/a.txt",
|
||||
relativePath: "a.txt",
|
||||
type: "file",
|
||||
size: 42,
|
||||
lastModified: 9,
|
||||
}]);
|
||||
assert.equal(entries[0].file, null);
|
||||
assert.equal(entries[0].localPath, "/tmp/a.txt");
|
||||
assert.equal(entries[0].size, 42);
|
||||
});
|
||||
+446
-98
@@ -242,6 +242,59 @@ export interface DropEntry {
|
||||
isDirectory: boolean;
|
||||
}
|
||||
|
||||
/** Local tree row from main-process `listLocalTree`. */
|
||||
export interface LocalTreeListEntry {
|
||||
localPath: string;
|
||||
relativePath: string;
|
||||
type: "file" | "directory";
|
||||
size: number;
|
||||
lastModified: number;
|
||||
}
|
||||
|
||||
/** Live scan counters while expanding a dropped folder. */
|
||||
export interface DropScanProgress {
|
||||
fileCount: number;
|
||||
directoryCount: number;
|
||||
entryCount: number;
|
||||
/** Optional UI label (e.g. root folder names). */
|
||||
label?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sync snapshot of a drop. DataTransfer must be read before any await —
|
||||
* after that, items/files may be empty.
|
||||
*/
|
||||
export interface CapturedDropRoot {
|
||||
name: string;
|
||||
isDirectory: boolean;
|
||||
localPath?: string;
|
||||
file?: File | null;
|
||||
size?: number;
|
||||
/** Folder-picker relative path when the File carries webkitRelativePath */
|
||||
relativePath?: string;
|
||||
/** webkit entry for Chromium walk fallback when no local path is available */
|
||||
fsEntry?: FileSystemEntry;
|
||||
}
|
||||
|
||||
export interface CapturedDropPayload {
|
||||
roots: CapturedDropRoot[];
|
||||
filesFallback: File[];
|
||||
}
|
||||
|
||||
export interface MaterializeDropOptions {
|
||||
listLocalTree?: (
|
||||
path: string,
|
||||
options?: {
|
||||
onProgress?: (progress: DropScanProgress) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
},
|
||||
) => Promise<LocalTreeListEntry[]>;
|
||||
onProgress?: (progress: DropScanProgress) => void;
|
||||
/** Cooperative cancel for webkit walks and native listLocalTree. */
|
||||
abortSignal?: AbortSignal;
|
||||
isCancelled?: () => boolean;
|
||||
}
|
||||
|
||||
export const getDropEntryLocalPath = (entry: DropEntry): string | undefined =>
|
||||
entry.localPath ?? (entry.file ? getPathForFile(entry.file) : undefined);
|
||||
|
||||
@@ -291,74 +344,133 @@ async function readAllDirectoryEntries(
|
||||
return allEntries;
|
||||
}
|
||||
|
||||
function joinLocalRelativePath(rootPath: string, relativePath: string): string {
|
||||
const normalizedRelative = relativePath.replace(/\\/g, "/");
|
||||
const parts = normalizedRelative.split("/");
|
||||
// relativePath is rooted at the drop root name; local path already points at that root.
|
||||
const nested = parts.length > 1 ? parts.slice(1).join("/") : "";
|
||||
if (!nested) return rootPath;
|
||||
const separator = rootPath.includes("\\") ? "\\" : "/";
|
||||
return rootPath + separator + nested.replace(/\//g, separator);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process file system entries iteratively (non-recursive) to handle large folders
|
||||
* Uses a queue-based approach to avoid stack overflow
|
||||
* @param rootEntries - The root entries to process
|
||||
* @returns Array of DropEntry objects with files and their relative paths
|
||||
* Process file system entries iteratively (non-recursive) to handle large folders.
|
||||
* Prefer reconstructing local paths from the drop root so we can skip per-file
|
||||
* `entry.file()` when Electron already exposed the folder path.
|
||||
*/
|
||||
export function isDropScanCancelledError(error: unknown): boolean {
|
||||
if (!error || typeof error !== "object") return false;
|
||||
const code = (error as { code?: string }).code;
|
||||
return code === "ERR_DROP_SCAN_CANCELLED" || code === "ERR_LOCAL_TREE_CANCELLED";
|
||||
}
|
||||
|
||||
function throwIfDropScanCancelled(options: {
|
||||
abortSignal?: AbortSignal;
|
||||
isCancelled?: () => boolean;
|
||||
}): void {
|
||||
if (options.abortSignal?.aborted || options.isCancelled?.()) {
|
||||
const error = new Error("Drop scan cancelled");
|
||||
(error as Error & { code?: string }).code = "ERR_DROP_SCAN_CANCELLED";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function processEntriesIteratively(
|
||||
rootEntries: FileSystemEntry[]
|
||||
rootEntries: FileSystemEntry[],
|
||||
options: {
|
||||
rootPathByName?: Map<string, string>;
|
||||
onProgress?: (progress: DropScanProgress) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
isCancelled?: () => boolean;
|
||||
} = {},
|
||||
): Promise<DropEntry[]> {
|
||||
const results: DropEntry[] = [];
|
||||
const rootPathByName = options.rootPathByName ?? new Map<string, string>();
|
||||
|
||||
// Queue of entries to process: [entry, basePath]
|
||||
// Index-based queue avoids O(n²) Array.shift on huge trees.
|
||||
const queue: Array<{ entry: FileSystemEntry; basePath: string }> = [];
|
||||
|
||||
// Initialize queue with root entries
|
||||
for (const entry of rootEntries) {
|
||||
queue.push({ entry, basePath: "" });
|
||||
}
|
||||
|
||||
let queueIndex = 0;
|
||||
let processedCount = 0;
|
||||
const YIELD_INTERVAL = 100; // Yield to main thread every N items
|
||||
let fileCount = 0;
|
||||
let directoryCount = 0;
|
||||
const YIELD_INTERVAL = 100;
|
||||
const PROGRESS_INTERVAL = 32;
|
||||
|
||||
while (queue.length > 0) {
|
||||
const { entry, basePath } = queue.shift()!;
|
||||
const reportProgress = (force = false) => {
|
||||
if (!options.onProgress) return;
|
||||
if (!force && processedCount % PROGRESS_INTERVAL !== 0) return;
|
||||
options.onProgress({
|
||||
fileCount,
|
||||
directoryCount,
|
||||
entryCount: fileCount + directoryCount,
|
||||
});
|
||||
};
|
||||
|
||||
while (queueIndex < queue.length) {
|
||||
throwIfDropScanCancelled(options);
|
||||
const { entry, basePath } = queue[queueIndex++];
|
||||
const relativePath = basePath ? `${basePath}/${entry.name}` : entry.name;
|
||||
const rootName = relativePath.split("/")[0] ?? entry.name;
|
||||
const rootLocalPath = rootPathByName.get(rootName);
|
||||
|
||||
if (entry.isFile) {
|
||||
const fileEntry = entry as FileSystemFileEntry;
|
||||
try {
|
||||
const file = await entryToFile(fileEntry);
|
||||
if (rootLocalPath) {
|
||||
// Native path is enough for stream upload; avoid Chromium File materialization.
|
||||
results.push({
|
||||
file,
|
||||
relativePath: basePath ? `${basePath}/${entry.name}` : entry.name,
|
||||
file: null,
|
||||
localPath: joinLocalRelativePath(rootLocalPath, relativePath),
|
||||
relativePath,
|
||||
isDirectory: false,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(`Failed to read file entry: ${entry.name}`, error);
|
||||
fileCount++;
|
||||
} else {
|
||||
try {
|
||||
const file = await entryToFile(fileEntry);
|
||||
results.push({
|
||||
file,
|
||||
relativePath,
|
||||
isDirectory: false,
|
||||
});
|
||||
fileCount++;
|
||||
} catch (error) {
|
||||
console.warn(`Failed to read file entry: ${entry.name}`, error);
|
||||
}
|
||||
}
|
||||
} else if (entry.isDirectory) {
|
||||
const dirEntry = entry as FileSystemDirectoryEntry;
|
||||
const currentPath = basePath ? `${basePath}/${entry.name}` : entry.name;
|
||||
|
||||
// Add directory entry
|
||||
results.push({
|
||||
file: null,
|
||||
relativePath: currentPath,
|
||||
localPath: rootLocalPath ? joinLocalRelativePath(rootLocalPath, relativePath) : undefined,
|
||||
relativePath,
|
||||
isDirectory: true,
|
||||
});
|
||||
directoryCount++;
|
||||
|
||||
try {
|
||||
const reader = dirEntry.createReader();
|
||||
const childEntries = await readAllDirectoryEntries(reader);
|
||||
|
||||
// Add child entries to the queue (not recursive!)
|
||||
for (const childEntry of childEntries) {
|
||||
queue.push({ entry: childEntry, basePath: currentPath });
|
||||
queue.push({ entry: childEntry, basePath: relativePath });
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`Failed to read directory: ${entry.name}`, error);
|
||||
}
|
||||
}
|
||||
|
||||
// Yield to main thread periodically to keep UI responsive
|
||||
processedCount++;
|
||||
reportProgress();
|
||||
if (processedCount % YIELD_INTERVAL === 0) {
|
||||
await new Promise<void>(resolve => setTimeout(resolve, 0));
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
}
|
||||
|
||||
reportProgress(true);
|
||||
return results;
|
||||
}
|
||||
|
||||
@@ -378,87 +490,323 @@ export function getPathForFile(file: File): string | undefined {
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a short label for the scanning task (folder names visible immediately). */
|
||||
export function formatDropScanLabel(roots: readonly CapturedDropRoot[]): string {
|
||||
const names = roots.map((root) => root.name).filter(Boolean);
|
||||
if (names.length === 0) return "Scanning files...";
|
||||
if (names.length === 1) return names[0];
|
||||
if (names.length === 2) return `${names[0]}, ${names[1]}`;
|
||||
return `${names[0]}, ${names[1]} +${names.length - 2}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all files and directories from a DataTransfer object
|
||||
* Supports both regular files and folders dropped from the OS
|
||||
*
|
||||
* Uses the webkitGetAsEntry() API for folder access, with fallback
|
||||
* to regular FileList for browsers that don't support it.
|
||||
*
|
||||
* @param dataTransfer - The DataTransfer object from a drop event
|
||||
* @returns Array of DropEntry objects with files and relative paths
|
||||
* Synchronously capture drop roots. Must run during the drop/paste event —
|
||||
* before any await — or DataTransfer becomes empty.
|
||||
*/
|
||||
export async function extractDropEntries(
|
||||
dataTransfer: DataTransfer
|
||||
): Promise<DropEntry[]> {
|
||||
export function captureDropPayload(dataTransfer: DataTransfer): CapturedDropPayload {
|
||||
const filesFallback: File[] = [];
|
||||
const files = dataTransfer.files;
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
filesFallback.push(files[i]);
|
||||
}
|
||||
|
||||
const roots: CapturedDropRoot[] = [];
|
||||
const items = dataTransfer.items;
|
||||
|
||||
// Build a map of file/folder name to path from the original files in DataTransfer.files
|
||||
const filePathMap = new Map<string, string>();
|
||||
const filesWithPath = dataTransfer.files;
|
||||
for (let i = 0; i < filesWithPath.length; i++) {
|
||||
const f = filesWithPath[i];
|
||||
const path = getPathForFile(f);
|
||||
if (path) {
|
||||
filePathMap.set(f.name, path);
|
||||
}
|
||||
}
|
||||
const relativePathForFile = (file: File): string | undefined => {
|
||||
const relative = (file as File & { webkitRelativePath?: string }).webkitRelativePath;
|
||||
return relative && relative.length > 0 ? relative : undefined;
|
||||
};
|
||||
|
||||
// Check if webkitGetAsEntry is supported (for folder access)
|
||||
if (items && items.length > 0 && typeof items[0].webkitGetAsEntry === 'function') {
|
||||
// Collect all entries first (getAsEntry must be called synchronously)
|
||||
const entries: FileSystemEntry[] = [];
|
||||
if (items && items.length > 0 && typeof items[0].webkitGetAsEntry === "function") {
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const item = items[i];
|
||||
if (item.kind === 'file') {
|
||||
const entry = item.webkitGetAsEntry();
|
||||
if (entry) {
|
||||
entries.push(entry);
|
||||
}
|
||||
if (item.kind !== "file") continue;
|
||||
|
||||
const entry = item.webkitGetAsEntry();
|
||||
const file = typeof item.getAsFile === "function" ? item.getAsFile() : (filesFallback[i] ?? null);
|
||||
const localPath = file ? getPathForFile(file) : undefined;
|
||||
const relativePath = file ? relativePathForFile(file) : undefined;
|
||||
|
||||
if (entry) {
|
||||
roots.push({
|
||||
name: entry.name,
|
||||
isDirectory: entry.isDirectory,
|
||||
localPath,
|
||||
file: entry.isFile ? file : null,
|
||||
size: file?.size,
|
||||
relativePath: entry.isFile ? relativePath : undefined,
|
||||
fsEntry: entry,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (file) {
|
||||
roots.push({
|
||||
name: file.name,
|
||||
isDirectory: false,
|
||||
localPath: getPathForFile(file),
|
||||
file,
|
||||
size: file.size,
|
||||
relativePath,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Process entries iteratively (non-recursive) to avoid stack overflow
|
||||
const results = await processEntriesIteratively(entries);
|
||||
if (results.length === 0) {
|
||||
return createDropEntriesFromFiles(dataTransfer.files);
|
||||
}
|
||||
|
||||
// Preserve local paths on DropEntry instead of mutating File.path. Modern
|
||||
// runtimes expose File.path as read-only when it exists.
|
||||
for (const result of results) {
|
||||
if (result.file) {
|
||||
// First try to get path directly from the file
|
||||
const directPath = getPathForFile(result.file);
|
||||
if (directPath) {
|
||||
result.localPath = directPath;
|
||||
} else {
|
||||
// Fallback: try to reconstruct from root folder path
|
||||
const pathParts = result.relativePath.split('/');
|
||||
const rootName = pathParts[0];
|
||||
const rootPath = filePathMap.get(rootName);
|
||||
|
||||
if (rootPath) {
|
||||
if (pathParts.length === 1) {
|
||||
// Root-level file: use the path directly
|
||||
result.localPath = rootPath;
|
||||
} else {
|
||||
// Nested file in a folder: construct full path
|
||||
// rootPath is the path to the root folder, we need to append the rest
|
||||
const restOfPath = pathParts.slice(1).join('/');
|
||||
const separator = rootPath.includes('\\') ? '\\' : '/';
|
||||
const fullPath = rootPath + separator + restOfPath.replace(/\//g, separator);
|
||||
result.localPath = fullPath;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
} else {
|
||||
// Fallback: use regular FileList (no folder support)
|
||||
// Files from FileList in Electron already have the 'path' property
|
||||
return createDropEntriesFromFiles(dataTransfer.files);
|
||||
for (const file of filesFallback) {
|
||||
roots.push({
|
||||
name: file.name,
|
||||
isDirectory: false,
|
||||
localPath: getPathForFile(file),
|
||||
file,
|
||||
size: file.size,
|
||||
relativePath: relativePathForFile(file),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { roots, filesFallback };
|
||||
}
|
||||
|
||||
/** Map main-process local tree rows into upload DropEntry records. */
|
||||
export function localTreeToDropEntries(tree: readonly LocalTreeListEntry[]): DropEntry[] {
|
||||
return tree.map((entry) => {
|
||||
if (entry.type === "directory") {
|
||||
return {
|
||||
file: null,
|
||||
localPath: entry.localPath,
|
||||
relativePath: entry.relativePath,
|
||||
isDirectory: true,
|
||||
// Do not use directory metadata size in conflict / compressed totals.
|
||||
};
|
||||
}
|
||||
return {
|
||||
file: null,
|
||||
localPath: entry.localPath,
|
||||
relativePath: entry.relativePath,
|
||||
isDirectory: false,
|
||||
size: entry.size,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function countDropEntries(entries: readonly DropEntry[]): DropScanProgress {
|
||||
let fileCount = 0;
|
||||
let directoryCount = 0;
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory) directoryCount += 1;
|
||||
else fileCount += 1;
|
||||
}
|
||||
return {
|
||||
fileCount,
|
||||
directoryCount,
|
||||
entryCount: fileCount + directoryCount,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Expand a captured drop into full DropEntry rows.
|
||||
* Prefer Electron `listLocalTree` (native fs) for directory roots with a local
|
||||
* path; fall back to Chromium FileSystemEntry walk only when necessary.
|
||||
*/
|
||||
export async function materializeDropEntries(
|
||||
payload: CapturedDropPayload,
|
||||
options: MaterializeDropOptions = {},
|
||||
): Promise<DropEntry[]> {
|
||||
const { listLocalTree, onProgress, abortSignal, isCancelled } = options;
|
||||
const results: DropEntry[] = [];
|
||||
let fileCount = 0;
|
||||
let directoryCount = 0;
|
||||
|
||||
const report = (label?: string, forcePartial?: DropScanProgress) => {
|
||||
if (!onProgress) return;
|
||||
if (forcePartial) {
|
||||
onProgress({
|
||||
...forcePartial,
|
||||
label,
|
||||
});
|
||||
return;
|
||||
}
|
||||
onProgress({
|
||||
fileCount,
|
||||
directoryCount,
|
||||
entryCount: fileCount + directoryCount,
|
||||
label,
|
||||
});
|
||||
};
|
||||
|
||||
const nativeDirectoryRoots: CapturedDropRoot[] = [];
|
||||
const webkitDirectoryRoots: CapturedDropRoot[] = [];
|
||||
const fileRoots: CapturedDropRoot[] = [];
|
||||
|
||||
for (const root of payload.roots) {
|
||||
if (root.isDirectory) {
|
||||
if (root.localPath && listLocalTree) {
|
||||
nativeDirectoryRoots.push(root);
|
||||
} else if (root.fsEntry) {
|
||||
webkitDirectoryRoots.push(root);
|
||||
} else {
|
||||
console.warn(`[SFTP] Skipping directory drop root without path or entry: ${root.name}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
fileRoots.push(root);
|
||||
}
|
||||
|
||||
throwIfDropScanCancelled({ abortSignal, isCancelled });
|
||||
|
||||
// Parallel native walks — one IPC per root folder, much faster than webkit.
|
||||
if (nativeDirectoryRoots.length > 0 && listLocalTree) {
|
||||
// Cumulative progress across roots: each walk reports its own counts.
|
||||
const partialByRoot = new Map<string, DropScanProgress>();
|
||||
const emitCombined = (label?: string) => {
|
||||
let files = fileCount;
|
||||
let dirs = directoryCount;
|
||||
for (const partial of partialByRoot.values()) {
|
||||
files += partial.fileCount;
|
||||
dirs += partial.directoryCount;
|
||||
}
|
||||
report(label, { fileCount: files, directoryCount: dirs, entryCount: files + dirs });
|
||||
};
|
||||
|
||||
// Local controller so a single root failure aborts sibling walks too.
|
||||
const siblingAbort = new AbortController();
|
||||
const stopSiblingScans = () => {
|
||||
try {
|
||||
siblingAbort.abort();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
};
|
||||
if (abortSignal) {
|
||||
if (abortSignal.aborted) stopSiblingScans();
|
||||
else abortSignal.addEventListener("abort", stopSiblingScans, { once: true });
|
||||
}
|
||||
const walkSignal = siblingAbort.signal;
|
||||
const walkCancelled = () => (
|
||||
walkSignal.aborted || abortSignal?.aborted === true || isCancelled?.() === true
|
||||
);
|
||||
|
||||
const walkPromises = nativeDirectoryRoots.map(async (root) => {
|
||||
throwIfDropScanCancelled({ abortSignal: walkSignal, isCancelled: walkCancelled });
|
||||
const tree = await listLocalTree(root.localPath!, {
|
||||
abortSignal: walkSignal,
|
||||
onProgress: (partial) => {
|
||||
partialByRoot.set(root.localPath!, partial);
|
||||
emitCombined(root.name);
|
||||
},
|
||||
});
|
||||
return { root, tree };
|
||||
});
|
||||
|
||||
let trees: Array<{ root: CapturedDropRoot; tree: LocalTreeListEntry[] }>;
|
||||
try {
|
||||
trees = await Promise.all(walkPromises);
|
||||
} catch (error) {
|
||||
stopSiblingScans();
|
||||
// Drain remaining native walks so retry does not pile more I/O on top.
|
||||
await Promise.allSettled(walkPromises);
|
||||
throw error;
|
||||
} finally {
|
||||
abortSignal?.removeEventListener("abort", stopSiblingScans);
|
||||
}
|
||||
|
||||
for (const { root, tree } of trees) {
|
||||
const entries = localTreeToDropEntries(tree);
|
||||
for (const entry of entries) {
|
||||
results.push(entry);
|
||||
if (entry.isDirectory) directoryCount += 1;
|
||||
else fileCount += 1;
|
||||
}
|
||||
partialByRoot.delete(root.localPath!);
|
||||
report(root.name);
|
||||
}
|
||||
}
|
||||
|
||||
for (const root of fileRoots) {
|
||||
results.push({
|
||||
file: root.file ?? null,
|
||||
localPath: root.localPath,
|
||||
relativePath: root.relativePath || root.name,
|
||||
isDirectory: false,
|
||||
size: root.size ?? root.file?.size,
|
||||
});
|
||||
fileCount += 1;
|
||||
}
|
||||
if (fileRoots.length > 0) {
|
||||
report();
|
||||
}
|
||||
|
||||
if (webkitDirectoryRoots.length > 0) {
|
||||
const rootPathByName = new Map<string, string>();
|
||||
for (const root of webkitDirectoryRoots) {
|
||||
if (root.localPath) rootPathByName.set(root.name, root.localPath);
|
||||
}
|
||||
const walked = await processEntriesIteratively(
|
||||
webkitDirectoryRoots.map((root) => root.fsEntry!).filter(Boolean),
|
||||
{
|
||||
rootPathByName,
|
||||
abortSignal,
|
||||
isCancelled,
|
||||
onProgress: (partial) => {
|
||||
report(undefined, {
|
||||
fileCount: fileCount + partial.fileCount,
|
||||
directoryCount: directoryCount + partial.directoryCount,
|
||||
entryCount: fileCount + directoryCount + partial.entryCount,
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Attach reconstructed paths when we only know the root path.
|
||||
for (const entry of walked) {
|
||||
if (!entry.localPath) {
|
||||
const rootName = entry.relativePath.split("/")[0];
|
||||
const rootPath = rootPathByName.get(rootName);
|
||||
if (rootPath) {
|
||||
entry.localPath = joinLocalRelativePath(rootPath, entry.relativePath);
|
||||
} else if (entry.file) {
|
||||
entry.localPath = getPathForFile(entry.file);
|
||||
}
|
||||
}
|
||||
results.push(entry);
|
||||
if (entry.isDirectory) directoryCount += 1;
|
||||
else fileCount += 1;
|
||||
}
|
||||
report();
|
||||
}
|
||||
|
||||
if (results.length === 0 && payload.filesFallback.length > 0) {
|
||||
const fallback = createDropEntriesFromFiles(payload.filesFallback);
|
||||
const counts = countDropEntries(fallback);
|
||||
report(undefined, counts);
|
||||
return fallback;
|
||||
}
|
||||
|
||||
report(undefined, { fileCount, directoryCount, entryCount: fileCount + directoryCount });
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all files and directories from a DataTransfer object.
|
||||
* Supports both regular files and folders dropped from the OS.
|
||||
*
|
||||
* Prefer Electron native tree walk when local paths are available; otherwise
|
||||
* use webkitGetAsEntry with a path-reconstruction fast path.
|
||||
*/
|
||||
export async function extractDropEntries(
|
||||
dataTransfer: DataTransfer,
|
||||
options: MaterializeDropOptions = {},
|
||||
): Promise<DropEntry[]> {
|
||||
const payload = captureDropPayload(dataTransfer);
|
||||
const bridge = netcattyBridge.get();
|
||||
return materializeDropEntries(payload, {
|
||||
listLocalTree: options.listLocalTree
|
||||
?? (bridge?.listLocalTree
|
||||
? (path, treeOptions) => bridge.listLocalTree!(path, treeOptions)
|
||||
: undefined),
|
||||
onProgress: options.onProgress,
|
||||
abortSignal: options.abortSignal,
|
||||
isCancelled: options.isCancelled,
|
||||
});
|
||||
}
|
||||
@@ -48,3 +48,48 @@ test("compressed folder uploads use reconstructed drop-entry paths", async () =>
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("compressed folder totals exclude directory entries", async () => {
|
||||
const previousWindow = globalThis.window;
|
||||
const started: Array<{ totalBytes: number; folderPath: string }> = [];
|
||||
Object.defineProperty(globalThis, "window", {
|
||||
configurable: true,
|
||||
value: {
|
||||
netcatty: {
|
||||
checkCompressedUploadSupport: async () => ({
|
||||
supported: true,
|
||||
localTar: true,
|
||||
remoteTar: true,
|
||||
}),
|
||||
startCompressedUpload: async (options: { totalBytes: number; folderPath: string; compressionId: string }) => {
|
||||
started.push({ totalBytes: options.totalBytes, folderPath: options.folderPath });
|
||||
return { compressionId: options.compressionId, success: true };
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
try {
|
||||
const results = await uploadFoldersCompressed(
|
||||
[["docs", [
|
||||
{ file: null, localPath: "/tmp/docs", relativePath: "docs", isDirectory: true, size: 4096 },
|
||||
{ file: null, localPath: "/tmp/docs/sub", relativePath: "docs/sub", isDirectory: true, size: 4096 },
|
||||
{ file: null, localPath: "/tmp/docs/a.txt", relativePath: "docs/a.txt", isDirectory: false, size: 10 },
|
||||
]]],
|
||||
"/remote",
|
||||
"sftp-1",
|
||||
);
|
||||
|
||||
assert.deepEqual(results, [{ fileName: "docs", success: true }]);
|
||||
assert.deepEqual(started, [{ totalBytes: 10, folderPath: "/tmp/docs" }]);
|
||||
} finally {
|
||||
if (previousWindow === undefined) {
|
||||
Reflect.deleteProperty(globalThis, "window");
|
||||
} else {
|
||||
Object.defineProperty(globalThis, "window", {
|
||||
configurable: true,
|
||||
value: previousWindow,
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
+17
-7
@@ -1,5 +1,7 @@
|
||||
import type { DropEntry } from "./sftpFileUtils";
|
||||
import { getDropEntryLocalPath } from "./sftpFileUtils";
|
||||
|
||||
const getDropEntrySize = (entry: DropEntry): number => entry.file?.size ?? entry.size ?? 0;
|
||||
import type { UploadCallbacks, UploadResult } from "./uploadService.types";
|
||||
import type { UploadController } from "./uploadController";
|
||||
|
||||
@@ -23,14 +25,17 @@ export async function uploadFoldersCompressed(
|
||||
break;
|
||||
}
|
||||
|
||||
// Get the local folder path from the first file in the folder
|
||||
const firstFile = entries.find(e => e.file);
|
||||
if (!firstFile?.file) {
|
||||
// Prefer any file-like entry with a resolvable local path (native tree scans
|
||||
// set localPath without a browser File handle).
|
||||
const firstFile = entries.find((entry) => (
|
||||
!entry.isDirectory && (!!entry.file || !!getDropEntryLocalPath(entry))
|
||||
));
|
||||
if (!firstFile) {
|
||||
// Empty folder - mark for fallback to regular upload which will create the directory
|
||||
results.push({ fileName: folderName, success: false, error: "Compressed upload not supported - fallback needed" });
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
const localFilePath = getDropEntryLocalPath(firstFile);
|
||||
if (!localFilePath) {
|
||||
results.push({ fileName: folderName, success: false, error: "Could not get local file path" });
|
||||
@@ -41,7 +46,10 @@ export async function uploadFoldersCompressed(
|
||||
// Use DropEntry.relativePath which works for both file input and drag-drop scenarios
|
||||
// For file input: webkitRelativePath is set (e.g., "folder/subdir/file.txt")
|
||||
// For drag-drop: DropEntry.relativePath contains the correct path from extractDropEntries
|
||||
const relativePath = firstFile.relativePath || (firstFile.file as File & { webkitRelativePath?: string }).webkitRelativePath || firstFile.file.name;
|
||||
const relativePath = firstFile.relativePath
|
||||
|| (firstFile.file as (File & { webkitRelativePath?: string }) | null)?.webkitRelativePath
|
||||
|| firstFile.file?.name
|
||||
|| folderName;
|
||||
|
||||
// Normalize path separators for cross-platform compatibility
|
||||
const normalizePathSeparators = (path: string) => path.replace(/\\/g, '/');
|
||||
@@ -114,7 +122,9 @@ export async function uploadFoldersCompressed(
|
||||
controller?.addActiveCompression(compressionId);
|
||||
|
||||
// Create a task for this folder compression
|
||||
const totalBytes = entries.reduce((sum, entry) => sum + (entry.file?.size || 0), 0);
|
||||
// Path-only drop entries (listLocalTree) carry size without a File handle.
|
||||
const fileEntries = entries.filter((entry) => !entry.isDirectory);
|
||||
const totalBytes = fileEntries.reduce((sum, entry) => sum + getDropEntrySize(entry), 0);
|
||||
taskId = compressionId;
|
||||
|
||||
if (callbacks?.onTaskCreated) {
|
||||
@@ -127,7 +137,7 @@ export async function uploadFoldersCompressed(
|
||||
totalBytes,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
fileCount: entries.length,
|
||||
fileCount: fileEntries.length,
|
||||
completedCount: 0,
|
||||
sourcePath: folderPath,
|
||||
controlKind: 'compressed-upload',
|
||||
|
||||
+34
-1
@@ -6,12 +6,31 @@ export class UploadController {
|
||||
private activeCompressionIds = new Set<string>();
|
||||
private currentTransferId = "";
|
||||
private bridge: UploadBridge | null = null;
|
||||
private cancelListeners = new Set<() => void>();
|
||||
|
||||
/**
|
||||
* Register a listener fired as soon as cancel() is requested (before async
|
||||
* cleanup). Used to abort in-flight local tree scans.
|
||||
*/
|
||||
addCancelListener(listener: () => void): () => void {
|
||||
this.cancelListeners.add(listener);
|
||||
return () => {
|
||||
this.cancelListeners.delete(listener);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancel all active uploads
|
||||
*/
|
||||
async cancel(): Promise<void> {
|
||||
this.cancelled = true;
|
||||
for (const listener of Array.from(this.cancelListeners)) {
|
||||
try {
|
||||
listener();
|
||||
} catch {
|
||||
// Ignore listener errors so cancel still drains transfers.
|
||||
}
|
||||
}
|
||||
|
||||
// Cancel all active compressed uploads
|
||||
const activeCompressionIds = Array.from(this.activeCompressionIds);
|
||||
@@ -73,10 +92,24 @@ export class UploadController {
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset controller state for new upload
|
||||
* Reset controller state for a brand-new upload session.
|
||||
* Prefer prepareForEntries when the controller already owns an external drop
|
||||
* (scan cancel listeners must stay attached until the drop settles).
|
||||
*/
|
||||
reset(): void {
|
||||
this.cancelled = false;
|
||||
this.cancelListeners.clear();
|
||||
this.activeFileTransferIds.clear();
|
||||
this.activeCompressionIds.clear();
|
||||
this.currentTransferId = "";
|
||||
}
|
||||
|
||||
/**
|
||||
* Soft prepare for entry upload without clearing cancel latches/listeners.
|
||||
* External drop flows attach scan/conflict cancel listeners before entries
|
||||
* are ready; a full reset would leave the scanning row uncancelable.
|
||||
*/
|
||||
prepareForEntries(): void {
|
||||
this.activeFileTransferIds.clear();
|
||||
this.activeCompressionIds.clear();
|
||||
this.currentTransferId = "";
|
||||
|
||||
+28
-10
@@ -37,7 +37,9 @@ import type { UploadBridge, UploadCallbacks, UploadConfig, UploadResult } from "
|
||||
const formatUploadError = (error: unknown): string =>
|
||||
error instanceof Error ? error.message : String(error);
|
||||
|
||||
const getDropEntrySize = (entry: DropEntry): number => entry.file?.size ?? entry.size ?? 0;
|
||||
const getDropEntrySize = (entry: DropEntry): number => (
|
||||
entry.isDirectory ? 0 : entry.file?.size ?? entry.size ?? 0
|
||||
);
|
||||
const getRootDropLocalPath = (rootName: string, entries: DropEntry[]): string | undefined => {
|
||||
const entry = entries.find((candidate) => getDropEntryLocalPath(candidate));
|
||||
const localPath = entry ? getDropEntryLocalPath(entry) : undefined;
|
||||
@@ -61,9 +63,10 @@ export interface UploadScanningTask {
|
||||
export function startUploadScanningTask(
|
||||
callbacks?: UploadCallbacks,
|
||||
taskId = crypto.randomUUID(),
|
||||
info?: { label?: string },
|
||||
): UploadScanningTask {
|
||||
let open = true;
|
||||
callbacks?.onScanningStart?.(taskId);
|
||||
callbacks?.onScanningStart?.(taskId, info);
|
||||
|
||||
const close = (settle: () => void) => {
|
||||
if (!open) return;
|
||||
@@ -160,11 +163,24 @@ export async function uploadFromDataTransfer(
|
||||
const scanningTask = startUploadScanningTask(callbacks);
|
||||
let entries: DropEntry[];
|
||||
try {
|
||||
entries = await extractDropEntries(dataTransfer);
|
||||
entries = await extractDropEntries(dataTransfer, {
|
||||
onProgress: (progress) => {
|
||||
callbacks?.onScanningProgress?.(scanningTask.taskId, progress);
|
||||
},
|
||||
isCancelled: () => controller?.isCancelled() === true,
|
||||
});
|
||||
} catch (error) {
|
||||
scanningTask.complete();
|
||||
if (controller?.isCancelled() || /cancel/i.test(error instanceof Error ? error.message : String(error))) {
|
||||
scanningTask.cancel();
|
||||
} else {
|
||||
scanningTask.fail(error);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (controller?.isCancelled()) {
|
||||
scanningTask.cancel();
|
||||
return [{ fileName: "", success: false, cancelled: true }];
|
||||
}
|
||||
scanningTask.complete();
|
||||
logger.debug(`[SFTP:perf] extractDropEntries — ${entries.length} entries — ${(performance.now() - scanT0).toFixed(0)}ms`);
|
||||
|
||||
@@ -668,14 +684,14 @@ async function uploadEntries(
|
||||
const isStandaloneFile = rootName.startsWith("__file__");
|
||||
if (isStandaloneFile) continue;
|
||||
|
||||
// Calculate total bytes for this folder
|
||||
// Calculate total bytes for this folder (path-only entries from listLocalTree
|
||||
// carry size without a browser File handle).
|
||||
let totalBytes = 0;
|
||||
let fileCount = 0;
|
||||
for (const entry of rootEntries) {
|
||||
if (!entry.isDirectory && entry.file) {
|
||||
totalBytes += entry.file.size;
|
||||
fileCount++;
|
||||
}
|
||||
if (!isUploadableFileEntry(entry)) continue;
|
||||
totalBytes += getDropEntrySize(entry);
|
||||
fileCount++;
|
||||
}
|
||||
|
||||
if (fileCount === 0) continue;
|
||||
@@ -1036,7 +1052,9 @@ export async function uploadEntriesDirect(
|
||||
}
|
||||
|
||||
if (controller) {
|
||||
controller.reset();
|
||||
// Keep cancel latches/listeners from the external drop scan so the user
|
||||
// can still cancel while pre-task work (stat / conflict / mkdir) runs.
|
||||
controller.prepareForEntries();
|
||||
controller.setBridge(config.bridge);
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,14 @@ export interface UploadCallbacks {
|
||||
/** Called when a task is cancelled */
|
||||
onTaskCancelled?: (taskId: string) => void;
|
||||
/** Called when scanning starts (for showing placeholder) */
|
||||
onScanningStart?: (taskId: string) => void;
|
||||
onScanningStart?: (taskId: string, info?: { label?: string }) => void;
|
||||
/** Live scan counters (file/dir totals) while enumerating a drop */
|
||||
onScanningProgress?: (taskId: string, progress: {
|
||||
fileCount: number;
|
||||
directoryCount: number;
|
||||
entryCount: number;
|
||||
label?: string;
|
||||
}) => void;
|
||||
/** Called when scanning ends */
|
||||
onScanningEnd?: (taskId: string) => void;
|
||||
/** Called when task name needs to be updated (for phase changes) */
|
||||
|
||||
@@ -28,3 +28,18 @@ test("ZMODEM drops use reconstructed paths without buffering the file", async ()
|
||||
}]);
|
||||
assert.equal(buffered, false);
|
||||
});
|
||||
|
||||
test("ZMODEM drops accept path-only files from native folder scans", async () => {
|
||||
const files = await buildZmodemDragDropFiles([{
|
||||
file: null,
|
||||
localPath: "/tmp/project/src/main.ts",
|
||||
relativePath: "project/src/main.ts",
|
||||
isDirectory: false,
|
||||
}]);
|
||||
|
||||
assert.deepEqual(files, [{
|
||||
path: "/tmp/project/src/main.ts",
|
||||
name: "main.ts",
|
||||
remoteName: "main.ts",
|
||||
}]);
|
||||
});
|
||||
@@ -60,23 +60,27 @@ export async function buildZmodemDragDropFiles(
|
||||
const files: ZmodemDragDropFile[] = [];
|
||||
|
||||
for (const entry of dropEntries) {
|
||||
if (entry.isDirectory || !entry.file) continue;
|
||||
if (entry.isDirectory) continue;
|
||||
|
||||
const remoteName = getZmodemRemoteName(entry.relativePath, entry.file.name);
|
||||
const fileName = entry.file?.name
|
||||
|| entry.relativePath.replace(/\\/g, "/").split("/").pop()
|
||||
|| entry.relativePath;
|
||||
const remoteName = getZmodemRemoteName(entry.relativePath, fileName);
|
||||
const localPath = getDropEntryLocalPath(entry);
|
||||
|
||||
if (localPath) {
|
||||
files.push({
|
||||
path: localPath,
|
||||
name: entry.file.name,
|
||||
name: fileName,
|
||||
remoteName,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.file) continue;
|
||||
const data = await entry.file.arrayBuffer();
|
||||
files.push({
|
||||
name: entry.file.name,
|
||||
name: fileName,
|
||||
remoteName,
|
||||
data,
|
||||
});
|
||||
|
||||
+26
-2
@@ -143,13 +143,37 @@ declare global {
|
||||
renameLocalFile?(oldPath: string, newPath: string): Promise<void>;
|
||||
mkdirLocal?(path: string): Promise<void>;
|
||||
statLocal?(path: string): Promise<SftpStatResult>;
|
||||
listLocalTree?(path: string): Promise<Array<{
|
||||
listLocalTree?(
|
||||
path: string,
|
||||
options?: {
|
||||
onProgress?: (progress: {
|
||||
fileCount: number;
|
||||
directoryCount: number;
|
||||
entryCount: number;
|
||||
}) => void;
|
||||
/** Stream discovered rows while the walk continues (edge-scan/upload). */
|
||||
onEntries?: (entries: Array<{
|
||||
localPath: string;
|
||||
relativePath: string;
|
||||
type: 'file' | 'directory';
|
||||
size: number;
|
||||
lastModified: number;
|
||||
}>) => void;
|
||||
/** Renderer-generated ID used by cancelLocalTreeScan. */
|
||||
scanId?: string;
|
||||
limits?: {
|
||||
maxDirectories?: number;
|
||||
maxEntries?: number;
|
||||
};
|
||||
},
|
||||
): Promise<Array<{
|
||||
localPath: string;
|
||||
relativePath: string;
|
||||
type: 'file' | 'directory';
|
||||
size: number;
|
||||
lastModified: number;
|
||||
}>>;
|
||||
}>>;
|
||||
cancelLocalTreeScan?(scanId: string): Promise<void>;
|
||||
getHomeDir?(): Promise<string>;
|
||||
listDrives?(): Promise<string[]>;
|
||||
getSystemInfo?(): Promise<{ username: string; hostname: string }>;
|
||||
|
||||
Reference in new issue
Block a user