mirror of
https://github.com/binaricat/Netcatty.git
synced 2026-11-12 01:00:54 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c12df8b166 | ||
|
|
85751a9fc5 | ||
|
|
f671f58b33 | ||
|
|
a609578d6c | ||
|
|
515adb8303 | ||
|
|
e549ecdcdb | ||
|
|
e3d8246fa3 | ||
|
|
86c64f48e3 | ||
|
|
cf2d1c9dee | ||
|
|
6ee48f4b25 | ||
|
|
0f8c93649b | ||
|
|
3d4cecfa3b | ||
|
|
e2cc824152 | ||
|
|
6ca4ad1ee1 | ||
|
|
e0c492128e | ||
|
|
aaed9ff7d4 | ||
|
|
a025db4452 | ||
|
|
94d0464ff2 | ||
|
|
bce474348e |
No files matched your search
@@ -560,6 +560,11 @@ export const enVaultMessages: Messages = {
|
||||
'settings.sftp.compressedUpload.enable': 'Enable folder compression',
|
||||
'settings.sftp.compressedUpload.enableDesc': 'Automatically compress folders using tar before transfer. Requires tar support on the server. Falls back to regular transfer if not available.',
|
||||
|
||||
'settings.sftp.folderPrescan.enable': 'Pre-scan folders before transfer',
|
||||
'settings.sftp.folderPrescan.enableDesc': 'Walk the full folder tree first so the transfer center shows a stable file total before bytes move. Slightly delays the first transfer on very large trees.',
|
||||
'settings.sftp.skipUnchanged.enable': 'Skip unchanged files',
|
||||
'settings.sftp.skipUnchanged.enableDesc': 'Skip files whose size and modification time already match the destination (rsync-style). Disabled when replacing a folder into a staging path.',
|
||||
|
||||
// Quick Switcher
|
||||
'qs.search.placeholder': 'Search hosts or tabs',
|
||||
'qs.jumpTo': 'Jump To',
|
||||
|
||||
@@ -597,6 +597,11 @@ export const ruVaultMessages: Messages = {
|
||||
'settings.sftp.compressedUpload.enable': 'Включить сжатие папок',
|
||||
'settings.sftp.compressedUpload.enableDesc': 'Автоматически сжимать папки с помощью tar перед передачей. Требует поддержки tar на сервере. Если она недоступна, будет использована обычная передача.',
|
||||
|
||||
'settings.sftp.folderPrescan.enable': 'Предварительное сканирование папок',
|
||||
'settings.sftp.folderPrescan.enableDesc': 'Сначала обойти всё дерево папок, чтобы центр передачи показал стабильное число файлов до начала передачи байтов. На очень больших деревьях немного откладывает первую передачу.',
|
||||
'settings.sftp.skipUnchanged.enable': 'Пропускать неизменённые файлы',
|
||||
'settings.sftp.skipUnchanged.enableDesc': 'Пропускать файлы, у которых размер и время изменения уже совпадают с назначением (в стиле rsync). Отключается при замене папки во временный путь.',
|
||||
|
||||
// Quick Switcher
|
||||
'qs.search.placeholder': 'Поиск хостов или вкладок',
|
||||
'qs.jumpTo': 'Перейти к',
|
||||
|
||||
@@ -206,6 +206,11 @@ export const zhCNTerminalMessages: Messages = {
|
||||
'settings.sftp.compressedUpload.enable': '启用文件夹压缩',
|
||||
'settings.sftp.compressedUpload.enableDesc': '自动使用 tar 压缩文件夹后再传输。需要服务器支持 tar 命令,不支持时自动回退到普通传输。',
|
||||
|
||||
'settings.sftp.folderPrescan.enable': '传输前预扫描文件夹',
|
||||
'settings.sftp.folderPrescan.enableDesc': '先完整遍历文件夹树,传输中心在开始传字节前就能显示稳定的文件总数。超大目录会略微推迟首个文件开传。',
|
||||
'settings.sftp.skipUnchanged.enable': '跳过未改文件',
|
||||
'settings.sftp.skipUnchanged.enableDesc': '当目标文件的大小与修改时间已与源一致时跳过传输(类似 rsync)。替换到暂存路径时自动禁用。',
|
||||
|
||||
// Settings > Terminal
|
||||
'settings.terminal.section.theme': '终端主题',
|
||||
'settings.terminal.themeModal.title': '选择主题',
|
||||
|
||||
@@ -206,6 +206,11 @@ export const zhTWTerminalMessages: Messages = {
|
||||
'settings.sftp.compressedUpload.enable': '啟用資料夾壓縮',
|
||||
'settings.sftp.compressedUpload.enableDesc': '自動使用 tar 壓縮資料夾後再傳輸。需要伺服器支援 tar 指令,不支援時自動回退到普通傳輸。',
|
||||
|
||||
'settings.sftp.folderPrescan.enable': '傳輸前預掃描資料夾',
|
||||
'settings.sftp.folderPrescan.enableDesc': '先完整遍歷資料夾樹,傳輸中心在開始傳位元組前就能顯示穩定的檔案總數。超大目錄會略微推遲首個檔案開傳。',
|
||||
'settings.sftp.skipUnchanged.enable': '跳過未改檔案',
|
||||
'settings.sftp.skipUnchanged.enableDesc': '當目標檔案的大小與修改時間已與來源一致時跳過傳輸(類似 rsync)。替換到暫存路徑時自動停用。',
|
||||
|
||||
// Settings > Terminal
|
||||
'settings.terminal.section.theme': '終端主題',
|
||||
'settings.terminal.themeModal.title': '選擇主題',
|
||||
|
||||
@@ -28,6 +28,8 @@ import {
|
||||
STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY,
|
||||
STORAGE_KEY_SSH_TRANSPORT_IDLE_TTL_MS,
|
||||
STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD,
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SHOW_ONLY_UNGROUPED_HOSTS_IN_ROOT,
|
||||
STORAGE_KEY_SHOW_RECENT_HOSTS,
|
||||
STORAGE_KEY_HOST_CLICK_BEHAVIOR,
|
||||
@@ -79,6 +81,8 @@ interface UseSettingsStorageSyncParams {
|
||||
sftpAutoSync: boolean;
|
||||
sftpShowHiddenFiles: boolean;
|
||||
sftpUseCompressedUpload: boolean;
|
||||
sftpFolderPrescan: boolean;
|
||||
sftpSkipUnchanged: boolean;
|
||||
sftpAutoOpenSidebar: boolean;
|
||||
sftpFollowTerminalCwd: boolean;
|
||||
sftpDefaultViewMode: 'list' | 'tree';
|
||||
@@ -125,6 +129,8 @@ interface UseSettingsStorageSyncParams {
|
||||
setSftpAutoSync: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpShowHiddenFiles: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpUseCompressedUpload: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpFolderPrescan: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpSkipUnchanged: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpAutoOpenSidebar: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpFollowTerminalCwd: Dispatch<SetStateAction<boolean>>;
|
||||
setSftpDefaultViewMode: Dispatch<SetStateAction<'list' | 'tree'>>;
|
||||
@@ -165,7 +171,7 @@ export function useSettingsStorageSync({
|
||||
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
|
||||
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
|
||||
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
|
||||
@@ -174,7 +180,7 @@ export function useSettingsStorageSync({
|
||||
setTerminalThemeId, setTerminalThemeDarkId, setTerminalThemeLightId,
|
||||
setFollowAppTerminalThemeState, setTerminalFontFamilyId, setTerminalFontSize,
|
||||
setSftpDoubleClickBehavior, setSftpAutoSync, setSftpShowHiddenFiles,
|
||||
setSftpUseCompressedUpload, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
|
||||
setSftpUseCompressedUpload, setSftpFolderPrescan, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
|
||||
setShowRecentHostsState, setHostClickBehaviorState, setShowOnlyUngroupedHostsInRootState, setShowSftpTabState, setShowHostTreeSidebarState, setTerminalSidePanelAutoOpenState, setTerminalSidePanelAutoOpenTabState, setShellOnlyTabNumberShortcutsState, setDisableTerminalFontZoomState, setRestorePreviousSessionState, setRestoreTerminalCwdState,
|
||||
setEditorWordWrapState, setSessionLogsEnabled, setSessionLogsDir, setSessionLogsFormat, setSessionLogsTimestampsEnabled, setSshDebugLogsEnabled, setSshDeepLinkEnabledState, setJmsDeepLinkEnabledState, setExplorerContextMenuEnabledState,
|
||||
setGlobalHotkeyEnabled, setWindowOpacity, setAppIconVariant, setAutoUpdateEnabled, setWorkspaceFocusStyleState,
|
||||
@@ -189,7 +195,7 @@ export function useSettingsStorageSync({
|
||||
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
|
||||
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
|
||||
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
|
||||
@@ -199,7 +205,7 @@ export function useSettingsStorageSync({
|
||||
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
|
||||
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
|
||||
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
|
||||
@@ -379,6 +385,18 @@ export function useSettingsStorageSync({
|
||||
setSftpUseCompressedUpload(newValue);
|
||||
}
|
||||
}
|
||||
if (e.key === STORAGE_KEY_SFTP_FOLDER_PRESCAN && e.newValue !== null) {
|
||||
const newValue = e.newValue === 'true';
|
||||
if (newValue !== s.sftpFolderPrescan) {
|
||||
setSftpFolderPrescan(newValue);
|
||||
}
|
||||
}
|
||||
if (e.key === STORAGE_KEY_SFTP_SKIP_UNCHANGED && e.newValue !== null) {
|
||||
const newValue = e.newValue === 'true';
|
||||
if (newValue !== s.sftpSkipUnchanged) {
|
||||
setSftpSkipUnchanged(newValue);
|
||||
}
|
||||
}
|
||||
// Sync SFTP auto-open sidebar setting from other windows
|
||||
if (e.key === STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR && e.newValue !== null) {
|
||||
const newValue = e.newValue === 'true';
|
||||
@@ -542,6 +560,8 @@ export function useSettingsStorageSync({
|
||||
setSftpTransferConcurrencyState,
|
||||
setSshTransportIdleTtlMsState,
|
||||
setSftpUseCompressedUpload,
|
||||
setSftpFolderPrescan,
|
||||
setSftpSkipUnchanged,
|
||||
setShowOnlyUngroupedHostsInRootState,
|
||||
setShowHostTreeSidebarState,
|
||||
setTerminalSidePanelAutoOpenState,
|
||||
|
||||
@@ -2,8 +2,15 @@ import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
|
||||
DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY,
|
||||
DEFAULT_SFTP_FOLDER_PRESCAN,
|
||||
DEFAULT_SFTP_SKIP_UNCHANGED,
|
||||
resolveSftpDirectoryListingConcurrency,
|
||||
resolveSftpFolderPrescanEnabled,
|
||||
resolveSftpSkipUnchangedEnabled,
|
||||
resolveSftpTransferConcurrency,
|
||||
runBoundedConcurrency,
|
||||
runSftpTransferWorkers,
|
||||
} from "./transferConcurrency";
|
||||
|
||||
@@ -12,6 +19,23 @@ test("defaults folder file transfers to two concurrent files", () => {
|
||||
assert.equal(DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY, 2);
|
||||
});
|
||||
|
||||
test("defaults directory listing fanout to four concurrent readdirs", () => {
|
||||
assert.equal(
|
||||
resolveSftpDirectoryListingConcurrency(() => null),
|
||||
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
|
||||
);
|
||||
assert.equal(DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY, 4);
|
||||
});
|
||||
|
||||
test("defaults folder pre-scan and skip-unchanged to enabled", () => {
|
||||
assert.equal(resolveSftpFolderPrescanEnabled(() => null), DEFAULT_SFTP_FOLDER_PRESCAN);
|
||||
assert.equal(resolveSftpSkipUnchangedEnabled(() => null), DEFAULT_SFTP_SKIP_UNCHANGED);
|
||||
assert.equal(DEFAULT_SFTP_FOLDER_PRESCAN, true);
|
||||
assert.equal(DEFAULT_SFTP_SKIP_UNCHANGED, true);
|
||||
assert.equal(resolveSftpFolderPrescanEnabled(() => false), false);
|
||||
assert.equal(resolveSftpSkipUnchangedEnabled(() => false), false);
|
||||
});
|
||||
|
||||
test("keeps explicit folder transfer concurrency within the supported range", () => {
|
||||
assert.equal(resolveSftpTransferConcurrency(() => 1), 1);
|
||||
assert.equal(resolveSftpTransferConcurrency(() => 16), 16);
|
||||
@@ -19,6 +43,19 @@ test("keeps explicit folder transfer concurrency within the supported range", ()
|
||||
assert.equal(resolveSftpTransferConcurrency(() => 17), DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY);
|
||||
});
|
||||
|
||||
test("keeps directory listing concurrency within the supported range", () => {
|
||||
assert.equal(resolveSftpDirectoryListingConcurrency(() => 1), 1);
|
||||
assert.equal(resolveSftpDirectoryListingConcurrency(() => 8), 8);
|
||||
assert.equal(
|
||||
resolveSftpDirectoryListingConcurrency(() => 0),
|
||||
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
|
||||
);
|
||||
assert.equal(
|
||||
resolveSftpDirectoryListingConcurrency(() => 9),
|
||||
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
|
||||
);
|
||||
});
|
||||
|
||||
test("limits default multi-file transfer scheduling to two concurrent workers", async () => {
|
||||
let active = 0;
|
||||
let maxActive = 0;
|
||||
@@ -33,6 +70,106 @@ test("limits default multi-file transfer scheduling to two concurrent workers",
|
||||
assert.equal(maxActive, 2);
|
||||
});
|
||||
|
||||
test("runBoundedConcurrency respects an explicit limit", async () => {
|
||||
let active = 0;
|
||||
let maxActive = 0;
|
||||
await runBoundedConcurrency([1, 2, 3, 4, 5, 6], 3, async () => {
|
||||
active += 1;
|
||||
maxActive = Math.max(maxActive, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
active -= 1;
|
||||
});
|
||||
assert.equal(maxActive, 3);
|
||||
});
|
||||
|
||||
test("runBoundedConcurrency drains siblings and stops new claims after an error", async () => {
|
||||
const started: number[] = [];
|
||||
const finished: number[] = [];
|
||||
let releaseSlow!: () => void;
|
||||
const slowGate = new Promise<void>((resolve) => {
|
||||
releaseSlow = resolve;
|
||||
});
|
||||
|
||||
const run = runBoundedConcurrency([0, 1, 2, 3, 4], 2, async (item) => {
|
||||
started.push(item);
|
||||
if (item === 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
throw new Error("boom");
|
||||
}
|
||||
if (item === 1) {
|
||||
await slowGate;
|
||||
}
|
||||
finished.push(item);
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
|
||||
let settledWhileSiblingRunning = false;
|
||||
await Promise.race([
|
||||
run.then(
|
||||
() => {
|
||||
settledWhileSiblingRunning = true;
|
||||
},
|
||||
() => {
|
||||
settledWhileSiblingRunning = true;
|
||||
},
|
||||
),
|
||||
new Promise((resolve) => setTimeout(resolve, 5)),
|
||||
]);
|
||||
assert.equal(
|
||||
settledWhileSiblingRunning,
|
||||
false,
|
||||
"must wait for in-flight siblings before propagating the error",
|
||||
);
|
||||
assert.ok(started.includes(1));
|
||||
assert.equal(finished.includes(1), false);
|
||||
|
||||
releaseSlow();
|
||||
await assert.rejects(run, /boom/);
|
||||
assert.ok(finished.includes(1), "in-flight sibling must finish before reject");
|
||||
assert.deepEqual(
|
||||
[...started].sort((a, b) => a - b),
|
||||
[0, 1],
|
||||
"must not claim additional queue items after the first error",
|
||||
);
|
||||
});
|
||||
|
||||
test("runSftpTransferWorkers drains siblings and stops new claims after an error", async () => {
|
||||
const started: number[] = [];
|
||||
const finished: number[] = [];
|
||||
let releaseSlow!: () => void;
|
||||
const slowGate = new Promise<void>((resolve) => {
|
||||
releaseSlow = resolve;
|
||||
});
|
||||
|
||||
const run = runSftpTransferWorkers([0, 1, 2, 3, 4], () => 2, async (item) => {
|
||||
started.push(item);
|
||||
if (item === 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
throw new Error("session lost");
|
||||
}
|
||||
if (item === 1) {
|
||||
await slowGate;
|
||||
}
|
||||
finished.push(item);
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
let settledEarly = false;
|
||||
await Promise.race([
|
||||
run.then(
|
||||
() => { settledEarly = true; },
|
||||
() => { settledEarly = true; },
|
||||
),
|
||||
new Promise((resolve) => setTimeout(resolve, 5)),
|
||||
]);
|
||||
assert.equal(settledEarly, false);
|
||||
releaseSlow();
|
||||
await assert.rejects(run, /session lost/);
|
||||
assert.ok(finished.includes(1));
|
||||
assert.deepEqual([...started].sort((a, b) => a - b), [0, 1]);
|
||||
});
|
||||
|
||||
test("beforeClaim runs before claiming the next queue index", async () => {
|
||||
const events: string[] = [];
|
||||
let paused = true;
|
||||
|
||||
@@ -2,6 +2,22 @@ export const DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY = 2;
|
||||
export const MIN_SFTP_FILE_TRANSFER_CONCURRENCY = 1;
|
||||
export const MAX_SFTP_FILE_TRANSFER_CONCURRENCY = 16;
|
||||
|
||||
/**
|
||||
* Bounded parallel directory listings while walking a folder tree.
|
||||
* SFTP has no recursive LIST; FileZilla/WinSCP still walk one dir at a time on
|
||||
* a single control channel. We pipeline several OPENDIR/READDIR requests so
|
||||
* wide trees discover total file counts much faster without a second full scan.
|
||||
* Keep this modest - listing shares the transfer SFTP session with file I/O.
|
||||
*/
|
||||
export const DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY = 4;
|
||||
export const MIN_SFTP_DIRECTORY_LISTING_CONCURRENCY = 1;
|
||||
export const MAX_SFTP_DIRECTORY_LISTING_CONCURRENCY = 8;
|
||||
|
||||
/** Default on: accurate folder totals before first byte (rsync --no-i-r style). */
|
||||
export const DEFAULT_SFTP_FOLDER_PRESCAN = true;
|
||||
/** Default on: skip size+mtime matches like rsync's generator. */
|
||||
export const DEFAULT_SFTP_SKIP_UNCHANGED = true;
|
||||
|
||||
export function resolveSftpTransferConcurrency(readStoredValue: () => number | null | undefined): number {
|
||||
const stored = readStoredValue();
|
||||
return stored != null &&
|
||||
@@ -11,6 +27,31 @@ export function resolveSftpTransferConcurrency(readStoredValue: () => number | n
|
||||
: DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY;
|
||||
}
|
||||
|
||||
export function resolveSftpDirectoryListingConcurrency(
|
||||
readStoredValue?: () => number | null | undefined,
|
||||
): number {
|
||||
const stored = readStoredValue?.();
|
||||
return stored != null &&
|
||||
stored >= MIN_SFTP_DIRECTORY_LISTING_CONCURRENCY &&
|
||||
stored <= MAX_SFTP_DIRECTORY_LISTING_CONCURRENCY
|
||||
? stored
|
||||
: DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY;
|
||||
}
|
||||
|
||||
export function resolveSftpFolderPrescanEnabled(
|
||||
readStoredValue: () => boolean | null | undefined,
|
||||
): boolean {
|
||||
const stored = readStoredValue();
|
||||
return stored == null ? DEFAULT_SFTP_FOLDER_PRESCAN : stored;
|
||||
}
|
||||
|
||||
export function resolveSftpSkipUnchangedEnabled(
|
||||
readStoredValue: () => boolean | null | undefined,
|
||||
): boolean {
|
||||
const stored = readStoredValue();
|
||||
return stored == null ? DEFAULT_SFTP_SKIP_UNCHANGED : stored;
|
||||
}
|
||||
|
||||
export async function runSftpTransferWorkers<T>(
|
||||
items: T[],
|
||||
readStoredConcurrency: () => number | null | undefined,
|
||||
@@ -26,17 +67,27 @@ export async function runSftpTransferWorkers<T>(
|
||||
): Promise<void> {
|
||||
const concurrency = resolveSftpTransferConcurrency(readStoredConcurrency);
|
||||
let nextIndex = 0;
|
||||
let failed = false;
|
||||
let firstError: unknown;
|
||||
|
||||
const runNext = async () => {
|
||||
while (nextIndex < items.length) {
|
||||
// Wait BEFORE claiming so pause does not leave a claimed-but-not-started
|
||||
// index that arms as soon as soft-drain finishes the previous file.
|
||||
if (options?.beforeClaim) {
|
||||
await options.beforeClaim();
|
||||
while (!failed && nextIndex < items.length) {
|
||||
try {
|
||||
// Wait BEFORE claiming so pause does not leave a claimed-but-not-started
|
||||
// index that arms as soon as soft-drain finishes the previous file.
|
||||
if (options?.beforeClaim) {
|
||||
await options.beforeClaim();
|
||||
}
|
||||
if (failed || nextIndex >= items.length) return;
|
||||
const index = nextIndex++;
|
||||
await worker(items[index], index);
|
||||
} catch (err) {
|
||||
if (!failed) {
|
||||
failed = true;
|
||||
firstError = err;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (nextIndex >= items.length) return;
|
||||
const index = nextIndex++;
|
||||
await worker(items[index], index);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -44,5 +95,47 @@ export async function runSftpTransferWorkers<T>(
|
||||
{ length: Math.min(concurrency, items.length) },
|
||||
() => runNext(),
|
||||
);
|
||||
// Settle every started worker before propagating - Promise.all would reject
|
||||
// while siblings keep claiming files after the caller releases leases.
|
||||
await Promise.all(workers);
|
||||
if (failed) throw firstError;
|
||||
}
|
||||
|
||||
/** Run workers over a queue with an explicit concurrency (not settings-backed). */
|
||||
export async function runBoundedConcurrency<T>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
worker: (item: T, index: number) => Promise<void>,
|
||||
options?: {
|
||||
beforeClaim?: () => Promise<void>;
|
||||
},
|
||||
): Promise<void> {
|
||||
const limit = Math.max(1, Math.min(Math.floor(concurrency) || 1, items.length || 1));
|
||||
if (items.length === 0) return;
|
||||
let nextIndex = 0;
|
||||
let failed = false;
|
||||
let firstError: unknown;
|
||||
const runNext = async () => {
|
||||
while (!failed && nextIndex < items.length) {
|
||||
try {
|
||||
if (options?.beforeClaim) {
|
||||
await options.beforeClaim();
|
||||
}
|
||||
// Re-check after beforeClaim: a sibling may have failed while we waited.
|
||||
if (failed || nextIndex >= items.length) return;
|
||||
const index = nextIndex++;
|
||||
await worker(items[index], index);
|
||||
} catch (err) {
|
||||
if (!failed) {
|
||||
failed = true;
|
||||
firstError = err;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
// Settle every started worker before propagating - Promise.all would reject
|
||||
// while siblings keep claiming directories / transferring after the caller cleans up.
|
||||
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => runNext()));
|
||||
if (failed) throw firstError;
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import type { SftpFileEntry } from "../../../domain/models";
|
||||
import { compareDirectoryTraversalPaths } from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { createSftpDirectoryTraversalBudget } from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { discoverTransferTree } from "./transferDirectoryDiscovery";
|
||||
|
||||
const directoryEntry = (name: string): SftpFileEntry => ({
|
||||
name,
|
||||
type: "directory",
|
||||
size: 0,
|
||||
sizeFormatted: "0 B",
|
||||
lastModified: 0,
|
||||
lastModifiedFormatted: "",
|
||||
});
|
||||
|
||||
const fileEntry = (name: string, size = 1, lastModified = 1_700_000_000): SftpFileEntry => ({
|
||||
name,
|
||||
type: "file",
|
||||
size,
|
||||
sizeFormatted: `${size} B`,
|
||||
lastModified,
|
||||
lastModifiedFormatted: "",
|
||||
});
|
||||
|
||||
test("discoverTransferTree returns a stable flat file list with bounded listings", async () => {
|
||||
const previousWindow = (globalThis as { window?: unknown }).window;
|
||||
(globalThis as { window?: unknown }).window = {
|
||||
netcatty: {
|
||||
realpathSftp: async (_sftpId: string, remotePath: string) => remotePath,
|
||||
},
|
||||
};
|
||||
|
||||
let active = 0;
|
||||
let maxActive = 0;
|
||||
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
|
||||
active += 1;
|
||||
maxActive = Math.max(maxActive, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1));
|
||||
active -= 1;
|
||||
if (path === "/source") {
|
||||
return [directoryEntry("a"), directoryEntry("b"), fileEntry("root.txt")];
|
||||
}
|
||||
if (path === "/source/a") return [fileEntry("a1.txt"), fileEntry("a2.txt")];
|
||||
if (path === "/source/b") return [fileEntry("b1.txt")];
|
||||
return [];
|
||||
};
|
||||
|
||||
try {
|
||||
const counts: number[] = [];
|
||||
const result = await discoverTransferTree({
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: false,
|
||||
sourceSftpId: "sftp-1",
|
||||
sourceEncoding: "auto",
|
||||
listLocalFiles: async () => [],
|
||||
listRemoteFiles,
|
||||
listingConcurrency: 2,
|
||||
onDiscoveredFiles: (total) => counts.push(total),
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
result.files.map((file) => file.sourcePath),
|
||||
[...result.files.map((file) => file.sourcePath)].sort((left, right) =>
|
||||
compareDirectoryTraversalPaths(left, right)
|
||||
),
|
||||
"pre-scan file order must match resume comparator",
|
||||
);
|
||||
assert.deepEqual(
|
||||
result.files.map((file) => file.sourcePath).sort(),
|
||||
["/source/a/a1.txt", "/source/a/a2.txt", "/source/b/b1.txt", "/source/root.txt"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
result.directories.map((dir) => dir.sourcePath).sort(),
|
||||
["/source", "/source/a", "/source/b"],
|
||||
);
|
||||
assert.equal(counts.at(-1), 4);
|
||||
assert.equal(result.omittedSymlinkDirectoryErrors, 0);
|
||||
assert.ok(maxActive >= 2, `expected parallel listings, got ${maxActive}`);
|
||||
assert.ok(maxActive <= 2, `listing concurrency exceeded: ${maxActive}`);
|
||||
} finally {
|
||||
(globalThis as { window?: unknown }).window = previousWindow;
|
||||
}
|
||||
});
|
||||
|
||||
test("discoverTransferTree skips symlink directory cycles across BFS waves", async () => {
|
||||
const previousWindow = (globalThis as { window?: unknown }).window;
|
||||
const listCalls = new Map<string, number>();
|
||||
(globalThis as { window?: unknown }).window = {
|
||||
netcatty: {
|
||||
realpathSftp: async (_sftpId: string, remotePath: string) => (
|
||||
remotePath === "/source/loop" ? "/source" : remotePath
|
||||
),
|
||||
},
|
||||
};
|
||||
|
||||
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
|
||||
listCalls.set(path, (listCalls.get(path) ?? 0) + 1);
|
||||
if (path === "/source") {
|
||||
return [
|
||||
directoryEntry("child"),
|
||||
{ ...directoryEntry("loop"), type: "symlink", linkTarget: "directory" },
|
||||
fileEntry("root.txt"),
|
||||
];
|
||||
}
|
||||
if (path === "/source/child") return [fileEntry("nested.txt")];
|
||||
throw new Error(`unexpected list of ${path}`);
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await discoverTransferTree({
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: false,
|
||||
sourceSftpId: "sftp-1",
|
||||
sourceEncoding: "auto",
|
||||
followSymlinks: true,
|
||||
listLocalFiles: async () => [],
|
||||
listRemoteFiles,
|
||||
listingConcurrency: 4,
|
||||
});
|
||||
|
||||
assert.equal(listCalls.has("/source/loop"), false);
|
||||
assert.equal(listCalls.get("/source"), 1);
|
||||
assert.equal(listCalls.get("/source/child"), 1);
|
||||
assert.deepEqual(
|
||||
result.files.map((file) => file.sourcePath).sort(),
|
||||
["/source/child/nested.txt", "/source/root.txt"],
|
||||
);
|
||||
assert.deepEqual(
|
||||
result.directories.map((dir) => dir.sourcePath).sort(),
|
||||
["/source", "/source/child"],
|
||||
"cyclic symlink dirs must not appear in the creation plan",
|
||||
);
|
||||
assert.equal(result.omittedSymlinkDirectoryErrors, 0);
|
||||
} finally {
|
||||
(globalThis as { window?: unknown }).window = previousWindow;
|
||||
}
|
||||
});
|
||||
|
||||
test("discoverTransferTree copies sibling symlink aliases to distinct targets", async () => {
|
||||
const previousWindow = (globalThis as { window?: unknown }).window;
|
||||
const listCalls = new Map<string, number>();
|
||||
(globalThis as { window?: unknown }).window = {
|
||||
netcatty: {
|
||||
realpathSftp: async (_sftpId: string, remotePath: string) => {
|
||||
if (remotePath === "/source/alias-a" || remotePath === "/source/alias-b") {
|
||||
return "/shared";
|
||||
}
|
||||
return remotePath;
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
|
||||
listCalls.set(path, (listCalls.get(path) ?? 0) + 1);
|
||||
if (path === "/source") {
|
||||
return [
|
||||
{ ...directoryEntry("alias-a"), type: "symlink", linkTarget: "directory" },
|
||||
{ ...directoryEntry("alias-b"), type: "symlink", linkTarget: "directory" },
|
||||
];
|
||||
}
|
||||
if (path === "/source/alias-a" || path === "/source/alias-b") {
|
||||
return [fileEntry("shared.txt")];
|
||||
}
|
||||
throw new Error(`unexpected list of ${path}`);
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await discoverTransferTree({
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: false,
|
||||
sourceSftpId: "sftp-1",
|
||||
sourceEncoding: "auto",
|
||||
followSymlinks: true,
|
||||
listLocalFiles: async () => [],
|
||||
listRemoteFiles,
|
||||
listingConcurrency: 4,
|
||||
});
|
||||
|
||||
assert.equal(listCalls.get("/source/alias-a"), 1);
|
||||
assert.equal(listCalls.get("/source/alias-b"), 1);
|
||||
assert.deepEqual(
|
||||
result.files.map((file) => file.targetPath).sort(),
|
||||
["/target/alias-a/shared.txt", "/target/alias-b/shared.txt"],
|
||||
);
|
||||
} finally {
|
||||
(globalThis as { window?: unknown }).window = previousWindow;
|
||||
}
|
||||
});
|
||||
|
||||
test("discoverTransferTree counts max-depth symlink directory omissions as errors", async () => {
|
||||
const previousWindow = (globalThis as { window?: unknown }).window;
|
||||
(globalThis as { window?: unknown }).window = {
|
||||
netcatty: {
|
||||
realpathSftp: async (_sftpId: string, remotePath: string) => remotePath,
|
||||
},
|
||||
};
|
||||
|
||||
const listRemoteFiles = async (_sftpId: string, path: string): Promise<SftpFileEntry[]> => {
|
||||
const depth = path === "/source"
|
||||
? 0
|
||||
: path.startsWith("/source/")
|
||||
? path.slice("/source/".length).split("/").filter(Boolean).length
|
||||
: -1;
|
||||
if (depth < 0 || depth > 32) throw new Error(`unexpected list of ${path}`);
|
||||
const nextLink = { ...directoryEntry("deep"), type: "symlink" as const, linkTarget: "directory" as const };
|
||||
if (depth === 0) {
|
||||
return [
|
||||
nextLink,
|
||||
{
|
||||
name: "link.txt",
|
||||
type: "symlink",
|
||||
linkTarget: "file",
|
||||
size: 3,
|
||||
sizeFormatted: "3 B",
|
||||
lastModified: 1_700_000_000,
|
||||
lastModifiedFormatted: "",
|
||||
},
|
||||
];
|
||||
}
|
||||
return [nextLink];
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await discoverTransferTree({
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: false,
|
||||
sourceSftpId: "sftp-1",
|
||||
sourceEncoding: "auto",
|
||||
followSymlinks: true,
|
||||
listLocalFiles: async () => [],
|
||||
listRemoteFiles,
|
||||
listingConcurrency: 2,
|
||||
});
|
||||
|
||||
assert.equal(result.omittedSymlinkDirectoryErrors, 1);
|
||||
assert.equal(result.files.length, 1);
|
||||
assert.equal(result.files[0]?.isSymlink, true);
|
||||
assert.equal(result.files[0]?.name, "link.txt");
|
||||
} finally {
|
||||
(globalThis as { window?: unknown }).window = previousWindow;
|
||||
}
|
||||
});
|
||||
|
||||
test("discoverTransferTree bounds local folder pre-scan memory", async () => {
|
||||
const listLocalFiles = async (path: string): Promise<SftpFileEntry[]> => {
|
||||
if (path === "/local") return [directoryEntry("a"), fileEntry("root.txt")];
|
||||
if (path === "/local/a") return [fileEntry("nested.txt")];
|
||||
return [];
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() => discoverTransferTree({
|
||||
sourcePath: "/local",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: true,
|
||||
sourceSftpId: null,
|
||||
sourceEncoding: "auto",
|
||||
listLocalFiles,
|
||||
listRemoteFiles: async () => [],
|
||||
listingConcurrency: 2,
|
||||
traversalBudget: createSftpDirectoryTraversalBudget({ maxDirectories: 1, maxEntries: 200_000 }),
|
||||
}),
|
||||
/Directory traversal directory limit exceeded/,
|
||||
);
|
||||
});
|
||||
|
||||
test("discoverTransferTree waits while paused between BFS waves", async () => {
|
||||
let paused = true;
|
||||
let listCalls = 0;
|
||||
let pauseWaits = 0;
|
||||
const listLocalFiles = async (path: string): Promise<SftpFileEntry[]> => {
|
||||
listCalls += 1;
|
||||
if (path === "/local") return [directoryEntry("a"), fileEntry("root.txt")];
|
||||
if (path === "/local/a") return [fileEntry("nested.txt")];
|
||||
return [];
|
||||
};
|
||||
|
||||
const discovery = discoverTransferTree({
|
||||
sourcePath: "/local",
|
||||
targetPath: "/target",
|
||||
sourceIsLocal: true,
|
||||
sourceSftpId: null,
|
||||
sourceEncoding: "auto",
|
||||
listLocalFiles,
|
||||
listRemoteFiles: async () => [],
|
||||
listingConcurrency: 1,
|
||||
waitWhilePaused: async () => {
|
||||
pauseWaits += 1;
|
||||
while (paused) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// First waitWhilePaused (wave start) should block before any listing.
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
assert.equal(listCalls, 0, "pause must block pre-scan listings");
|
||||
assert.ok(pauseWaits >= 1, "expected waitWhilePaused to be invoked");
|
||||
|
||||
paused = false;
|
||||
const result = await discovery;
|
||||
assert.equal(result.files.length, 2);
|
||||
assert.ok(listCalls >= 2);
|
||||
assert.ok(pauseWaits >= 2, "expected pause checks across waves/listings");
|
||||
});
|
||||
@@ -0,0 +1,279 @@
|
||||
import type { SftpFileEntry, SftpFilenameEncoding } from "../../../domain/models";
|
||||
import {
|
||||
accountSftpDirectoryEntries,
|
||||
claimSftpDirectoryVisit,
|
||||
compareDirectoryTraversalPaths,
|
||||
createSftpDirectoryBranchAncestors,
|
||||
createSftpDirectoryTraversalBudget,
|
||||
releaseSftpDirectoryVisit,
|
||||
shouldFollowSftpSymlinkDirectory,
|
||||
type SftpDirectoryTraversalBudget,
|
||||
} from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { netcattyBridge } from "../../../infrastructure/services/netcattyBridge";
|
||||
import { logger } from "../../../lib/logger";
|
||||
import {
|
||||
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
|
||||
resolveSftpDirectoryListingConcurrency,
|
||||
runBoundedConcurrency,
|
||||
} from "./transferConcurrency";
|
||||
import { joinPath, joinTransferTargetPath } from "./utils";
|
||||
|
||||
export type DiscoveredTransferFile = {
|
||||
name: string;
|
||||
sourcePath: string;
|
||||
targetPath: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
/**
|
||||
* True when the listing entry was a symlink. Link-node size/mtime must not
|
||||
* drive skip-unchanged; transfer follows the target bytes.
|
||||
*/
|
||||
isSymlink?: boolean;
|
||||
};
|
||||
|
||||
export type DiscoveredTransferDirectory = {
|
||||
sourcePath: string;
|
||||
targetPath: string;
|
||||
};
|
||||
|
||||
export type DiscoverTransferTreeResult = {
|
||||
files: DiscoveredTransferFile[];
|
||||
directories: DiscoveredTransferDirectory[];
|
||||
/** Symlink directories omitted at max follow depth (incomplete tree). */
|
||||
omittedSymlinkDirectoryErrors: number;
|
||||
};
|
||||
|
||||
type ListingGate = {
|
||||
run: <T>(fn: () => Promise<T>) => Promise<T>;
|
||||
};
|
||||
|
||||
function createListingGate(concurrency = DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY): ListingGate {
|
||||
const limit = Math.max(1, Math.floor(concurrency) || 1);
|
||||
let active = 0;
|
||||
const waiters: Array<() => void> = [];
|
||||
const acquire = async () => {
|
||||
if (active < limit) {
|
||||
active += 1;
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
waiters.push(() => {
|
||||
active += 1;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
const release = () => {
|
||||
active = Math.max(0, active - 1);
|
||||
const next = waiters.shift();
|
||||
if (next) next();
|
||||
};
|
||||
return {
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
await acquire();
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function getEntrySize(entry: SftpFileEntry): number {
|
||||
if (typeof entry.size === "string") {
|
||||
const parsed = parseInt(entry.size, 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
|
||||
}
|
||||
return typeof entry.size === "number" && entry.size > 0 ? entry.size : 0;
|
||||
}
|
||||
|
||||
export type DiscoverTransferTreeOptions = {
|
||||
sourcePath: string;
|
||||
targetPath: string;
|
||||
sourceIsLocal: boolean;
|
||||
sourceSftpId: string | null;
|
||||
sourceEncoding: SftpFilenameEncoding;
|
||||
followSymlinks?: boolean;
|
||||
listLocalFiles: (path: string) => Promise<SftpFileEntry[]>;
|
||||
listRemoteFiles: (sftpId: string, path: string, encoding?: SftpFilenameEncoding) => Promise<SftpFileEntry[]>;
|
||||
shouldAbort?: () => boolean;
|
||||
/** Honor transfer-center pause between BFS waves / listings. */
|
||||
waitWhilePaused?: () => Promise<void>;
|
||||
onDiscoveredFiles?: (totalFiles: number) => void;
|
||||
listingConcurrency?: number;
|
||||
traversalBudget?: SftpDirectoryTraversalBudget;
|
||||
};
|
||||
|
||||
/**
|
||||
* Full-tree discovery (rsync --no-inc-recursive style): walk every directory
|
||||
* before transfer starts so the UI can show a stable file total.
|
||||
* Listings are bounded-parallel; nested walks share one semaphore.
|
||||
*/
|
||||
export async function discoverTransferTree(
|
||||
options: DiscoverTransferTreeOptions,
|
||||
): Promise<DiscoverTransferTreeResult> {
|
||||
const followSymlinks = options.followSymlinks === true;
|
||||
const listingConcurrency = options.listingConcurrency
|
||||
?? resolveSftpDirectoryListingConcurrency();
|
||||
const listingGate = createListingGate(listingConcurrency);
|
||||
const traversal = options.traversalBudget ?? createSftpDirectoryTraversalBudget();
|
||||
const files: DiscoveredTransferFile[] = [];
|
||||
const directories: DiscoveredTransferDirectory[] = [
|
||||
{ sourcePath: options.sourcePath, targetPath: options.targetPath },
|
||||
];
|
||||
let omittedSymlinkDirectoryErrors = 0;
|
||||
|
||||
type DirJob = {
|
||||
sourcePath: string;
|
||||
targetPath: string;
|
||||
symlinkDepth: number;
|
||||
/** Ancestors on this BFS branch (copied for parallel sibling aliases). */
|
||||
branchAncestors: Set<string>;
|
||||
/** False for the root (already in `directories`); true for accepted children. */
|
||||
includeInCreationPlan: boolean;
|
||||
};
|
||||
|
||||
const queue: DirJob[] = [{
|
||||
sourcePath: options.sourcePath,
|
||||
targetPath: options.targetPath,
|
||||
symlinkDepth: 0,
|
||||
branchAncestors: createSftpDirectoryBranchAncestors(),
|
||||
includeInCreationPlan: false,
|
||||
}];
|
||||
|
||||
const publishCount = () => {
|
||||
options.onDiscoveredFiles?.(files.length);
|
||||
};
|
||||
|
||||
const processDir = async (job: DirJob): Promise<DirJob[]> => {
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
await options.waitWhilePaused?.();
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
let claimedCanonicalPath: string | null = null;
|
||||
try {
|
||||
let canonicalPath = job.sourcePath;
|
||||
if (!options.sourceIsLocal && options.sourceSftpId) {
|
||||
const bridge = netcattyBridge.get();
|
||||
canonicalPath = await bridge?.realpathSftp?.(
|
||||
options.sourceSftpId,
|
||||
job.sourcePath,
|
||||
options.sourceEncoding,
|
||||
).catch(() => job.sourcePath) ?? job.sourcePath;
|
||||
}
|
||||
// Bound both remote and local pre-scan trees (Codex P1 on 86c64f48).
|
||||
claimedCanonicalPath = claimSftpDirectoryVisit(
|
||||
traversal,
|
||||
canonicalPath,
|
||||
job.branchAncestors,
|
||||
);
|
||||
// Cycle / already visited: do not create an empty target dir for this path.
|
||||
if (!claimedCanonicalPath) return [];
|
||||
|
||||
// Only record directories after the visit is accepted so cyclic symlink
|
||||
// children (e.g. /source/loop -> /source) are omitted from mkdir plans.
|
||||
if (job.includeInCreationPlan) {
|
||||
directories.push({ sourcePath: job.sourcePath, targetPath: job.targetPath });
|
||||
}
|
||||
|
||||
const listed = await listingGate.run(async () => {
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
await options.waitWhilePaused?.();
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
if (options.sourceIsLocal) {
|
||||
return options.listLocalFiles(job.sourcePath);
|
||||
}
|
||||
if (!options.sourceSftpId) throw new Error("No source connection");
|
||||
return options.listRemoteFiles(
|
||||
options.sourceSftpId,
|
||||
job.sourcePath,
|
||||
options.sourceEncoding,
|
||||
);
|
||||
});
|
||||
|
||||
const filtered = listed.filter((entry) => entry.name !== "." && entry.name !== "..");
|
||||
accountSftpDirectoryEntries(traversal, filtered.length);
|
||||
|
||||
const childDirs: DirJob[] = [];
|
||||
const regularFiles: SftpFileEntry[] = [];
|
||||
for (const entry of filtered) {
|
||||
if (entry.type === "directory") {
|
||||
childDirs.push({
|
||||
sourcePath: joinPath(job.sourcePath, entry.name),
|
||||
targetPath: joinTransferTargetPath(job.targetPath, entry.name),
|
||||
symlinkDepth: job.symlinkDepth,
|
||||
branchAncestors: createSftpDirectoryBranchAncestors(job.branchAncestors),
|
||||
includeInCreationPlan: true,
|
||||
});
|
||||
} else if (
|
||||
followSymlinks
|
||||
&& entry.type === "symlink"
|
||||
&& entry.linkTarget === "directory"
|
||||
) {
|
||||
if (shouldFollowSftpSymlinkDirectory(job.symlinkDepth)) {
|
||||
childDirs.push({
|
||||
sourcePath: joinPath(job.sourcePath, entry.name),
|
||||
targetPath: joinTransferTargetPath(job.targetPath, entry.name),
|
||||
symlinkDepth: job.symlinkDepth + 1,
|
||||
branchAncestors: createSftpDirectoryBranchAncestors(job.branchAncestors),
|
||||
includeInCreationPlan: true,
|
||||
});
|
||||
} else {
|
||||
omittedSymlinkDirectoryErrors += 1;
|
||||
logger.warn(
|
||||
`[SFTP] Skipping symlink directory at max depth during pre-scan: ${joinPath(job.sourcePath, entry.name)}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
regularFiles.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
childDirs.sort((left, right) => left.sourcePath.localeCompare(right.sourcePath));
|
||||
regularFiles.sort((left, right) => left.name.localeCompare(right.name));
|
||||
|
||||
for (const entry of regularFiles) {
|
||||
files.push({
|
||||
name: entry.name,
|
||||
sourcePath: joinPath(job.sourcePath, entry.name),
|
||||
targetPath: joinTransferTargetPath(job.targetPath, entry.name),
|
||||
size: getEntrySize(entry),
|
||||
lastModified: typeof entry.lastModified === "number" ? entry.lastModified : 0,
|
||||
isSymlink: entry.type === "symlink",
|
||||
});
|
||||
}
|
||||
publishCount();
|
||||
return childDirs;
|
||||
} finally {
|
||||
if (claimedCanonicalPath) {
|
||||
releaseSftpDirectoryVisit(traversal, claimedCanonicalPath, job.branchAncestors);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// BFS waves: each wave lists up to listingConcurrency directories in parallel.
|
||||
while (queue.length > 0) {
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
await options.waitWhilePaused?.();
|
||||
if (options.shouldAbort?.()) throw new Error("Transfer cancelled");
|
||||
const wave = queue.splice(0, queue.length);
|
||||
const nested: DirJob[][] = new Array(wave.length);
|
||||
await runBoundedConcurrency(
|
||||
wave,
|
||||
listingConcurrency,
|
||||
async (job, index) => {
|
||||
nested[index] = await processDir(job);
|
||||
},
|
||||
);
|
||||
for (const children of nested) {
|
||||
if (!children) continue;
|
||||
for (const child of children) queue.push(child);
|
||||
}
|
||||
}
|
||||
|
||||
// Match dedicated-resume / transfer-center ordering so directoryEntryIndex and
|
||||
// manifest checkpoints stay valid across interrupt + rebuild.
|
||||
files.sort((left, right) => compareDirectoryTraversalPaths(left.sourcePath, right.sourcePath));
|
||||
|
||||
return { files, directories, omittedSymlinkDirectoryErrors };
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import type { MutableRefObject } from "react";
|
||||
|
||||
import type { TransferTask } from "../../../domain/models";
|
||||
import { STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY } from "../../../infrastructure/config/storageKeys";
|
||||
import { transferDiscoveredFiles } from "./transferDirectoryFileBatch";
|
||||
|
||||
const memoryStore = new Map<string, string>();
|
||||
(globalThis as { localStorage?: Storage }).localStorage = {
|
||||
getItem: (key: string) => memoryStore.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => {
|
||||
memoryStore.set(key, String(value));
|
||||
},
|
||||
removeItem: (key: string) => {
|
||||
memoryStore.delete(key);
|
||||
},
|
||||
clear: () => memoryStore.clear(),
|
||||
key: () => null,
|
||||
length: 0,
|
||||
} as Storage;
|
||||
memoryStore.set(STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY, "1");
|
||||
|
||||
const rootTask = (): TransferTask => ({
|
||||
id: "root",
|
||||
fileName: "source",
|
||||
sourcePath: "/source",
|
||||
targetPath: "/target",
|
||||
sourceConnectionId: "local",
|
||||
targetConnectionId: "local",
|
||||
direction: "upload",
|
||||
status: "transferring",
|
||||
totalBytes: 1,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
startTime: 0,
|
||||
isDirectory: true,
|
||||
});
|
||||
|
||||
test("transferDiscoveredFiles re-stats source before skip-unchanged", async () => {
|
||||
const transfersRef = { current: [rootTask()] } as MutableRefObject<TransferTask[]>;
|
||||
const cancelledTasksRef = { current: new Set<string>() } as MutableRefObject<Set<string>>;
|
||||
const activeChildIdsRef = {
|
||||
current: new Map<string, Set<string>>(),
|
||||
} as MutableRefObject<Map<string, Set<string>>>;
|
||||
let transfers = transfersRef.current;
|
||||
let transferCalls = 0;
|
||||
let sourceStatCalls = 0;
|
||||
|
||||
const staleMtime = 1_700_000_000_000;
|
||||
const freshMtime = 1_700_000_100_000;
|
||||
|
||||
await transferDiscoveredFiles({
|
||||
rootTask: rootTask(),
|
||||
files: [{
|
||||
name: "a.txt",
|
||||
sourcePath: "/source/a.txt",
|
||||
targetPath: "/target/a.txt",
|
||||
size: 10,
|
||||
lastModified: staleMtime,
|
||||
}],
|
||||
sourceSftpId: null,
|
||||
targetSftpId: null,
|
||||
sourceIsLocal: true,
|
||||
targetIsLocal: true,
|
||||
sourceEncoding: "auto",
|
||||
targetEncoding: "auto",
|
||||
rootTaskId: "root",
|
||||
skipUnchanged: true,
|
||||
startEntryIndex: 0,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers: (updater) => {
|
||||
transfers = typeof updater === "function" ? updater(transfers) : updater;
|
||||
transfersRef.current = transfers;
|
||||
},
|
||||
waitWhileTransferPaused: async () => {},
|
||||
isPauseLatched: () => false,
|
||||
transferFile: async () => {
|
||||
transferCalls += 1;
|
||||
},
|
||||
tryStatTarget: async () => ({
|
||||
size: 10,
|
||||
lastModified: staleMtime,
|
||||
type: "file",
|
||||
}),
|
||||
tryStatSource: async () => {
|
||||
sourceStatCalls += 1;
|
||||
return {
|
||||
size: 10,
|
||||
lastModified: freshMtime,
|
||||
type: "file",
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(sourceStatCalls, 1, "must re-stat source before skip check");
|
||||
assert.equal(transferCalls, 1, "stale listing must not skip a changed source");
|
||||
const child = transfers.find((row) => row.parentTaskId === "root");
|
||||
assert.ok(child);
|
||||
assert.equal(child?.status, "completed");
|
||||
assert.equal(child?.sourceLastModified, freshMtime);
|
||||
});
|
||||
|
||||
test("transferDiscoveredFiles does not skip when source re-stat fails", async () => {
|
||||
const transfersRef = { current: [rootTask()] } as MutableRefObject<TransferTask[]>;
|
||||
const cancelledTasksRef = { current: new Set<string>() } as MutableRefObject<Set<string>>;
|
||||
const activeChildIdsRef = {
|
||||
current: new Map<string, Set<string>>(),
|
||||
} as MutableRefObject<Map<string, Set<string>>>;
|
||||
let transfers = transfersRef.current;
|
||||
let transferCalls = 0;
|
||||
let transferredTotalBytes: number | undefined;
|
||||
let transferredSourceLastModified: number | undefined;
|
||||
const mtime = 1_700_000_000_000;
|
||||
|
||||
await transferDiscoveredFiles({
|
||||
rootTask: rootTask(),
|
||||
files: [{
|
||||
name: "a.txt",
|
||||
sourcePath: "/source/a.txt",
|
||||
targetPath: "/target/a.txt",
|
||||
size: 10,
|
||||
lastModified: mtime,
|
||||
}],
|
||||
sourceSftpId: null,
|
||||
targetSftpId: null,
|
||||
sourceIsLocal: true,
|
||||
targetIsLocal: true,
|
||||
sourceEncoding: "auto",
|
||||
targetEncoding: "auto",
|
||||
rootTaskId: "root",
|
||||
skipUnchanged: true,
|
||||
startEntryIndex: 0,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers: (updater) => {
|
||||
transfers = typeof updater === "function" ? updater(transfers) : updater;
|
||||
transfersRef.current = transfers;
|
||||
},
|
||||
waitWhileTransferPaused: async () => {},
|
||||
isPauseLatched: () => false,
|
||||
transferFile: async (task) => {
|
||||
transferCalls += 1;
|
||||
transferredTotalBytes = task.totalBytes;
|
||||
transferredSourceLastModified = task.sourceLastModified;
|
||||
},
|
||||
tryStatTarget: async () => ({
|
||||
size: 10,
|
||||
lastModified: mtime,
|
||||
type: "file",
|
||||
}),
|
||||
tryStatSource: async () => null,
|
||||
});
|
||||
|
||||
assert.equal(transferCalls, 1, "missing fresh source meta must not skip");
|
||||
assert.equal(transferredTotalBytes, 0, "must not pass stale listing size after re-stat failure");
|
||||
assert.equal(
|
||||
transferredSourceLastModified,
|
||||
undefined,
|
||||
"must clear stale listing mtime after re-stat failure",
|
||||
);
|
||||
});
|
||||
|
||||
test("transferDiscoveredFiles skips only when fresh source still matches", async () => {
|
||||
const transfersRef = { current: [rootTask()] } as MutableRefObject<TransferTask[]>;
|
||||
const cancelledTasksRef = { current: new Set<string>() } as MutableRefObject<Set<string>>;
|
||||
const activeChildIdsRef = {
|
||||
current: new Map<string, Set<string>>(),
|
||||
} as MutableRefObject<Map<string, Set<string>>>;
|
||||
let transfers = transfersRef.current;
|
||||
let transferCalls = 0;
|
||||
const mtime = 1_700_000_000_000;
|
||||
|
||||
await transferDiscoveredFiles({
|
||||
rootTask: rootTask(),
|
||||
files: [{
|
||||
name: "a.txt",
|
||||
sourcePath: "/source/a.txt",
|
||||
targetPath: "/target/a.txt",
|
||||
size: 10,
|
||||
lastModified: mtime - 60_000,
|
||||
}],
|
||||
sourceSftpId: null,
|
||||
targetSftpId: null,
|
||||
sourceIsLocal: true,
|
||||
targetIsLocal: true,
|
||||
sourceEncoding: "auto",
|
||||
targetEncoding: "auto",
|
||||
rootTaskId: "root",
|
||||
skipUnchanged: true,
|
||||
startEntryIndex: 0,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers: (updater) => {
|
||||
transfers = typeof updater === "function" ? updater(transfers) : updater;
|
||||
transfersRef.current = transfers;
|
||||
},
|
||||
waitWhileTransferPaused: async () => {},
|
||||
isPauseLatched: () => false,
|
||||
transferFile: async () => {
|
||||
transferCalls += 1;
|
||||
},
|
||||
tryStatTarget: async () => ({
|
||||
size: 10,
|
||||
lastModified: mtime,
|
||||
type: "file",
|
||||
}),
|
||||
tryStatSource: async () => ({
|
||||
size: 10,
|
||||
lastModified: mtime,
|
||||
type: "file",
|
||||
}),
|
||||
});
|
||||
|
||||
assert.equal(transferCalls, 0, "matching fresh identity must skip transfer");
|
||||
const child = transfers.find((row) => row.parentTaskId === "root");
|
||||
assert.equal(child?.status, "completed");
|
||||
assert.equal(child?.transferredBytes, 10);
|
||||
});
|
||||
@@ -0,0 +1,354 @@
|
||||
import type { Dispatch, MutableRefObject, SetStateAction } from "react";
|
||||
import type {
|
||||
SftpFilenameEncoding,
|
||||
TransferStatus,
|
||||
TransferTask,
|
||||
} from "../../../domain/models";
|
||||
import { createDirectoryEntryIdentity } from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { isUnchangedTransferCandidate } from "../../../domain/sftpTransferSkip";
|
||||
import { STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY } from "../../../infrastructure/config/storageKeys";
|
||||
import { localStorageAdapter } from "../../../infrastructure/persistence/localStorageAdapter";
|
||||
import { logger } from "../../../lib/logger";
|
||||
import { isSessionError } from "./errors";
|
||||
import type { DiscoveredTransferFile } from "./transferDirectoryDiscovery";
|
||||
import { runSftpTransferWorkers } from "./transferConcurrency";
|
||||
import { isTransferCancelledError } from "./transferRetry";
|
||||
|
||||
type TransferOneFile = (
|
||||
task: TransferTask,
|
||||
sourceSftpId: string | null,
|
||||
targetSftpId: string | null,
|
||||
sourceIsLocal: boolean,
|
||||
targetIsLocal: boolean,
|
||||
sourceEncoding: SftpFilenameEncoding,
|
||||
targetEncoding: SftpFilenameEncoding,
|
||||
rootTaskId: string,
|
||||
sameHost?: boolean,
|
||||
) => Promise<void>;
|
||||
|
||||
export type TransferDiscoveredFilesParams = {
|
||||
rootTask: TransferTask;
|
||||
files: DiscoveredTransferFile[];
|
||||
sourceSftpId: string | null;
|
||||
targetSftpId: string | null;
|
||||
sourceIsLocal: boolean;
|
||||
targetIsLocal: boolean;
|
||||
sourceEncoding: SftpFilenameEncoding;
|
||||
targetEncoding: SftpFilenameEncoding;
|
||||
rootTaskId: string;
|
||||
sameHost?: boolean;
|
||||
skipUnchanged: boolean;
|
||||
startEntryIndex: number;
|
||||
cancelledTasksRef: MutableRefObject<Set<string>>;
|
||||
activeChildIdsRef: MutableRefObject<Map<string, Set<string>>>;
|
||||
transfersRef: MutableRefObject<TransferTask[]>;
|
||||
setTransfers: Dispatch<SetStateAction<TransferTask[]>>;
|
||||
waitWhileTransferPaused: (rootTaskId: string, taskId?: string) => Promise<void>;
|
||||
isPauseLatched: (rootTaskId: string, taskId?: string) => boolean;
|
||||
transferFile: TransferOneFile;
|
||||
tryStatTarget: (
|
||||
targetPath: string,
|
||||
) => Promise<{ size: number; lastModified: number; type: string } | null>;
|
||||
/** Fresh source identity immediately before skip-unchanged (Codex P1). */
|
||||
tryStatSource: (
|
||||
sourcePath: string,
|
||||
) => Promise<{ size: number; lastModified: number; type: string } | null>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Transfer a flat pre-scanned file list with the same child-task bookkeeping as
|
||||
* the interleaved directory walk.
|
||||
*/
|
||||
export async function transferDiscoveredFiles(
|
||||
params: TransferDiscoveredFilesParams,
|
||||
): Promise<number> {
|
||||
const {
|
||||
rootTask,
|
||||
files,
|
||||
sourceSftpId,
|
||||
targetSftpId,
|
||||
sourceIsLocal,
|
||||
targetIsLocal,
|
||||
sourceEncoding,
|
||||
targetEncoding,
|
||||
rootTaskId,
|
||||
sameHost,
|
||||
skipUnchanged,
|
||||
startEntryIndex,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers,
|
||||
waitWhileTransferPaused,
|
||||
isPauseLatched,
|
||||
transferFile,
|
||||
tryStatTarget,
|
||||
tryStatSource,
|
||||
} = params;
|
||||
|
||||
if (files.length === 0) return 0;
|
||||
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? { ...candidate, phase: "transferring" }
|
||||
: candidate));
|
||||
|
||||
const errors: Error[] = [];
|
||||
let sessionLostError: Error | null = null;
|
||||
|
||||
await runSftpTransferWorkers(
|
||||
files,
|
||||
() => localStorageAdapter.readNumber(STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY),
|
||||
async (file, fileIndex) => {
|
||||
if (sessionLostError) throw sessionLostError;
|
||||
if (cancelledTasksRef.current.has(rootTask.id) || cancelledTasksRef.current.has(rootTaskId)) {
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
|
||||
const fileSize = file.size;
|
||||
const sourcePath = file.sourcePath;
|
||||
const targetPath = file.targetPath;
|
||||
let skipSourceSize = fileSize;
|
||||
let skipSourceLastModified = file.lastModified;
|
||||
const directoryEntryIndex = startEntryIndex + fileIndex;
|
||||
let directoryEntryIdentity = createDirectoryEntryIdentity({
|
||||
sourcePath,
|
||||
targetPath,
|
||||
size: fileSize,
|
||||
lastModified: file.lastModified,
|
||||
});
|
||||
const persistedChild = transfersRef.current.find((candidate) => (
|
||||
candidate.parentTaskId === rootTaskId
|
||||
&& candidate.sourcePath === sourcePath
|
||||
&& candidate.targetPath === targetPath
|
||||
));
|
||||
if (persistedChild?.status === "completed") return;
|
||||
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
if (cancelledTasksRef.current.has(rootTask.id) || cancelledTasksRef.current.has(rootTaskId)) {
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
if (isPauseLatched(rootTaskId)) {
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
}
|
||||
|
||||
// Symlink listing attrs are the link node; transfer follows target bytes.
|
||||
// Re-stat regular files so skip and transfer both use current size/mtime
|
||||
// (Codex P1: pre-scan listing attrs can go stale before transfer starts).
|
||||
let freshSourceOk = false;
|
||||
if (!file.isSymlink) {
|
||||
const freshSource = await tryStatSource(sourcePath);
|
||||
if (freshSource && freshSource.type !== "directory") {
|
||||
skipSourceSize = freshSource.size;
|
||||
skipSourceLastModified = freshSource.lastModified;
|
||||
directoryEntryIdentity = createDirectoryEntryIdentity({
|
||||
sourcePath,
|
||||
targetPath,
|
||||
size: skipSourceSize,
|
||||
lastModified: skipSourceLastModified,
|
||||
});
|
||||
freshSourceOk = true;
|
||||
} else {
|
||||
// Do not pass stale pre-scan size as an explicit snapshot (Codex P2).
|
||||
// Child totalBytes 0 is omitted at startStreamTransfer via `|| undefined`,
|
||||
// so the dedicated transfer session re-stats the source itself.
|
||||
skipSourceSize = 0;
|
||||
skipSourceLastModified = 0;
|
||||
}
|
||||
}
|
||||
if (skipUnchanged && !file.isSymlink && freshSourceOk) {
|
||||
const existing = await tryStatTarget(targetPath);
|
||||
if (
|
||||
existing
|
||||
&& existing.type !== "directory"
|
||||
&& isUnchangedTransferCandidate(
|
||||
{ size: skipSourceSize, lastModified: skipSourceLastModified, mtimeUnit: "ms" },
|
||||
{ size: existing.size, lastModified: existing.lastModified, mtimeUnit: "ms" },
|
||||
)
|
||||
) {
|
||||
const skippedId = persistedChild?.id ?? crypto.randomUUID();
|
||||
setTransfers((prev) => {
|
||||
const hasChild = prev.some((row) => row.id === skippedId);
|
||||
const next = hasChild
|
||||
? prev.map((row) => row.id === skippedId
|
||||
? {
|
||||
...row,
|
||||
status: "completed" as TransferStatus,
|
||||
transferredBytes: skipSourceSize,
|
||||
totalBytes: skipSourceSize,
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
error: undefined,
|
||||
}
|
||||
: row)
|
||||
: [...prev, {
|
||||
...rootTask,
|
||||
id: skippedId,
|
||||
fileName: file.name,
|
||||
originalFileName: file.name,
|
||||
sourcePath,
|
||||
targetPath,
|
||||
isDirectory: false,
|
||||
progressMode: "bytes" as const,
|
||||
parentTaskId: rootTaskId,
|
||||
totalBytes: skipSourceSize,
|
||||
transferredBytes: skipSourceSize,
|
||||
sourceLastModified: skipSourceLastModified,
|
||||
directoryEntryIndex,
|
||||
directoryEntryIdentity,
|
||||
status: "completed" as TransferStatus,
|
||||
speed: 0,
|
||||
startTime: Date.now(),
|
||||
endTime: Date.now(),
|
||||
}];
|
||||
return next.map((row) => {
|
||||
if (row.id !== rootTaskId) return row;
|
||||
if (row.status === "paused" || row.status === "pausing" || isPauseLatched(rootTaskId)) {
|
||||
return { ...row, speed: 0 };
|
||||
}
|
||||
return { ...row, transferredBytes: row.transferredBytes + 1 };
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const fileId = persistedChild?.id ?? crypto.randomUUID();
|
||||
if (!activeChildIdsRef.current.has(rootTaskId)) {
|
||||
activeChildIdsRef.current.set(rootTaskId, new Set());
|
||||
}
|
||||
activeChildIdsRef.current.get(rootTaskId)!.add(fileId);
|
||||
|
||||
const childTask: TransferTask = {
|
||||
...rootTask,
|
||||
...persistedChild,
|
||||
id: fileId,
|
||||
fileName: file.name,
|
||||
originalFileName: file.name,
|
||||
sourcePath,
|
||||
targetPath,
|
||||
isDirectory: false,
|
||||
progressMode: "bytes",
|
||||
parentTaskId: rootTaskId,
|
||||
totalBytes: skipSourceSize,
|
||||
sourceLastModified: skipSourceLastModified || undefined,
|
||||
directoryEntryIndex,
|
||||
directoryEntryIdentity,
|
||||
retryable: rootTask.retryable,
|
||||
lifecycleEpoch: undefined,
|
||||
phase: undefined,
|
||||
pauseUnavailableReason: undefined,
|
||||
};
|
||||
|
||||
setTransfers((prev) => persistedChild
|
||||
? prev.map((candidate) => candidate.id === fileId ? {
|
||||
...childTask,
|
||||
status: "queued" as TransferStatus,
|
||||
speed: 0,
|
||||
error: undefined,
|
||||
endTime: undefined,
|
||||
lifecycleEpoch: undefined,
|
||||
} : candidate)
|
||||
: [...prev, {
|
||||
...childTask,
|
||||
status: "queued" as TransferStatus,
|
||||
transferredBytes: 0,
|
||||
speed: 0,
|
||||
startTime: Date.now(),
|
||||
lifecycleEpoch: undefined,
|
||||
}]);
|
||||
|
||||
try {
|
||||
await transferFile(
|
||||
childTask,
|
||||
sourceSftpId,
|
||||
targetSftpId,
|
||||
sourceIsLocal,
|
||||
targetIsLocal,
|
||||
sourceEncoding,
|
||||
targetEncoding,
|
||||
rootTaskId,
|
||||
sameHost,
|
||||
);
|
||||
|
||||
activeChildIdsRef.current.get(rootTaskId)?.delete(fileId);
|
||||
setTransfers((prev) => {
|
||||
const parentRow = prev.find((row) => row.id === rootTaskId);
|
||||
const parentFrozen = !!parentRow && (
|
||||
parentRow.status === "paused"
|
||||
|| parentRow.status === "pausing"
|
||||
|| isPauseLatched(rootTaskId)
|
||||
);
|
||||
return prev.map((t) => {
|
||||
if (t.id === fileId) {
|
||||
return {
|
||||
...t,
|
||||
status: "completed" as TransferStatus,
|
||||
endTime: Date.now(),
|
||||
transferredBytes: t.totalBytes,
|
||||
};
|
||||
}
|
||||
if (t.id === rootTaskId) {
|
||||
if (parentFrozen) return { ...t, speed: 0 };
|
||||
return { ...t, transferredBytes: t.transferredBytes + 1, speed: t.speed };
|
||||
}
|
||||
return t;
|
||||
});
|
||||
});
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
} catch (err) {
|
||||
activeChildIdsRef.current.get(rootTaskId)?.delete(fileId);
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
if (isTransferCancelledError(err)) {
|
||||
setTransfers((prev) => prev.map((t) => (
|
||||
t.id === fileId
|
||||
? { ...t, status: "cancelled" as TransferStatus, error: undefined, endTime: Date.now() }
|
||||
: t
|
||||
)));
|
||||
errors.push(err instanceof Error ? err : new Error(message));
|
||||
return;
|
||||
}
|
||||
setTransfers((prev) => prev.map((t) => (
|
||||
t.id === fileId
|
||||
? { ...t, status: "failed" as TransferStatus, error: message }
|
||||
: t
|
||||
)));
|
||||
if (isSessionError(err) && !sessionLostError) {
|
||||
sessionLostError = err instanceof Error ? err : new Error(message);
|
||||
setTransfers((prev) => prev.map((t) => (
|
||||
t.parentTaskId === rootTaskId
|
||||
&& (t.status === "queued" || t.status === "pending")
|
||||
? {
|
||||
...t,
|
||||
status: "failed" as TransferStatus,
|
||||
error: "SFTP session lost - reconnect and resume remaining files",
|
||||
speed: 0,
|
||||
endTime: Date.now(),
|
||||
}
|
||||
: t
|
||||
)));
|
||||
}
|
||||
errors.push(err instanceof Error ? err : new Error(message));
|
||||
if (sessionLostError) throw sessionLostError;
|
||||
if (isPauseLatched(rootTaskId)) {
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
beforeClaim: async () => {
|
||||
if (sessionLostError) return;
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
},
|
||||
},
|
||||
).catch((err) => {
|
||||
if (sessionLostError || isTransferCancelledError(err)) return;
|
||||
throw err;
|
||||
});
|
||||
|
||||
if (sessionLostError) {
|
||||
logger.warn("[SFTP] Directory transfer stopped early: session lost", sessionLostError.message);
|
||||
} else if (errors.length > 0) {
|
||||
logger.debug?.("[SFTP] Some files in directory transfer failed", errors);
|
||||
}
|
||||
return errors.length;
|
||||
}
|
||||
@@ -146,7 +146,9 @@ test("directory transfer discovers each directory once with bounded listing conc
|
||||
assert.equal(listCalls.size, directoryCount + 1);
|
||||
assert.ok(Array.from(listCalls.values()).every((count) => count === 1));
|
||||
assert.equal(listCalls.has("/source/loop"), false, "canonical symlink cycles must not be listed");
|
||||
assert.equal(maxActiveListings, 1);
|
||||
// Bounded parallel discovery (default 4) - never unbounded fan-out.
|
||||
assert.ok(maxActiveListings >= 2, `expected parallel listings, got ${maxActiveListings}`);
|
||||
assert.ok(maxActiveListings <= 4, `listing concurrency exceeded budget: ${maxActiveListings}`);
|
||||
assert.equal(tasks.find((task) => task.id === root.id)?.totalBytes, directoryCount);
|
||||
} finally {
|
||||
await act(async () => {
|
||||
|
||||
@@ -9,11 +9,26 @@ import {
|
||||
type SftpDirectoryTraversalBudget,
|
||||
shouldFollowSftpSymlinkDirectory,
|
||||
} from "../../../domain/sftpDirectoryCheckpoint";
|
||||
import { STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY } from "../../../infrastructure/config/storageKeys";
|
||||
import { isUnchangedTransferCandidate } from "../../../domain/sftpTransferSkip";
|
||||
import {
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY,
|
||||
} from "../../../infrastructure/config/storageKeys";
|
||||
import { localStorageAdapter } from "../../../infrastructure/persistence/localStorageAdapter";
|
||||
import { netcattyBridge } from "../../../infrastructure/services/netcattyBridge";
|
||||
import { logger } from "../../../lib/logger";
|
||||
import { runSftpTransferWorkers } from "./transferConcurrency";
|
||||
import {
|
||||
DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY,
|
||||
resolveSftpDirectoryListingConcurrency,
|
||||
resolveSftpFolderPrescanEnabled,
|
||||
resolveSftpSkipUnchangedEnabled,
|
||||
runSftpTransferWorkers,
|
||||
} from "./transferConcurrency";
|
||||
import {
|
||||
discoverTransferTree,
|
||||
} from "./transferDirectoryDiscovery";
|
||||
import { transferDiscoveredFiles } from "./transferDirectoryFileBatch";
|
||||
import { getSftpTransferResourceKeys, globalSftpTransferScheduler } from "./globalTransferScheduler";
|
||||
import { resolveDedicatedStreamEndpointIds } from "../../../domain/sftpDedicatedStreamPolicy";
|
||||
import { isSessionError } from "./errors";
|
||||
@@ -31,6 +46,49 @@ import {
|
||||
import { isTransferOrRootCancelled } from "./transferCancelLatch";
|
||||
import { joinPath, joinTransferTargetPath } from "./utils";
|
||||
|
||||
function createDirectoryListingGate(concurrency = DEFAULT_SFTP_DIRECTORY_LISTING_CONCURRENCY) {
|
||||
const limit = Math.max(1, Math.floor(concurrency) || 1);
|
||||
let active = 0;
|
||||
const waiters: Array<() => void> = [];
|
||||
const acquire = async () => {
|
||||
if (active < limit) {
|
||||
active += 1;
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve) => {
|
||||
waiters.push(() => {
|
||||
active += 1;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
};
|
||||
const release = () => {
|
||||
active = Math.max(0, active - 1);
|
||||
const next = waiters.shift();
|
||||
if (next) next();
|
||||
};
|
||||
return {
|
||||
get active() {
|
||||
return active;
|
||||
},
|
||||
async run<T>(fn: () => Promise<T>): Promise<T> {
|
||||
await acquire();
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type DirectoryDiscoveryProgress = {
|
||||
discoveredFiles: number;
|
||||
nextEntryIndex: number;
|
||||
/** Tree-wide semaphore so nested parallel walks cannot multiply listings. */
|
||||
listingGate?: ReturnType<typeof createDirectoryListingGate>;
|
||||
};
|
||||
|
||||
function isCancelledLocalOrGlobal(
|
||||
cancelledTasksRef: { current: Set<string> },
|
||||
rootTaskId: string,
|
||||
@@ -45,6 +103,68 @@ function isCancelledLocalOrGlobal(
|
||||
return false;
|
||||
}
|
||||
|
||||
function readFolderPrescanEnabled(): boolean {
|
||||
return resolveSftpFolderPrescanEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN));
|
||||
}
|
||||
|
||||
function readSkipUnchangedEnabled(): boolean {
|
||||
return resolveSftpSkipUnchangedEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED));
|
||||
}
|
||||
|
||||
async function ensureTransferDirectory(
|
||||
targetPath: string,
|
||||
targetIsLocal: boolean,
|
||||
targetSftpId: string | null,
|
||||
targetEncoding: SftpFilenameEncoding,
|
||||
): Promise<void> {
|
||||
if (targetIsLocal) {
|
||||
try {
|
||||
await netcattyBridge.get()?.mkdirLocal?.(targetPath);
|
||||
} catch (mkdirErr: unknown) {
|
||||
const isEEXIST = mkdirErr instanceof Error && mkdirErr.message.includes("EEXIST");
|
||||
if (!isEEXIST) throw mkdirErr;
|
||||
const stat = await netcattyBridge.get()?.statLocal?.(targetPath);
|
||||
if (stat && stat.type !== "directory") {
|
||||
throw new Error(`Target path exists as a file: ${targetPath}`);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (targetSftpId) {
|
||||
await netcattyBridge.get()?.mkdirSftp(targetSftpId, targetPath, targetEncoding);
|
||||
}
|
||||
}
|
||||
|
||||
async function tryStatTransferPath(
|
||||
filePath: string,
|
||||
isLocal: boolean,
|
||||
sftpId: string | null,
|
||||
encoding: SftpFilenameEncoding,
|
||||
): Promise<{ size: number; lastModified: number; type: string } | null> {
|
||||
try {
|
||||
if (isLocal) {
|
||||
const stat = await netcattyBridge.get()?.statLocal?.(filePath);
|
||||
if (!stat || stat.type === "directory") return null;
|
||||
return { size: Number(stat.size) || 0, lastModified: Number(stat.lastModified) || 0, type: stat.type };
|
||||
}
|
||||
if (!sftpId) return null;
|
||||
const stat = await netcattyBridge.get()?.statSftp?.(sftpId, filePath, encoding);
|
||||
if (!stat || stat.type === "directory") return null;
|
||||
return { size: Number(stat.size) || 0, lastModified: Number(stat.lastModified) || 0, type: stat.type };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function tryStatTransferTarget(
|
||||
targetPath: string,
|
||||
targetIsLocal: boolean,
|
||||
targetSftpId: string | null,
|
||||
targetEncoding: SftpFilenameEncoding,
|
||||
): Promise<{ size: number; lastModified: number; type: string } | null> {
|
||||
return tryStatTransferPath(targetPath, targetIsLocal, targetSftpId, targetEncoding);
|
||||
}
|
||||
|
||||
export type AcquireTransferSessionFn = (
|
||||
hostId: string,
|
||||
transferId: string,
|
||||
@@ -449,7 +569,7 @@ export function useSftpDirectoryTransferOps({
|
||||
sameHost?: boolean,
|
||||
symlinkDepth = 0,
|
||||
followSymlinks = false, // Only true for downloadToLocal — uploads/copies treat symlinks as files
|
||||
discoveryProgress?: { discoveredFiles: number; nextEntryIndex: number },
|
||||
discoveryProgress?: DirectoryDiscoveryProgress,
|
||||
traversalBudget?: SftpDirectoryTraversalBudget,
|
||||
) => {
|
||||
// Check if task or root task was cancelled before starting
|
||||
@@ -457,8 +577,112 @@ export function useSftpDirectoryTransferOps({
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
|
||||
// Root-only pre-scan (rsync --no-inc-recursive): discover the full tree first
|
||||
// so the transfer center can show a stable file total before bytes move.
|
||||
if (!discoveryProgress && readFolderPrescanEnabled()) {
|
||||
const skipUnchanged = readSkipUnchangedEnabled()
|
||||
&& !task.replaceExistingTarget
|
||||
&& !String(task.targetPath).includes(".netcatty-");
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? {
|
||||
...candidate,
|
||||
phase: "scanning",
|
||||
totalBytes: Math.max(candidate.transferredBytes, candidate.totalBytes || 0),
|
||||
}
|
||||
: candidate));
|
||||
|
||||
const traversal = traversalBudget ?? createSftpDirectoryTraversalBudget();
|
||||
const discovered = await discoverTransferTree({
|
||||
sourcePath: task.sourcePath,
|
||||
targetPath: task.targetPath,
|
||||
sourceIsLocal,
|
||||
sourceSftpId,
|
||||
sourceEncoding,
|
||||
followSymlinks,
|
||||
listLocalFiles,
|
||||
listRemoteFiles,
|
||||
traversalBudget: traversal,
|
||||
shouldAbort: () => (
|
||||
cancelledTasksRef.current.has(task.id)
|
||||
|| cancelledTasksRef.current.has(rootTaskId)
|
||||
),
|
||||
waitWhilePaused: async () => {
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
},
|
||||
onDiscoveredFiles: (totalFiles) => {
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? {
|
||||
...candidate,
|
||||
phase: "scanning",
|
||||
totalBytes: Math.max(totalFiles, candidate.transferredBytes),
|
||||
}
|
||||
: candidate));
|
||||
},
|
||||
});
|
||||
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? {
|
||||
...candidate,
|
||||
totalBytes: Math.max(discovered.files.length, candidate.transferredBytes),
|
||||
}
|
||||
: candidate));
|
||||
|
||||
// Create directories breadth-first so parents exist before children.
|
||||
for (const dir of discovered.directories) {
|
||||
if (cancelledTasksRef.current.has(task.id) || cancelledTasksRef.current.has(rootTaskId)) {
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
await ensureTransferDirectory(dir.targetPath, targetIsLocal, targetSftpId, targetEncoding);
|
||||
}
|
||||
|
||||
const fileErrors = await transferDiscoveredFiles({
|
||||
rootTask: task,
|
||||
files: discovered.files,
|
||||
sourceSftpId,
|
||||
targetSftpId,
|
||||
sourceIsLocal,
|
||||
targetIsLocal,
|
||||
sourceEncoding,
|
||||
targetEncoding,
|
||||
rootTaskId,
|
||||
sameHost,
|
||||
skipUnchanged,
|
||||
startEntryIndex: 0,
|
||||
cancelledTasksRef,
|
||||
activeChildIdsRef,
|
||||
transfersRef,
|
||||
setTransfers,
|
||||
waitWhileTransferPaused,
|
||||
isPauseLatched,
|
||||
transferFile,
|
||||
tryStatTarget: (targetPath) => tryStatTransferTarget(
|
||||
targetPath,
|
||||
targetIsLocal,
|
||||
targetSftpId,
|
||||
targetEncoding,
|
||||
),
|
||||
tryStatSource: (sourcePath) => tryStatTransferPath(
|
||||
sourcePath,
|
||||
sourceIsLocal,
|
||||
sourceSftpId,
|
||||
sourceEncoding,
|
||||
),
|
||||
});
|
||||
// Match interleaved walk: max-depth symlink omissions count as failures.
|
||||
return fileErrors + discovered.omittedSymlinkDirectoryErrors;
|
||||
}
|
||||
|
||||
let totalErrors = 0;
|
||||
const progress = discoveryProgress ?? { discoveredFiles: 0, nextEntryIndex: 0 };
|
||||
const progress = discoveryProgress ?? {
|
||||
discoveredFiles: 0,
|
||||
nextEntryIndex: 0,
|
||||
listingGate: createDirectoryListingGate(resolveSftpDirectoryListingConcurrency()),
|
||||
};
|
||||
if (!progress.listingGate) {
|
||||
progress.listingGate = createDirectoryListingGate(resolveSftpDirectoryListingConcurrency());
|
||||
}
|
||||
const listingGate = progress.listingGate;
|
||||
const traversal = traversalBudget ?? createSftpDirectoryTraversalBudget();
|
||||
let claimedCanonicalPath: string | null = null;
|
||||
let regularFiles: SftpFileEntry[] = [];
|
||||
@@ -500,13 +724,16 @@ export function useSftpDirectoryTransferOps({
|
||||
}
|
||||
|
||||
let files: SftpFileEntry[];
|
||||
if (sourceIsLocal) {
|
||||
files = await listLocalFiles(task.sourcePath);
|
||||
} else if (sourceSftpId) {
|
||||
files = await listRemoteFiles(sourceSftpId, task.sourcePath, sourceEncoding);
|
||||
} else {
|
||||
// Tree-wide listing gate: parallel siblings share one budget.
|
||||
files = await listingGate.run(async () => {
|
||||
if (sourceIsLocal) {
|
||||
return listLocalFiles(task.sourcePath);
|
||||
}
|
||||
if (sourceSftpId) {
|
||||
return listRemoteFiles(sourceSftpId, task.sourcePath, sourceEncoding);
|
||||
}
|
||||
throw new Error("No source connection");
|
||||
}
|
||||
});
|
||||
|
||||
// Filter both "." and ".." — some SFTP servers include "." in readdir
|
||||
const filtered = files.filter((f) => f.name !== ".." && f.name !== ".");
|
||||
@@ -536,26 +763,23 @@ export function useSftpDirectoryTransferOps({
|
||||
regularFiles.sort((left, right) => left.name.localeCompare(right.name));
|
||||
|
||||
// Directory progress is discovered by the same traversal that performs
|
||||
// the transfer. This avoids a second full-tree list pass and lets the UI
|
||||
// grow the total incrementally without flooding the server at startup.
|
||||
// the transfer (single pass - no separate countDirectoryFiles walk).
|
||||
// Process subdirectories sequentially so directoryEntryIndex / manifest
|
||||
// order stay deterministic for resume, and sibling symlink aliases can
|
||||
// claim the same canonical path after the prior branch releases.
|
||||
progress.discoveredFiles += regularFiles.length;
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? {
|
||||
...candidate,
|
||||
totalBytes: Math.max(progress.discoveredFiles, candidate.transferredBytes),
|
||||
phase: "scanning",
|
||||
}
|
||||
: candidate));
|
||||
|
||||
// Process subdirectories sequentially to avoid unbounded concurrent SFTP
|
||||
// requests from nested Promise.all + worker pools across the tree.
|
||||
// File-level concurrency within each directory is still governed by the
|
||||
// shared SFTP transfer worker scheduler below.
|
||||
for (const dir of dirs) {
|
||||
if (cancelledTasksRef.current.has(task.id) || cancelledTasksRef.current.has(rootTaskId)) {
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
// Pause between subfolders — otherwise the walk enters the next tree while
|
||||
// the user thinks the whole folder transfer is paused.
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
|
||||
const childTask: TransferTask = {
|
||||
@@ -571,7 +795,7 @@ export function useSftpDirectoryTransferOps({
|
||||
};
|
||||
|
||||
const isSymlink = dir.type === "symlink";
|
||||
const subdirErrors = await transferDirectory(
|
||||
totalErrors += await transferDirectory(
|
||||
childTask,
|
||||
sourceSftpId,
|
||||
targetSftpId,
|
||||
@@ -586,7 +810,6 @@ export function useSftpDirectoryTransferOps({
|
||||
progress,
|
||||
traversal,
|
||||
);
|
||||
totalErrors += subdirErrors;
|
||||
}
|
||||
} finally {
|
||||
// Release on success, cancellation, and traversal errors.
|
||||
@@ -597,6 +820,12 @@ export function useSftpDirectoryTransferOps({
|
||||
|
||||
// Transfer files in parallel with concurrency limit
|
||||
if (regularFiles.length > 0) {
|
||||
setTransfers((prev) => prev.map((candidate) => candidate.id === rootTaskId
|
||||
? {
|
||||
...candidate,
|
||||
phase: "transferring",
|
||||
}
|
||||
: candidate));
|
||||
const errors: Error[] = [];
|
||||
// If the SFTP session dies mid-directory, stop queueing more files
|
||||
// (remaining workers will still finish their current item).
|
||||
@@ -616,8 +845,10 @@ export function useSftpDirectoryTransferOps({
|
||||
const fileSize = getEntrySize(file);
|
||||
const sourcePath = joinPath(task.sourcePath, file.name);
|
||||
const targetPath = joinTransferTargetPath(task.targetPath, file.name);
|
||||
let transferSize = fileSize;
|
||||
let transferLastModified = file.lastModified;
|
||||
const directoryEntryIndex = directoryEntryBase + fileIndex;
|
||||
const directoryEntryIdentity = createDirectoryEntryIdentity({
|
||||
let directoryEntryIdentity = createDirectoryEntryIdentity({
|
||||
sourcePath,
|
||||
targetPath,
|
||||
size: fileSize,
|
||||
@@ -642,6 +873,104 @@ export function useSftpDirectoryTransferOps({
|
||||
if (isPauseLatched(rootTaskId)) {
|
||||
await waitWhileTransferPaused(rootTaskId);
|
||||
}
|
||||
|
||||
// Symlink listing attrs are the link node; transfer follows target bytes.
|
||||
// For regular files, re-stat so skip and transfer use current size/mtime
|
||||
// (Codex P1: listing attrs can go stale during a long interleaved walk).
|
||||
let freshSourceOk = false;
|
||||
if (file.type !== "symlink") {
|
||||
const freshSource = await tryStatTransferPath(
|
||||
sourcePath,
|
||||
sourceIsLocal,
|
||||
sourceSftpId,
|
||||
sourceEncoding,
|
||||
);
|
||||
if (freshSource && freshSource.type !== "directory") {
|
||||
transferSize = freshSource.size;
|
||||
transferLastModified = freshSource.lastModified;
|
||||
directoryEntryIdentity = createDirectoryEntryIdentity({
|
||||
sourcePath,
|
||||
targetPath,
|
||||
size: transferSize,
|
||||
lastModified: transferLastModified,
|
||||
});
|
||||
freshSourceOk = true;
|
||||
} else {
|
||||
// Do not pass stale listing size as an explicit snapshot: the
|
||||
// dedicated transfer session may still read a grown file (Codex P2).
|
||||
// Child totalBytes 0 is omitted at startStreamTransfer via
|
||||
// `|| undefined`, so the bridge re-stats instead of truncating.
|
||||
transferSize = 0;
|
||||
transferLastModified = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const skipUnchanged = readSkipUnchangedEnabled()
|
||||
&& !task.replaceExistingTarget
|
||||
&& !String(task.targetPath).includes(".netcatty-");
|
||||
// Missing fresh metadata must not fall back to listing attrs for skip.
|
||||
if (skipUnchanged && file.type !== "symlink" && freshSourceOk) {
|
||||
const existing = await tryStatTransferTarget(
|
||||
targetPath,
|
||||
targetIsLocal,
|
||||
targetSftpId,
|
||||
targetEncoding,
|
||||
);
|
||||
if (
|
||||
existing
|
||||
&& existing.type !== "directory"
|
||||
&& isUnchangedTransferCandidate(
|
||||
{ size: transferSize, lastModified: transferLastModified, mtimeUnit: "ms" },
|
||||
{ size: existing.size, lastModified: existing.lastModified, mtimeUnit: "ms" },
|
||||
)
|
||||
) {
|
||||
const skippedId = persistedChild?.id ?? crypto.randomUUID();
|
||||
setTransfers((prev) => {
|
||||
const hasChild = prev.some((row) => row.id === skippedId);
|
||||
const base = hasChild
|
||||
? prev.map((row) => row.id === skippedId
|
||||
? {
|
||||
...row,
|
||||
status: "completed" as TransferStatus,
|
||||
transferredBytes: transferSize,
|
||||
totalBytes: transferSize,
|
||||
endTime: Date.now(),
|
||||
speed: 0,
|
||||
error: undefined,
|
||||
}
|
||||
: row)
|
||||
: [...prev, {
|
||||
...task,
|
||||
id: skippedId,
|
||||
fileName: file.name,
|
||||
originalFileName: file.name,
|
||||
sourcePath,
|
||||
targetPath,
|
||||
isDirectory: false,
|
||||
progressMode: "bytes" as const,
|
||||
parentTaskId: rootTaskId,
|
||||
totalBytes: transferSize,
|
||||
transferredBytes: transferSize,
|
||||
sourceLastModified: transferLastModified,
|
||||
directoryEntryIndex,
|
||||
directoryEntryIdentity,
|
||||
status: "completed" as TransferStatus,
|
||||
speed: 0,
|
||||
startTime: Date.now(),
|
||||
endTime: Date.now(),
|
||||
}];
|
||||
return base.map((row) => {
|
||||
if (row.id !== rootTaskId) return row;
|
||||
if (row.status === "paused" || row.status === "pausing" || isPauseLatched(rootTaskId)) {
|
||||
return { ...row, speed: 0 };
|
||||
}
|
||||
return { ...row, transferredBytes: row.transferredBytes + 1 };
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const fileId = persistedChild?.id ?? crypto.randomUUID();
|
||||
|
||||
// Track child ID outside React state for immediate cancellation visibility
|
||||
@@ -661,11 +990,11 @@ export function useSftpDirectoryTransferOps({
|
||||
isDirectory: false,
|
||||
progressMode: "bytes",
|
||||
parentTaskId: rootTaskId,
|
||||
totalBytes: fileSize,
|
||||
sourceLastModified: file.lastModified,
|
||||
totalBytes: transferSize,
|
||||
sourceLastModified: transferLastModified || undefined,
|
||||
directoryEntryIndex,
|
||||
directoryEntryIdentity,
|
||||
// Inherit retryable from parent — downloadToLocal sets retryable: false
|
||||
// Inherit retryable from parent - downloadToLocal sets retryable: false
|
||||
// because "local" targetConnectionId can't be resolved by retryTransfer
|
||||
retryable: task.retryable,
|
||||
// New/restarted child streams arm at bridge lifecycleEpoch 0. Never
|
||||
@@ -773,7 +1102,7 @@ export function useSftpDirectoryTransferOps({
|
||||
? {
|
||||
...t,
|
||||
status: "failed" as TransferStatus,
|
||||
error: "SFTP session lost — reconnect and resume remaining files",
|
||||
error: "SFTP session lost - reconnect and resume remaining files",
|
||||
speed: 0,
|
||||
endTime: Date.now(),
|
||||
}
|
||||
|
||||
@@ -35,6 +35,8 @@ import {
|
||||
STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR,
|
||||
STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD,
|
||||
STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY,
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SSH_TRANSPORT_IDLE_TTL_MS,
|
||||
STORAGE_KEY_SFTP_TRANSFER_POOL_IDLE_TTL_MS,
|
||||
STORAGE_KEY_SFTP_DEFAULT_VIEW_MODE,
|
||||
@@ -84,7 +86,11 @@ import { DEFAULT_UI_FONT_ID, withWindowsEmojiFallback } from '../../infrastructu
|
||||
import { uiFontStore, useUIFontsLoaded } from './uiFontStore';
|
||||
import { localStorageAdapter } from '../../infrastructure/persistence/localStorageAdapter';
|
||||
import { netcattyBridge } from '../../infrastructure/services/netcattyBridge';
|
||||
import { resolveSftpTransferConcurrency } from './sftp/transferConcurrency';
|
||||
import {
|
||||
resolveSftpTransferConcurrency,
|
||||
resolveSftpFolderPrescanEnabled,
|
||||
resolveSftpSkipUnchangedEnabled,
|
||||
} from './sftp/transferConcurrency';
|
||||
import { resolveSshTransportIdleTtlMs } from '../../infrastructure/config/sshTransportIdleTtl';
|
||||
import {
|
||||
DEFAULT_ACCENT_MODE,
|
||||
@@ -420,6 +426,12 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
// Folder transfer concurrency is renderer-only (runSftpTransferWorkers).
|
||||
// Do not push it into main-process host admission - that made multi-select
|
||||
// top-level files queue against each other and against folder children.
|
||||
const [sftpFolderPrescan, setSftpFolderPrescan] = useState<boolean>(() =>
|
||||
resolveSftpFolderPrescanEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN)),
|
||||
);
|
||||
const [sftpSkipUnchanged, setSftpSkipUnchanged] = useState<boolean>(() =>
|
||||
resolveSftpSkipUnchangedEnabled(() => localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED)),
|
||||
);
|
||||
|
||||
const [sshTransportIdleTtlMs, setSshTransportIdleTtlMsState] = useState<number>(() => {
|
||||
// Prefer the new SSH transport key; if absent (upgrade), migrate the legacy
|
||||
@@ -890,6 +902,10 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
if (storedHidden === 'true' || storedHidden === 'false') setSftpShowHiddenFiles(storedHidden === 'true');
|
||||
const storedCompress = readStoredString(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD);
|
||||
if (storedCompress === 'true' || storedCompress === 'false') setSftpUseCompressedUpload(storedCompress === 'true');
|
||||
const storedFolderPrescan = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN);
|
||||
if (storedFolderPrescan != null) setSftpFolderPrescan(storedFolderPrescan);
|
||||
const storedSkipUnchanged = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED);
|
||||
if (storedSkipUnchanged != null) setSftpSkipUnchanged(storedSkipUnchanged);
|
||||
const storedAutoOpenSidebar = readStoredString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR);
|
||||
if (storedAutoOpenSidebar === 'true' || storedAutoOpenSidebar === 'false') setSftpAutoOpenSidebar(storedAutoOpenSidebar === 'true');
|
||||
const storedFollowTerminalCwd = readStoredString(STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD);
|
||||
@@ -1121,7 +1137,7 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
customCSS, uiFontFamilyId, hotkeyScheme, uiLanguage,
|
||||
terminalThemeId, followAppTerminalTheme, terminalFontFamilyId, terminalFontSize,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom, restorePreviousSession, restoreTerminalCwd,
|
||||
editorWordWrap, sessionLogsEnabled, sessionLogsDir, sessionLogsFormat, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
globalHotkeyEnabled, autoUpdateEnabled, windowOpacity, appIconVariant,
|
||||
@@ -1131,7 +1147,7 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
setTerminalThemeId, setTerminalThemeDarkId, setTerminalThemeLightId,
|
||||
setFollowAppTerminalThemeState, setTerminalFontFamilyId, setTerminalFontSize: applyIncomingTerminalFontSize,
|
||||
setSftpDoubleClickBehavior, setSftpAutoSync, setSftpShowHiddenFiles,
|
||||
setSftpUseCompressedUpload, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
|
||||
setSftpUseCompressedUpload, setSftpFolderPrescan, setSftpSkipUnchanged, setSftpAutoOpenSidebar, setSftpFollowTerminalCwd, setSftpDefaultViewMode,
|
||||
setShowRecentHostsState, setHostClickBehaviorState, setShowOnlyUngroupedHostsInRootState, setShowSftpTabState, setShowHostTreeSidebarState, setTerminalSidePanelAutoOpenState, setTerminalSidePanelAutoOpenTabState, setShellOnlyTabNumberShortcutsState, setDisableTerminalFontZoomState, setRestorePreviousSessionState, setRestoreTerminalCwdState,
|
||||
setEditorWordWrapState, setSessionLogsEnabled, setSessionLogsDir, setSessionLogsFormat, setSessionLogsTimestampsEnabled, setSshDebugLogsEnabled, setSshDeepLinkEnabledState: applyIncomingSshDeepLinkEnabled, setJmsDeepLinkEnabledState: applyIncomingJmsDeepLinkEnabled, setExplorerContextMenuEnabledState: applyIncomingExplorerContextMenuEnabled,
|
||||
setGlobalHotkeyEnabled, setWindowOpacity: applyIncomingWindowOpacity, setAppIconVariant, setAutoUpdateEnabled, setWorkspaceFocusStyleState,
|
||||
@@ -1359,6 +1375,18 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
notifySettingsChanged(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD, sftpUseCompressedUpload);
|
||||
}, [sftpUseCompressedUpload, notifySettingsChanged]);
|
||||
|
||||
useEffect(() => {
|
||||
localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN, sftpFolderPrescan);
|
||||
if (!persistMountedRef.current) return;
|
||||
notifySettingsChanged(STORAGE_KEY_SFTP_FOLDER_PRESCAN, sftpFolderPrescan);
|
||||
}, [sftpFolderPrescan, notifySettingsChanged]);
|
||||
|
||||
useEffect(() => {
|
||||
localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED, sftpSkipUnchanged);
|
||||
if (!persistMountedRef.current) return;
|
||||
notifySettingsChanged(STORAGE_KEY_SFTP_SKIP_UNCHANGED, sftpSkipUnchanged);
|
||||
}, [sftpSkipUnchanged, notifySettingsChanged]);
|
||||
|
||||
// Persist SFTP auto-open sidebar setting
|
||||
useEffect(() => {
|
||||
localStorageAdapter.writeString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR, sftpAutoOpenSidebar ? 'true' : 'false');
|
||||
@@ -1744,6 +1772,10 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
setSftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload,
|
||||
setSftpUseCompressedUpload,
|
||||
sftpFolderPrescan,
|
||||
setSftpFolderPrescan,
|
||||
sftpSkipUnchanged,
|
||||
setSftpSkipUnchanged,
|
||||
sftpAutoOpenSidebar,
|
||||
setSftpAutoOpenSidebar,
|
||||
sftpFollowTerminalCwd,
|
||||
@@ -1828,7 +1860,7 @@ export const useSettingsState = (options: { enableSettingsSync?: boolean; enable
|
||||
uiFontFamilyId, uiLanguage, customCSS,
|
||||
terminalThemeId, terminalFontFamilyId, terminalFontSize, terminalSettings,
|
||||
customKeyBindings, editorWordWrap,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles, sftpUseCompressedUpload, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
sftpDoubleClickBehavior, sftpAutoSync, sftpShowHiddenFiles, sftpUseCompressedUpload, sftpFolderPrescan, sftpSkipUnchanged, sftpAutoOpenSidebar, sftpFollowTerminalCwd, sftpDefaultViewMode,
|
||||
showRecentHosts, hostClickBehavior, showOnlyUngroupedHostsInRoot, showSftpTab, showHostTreeSidebar, terminalSidePanelAutoOpen, terminalSidePanelAutoOpenTab, shellOnlyTabNumberShortcuts, disableTerminalFontZoom,
|
||||
customThemes, workspaceFocusStyle, sessionLogsTimestampsEnabled, sshDebugLogsEnabled, sshDeepLinkEnabled, jmsDeepLinkEnabled, explorerContextMenuEnabled,
|
||||
]),
|
||||
|
||||
@@ -74,6 +74,8 @@ import {
|
||||
STORAGE_KEY_SFTP_AUTO_SYNC,
|
||||
STORAGE_KEY_SFTP_SHOW_HIDDEN_FILES,
|
||||
STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD,
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR,
|
||||
STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD,
|
||||
STORAGE_KEY_SFTP_DEFAULT_VIEW_MODE,
|
||||
@@ -286,6 +288,8 @@ export const SYNCABLE_SETTING_STORAGE_KEYS = [
|
||||
STORAGE_KEY_SFTP_AUTO_SYNC,
|
||||
STORAGE_KEY_SFTP_SHOW_HIDDEN_FILES,
|
||||
STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD,
|
||||
STORAGE_KEY_SFTP_FOLDER_PRESCAN,
|
||||
STORAGE_KEY_SFTP_SKIP_UNCHANGED,
|
||||
STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR,
|
||||
STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD,
|
||||
STORAGE_KEY_SFTP_DEFAULT_VIEW_MODE,
|
||||
@@ -490,6 +494,10 @@ export function collectSyncableSettings(): SyncPayload['settings'] {
|
||||
if (hidden === 'true' || hidden === 'false') settings.sftpShowHiddenFiles = hidden === 'true';
|
||||
const compress = localStorageAdapter.readString(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD);
|
||||
if (compress === 'true' || compress === 'false') settings.sftpUseCompressedUpload = compress === 'true';
|
||||
const folderPrescan = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN);
|
||||
if (folderPrescan != null) settings.sftpFolderPrescan = folderPrescan;
|
||||
const skipUnchanged = localStorageAdapter.readBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED);
|
||||
if (skipUnchanged != null) settings.sftpSkipUnchanged = skipUnchanged;
|
||||
const autoOpenSidebar = localStorageAdapter.readString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR);
|
||||
if (autoOpenSidebar === 'true' || autoOpenSidebar === 'false') settings.sftpAutoOpenSidebar = autoOpenSidebar === 'true';
|
||||
const followTerminalCwd = localStorageAdapter.readString(STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD);
|
||||
@@ -726,6 +734,8 @@ async function applySyncableSettings(settings: NonNullable<SyncPayload['settings
|
||||
if (settings.sftpAutoSync != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_AUTO_SYNC, String(settings.sftpAutoSync));
|
||||
if (settings.sftpShowHiddenFiles != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_SHOW_HIDDEN_FILES, String(settings.sftpShowHiddenFiles));
|
||||
if (settings.sftpUseCompressedUpload != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_USE_COMPRESSED_UPLOAD, String(settings.sftpUseCompressedUpload));
|
||||
if (settings.sftpFolderPrescan != null) localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_FOLDER_PRESCAN, settings.sftpFolderPrescan);
|
||||
if (settings.sftpSkipUnchanged != null) localStorageAdapter.writeBoolean(STORAGE_KEY_SFTP_SKIP_UNCHANGED, settings.sftpSkipUnchanged);
|
||||
if (settings.sftpAutoOpenSidebar != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_AUTO_OPEN_SIDEBAR, String(settings.sftpAutoOpenSidebar));
|
||||
if (settings.sftpFollowTerminalCwd != null) localStorageAdapter.writeString(STORAGE_KEY_SFTP_FOLLOW_TERMINAL_CWD, String(settings.sftpFollowTerminalCwd));
|
||||
if (settings.sftpDefaultViewMode != null) {
|
||||
|
||||
@@ -40,6 +40,8 @@ export default function SettingsFileAssociationsTab() {
|
||||
sftpAutoSync, setSftpAutoSync,
|
||||
sftpShowHiddenFiles, setSftpShowHiddenFiles,
|
||||
sftpUseCompressedUpload, setSftpUseCompressedUpload,
|
||||
sftpFolderPrescan, setSftpFolderPrescan,
|
||||
sftpSkipUnchanged, setSftpSkipUnchanged,
|
||||
sftpAutoOpenSidebar, setSftpAutoOpenSidebar,
|
||||
sftpFollowTerminalCwd, setSftpFollowTerminalCwd,
|
||||
sftpDefaultViewMode, setSftpDefaultViewMode,
|
||||
@@ -209,6 +211,18 @@ export default function SettingsFileAssociationsTab() {
|
||||
</span>
|
||||
</div>
|
||||
</SettingRow>
|
||||
<SettingRow
|
||||
label={t('settings.sftp.folderPrescan.enable')}
|
||||
description={t('settings.sftp.folderPrescan.enableDesc')}
|
||||
>
|
||||
<Toggle checked={sftpFolderPrescan} onChange={setSftpFolderPrescan} />
|
||||
</SettingRow>
|
||||
<SettingRow
|
||||
label={t('settings.sftp.skipUnchanged.enable')}
|
||||
description={t('settings.sftp.skipUnchanged.enableDesc')}
|
||||
>
|
||||
<Toggle checked={sftpSkipUnchanged} onChange={setSftpSkipUnchanged} />
|
||||
</SettingRow>
|
||||
<SettingRow
|
||||
label={t('settings.ssh.transportIdleTtl')}
|
||||
description={t('settings.ssh.transportIdleTtl.desc')}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
import type { SftpFileEntry } from "../../../types";
|
||||
import type { TransferStatus } from "../../../domain/models";
|
||||
import { getParentPath, joinPath as joinFsPath } from "../../../application/state/sftp/utils";
|
||||
import {
|
||||
DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY,
|
||||
runBoundedConcurrency,
|
||||
} from "../../../application/state/sftp/transferConcurrency";
|
||||
import { logger } from "../../../lib/logger";
|
||||
import { toast } from "../../ui/toast";
|
||||
import { getFileExtension, getLanguageId, FileOpenerType, SystemAppInfo } from "../../../lib/sftpFileUtils";
|
||||
@@ -12,6 +17,14 @@ import type { TextEditorModalSnapshot } from "../../TextEditorModal";
|
||||
import type { UseSftpViewFileOpsParams, UseSftpViewFileOpsResult } from "./useSftpViewFileOps.types";
|
||||
import { assertSftpFileFitsBuiltinEditor } from "../sftpEditorFileLimits";
|
||||
|
||||
/** Local multi-select blob downloads read whole files into ArrayBuffers. */
|
||||
const LOCAL_BLOB_DOWNLOAD_CONCURRENCY = 1;
|
||||
/**
|
||||
* Multi-select roots each run folder pre-scan before the transfer scheduler.
|
||||
* Bound them so listing fan-out cannot multiply by selection size.
|
||||
*/
|
||||
const MULTI_SELECT_ROOT_DOWNLOAD_CONCURRENCY = DEFAULT_SFTP_FILE_TRANSFER_CONCURRENCY;
|
||||
|
||||
export const useSftpViewFileOps = ({
|
||||
sftpRef,
|
||||
behaviorRef,
|
||||
@@ -538,9 +551,14 @@ export const useSftpViewFileOps = ({
|
||||
if (!pane.connection) return;
|
||||
|
||||
if (pane.connection.isLocal) {
|
||||
for (const file of files) {
|
||||
await handleDownloadFileForSide(side, file);
|
||||
}
|
||||
// Sequential: each local download materializes a full ArrayBuffer.
|
||||
await runBoundedConcurrency(
|
||||
files,
|
||||
LOCAL_BLOB_DOWNLOAD_CONCURRENCY,
|
||||
async (file) => {
|
||||
await handleDownloadFileForSide(side, file);
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -558,27 +576,42 @@ export const useSftpViewFileOps = ({
|
||||
const selectedDirectory = await selectDirectory(t("sftp.context.download"));
|
||||
if (!selectedDirectory) return;
|
||||
|
||||
// Sequential enqueue; each file uses dedicated transfer-pool sessions
|
||||
// via downloadToLocal so the browse connection stays responsive.
|
||||
for (const file of files) {
|
||||
const sourcePath = sftpRef.current.joinPath(pane.connection.currentPath, file.name);
|
||||
const targetPath = joinFsPath(selectedDirectory, file.name);
|
||||
const isDirectory = isNavigableDirectory(file);
|
||||
const fileSize = typeof file.size === "string" ? parseInt(file.size, 10) || 0 : (file.size || 0);
|
||||
// Bound root jobs: directory roots run full pre-scan before the scheduler,
|
||||
// so unbounded Promise.allSettled would multiply LIST concurrency.
|
||||
const results: Array<PromiseSettledResult<{ file: SftpFileEntry; status: TransferStatus }>> = [];
|
||||
await runBoundedConcurrency(
|
||||
files,
|
||||
MULTI_SELECT_ROOT_DOWNLOAD_CONCURRENCY,
|
||||
async (file, index) => {
|
||||
try {
|
||||
const sourcePath = sftpRef.current.joinPath(pane.connection.currentPath, file.name);
|
||||
const targetPath = joinFsPath(selectedDirectory, file.name);
|
||||
const isDirectory = isNavigableDirectory(file);
|
||||
const fileSize = typeof file.size === "string" ? parseInt(file.size, 10) || 0 : (file.size || 0);
|
||||
|
||||
try {
|
||||
const status = await sftpRef.current.downloadToLocal({
|
||||
fileName: file.name,
|
||||
sourcePath,
|
||||
targetPath,
|
||||
sftpId,
|
||||
connectionId: pane.connection.id,
|
||||
sourceHostId: pane.connection.hostId,
|
||||
sourceHostLabel: pane.connection.hostLabel,
|
||||
sourceEncoding: pane.filenameEncoding,
|
||||
isDirectory,
|
||||
totalBytes: isDirectory ? undefined : fileSize,
|
||||
});
|
||||
const status = await sftpRef.current.downloadToLocal({
|
||||
fileName: file.name,
|
||||
sourcePath,
|
||||
targetPath,
|
||||
sftpId,
|
||||
connectionId: pane.connection.id,
|
||||
sourceHostId: pane.connection.hostId,
|
||||
sourceHostLabel: pane.connection.hostLabel,
|
||||
sourceEncoding: pane.filenameEncoding,
|
||||
isDirectory,
|
||||
totalBytes: isDirectory ? undefined : fileSize,
|
||||
});
|
||||
results[index] = { status: "fulfilled", value: { file, status } };
|
||||
} catch (reason) {
|
||||
results[index] = { status: "rejected", reason };
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
for (const result of results) {
|
||||
if (!result) continue;
|
||||
if (result.status === "fulfilled") {
|
||||
const { file, status } = result.value;
|
||||
if (status === "completed") {
|
||||
toast.success(`${t("sftp.context.download")}: ${file.name}`, "SFTP");
|
||||
} else if (status === "failed") {
|
||||
@@ -586,9 +619,9 @@ export const useSftpViewFileOps = ({
|
||||
} else if (status === "attention") {
|
||||
toast.error(`${file.name}: another transfer for this path is already in progress`, "SFTP");
|
||||
}
|
||||
} catch (e) {
|
||||
logger.error("[SftpView] Failed to download file:", e);
|
||||
const errorMessage = e instanceof Error ? e.message : String(e);
|
||||
} else {
|
||||
logger.error("[SftpView] Failed to download file:", result.reason);
|
||||
const errorMessage = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
||||
const isCancelError = errorMessage.includes("cancelled") || errorMessage.includes("canceled");
|
||||
if (!isCancelError) toast.error(errorMessage || t("sftp.error.downloadFailed"), "SFTP");
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
claimSftpDirectoryVisit,
|
||||
createDirectoryManifestAccumulator,
|
||||
createEmptyDirectoryResumeCheckpoint,
|
||||
createSftpDirectoryBranchAncestors,
|
||||
createSftpDirectoryTraversalBudget,
|
||||
EMPTY_DIRECTORY_MANIFEST_HASH,
|
||||
isValidDirectoryResumeCheckpoint,
|
||||
@@ -91,6 +92,24 @@ test("remote directory traversal rejects ancestor cycles but allows sibling alia
|
||||
assert.equal(budget.activeCanonicalDirectories.size, 0);
|
||||
});
|
||||
|
||||
test("parallel sibling branches may both claim the same canonical alias", () => {
|
||||
const budget = createSftpDirectoryTraversalBudget({ maxDirectories: 8, maxEntries: 8 });
|
||||
const rootAncestors = createSftpDirectoryBranchAncestors();
|
||||
const root = claimSftpDirectoryVisit(budget, "/srv/root", rootAncestors);
|
||||
assert.ok(root);
|
||||
const branchA = createSftpDirectoryBranchAncestors(rootAncestors);
|
||||
const branchB = createSftpDirectoryBranchAncestors(rootAncestors);
|
||||
const aliasA = claimSftpDirectoryVisit(budget, "/srv/shared", branchA);
|
||||
const aliasB = claimSftpDirectoryVisit(budget, "/srv/shared", branchB);
|
||||
assert.ok(aliasA);
|
||||
assert.ok(aliasB);
|
||||
assert.equal(claimSftpDirectoryVisit(budget, "/srv/root", branchA), null);
|
||||
assert.equal(claimSftpDirectoryVisit(budget, "/srv/root", branchB), null);
|
||||
releaseSftpDirectoryVisit(budget, aliasA, branchA);
|
||||
releaseSftpDirectoryVisit(budget, aliasB, branchB);
|
||||
releaseSftpDirectoryVisit(budget, root, rootAncestors);
|
||||
});
|
||||
|
||||
test("remote directory traversal releases sibling visits without retaining history", () => {
|
||||
const budget = createSftpDirectoryTraversalBudget({ maxDirectories: 50_000 });
|
||||
const root = claimSftpDirectoryVisit(budget, "/srv/root");
|
||||
|
||||
@@ -11,6 +11,10 @@ export const MAX_SFTP_DIRECTORY_TRAVERSAL_DIRECTORIES = 50_000;
|
||||
export const MAX_SFTP_DIRECTORY_TRAVERSAL_ENTRIES = 200_000;
|
||||
|
||||
export interface SftpDirectoryTraversalBudget {
|
||||
/**
|
||||
* Default branch ancestor set for sequential callers. Parallel walks must
|
||||
* pass a per-branch Set into claim/release instead of sharing this one.
|
||||
*/
|
||||
activeCanonicalDirectories: Set<string>;
|
||||
visitedDirectories: number;
|
||||
visitedEntries: number;
|
||||
@@ -35,30 +39,40 @@ export function normalizeSftpCanonicalDirectoryPath(canonicalPath: string): stri
|
||||
return canonicalPath.replace(/\\/g, "/").replace(/\/+$/, "") || "/";
|
||||
}
|
||||
|
||||
/** Copy parent ancestors for a parallel recursive branch (sibling aliases). */
|
||||
export function createSftpDirectoryBranchAncestors(
|
||||
parent?: ReadonlySet<string> | null,
|
||||
): Set<string> {
|
||||
return parent ? new Set(parent) : new Set();
|
||||
}
|
||||
|
||||
/**
|
||||
* Claims one directory against the global work budget. Active directories are
|
||||
* shared by one sequential traversal and released when each recursive frame
|
||||
* exits, so sibling aliases remain valid without copying the ancestor set.
|
||||
* Claims one directory against the global work budget. Cycle detection uses
|
||||
* `branchAncestors` (defaults to the budget's sequential set). Parallel sibling
|
||||
* walks must each pass their own branch set so two aliases of the same
|
||||
* canonical path can both be copied.
|
||||
*/
|
||||
export function claimSftpDirectoryVisit(
|
||||
budget: SftpDirectoryTraversalBudget,
|
||||
canonicalPath: string,
|
||||
branchAncestors: Set<string> = budget.activeCanonicalDirectories,
|
||||
): string | null {
|
||||
const normalized = normalizeSftpCanonicalDirectoryPath(canonicalPath);
|
||||
if (budget.activeCanonicalDirectories.has(normalized)) return null;
|
||||
if (branchAncestors.has(normalized)) return null;
|
||||
if (budget.visitedDirectories >= budget.maxDirectories) {
|
||||
throw new Error(`Remote directory traversal directory limit exceeded (${budget.maxDirectories})`);
|
||||
throw new Error(`Directory traversal directory limit exceeded (${budget.maxDirectories})`);
|
||||
}
|
||||
budget.visitedDirectories += 1;
|
||||
budget.activeCanonicalDirectories.add(normalized);
|
||||
branchAncestors.add(normalized);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function releaseSftpDirectoryVisit(
|
||||
budget: SftpDirectoryTraversalBudget,
|
||||
_budget: SftpDirectoryTraversalBudget,
|
||||
claimedCanonicalPath: string,
|
||||
branchAncestors: Set<string> = _budget.activeCanonicalDirectories,
|
||||
): void {
|
||||
budget.activeCanonicalDirectories.delete(claimedCanonicalPath);
|
||||
branchAncestors.delete(claimedCanonicalPath);
|
||||
}
|
||||
|
||||
export function accountSftpDirectoryEntries(
|
||||
@@ -67,7 +81,7 @@ export function accountSftpDirectoryEntries(
|
||||
): void {
|
||||
const next = budget.visitedEntries + Math.max(0, Number(count) || 0);
|
||||
if (next > budget.maxEntries) {
|
||||
throw new Error(`Remote directory traversal entry limit exceeded (${budget.maxEntries})`);
|
||||
throw new Error(`Directory traversal entry limit exceeded (${budget.maxEntries})`);
|
||||
}
|
||||
budget.visitedEntries = next;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
isUnchangedTransferCandidate,
|
||||
normalizeTransferMtimeSeconds,
|
||||
} from "./sftpTransferSkip";
|
||||
|
||||
test("normalizeTransferMtimeSeconds accepts seconds and milliseconds", () => {
|
||||
assert.equal(normalizeTransferMtimeSeconds(1_700_000_000), 1_700_000_000);
|
||||
assert.equal(normalizeTransferMtimeSeconds(1_700_000_000_500), 1_700_000_000);
|
||||
// Year 2000 in milliseconds must not be treated as epoch seconds.
|
||||
assert.equal(normalizeTransferMtimeSeconds(946_684_800_000), 946_684_800);
|
||||
assert.equal(normalizeTransferMtimeSeconds(946_684_800), 946_684_800);
|
||||
// Adapter-declared units cover early-1970 ms without magnitude cutoffs.
|
||||
assert.equal(normalizeTransferMtimeSeconds(2_678_400_000, "ms"), 2_678_400);
|
||||
assert.equal(normalizeTransferMtimeSeconds(2_678_400, "s"), 2_678_400);
|
||||
assert.equal(normalizeTransferMtimeSeconds(60_000_000_000), 60_000_000);
|
||||
});
|
||||
|
||||
test("isUnchangedTransferCandidate requires matching size and second mtime", () => {
|
||||
assert.equal(
|
||||
isUnchangedTransferCandidate(
|
||||
{ size: 10, lastModified: 1_700_000_000_200, mtimeUnit: "ms" },
|
||||
{ size: 10, lastModified: 1_700_000_000_000, mtimeUnit: "ms" },
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isUnchangedTransferCandidate(
|
||||
{ size: 10, lastModified: 946_684_800_123, mtimeUnit: "ms" },
|
||||
{ size: 10, lastModified: 946_684_800_000, mtimeUnit: "ms" },
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isUnchangedTransferCandidate(
|
||||
{ size: 10, lastModified: 2_678_400_000, mtimeUnit: "ms" },
|
||||
{ size: 10, lastModified: 2_678_400_000, mtimeUnit: "ms" },
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isUnchangedTransferCandidate(
|
||||
{ size: 10, lastModified: 1_700_000_000, mtimeUnit: "ms" },
|
||||
{ size: 11, lastModified: 1_700_000_000, mtimeUnit: "ms" },
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
isUnchangedTransferCandidate(
|
||||
{ size: 10, lastModified: 1_700_000_000_000, mtimeUnit: "ms" },
|
||||
{ size: 10, lastModified: 1_700_000_001_000, mtimeUnit: "ms" },
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
isUnchangedTransferCandidate(
|
||||
{ size: 0, lastModified: 1_700_000_000_000, mtimeUnit: "ms" },
|
||||
{ size: 0, lastModified: 1_700_000_000_000, mtimeUnit: "ms" },
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isUnchangedTransferCandidate(
|
||||
{ size: 10, lastModified: 0, mtimeUnit: "ms" },
|
||||
{ size: 10, lastModified: 0, mtimeUnit: "ms" },
|
||||
),
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* rsync-style "generator" skip: when size and mtime already match the target,
|
||||
* the file does not need to be transferred again.
|
||||
*
|
||||
* SFTP mtimes are typically second-precision; compare on whole seconds so a
|
||||
* local millisecond timestamp does not defeat an otherwise identical remote.
|
||||
*/
|
||||
|
||||
export type TransferMtimeUnit = "ms" | "s";
|
||||
|
||||
export interface TransferSkipIdentity {
|
||||
size: number;
|
||||
lastModified: number;
|
||||
/**
|
||||
* Preferred: adapters declare the unit. Local fs and Netcatty SFTP stats use
|
||||
* milliseconds (`Date` / `mtime * 1000`). Raw ssh2 attrs use seconds.
|
||||
* When omitted, a magnitude heuristic is used only as a legacy fallback.
|
||||
*/
|
||||
mtimeUnit?: TransferMtimeUnit;
|
||||
}
|
||||
|
||||
export function normalizeTransferMtimeSeconds(
|
||||
lastModified: number,
|
||||
mtimeUnit?: TransferMtimeUnit,
|
||||
): number {
|
||||
if (!Number.isFinite(lastModified) || lastModified <= 0) return 0;
|
||||
if (mtimeUnit === "ms") return Math.floor(lastModified / 1000);
|
||||
if (mtimeUnit === "s") return Math.floor(lastModified);
|
||||
// Legacy heuristic: magnitudes at/above 1e10 (~year 2286 as seconds) are ms.
|
||||
// Prefer passing mtimeUnit from adapters so early-1970 ms values are exact.
|
||||
return lastModified >= 1e10 ? Math.floor(lastModified / 1000) : Math.floor(lastModified);
|
||||
}
|
||||
|
||||
export function isUnchangedTransferCandidate(
|
||||
source: TransferSkipIdentity,
|
||||
target: TransferSkipIdentity,
|
||||
): boolean {
|
||||
const sourceSize = Number(source.size);
|
||||
const targetSize = Number(target.size);
|
||||
if (!Number.isFinite(sourceSize) || !Number.isFinite(targetSize)) return false;
|
||||
if (sourceSize !== targetSize) return false;
|
||||
if (sourceSize < 0 || targetSize < 0) return false;
|
||||
const sourceMtime = normalizeTransferMtimeSeconds(source.lastModified, source.mtimeUnit);
|
||||
const targetMtime = normalizeTransferMtimeSeconds(target.lastModified, target.mtimeUnit);
|
||||
if (sourceMtime <= 0 || targetMtime <= 0) return false;
|
||||
return sourceMtime === targetMtime;
|
||||
}
|
||||
@@ -324,6 +324,8 @@ export interface SyncPayload {
|
||||
sftpAutoSync?: boolean;
|
||||
sftpShowHiddenFiles?: boolean;
|
||||
sftpUseCompressedUpload?: boolean;
|
||||
sftpFolderPrescan?: boolean;
|
||||
sftpSkipUnchanged?: boolean;
|
||||
sftpAutoOpenSidebar?: boolean;
|
||||
sftpFollowTerminalCwd?: boolean;
|
||||
sftpDefaultViewMode?: 'list' | 'tree';
|
||||
|
||||
+493
-250
@@ -1004,6 +1004,107 @@ async function promoteLocalTransfer(stagedPath, targetPath, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the source mtime to the committed destination so skip-unchanged
|
||||
* (size + mtime) can match on a later folder transfer. Best-effort: never
|
||||
* fails the transfer if utimes/setstat is unsupported or times out.
|
||||
*/
|
||||
async function awaitBestEffortBounded(promise, timeoutMs, label) {
|
||||
let timer = null;
|
||||
try {
|
||||
await Promise.race([
|
||||
promise,
|
||||
new Promise((_, reject) => {
|
||||
timer = setTimeout(
|
||||
() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)),
|
||||
timeoutMs,
|
||||
);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function preserveTransferredDestinationMtime(transfer, options = {}) {
|
||||
try {
|
||||
// Only use the pre-transfer soft identity. Re-statting after commit can pick
|
||||
// up a rewritten same-size source and stamp a newer mtime onto older bytes
|
||||
// (Codex P1 on non-resumable / SCP paths).
|
||||
const mtimeMs = Number(transfer?.sourceSoftIdentity?.mtimeMs);
|
||||
if (!Number.isFinite(mtimeMs) || mtimeMs <= 0) return;
|
||||
|
||||
// Compare / skip logic uses whole seconds; stamp the destination the same way.
|
||||
const mtimeSec = Math.floor(mtimeMs / 1000);
|
||||
if (mtimeSec <= 0) return;
|
||||
const when = new Date(mtimeSec * 1000);
|
||||
// Best-effort stamp must not pin sendComplete / scheduler forever when the
|
||||
// server stops answering metadata requests (Codex P2).
|
||||
const mtimeTimeoutMs = Number(options.timeoutMs) > 0
|
||||
? Number(options.timeoutMs)
|
||||
: 15_000;
|
||||
|
||||
if (transfer.targetType === "local" && transfer.targetPath) {
|
||||
await awaitBestEffortBounded(
|
||||
fs.promises.utimes(transfer.targetPath, when, when),
|
||||
mtimeTimeoutMs,
|
||||
"Destination utimes",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (transfer.targetType !== "sftp" || !transfer.targetSftpId || !transfer.targetPath) {
|
||||
return;
|
||||
}
|
||||
const client = sftpClients.get(transfer.targetSftpId);
|
||||
if (!client) return;
|
||||
|
||||
if (isScpModeClient(client)) {
|
||||
// SCP has no SETSTAT; best-effort touch via the SSH session.
|
||||
const sshClient = client.client;
|
||||
if (!sshClient || typeof sshClient.exec !== "function") return;
|
||||
const escaped = String(transfer.targetPath).replace(/'/g, "'\\''");
|
||||
const command = `touch -d @${mtimeSec} -- '${escaped}' 2>/dev/null || `
|
||||
+ `touch -t "$(date -u -r ${mtimeSec} +%Y%m%d%H%M.%S 2>/dev/null `
|
||||
+ `|| date -u -d @${mtimeSec} +%Y%m%d%H%M.%S 2>/dev/null)" -- '${escaped}' 2>/dev/null `
|
||||
+ `|| true`;
|
||||
await executeBoundedSshCommand(sshClient, command, { runTimeoutMs: mtimeTimeoutMs });
|
||||
return;
|
||||
}
|
||||
|
||||
await requireSftpChannel(client);
|
||||
const encoded = encodePathForSession(
|
||||
transfer.targetSftpId,
|
||||
transfer.targetPath,
|
||||
transfer.targetEncoding,
|
||||
);
|
||||
if (typeof client.setStat === "function") {
|
||||
await awaitBestEffortBounded(
|
||||
client.setStat(encoded, { mtime: mtimeSec, atime: mtimeSec }),
|
||||
mtimeTimeoutMs,
|
||||
"Destination setStat",
|
||||
);
|
||||
return;
|
||||
}
|
||||
const sftp = client.sftp;
|
||||
if (!sftp || typeof sftp.setstat !== "function") return;
|
||||
await awaitBestEffortBounded(
|
||||
new Promise((resolve, reject) => {
|
||||
sftp.setstat(encoded, { mtime: mtimeSec, atime: mtimeSec }, (err) => (
|
||||
err ? reject(err) : resolve()
|
||||
));
|
||||
}),
|
||||
mtimeTimeoutMs,
|
||||
"Destination setstat",
|
||||
);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
"[transferBridge] failed to preserve destination mtime:",
|
||||
err?.message || String(err),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function inspectLocalPromotionTarget(targetPath) {
|
||||
const absoluteTargetPath = path.resolve(targetPath);
|
||||
const visitedLinkStates = new Set();
|
||||
@@ -1580,6 +1681,98 @@ async function prepareStreamFallbackAfterRangeFailure(transfer, client) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for a prior truncating WRITE OPEN's published gate before fastPut.
|
||||
* Cancelable and time-bounded so a dead isolated OPEN that never callbacks
|
||||
* cannot hang strategy fallback forever (#2755). On timeout/cancel the waiter
|
||||
* fails closed without resolving the published transfer gate (clearing that
|
||||
* poison early lets a late truncating OPEN wipe an in-place destination after
|
||||
* sendComplete; Codex P1 on e2cc8241), and also poisons the shared path map so
|
||||
* later same-path waiters fail promptly (#2755 / Codex P2 on dca41093).
|
||||
*/
|
||||
async function waitForPendingWriteOpenPathGate(transfer, options = {}) {
|
||||
if (transfer?.noTransferFallback || transfer?.inPlaceWriteOpenPoisoned) {
|
||||
const err = new Error("In-place write OPEN poison is still held; refusing same-path fallback");
|
||||
err.noTransferFallback = true;
|
||||
throw err;
|
||||
}
|
||||
const gate = transfer?.pendingWriteOpenPathGate;
|
||||
if (typeof gate?.then !== "function") return;
|
||||
// Default 2s matches fastPut's fail-closed wait (#2755): hanging forever on a
|
||||
// dead OPEN pins the transfer/lease; longer waits only delay the poison path.
|
||||
const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : 2_000;
|
||||
const timeoutMessage = typeof options.timeoutMessage === "string" && options.timeoutMessage
|
||||
? options.timeoutMessage
|
||||
: "Timed out waiting for prior write OPEN to settle before fastPut";
|
||||
const poisonGate = () => {
|
||||
// Fail closed for later same-path waiters: do not resolve/clear the published
|
||||
// transfer gate from this waiter (Codex P1 on e2cc8241), but do poison the
|
||||
// shared path map so a successor that replaced the entry cannot OPEN while a
|
||||
// truncating OPEN may still land (#2755 / Codex P2 on dca41093).
|
||||
try {
|
||||
transfer._failPendingWriteOpenPathGate?.(
|
||||
createPoisonedWriteOpenPathGateError(
|
||||
"Prior write OPEN never settled; path gate is fail-closed",
|
||||
),
|
||||
);
|
||||
} catch { /* ignore */ }
|
||||
};
|
||||
await new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
let timer = null;
|
||||
const previousAbort = transfer.abort;
|
||||
const cleanup = () => {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
timer = null;
|
||||
}
|
||||
transfer.signal?.removeEventListener?.("abort", onAbort);
|
||||
if (transfer.abort === abortWait) transfer.abort = previousAbort;
|
||||
};
|
||||
const finish = (fn, value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
fn(value);
|
||||
};
|
||||
const onAbort = () => {
|
||||
transfer.cancelled = true;
|
||||
// Same fail-closed poison as the timeout path: cancel must not leave later
|
||||
// same-path uploads hanging on waitForPrior (Codex P2).
|
||||
poisonGate();
|
||||
finish(reject, new Error("Transfer cancelled"));
|
||||
};
|
||||
const abortWait = () => {
|
||||
try { previousAbort?.(); } catch { /* ignore */ }
|
||||
onAbort();
|
||||
};
|
||||
transfer.abort = abortWait;
|
||||
transfer.signal?.addEventListener?.("abort", onAbort, { once: true });
|
||||
if (transfer.cancelled || transfer.signal?.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
timer = setTimeout(() => {
|
||||
const err = new Error(timeoutMessage);
|
||||
err.noTransferFallback = true;
|
||||
poisonGate();
|
||||
finish(reject, err);
|
||||
}, timeoutMs);
|
||||
// Only a successful gate release unblocks safely. A poisoned/rejected gate
|
||||
// must fail this waiter closed; do not proceed to another writer.
|
||||
Promise.resolve(gate).then(
|
||||
() => finish(resolve),
|
||||
(err) => finish(
|
||||
reject,
|
||||
err instanceof Error ? err : new Error(String(err?.message || err || "path gate failed")),
|
||||
),
|
||||
);
|
||||
});
|
||||
if (transfer.cancelled || transfer.signal?.aborted) {
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
}
|
||||
|
||||
function getIsolatedDownloadChannelPool(client) {
|
||||
let pool = isolatedDownloadChannelPools.get(client);
|
||||
if (!pool) {
|
||||
@@ -2037,6 +2230,14 @@ async function uploadFile(
|
||||
const initialSource = (transfer.resumable || !transfer.sourceIsOwnedTemp)
|
||||
? await fs.promises.stat(originalLocalPath)
|
||||
: null;
|
||||
|
||||
/** @type {Error | null} */
|
||||
let lastPipelineError = null;
|
||||
const rememberPipelineError = (err) => {
|
||||
if (err && typeof err === "object") lastPipelineError = err;
|
||||
else lastPipelineError = new Error(String(err || "SFTP upload failed"));
|
||||
};
|
||||
|
||||
if (!transfer.sourceIsOwnedTemp) {
|
||||
const digestId = crypto.createHash("sha256")
|
||||
.update(String(transfer.transferId || "upload"))
|
||||
@@ -2047,15 +2248,32 @@ async function uploadFile(
|
||||
"ranges.sha256",
|
||||
);
|
||||
transfer.sourceDigestPath = digestPath;
|
||||
await createUploadDigestBaseline(
|
||||
originalLocalPath,
|
||||
digestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
);
|
||||
const previousPhase = transfer.phase;
|
||||
transfer.phase = "verifying";
|
||||
transfer.publishCurrentProgress?.();
|
||||
try {
|
||||
await createUploadDigestBaseline(
|
||||
originalLocalPath,
|
||||
digestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
() => {
|
||||
// Keep remote transferredBytes at the resume checkpoint while hashing
|
||||
// so large uploads do not look stuck at 0 B with no status (#2712).
|
||||
sendProgress(Math.max(0, Number(transfer.checkpointBytes) || 0), fileSize, {
|
||||
force: true,
|
||||
});
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
transfer.phase = previousPhase || "transferring";
|
||||
if (!transfer.cancelled && !transfer.signal?.aborted) {
|
||||
transfer.publishCurrentProgress?.();
|
||||
}
|
||||
}
|
||||
if (isTransferCancelled(transfer)) throw new Error("Transfer cancelled");
|
||||
const sourceAfterBaseline = await fs.promises.stat(originalLocalPath);
|
||||
// Digest was just built + verified; only size/content matter from here.
|
||||
// Digest was just built; only size/content matter from here.
|
||||
assertSourceMetadataUnchanged(initialSource, sourceAfterBaseline, fileSize, {
|
||||
contentVerifiedSeparately: true,
|
||||
});
|
||||
@@ -2095,16 +2313,9 @@ async function uploadFile(
|
||||
}
|
||||
};
|
||||
|
||||
/** @type {Error | null} */
|
||||
let lastPipelineError = null;
|
||||
const rememberPipelineError = (err) => {
|
||||
if (err && typeof err === "object") lastPipelineError = err;
|
||||
else lastPipelineError = new Error(String(err || "SFTP upload failed"));
|
||||
};
|
||||
|
||||
// Prefer an isolated SFTP channel so cancellation cannot kill the browse session.
|
||||
let isolated = null;
|
||||
if (!client.__netcattySudoMode) {
|
||||
let isolated = null;
|
||||
try {
|
||||
isolated = await openIsolatedSftpChannel(client, transfer?.signal);
|
||||
} catch (err) {
|
||||
@@ -2114,225 +2325,178 @@ async function uploadFile(
|
||||
err.message || String(err),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (isolated && transfer.cancelled) {
|
||||
try { isolated.end?.(); } catch { /* ignore */ }
|
||||
isolated = null;
|
||||
throw new Error("Transfer cancelled");
|
||||
}
|
||||
|
||||
if (isolated) {
|
||||
let concurrentIsolatedOk = false;
|
||||
try {
|
||||
transfer.uploadStrategy = "concurrent-isolated";
|
||||
logTransferDiag(transfer, "strategy", {
|
||||
strategy: "concurrent-isolated",
|
||||
fields: {
|
||||
chunk: formatDiagBytes(TRANSFER_CHUNK_SIZE),
|
||||
concurrency: UPLOAD_TRANSFER_CONCURRENCY,
|
||||
},
|
||||
});
|
||||
await uploadFileConcurrent(
|
||||
localPath,
|
||||
remotePath,
|
||||
isolated,
|
||||
fileSize,
|
||||
transfer,
|
||||
sendProgress,
|
||||
{ disposeChannel: true, onBytesCommitted, generatedStagePath },
|
||||
);
|
||||
concurrentIsolatedOk = true;
|
||||
} catch (err) {
|
||||
// uploadFileConcurrent ends the isolated channel itself.
|
||||
isolated = null;
|
||||
if (transfer.cancelled) throw err;
|
||||
if (err?.noTransferFallback) throw err;
|
||||
rememberPipelineError(err);
|
||||
if (transfer.resumable) {
|
||||
await prepareUploadFallbackCheckpoint(transfer, client, fileSize, sendProgress);
|
||||
} else {
|
||||
transfer.checkpointBytes = 0;
|
||||
if (isolated) {
|
||||
let concurrentIsolatedOk = false;
|
||||
try {
|
||||
transfer.uploadStrategy = "concurrent-isolated";
|
||||
logTransferDiag(transfer, "strategy", {
|
||||
strategy: "concurrent-isolated",
|
||||
fields: {
|
||||
chunk: formatDiagBytes(TRANSFER_CHUNK_SIZE),
|
||||
concurrency: UPLOAD_TRANSFER_CONCURRENCY,
|
||||
},
|
||||
});
|
||||
await uploadFileConcurrent(
|
||||
localPath,
|
||||
remotePath,
|
||||
isolated,
|
||||
fileSize,
|
||||
transfer,
|
||||
sendProgress,
|
||||
{ disposeChannel: true, onBytesCommitted, generatedStagePath },
|
||||
);
|
||||
concurrentIsolatedOk = true;
|
||||
} catch (err) {
|
||||
// uploadFileConcurrent ends the isolated channel itself.
|
||||
isolated = null;
|
||||
if (transfer.cancelled) throw err;
|
||||
if (err?.noTransferFallback || transfer.noTransferFallback) {
|
||||
if (err && typeof err === "object" && transfer.noTransferFallback) {
|
||||
err.noTransferFallback = true;
|
||||
}
|
||||
console.warn(
|
||||
"[transferBridge] concurrent isolated upload failed, trying next pipelined strategy:",
|
||||
err?.message || String(err),
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
// Verification errors must not fall through into other strategies.
|
||||
if (concurrentIsolatedOk) {
|
||||
await finishSuccessfulUpload();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isolated) {
|
||||
try {
|
||||
isolated = await openIsolatedSftpChannel(client, transfer?.signal);
|
||||
} catch (err) {
|
||||
rememberPipelineError(err);
|
||||
console.warn(
|
||||
"[transferBridge] Failed to reopen isolated SFTP channel for fastPut:",
|
||||
err.message || String(err),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// fastPut always truncates and rewrites from offset 0 — skip when we
|
||||
// already have a durable resume checkpoint from a prior concurrent attempt.
|
||||
// fastPut is not pause-aware; do not advertise pause while it runs.
|
||||
const hasResumeCheckpoint = Math.max(0, Number(transfer.checkpointBytes) || 0) > 0;
|
||||
if (
|
||||
isolated
|
||||
&& typeof isolated.fastPut === "function"
|
||||
&& !hasResumeCheckpoint
|
||||
&& !transfer.resumable
|
||||
) {
|
||||
let fastPutOk = false;
|
||||
let fastPutSourcePath = localPath;
|
||||
let fastPutSnapshotPath = null;
|
||||
try {
|
||||
// Wait for any prior/in-flight write OPEN on this path (including our
|
||||
// own concurrent attempt's still-pending OPEN) before fastPut truncates
|
||||
// the same stage (Codex P1 on 7872a304). The wait must be cancelable and
|
||||
// bounded: an isolated OPEN that never callbacks after channel death
|
||||
// leaves pendingWriteOpenPathGate unresolved by design; hanging forever
|
||||
// (or ignoring cancel because uploadFileConcurrent cleared abort) pins
|
||||
// the transfer/lease (#2755 / Codex P2 on 667e9115). Fail closed on
|
||||
// timeout — skip fastPut rather than race a still-pending truncate.
|
||||
if (typeof transfer.pendingWriteOpenPathGate?.then === "function") {
|
||||
const pendingGate = transfer.pendingWriteOpenPathGate;
|
||||
// Local cancel race (not runTransferCancelablePreflight): cancel must
|
||||
// clear the timeout so an abandoned 2s reject cannot fire as an
|
||||
// unhandled rejection after the outer wait already settled.
|
||||
await new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
const previousAbort = transfer.abort;
|
||||
const signal = transfer.signal;
|
||||
const finish = (fn, value) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
try { signal?.removeEventListener?.("abort", onAbort); } catch { /* ignore */ }
|
||||
if (transfer.abort === onAbort) transfer.abort = previousAbort;
|
||||
fn(value);
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
// Fail closed: do not fall through to another writer on the
|
||||
// same stage while a truncating OPEN may still land. Poison the
|
||||
// shared path gate so a later same-path upload fails promptly
|
||||
// instead of awaiting the unresolved entry forever (Codex P2).
|
||||
const err = new Error(
|
||||
"Timed out waiting for prior write OPEN to settle before fastPut",
|
||||
);
|
||||
err.noTransferFallback = true;
|
||||
try {
|
||||
transfer._failPendingWriteOpenPathGate?.(
|
||||
createPoisonedWriteOpenPathGateError(
|
||||
"Prior write OPEN never settled; path gate is fail-closed",
|
||||
),
|
||||
);
|
||||
} catch { /* ignore */ }
|
||||
finish(reject, err);
|
||||
}, 2000);
|
||||
const onAbort = () => {
|
||||
try { previousAbort?.(); } catch { /* ignore */ }
|
||||
transfer.cancelled = true;
|
||||
// Same fail-closed poison as the timeout path: a dead prior OPEN
|
||||
// leaves the shared gate unresolved, and cancel must not leave
|
||||
// later same-path uploads hanging on waitForPrior (Codex P2).
|
||||
try {
|
||||
transfer._failPendingWriteOpenPathGate?.(
|
||||
createPoisonedWriteOpenPathGateError(
|
||||
"Prior write OPEN never settled; path gate is fail-closed",
|
||||
),
|
||||
);
|
||||
} catch { /* ignore */ }
|
||||
finish(reject, new Error("Transfer cancelled"));
|
||||
};
|
||||
if (transfer.cancelled || signal?.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
transfer.abort = onAbort;
|
||||
try { signal?.addEventListener?.("abort", onAbort, { once: true }); } catch { /* ignore */ }
|
||||
pendingGate.then(
|
||||
() => finish(resolve),
|
||||
(err) => finish(
|
||||
reject,
|
||||
err instanceof Error ? err : new Error(String(err?.message || err)),
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
if (transfer.cancelled) throw new Error("Transfer cancelled");
|
||||
transfer.uploadStrategy = "fastPut-isolated";
|
||||
logTransferDiag(transfer, "strategy", { strategy: "fastPut-isolated" });
|
||||
transfer.pauseSupported = false;
|
||||
transfer.pauseUnavailableReason = "Pause is unavailable during fastPut upload";
|
||||
if (!transfer.sourceIsOwnedTemp) {
|
||||
const snapshotId = crypto.createHash("sha256")
|
||||
.update(String(transfer.transferId || localPath))
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
fastPutSnapshotPath = tempDirBridge.getTransferTempFilePath(
|
||||
`upload-source-${snapshotId}`,
|
||||
"snapshot.bin",
|
||||
);
|
||||
await createVerifiedUploadSnapshot(
|
||||
originalLocalPath,
|
||||
fastPutSnapshotPath,
|
||||
transfer.sourceDigestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
);
|
||||
fastPutSourcePath = fastPutSnapshotPath;
|
||||
}
|
||||
sendProgress(Math.max(0, Number(transfer.checkpointBytes) || 0), fileSize, {
|
||||
force: true,
|
||||
});
|
||||
await uploadViaFastPut(
|
||||
fastPutSourcePath,
|
||||
remotePath,
|
||||
isolated,
|
||||
fileSize,
|
||||
transfer,
|
||||
sendProgress,
|
||||
{ disposeChannel: true },
|
||||
);
|
||||
fastPutOk = true;
|
||||
} catch (err) {
|
||||
// Gate-wait / snapshot / fastPut failure: end the reopened isolated
|
||||
// channel before nulling. Rethrow paths skip the post-block else-if
|
||||
// end, and fallthrough also clears isolated — either way we must not
|
||||
// leak the SSH subsystem opened for this attempt (#2755 Bugbot).
|
||||
if (isolated && typeof isolated.end === "function") {
|
||||
try { isolated.end(); } catch { /* ignore */ }
|
||||
}
|
||||
isolated = null;
|
||||
// Restore pause capability for subsequent pause-aware strategies.
|
||||
transfer.pauseSupported = Boolean(transfer.resumable);
|
||||
transfer.pauseUnavailableReason = transfer.resumable
|
||||
? undefined
|
||||
: transfer.pauseUnavailableReason;
|
||||
if (transfer.cancelled) throw err;
|
||||
// Source-change / hard safety errors must not be retried on another path.
|
||||
if (err?.noTransferFallback || err?.sourceChanged) throw err;
|
||||
rememberPipelineError(err);
|
||||
// fastPut progress is not a durable contiguous checkpoint — reset so
|
||||
// concurrent-shared does not resume past holes left by the failed put.
|
||||
rememberPipelineError(err);
|
||||
if (transfer.resumable) {
|
||||
await prepareUploadFallbackCheckpoint(transfer, client, fileSize, sendProgress);
|
||||
} else {
|
||||
transfer.checkpointBytes = 0;
|
||||
sendProgress(0, fileSize, { force: true, checkpointBytes: 0 });
|
||||
console.warn(
|
||||
"[transferBridge] isolated fastPut failed, trying next pipelined strategy:",
|
||||
err?.message || String(err),
|
||||
);
|
||||
} finally {
|
||||
if (fastPutSnapshotPath) {
|
||||
await fs.promises.rm(fastPutSnapshotPath, { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
if (fastPutOk) {
|
||||
onBytesCommitted?.();
|
||||
await finishSuccessfulUpload();
|
||||
return;
|
||||
}
|
||||
} else if (isolated && typeof isolated.end === "function") {
|
||||
try { isolated.end(); } catch { /* ignore */ }
|
||||
console.warn(
|
||||
"[transferBridge] concurrent isolated upload failed, trying next pipelined strategy:",
|
||||
err?.message || String(err),
|
||||
);
|
||||
}
|
||||
// Verification errors must not fall through into other strategies.
|
||||
if (concurrentIsolatedOk) {
|
||||
await finishSuccessfulUpload();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (!isolated && !client.__netcattySudoMode) {
|
||||
try {
|
||||
isolated = await openIsolatedSftpChannel(client, transfer?.signal);
|
||||
} catch (err) {
|
||||
rememberPipelineError(err);
|
||||
console.warn(
|
||||
"[transferBridge] Failed to reopen isolated SFTP channel for fastPut:",
|
||||
err.message || String(err),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// fastPut always truncates and rewrites from offset 0; skip when we
|
||||
// already have a durable resume checkpoint from a prior concurrent attempt.
|
||||
// fastPut is not pause-aware; do not advertise pause while it runs.
|
||||
// In-place OPEN poison is terminal for this transfer: do not wait on/race
|
||||
// the unreleased path gate with another same-path strategy (Codex P1 on 3d4cecfa).
|
||||
if (transfer.noTransferFallback) {
|
||||
const cause = lastPipelineError || new Error("SFTP pipelined upload failed");
|
||||
if (typeof cause === "object" && cause) cause.noTransferFallback = true;
|
||||
throw cause;
|
||||
}
|
||||
const hasResumeCheckpoint = Math.max(0, Number(transfer.checkpointBytes) || 0) > 0;
|
||||
if (
|
||||
isolated
|
||||
&& typeof isolated.fastPut === "function"
|
||||
&& !hasResumeCheckpoint
|
||||
&& !transfer.resumable
|
||||
) {
|
||||
let fastPutOk = false;
|
||||
let fastPutSourcePath = localPath;
|
||||
let fastPutSnapshotPath = null;
|
||||
try {
|
||||
// Wait for any prior/in-flight write OPEN on this path (including our
|
||||
// own concurrent attempt's still-pending OPEN) before fastPut truncates
|
||||
// the same stage (Codex P1 on 7872a304). Bounded + cancelable: a dead
|
||||
// isolated OPEN never settles the path gate; hang or ignore cancel pins
|
||||
// the transfer/lease (#2755 / Codex P2 on 667e9115).
|
||||
await waitForPendingWriteOpenPathGate(transfer, {
|
||||
timeoutMs: 2_000,
|
||||
timeoutMessage: "Timed out waiting for prior write OPEN to settle before fastPut",
|
||||
});
|
||||
if (transfer.cancelled) throw new Error("Transfer cancelled");
|
||||
transfer.uploadStrategy = "fastPut-isolated";
|
||||
logTransferDiag(transfer, "strategy", { strategy: "fastPut-isolated" });
|
||||
transfer.pauseSupported = false;
|
||||
transfer.pauseUnavailableReason = "Pause is unavailable during fastPut upload";
|
||||
if (!transfer.sourceIsOwnedTemp) {
|
||||
const snapshotId = crypto.createHash("sha256")
|
||||
.update(String(transfer.transferId || localPath))
|
||||
.digest("hex")
|
||||
.slice(0, 16);
|
||||
fastPutSnapshotPath = tempDirBridge.getTransferTempFilePath(
|
||||
`upload-source-${snapshotId}`,
|
||||
"snapshot.bin",
|
||||
);
|
||||
await createVerifiedUploadSnapshot(
|
||||
originalLocalPath,
|
||||
fastPutSnapshotPath,
|
||||
transfer.sourceDigestPath,
|
||||
fileSize,
|
||||
transfer,
|
||||
);
|
||||
fastPutSourcePath = fastPutSnapshotPath;
|
||||
}
|
||||
sendProgress(Math.max(0, Number(transfer.checkpointBytes) || 0), fileSize, {
|
||||
force: true,
|
||||
});
|
||||
await uploadViaFastPut(
|
||||
fastPutSourcePath,
|
||||
remotePath,
|
||||
isolated,
|
||||
fileSize,
|
||||
transfer,
|
||||
sendProgress,
|
||||
{ disposeChannel: true },
|
||||
);
|
||||
fastPutOk = true;
|
||||
} catch (err) {
|
||||
// Gate-wait / snapshot / fastPut failure: end the reopened isolated
|
||||
// channel before nulling. Rethrow paths skip the post-block else-if
|
||||
// end, and fallthrough also clears isolated; either way we must not
|
||||
// leak the SSH subsystem opened for this attempt (#2755 Bugbot).
|
||||
if (isolated && typeof isolated.end === "function") {
|
||||
try { isolated.end(); } catch { /* ignore */ }
|
||||
}
|
||||
isolated = null;
|
||||
// Restore pause capability for subsequent pause-aware strategies.
|
||||
transfer.pauseSupported = Boolean(transfer.resumable);
|
||||
transfer.pauseUnavailableReason = transfer.resumable
|
||||
? undefined
|
||||
: transfer.pauseUnavailableReason;
|
||||
if (transfer.cancelled) throw err;
|
||||
// Source-change / hard safety errors must not be retried on another path.
|
||||
if (err?.noTransferFallback || err?.sourceChanged) throw err;
|
||||
rememberPipelineError(err);
|
||||
// fastPut progress is not a durable contiguous checkpoint; reset so
|
||||
// concurrent-shared does not resume past holes left by the failed put.
|
||||
transfer.checkpointBytes = 0;
|
||||
sendProgress(0, fileSize, { force: true, checkpointBytes: 0 });
|
||||
console.warn(
|
||||
"[transferBridge] isolated fastPut failed, trying next pipelined strategy:",
|
||||
err?.message || String(err),
|
||||
);
|
||||
} finally {
|
||||
if (fastPutSnapshotPath) {
|
||||
await fs.promises.rm(fastPutSnapshotPath, { force: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
if (fastPutOk) {
|
||||
onBytesCommitted?.();
|
||||
await finishSuccessfulUpload();
|
||||
return;
|
||||
}
|
||||
} else if (isolated && typeof isolated.end === "function") {
|
||||
try { isolated.end(); } catch { /* ignore */ }
|
||||
}
|
||||
|
||||
// Concurrent WRITEs on the shared browse channel — still pipelined, does not
|
||||
@@ -2562,14 +2726,59 @@ function openSftpHandleForTransfer(sftp, filePath, flags, transfer, options = {}
|
||||
let drainForceTimer = null;
|
||||
const previousAbort = transfer.abort;
|
||||
let pathGateReleased = false;
|
||||
// True when we released the path gate without an OPEN callback (timeout
|
||||
// after channel error / cancel). A late truncating OPEN can then wipe
|
||||
// same-transfer fastPut/shared fallback bytes - invalidate that attempt.
|
||||
let pathGateForceReleased = false;
|
||||
|
||||
const releasePathGate = () => {
|
||||
const releasePathGate = (options = {}) => {
|
||||
if (!pathGate || pathGateReleased) return;
|
||||
pathGateReleased = true;
|
||||
if (options.forced === true) pathGateForceReleased = true;
|
||||
pathGate.release();
|
||||
try { transfer._resolvePendingWriteOpenPathGate?.(); } catch { /* ignore */ }
|
||||
};
|
||||
|
||||
// In-place truncating OPEN must stay poisoned until the OPEN callback
|
||||
// settles. Force-releasing lets fastPut/shared fallback finish, then a late
|
||||
// OPEN truncates the already-reported destination and invalidation cannot
|
||||
// restore it after sendComplete (Codex P1 on e2cc8241). Generated stages
|
||||
// still force-release: promotion checks staleOpenTruncatedStage.
|
||||
const mayForceReleasePathGate = () => !(
|
||||
isTruncatingOpenNow() && !generatedStagePath
|
||||
);
|
||||
|
||||
// When we keep in-place poison without force-release, same-path strategy
|
||||
// fallbacks must not run: concurrent-shared would wait forever on the
|
||||
// unreleased prior gate (Codex P1 on 3d4cecfa). Also poison the shared path
|
||||
// map so later same-path waiters fail promptly (Codex P2 on dca41093).
|
||||
const markInPlaceOpenPoisonTerminal = () => {
|
||||
try {
|
||||
transfer.noTransferFallback = true;
|
||||
transfer.inPlaceWriteOpenPoisoned = true;
|
||||
} catch { /* ignore */ }
|
||||
try {
|
||||
pathGate?.fail?.(createPoisonedWriteOpenPathGateError(
|
||||
"Prior write OPEN never settled; path gate is fail-closed",
|
||||
));
|
||||
} catch { /* ignore */ }
|
||||
};
|
||||
|
||||
const invalidateSameTransferAfterForcedGateRelease = () => {
|
||||
if (!pathGateForceReleased || !isTruncatingOpenNow() || !transfer) return;
|
||||
const transferId = transfer.transferId;
|
||||
const active = transferId != null && transferId !== ""
|
||||
? activeTransfers.get(transferId)
|
||||
: null;
|
||||
const target = active || transfer;
|
||||
target.staleOpenTruncatedStage = true;
|
||||
if (target.sharedWriteOpenAccepted === true) {
|
||||
try { target.abort?.(); } catch { /* ignore */ }
|
||||
} else {
|
||||
try { target.checkpointBytes = 0; } catch { /* ignore */ }
|
||||
}
|
||||
};
|
||||
|
||||
const clearDrainForceTimer = () => {
|
||||
if (!drainForceTimer) return;
|
||||
clearTimeout(drainForceTimer);
|
||||
@@ -2597,7 +2806,7 @@ function openSftpHandleForTransfer(sftp, filePath, flags, transfer, options = {}
|
||||
drainForceTimer = setTimeout(() => {
|
||||
drainForceTimer = null;
|
||||
// Free the transfer/lease only. Path gate stays held until the OPEN
|
||||
// callback actually settles (or never, if the channel is dead — safer
|
||||
// callback actually settles (or never, if the channel is dead - safer
|
||||
// than reusing a path that a stale truncating OPEN can still wipe).
|
||||
completeSharedWriteDrain();
|
||||
}, 2000);
|
||||
@@ -2684,6 +2893,9 @@ function openSftpHandleForTransfer(sftp, filePath, flags, transfer, options = {}
|
||||
// after drain force-complete (Codex P2 on 1a8cac20).
|
||||
const finishLateSharedWriteOpen = (handle) => {
|
||||
invalidateRetryStageIfStaleOpen();
|
||||
// Same-transfer fallback may already be writing after a forced gate
|
||||
// release; this late truncating OPEN wiped those bytes (Codex P1).
|
||||
invalidateSameTransferAfterForcedGateRelease();
|
||||
const afterClose = () => {
|
||||
const finish = () => {
|
||||
completeSharedWriteDrain();
|
||||
@@ -2722,9 +2934,10 @@ function openSftpHandleForTransfer(sftp, filePath, flags, transfer, options = {}
|
||||
transfer.cancelled = true;
|
||||
// Any write OPEN (shared or isolated) can truncate/create the remote path.
|
||||
// A settle timeout unblocks the transfer UX; the path gate stays held until
|
||||
// the OPEN callback finishes (or isolated force-release after end()) so a
|
||||
// same-path retry cannot race a still-in-flight truncating OPEN (Codex P1
|
||||
// on 76015575). Shared drain stays pending until callback/force-complete.
|
||||
// the OPEN callback finishes so a same-path retry cannot race a still-
|
||||
// in-flight truncating OPEN (Codex P1 on 76015575 / c30e1734). Shared drain
|
||||
// stays pending until callback/force-complete. In-place truncating OPEN
|
||||
// also marks this transfer terminal and poisons the shared barrier.
|
||||
if (isWriteOpen && !settled) {
|
||||
try { abortChannel?.(); } catch { /* ignore */ }
|
||||
if (!openDrainTimer) {
|
||||
@@ -2732,11 +2945,11 @@ function openSftpHandleForTransfer(sftp, filePath, flags, transfer, options = {}
|
||||
settle(reject, new Error("Transfer cancelled"));
|
||||
if (trackSharedWriteDrain) {
|
||||
armSharedWriteDrainForceComplete();
|
||||
} else if (!mayForceReleasePathGate()) {
|
||||
markInPlaceOpenPoisonTerminal();
|
||||
completeSharedWriteDrain();
|
||||
}
|
||||
// Isolated write: keep path gate until OPEN callback too. Force-
|
||||
// releasing after sftp.end() lets a same-path retry finish, then a
|
||||
// late truncating OPEN can wipe its destination with no live
|
||||
// activeTransfers entry to invalidate (Codex P1 on c30e1734).
|
||||
// Late OPEN after this still closes/unlinks via finishLateSharedWriteOpen.
|
||||
}, 2000);
|
||||
}
|
||||
return;
|
||||
@@ -2751,12 +2964,19 @@ function openSftpHandleForTransfer(sftp, filePath, flags, transfer, options = {}
|
||||
|
||||
const onOpenChannelError = (error) => {
|
||||
settle(reject, error || new Error("SFTP channel error"));
|
||||
// Write OPEN: keep path gate until OPEN callback (or isolated force-release).
|
||||
// Shared drain also stays pending until callback/force-complete.
|
||||
// Write OPEN: keep path gate until OPEN callback (or shared drain force).
|
||||
// Isolated writes never force-release here: a late truncating OPEN can wipe
|
||||
// a same-path fallback destination after sendComplete (#2755 / Codex P1 on
|
||||
// c30e1734). Generated-stage hang is bounded by fastPut's gate wait timeout
|
||||
// (noTransferFallback). In-place truncating OPEN marks the transfer
|
||||
// terminal so strategy fallbacks do not wait forever (Codex P1 on 3d4cecfa).
|
||||
if (trackSharedWriteDrain) {
|
||||
armSharedWriteDrainForceComplete();
|
||||
} else if (isWriteOpen) {
|
||||
// Isolated write: hold path gate until OPEN callback (same as cancel).
|
||||
if (!mayForceReleasePathGate()) {
|
||||
markInPlaceOpenPoisonTerminal();
|
||||
completeSharedWriteDrain();
|
||||
}
|
||||
} else {
|
||||
completeSharedWriteDrain();
|
||||
releasePathGate();
|
||||
@@ -3201,7 +3421,7 @@ async function assertUploadDigestCapacity(digestPath, fileSize) {
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer) {
|
||||
async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer, onProgress = null) {
|
||||
let sourceHandle = null;
|
||||
let digestHandle = null;
|
||||
try {
|
||||
@@ -3228,6 +3448,7 @@ async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, tran
|
||||
}
|
||||
position += length;
|
||||
chunkIndex += digestCount;
|
||||
onProgress?.(position, fileSize);
|
||||
}
|
||||
} finally {
|
||||
await sourceHandle?.close().catch(() => {});
|
||||
@@ -3235,7 +3456,7 @@ async function verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, tran
|
||||
}
|
||||
}
|
||||
|
||||
async function createUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer) {
|
||||
async function createUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer, onProgress = null) {
|
||||
// A crashed attempt may have left this transfer's old baseline behind. It is
|
||||
// fully replaceable and its blocks must be reclaimable before capacity is
|
||||
// evaluated for the new baseline.
|
||||
@@ -3278,6 +3499,9 @@ async function createUploadDigestBaseline(sourcePath, digestPath, fileSize, tran
|
||||
}
|
||||
position += length;
|
||||
digestPosition += digests.length;
|
||||
// Report scan progress so large uploads do not look stuck at 0 B/s while
|
||||
// hashing (#2712 / #2556). Bytes here are local read progress, not remote.
|
||||
onProgress?.(position, fileSize);
|
||||
}
|
||||
completed = true;
|
||||
} finally {
|
||||
@@ -3286,7 +3510,9 @@ async function createUploadDigestBaseline(sourcePath, digestPath, fileSize, tran
|
||||
if (!completed) await fs.promises.rm(digestPath, { force: true }).catch(() => {});
|
||||
}
|
||||
|
||||
await verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer);
|
||||
// Confirm the sidecar matches the source before any remote WRITE. Progress
|
||||
// callbacks keep the UI in "verifying" instead of looking stuck (#2712).
|
||||
await verifyUploadDigestBaseline(sourcePath, digestPath, fileSize, transfer, onProgress);
|
||||
}
|
||||
|
||||
async function createVerifiedUploadSnapshot(
|
||||
@@ -4895,7 +5121,9 @@ async function startTransferNow(event, payload, onProgress) {
|
||||
mtimeMs = Number.isFinite(st.mtimeMs) ? st.mtimeMs
|
||||
: (Number.isFinite(st.mtime) ? st.mtime * 1000 : undefined);
|
||||
} else {
|
||||
await requireSftpChannel(client);
|
||||
// Race channel reopen against cancel (Codex P2): a dead sftp handle can
|
||||
// sit in requireSftpChannel for ~10s; cancel must not wait that out.
|
||||
await requireSftpChannel(client, { signal: transfer.signal });
|
||||
const encoded = encodePathForSession(sourceSftpId, sourcePath, sourceEncoding);
|
||||
const st = await client.stat(encoded);
|
||||
size = st.size;
|
||||
@@ -4918,8 +5146,16 @@ async function startTransferNow(event, payload, onProgress) {
|
||||
transfer.captureSourceSoftIdentity = async () => {
|
||||
try {
|
||||
transfer.sourceSoftIdentity = await readSourceSoftIdentity();
|
||||
} catch {
|
||||
// Best-effort baseline for soft resume.
|
||||
} catch (error) {
|
||||
// Propagate cancel/abort so preflight racing can settle; other failures
|
||||
// stay best-effort (soft resume simply lacks a baseline).
|
||||
if (
|
||||
transfer.cancelled
|
||||
|| transfer.signal?.aborted
|
||||
|| /cancel|abort/i.test(String(error?.message || error))
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -5153,10 +5389,13 @@ async function startTransferNow(event, payload, onProgress) {
|
||||
// and soft resume compare against the original plan, not a grown remote.
|
||||
transfer.totalBytes = fileSize;
|
||||
|
||||
// Baseline for soft resume (size + mtime + head sample). Full SHA-256 remains
|
||||
// for hard reconnect / crash recovery.
|
||||
if (transfer.resumable && typeof transfer.captureSourceSoftIdentity === "function") {
|
||||
void transfer.captureSourceSoftIdentity();
|
||||
// Baseline for soft resume and destination mtime preserve (size + mtime +
|
||||
// head sample). Capture before bytes move so non-resumable / SCP paths stamp
|
||||
// the pre-transfer identity (Codex P1). Full SHA-256 remains for hard
|
||||
// reconnect / crash recovery. Run through cancelable preflight so a hung
|
||||
// SFTP reopen cannot outlive cancel (Codex P2).
|
||||
if (typeof transfer.captureSourceSoftIdentity === "function") {
|
||||
await runCancelablePreflight(() => transfer.captureSourceSoftIdentity());
|
||||
}
|
||||
|
||||
const sourceClient = sourceType === "sftp" ? sftpClients.get(sourceSftpId) : null;
|
||||
@@ -5727,6 +5966,8 @@ async function startTransferNow(event, payload, onProgress) {
|
||||
}
|
||||
|
||||
sendProgress(fileSize, fileSize);
|
||||
// Stamp destination mtime from the source so skip-unchanged can match later.
|
||||
await preserveTransferredDestinationMtime(transfer);
|
||||
logTransferDiag(transfer, "done", {
|
||||
transferred: fileSize,
|
||||
total: fileSize,
|
||||
@@ -6708,6 +6949,8 @@ module.exports = {
|
||||
releaseSftpTransferSession,
|
||||
listTransferSftpIds,
|
||||
_promoteLocalTransferForTests: promoteLocalTransfer,
|
||||
_preserveTransferredDestinationMtimeForTests: preserveTransferredDestinationMtime,
|
||||
_waitForPendingWriteOpenPathGateForTests: waitForPendingWriteOpenPathGate,
|
||||
_stableLocalFileIdentityForTests: stableLocalFileIdentity,
|
||||
_getWorkerTransferLifecycleEpochCountForTests: () => workerTransferLifecycleEpochs.size,
|
||||
_setWorkerTransferLifecycleEpochForTests: (transferId, epoch) => {
|
||||
|
||||
@@ -5342,7 +5342,10 @@ test("later same-path upload fails promptly after prior write OPEN gate is poiso
|
||||
);
|
||||
}),
|
||||
]);
|
||||
assert.match(firstResult.error || "", /Timed out waiting for prior write OPEN to settle before fastPut/i);
|
||||
assert.match(
|
||||
firstResult.error || "",
|
||||
/isolated SFTP channel died during OPEN|In-place write OPEN poison|Timed out waiting for prior write OPEN/i,
|
||||
);
|
||||
|
||||
// Later same-path in-place upload must fail promptly — not hang on the
|
||||
// unresolved truncatingSharedWriteOpenGates entry (Codex P2).
|
||||
@@ -5379,7 +5382,7 @@ test("later same-path upload fails promptly after prior write OPEN gate is poiso
|
||||
assert.ok(laterResult.error, "expected later upload to fail closed on unresolved path gate");
|
||||
assert.match(
|
||||
laterResult.error,
|
||||
/Timed out waiting for prior write OPEN|prior write OPEN never settled|write OPEN path gate/i,
|
||||
/Timed out waiting for prior write OPEN|prior write OPEN never settled|path gate is fail-closed|write OPEN path gate|In-place write OPEN poison/i,
|
||||
);
|
||||
assert.equal(
|
||||
laterWriteOpens,
|
||||
@@ -5512,13 +5515,9 @@ test("path gate poison survives a same-path waiter queued during fastPut timeout
|
||||
assert.ok(openReady, "expected first in-place write OPEN to stall");
|
||||
firstIsolated.emit("error", new Error("isolated SFTP channel died during OPEN"));
|
||||
|
||||
// Wait until first transfer has reopened for fastPut (second isolated channel)
|
||||
// and is inside the 2s pendingWriteOpenPathGate wait.
|
||||
const fastPutWaiting = await waitUntil(() => isolatedChannelCount >= 2, 2000);
|
||||
assert.ok(fastPutWaiting, "expected fastPut reopen while prior OPEN gate pending");
|
||||
|
||||
// Queue a second same-path upload during the timeout window so it replaces the
|
||||
// map entry before fail() runs.
|
||||
// In-place OPEN poison is terminal immediately (no fastPut wait). Queue a
|
||||
// second same-path upload while the first is settling so fail() must still
|
||||
// leave a fail-closed barrier for a third upload.
|
||||
const secondRunning = transferBridge.startTransfer(
|
||||
{ sender: createSender() },
|
||||
{
|
||||
@@ -5540,7 +5539,10 @@ test("path gate poison survives a same-path waiter queued during fastPut timeout
|
||||
setTimeout(() => reject(new Error("first transfer hung on gate timeout")), 5000);
|
||||
}),
|
||||
]);
|
||||
assert.match(firstResult.error || "", /Timed out waiting for prior write OPEN to settle before fastPut/i);
|
||||
assert.match(
|
||||
firstResult.error || "",
|
||||
/isolated SFTP channel died during OPEN|In-place write OPEN poison|Timed out waiting for prior write OPEN/i,
|
||||
);
|
||||
|
||||
const secondResult = await Promise.race([
|
||||
secondRunning,
|
||||
@@ -5549,7 +5551,10 @@ test("path gate poison survives a same-path waiter queued during fastPut timeout
|
||||
}),
|
||||
]);
|
||||
assert.ok(secondResult.error, "expected second waiter to fail closed after poison");
|
||||
assert.match(secondResult.error, /prior write OPEN never settled|path gate is fail-closed/i);
|
||||
assert.match(
|
||||
secondResult.error,
|
||||
/prior write OPEN never settled|path gate is fail-closed|In-place write OPEN poison/i,
|
||||
);
|
||||
|
||||
const thirdStartedAt = Date.now();
|
||||
const thirdRunning = transferBridge.startTransfer(
|
||||
@@ -5723,9 +5728,6 @@ test("path gate poison clears after late original OPEN settles past successor",
|
||||
assert.ok(openReady, "expected first in-place write OPEN to stall");
|
||||
firstIsolated.emit("error", new Error("isolated SFTP channel died during OPEN"));
|
||||
|
||||
const fastPutWaiting = await waitUntil(() => isolatedChannelCount >= 2, 2000);
|
||||
assert.ok(fastPutWaiting, "expected fastPut reopen while prior OPEN gate pending");
|
||||
|
||||
const secondRunning = transferBridge.startTransfer(
|
||||
{ sender: createSender() },
|
||||
{
|
||||
@@ -5747,7 +5749,10 @@ test("path gate poison clears after late original OPEN settles past successor",
|
||||
setTimeout(() => reject(new Error("first transfer hung on gate timeout")), 5000);
|
||||
}),
|
||||
]);
|
||||
assert.match(firstResult.error || "", /Timed out waiting for prior write OPEN to settle before fastPut/i);
|
||||
assert.match(
|
||||
firstResult.error || "",
|
||||
/isolated SFTP channel died during OPEN|In-place write OPEN poison|Timed out waiting for prior write OPEN/i,
|
||||
);
|
||||
|
||||
await Promise.race([
|
||||
secondRunning,
|
||||
@@ -5919,16 +5924,14 @@ test("later same-path upload fails promptly after cancel during unresolved OPEN
|
||||
|
||||
const openReady = await waitUntil(() => firstOpenStarted, 2000);
|
||||
assert.ok(openReady, "expected first in-place write OPEN to stall");
|
||||
firstIsolated.emit("error", new Error("isolated SFTP channel died during OPEN"));
|
||||
|
||||
const reopened = await waitUntil(() => isolatedChannelCount >= 2, 2000);
|
||||
assert.ok(reopened, "expected fastPut reopen while prior OPEN gate pending");
|
||||
|
||||
// Cancel while the in-place OPEN is still unsettled. After the settle timeout
|
||||
// the path gate is poisoned so later same-path uploads fail promptly.
|
||||
await transferBridge.cancelTransfer(null, { transferId: firstTransferId });
|
||||
const firstResult = await Promise.race([
|
||||
firstRunning,
|
||||
new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error("first transfer hung on cancel during gate wait")), 1500);
|
||||
setTimeout(() => reject(new Error("first transfer hung on cancel during OPEN")), 5000);
|
||||
}),
|
||||
]);
|
||||
assert.match(firstResult.error || "", /cancel/i);
|
||||
@@ -6105,7 +6108,10 @@ test("path gate poison clears after owning isolated transport ends", async (t) =
|
||||
setTimeout(() => reject(new Error("first transfer hung on gate timeout")), 5000);
|
||||
}),
|
||||
]);
|
||||
assert.match(firstResult.error || "", /Timed out waiting for prior write OPEN to settle before fastPut/i);
|
||||
assert.match(
|
||||
firstResult.error || "",
|
||||
/isolated SFTP channel died during OPEN|In-place write OPEN poison|Timed out waiting for prior write OPEN/i,
|
||||
);
|
||||
|
||||
// Immediately after poison the barrier must still fail closed.
|
||||
const midRunning = transferBridge.startTransfer(
|
||||
@@ -11263,3 +11269,369 @@ test("local promotion succeeds when backup still matches validated identity", as
|
||||
transferBridge._stableLocalFileIdentityForTests(originalStat),
|
||||
);
|
||||
});
|
||||
|
||||
test("preserveTransferredDestinationMtime stamps local targets from sourceSoftIdentity", async (t) => {
|
||||
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-preserve-mtime-"));
|
||||
t.after(async () => fs.promises.rm(tempDir, { recursive: true, force: true }));
|
||||
|
||||
const targetPath = path.join(tempDir, "copied.bin");
|
||||
await fs.promises.writeFile(targetPath, Buffer.from("payload"));
|
||||
const before = await fs.promises.stat(targetPath);
|
||||
const sourceMtimeMs = 1_700_000_000_000; // 2023-11-14T22:13:20.000Z
|
||||
|
||||
await transferBridge._preserveTransferredDestinationMtimeForTests({
|
||||
targetType: "local",
|
||||
targetPath,
|
||||
sourceSoftIdentity: { size: 7, mtimeMs: sourceMtimeMs },
|
||||
});
|
||||
|
||||
const after = await fs.promises.stat(targetPath);
|
||||
assert.equal(Math.floor(after.mtimeMs / 1000), Math.floor(sourceMtimeMs / 1000));
|
||||
assert.notEqual(Math.floor(after.mtimeMs / 1000), Math.floor(before.mtimeMs / 1000));
|
||||
});
|
||||
|
||||
test("preserveTransferredDestinationMtime times out hanging remote setStat", async () => {
|
||||
let setStatStarted = false;
|
||||
const hangingClient = {
|
||||
sftp: {
|
||||
readdir() {},
|
||||
stat() {},
|
||||
mkdir() {},
|
||||
unlink() {},
|
||||
},
|
||||
async setStat() {
|
||||
setStatStarted = true;
|
||||
await new Promise(() => {});
|
||||
},
|
||||
};
|
||||
transferBridge.init({ sftpClients: new Map([["target", hangingClient]]) });
|
||||
|
||||
const startedAt = Date.now();
|
||||
await transferBridge._preserveTransferredDestinationMtimeForTests({
|
||||
targetType: "sftp",
|
||||
targetSftpId: "target",
|
||||
targetPath: "/tmp/mtime-hang.bin",
|
||||
sourceSoftIdentity: { size: 1, mtimeMs: 1_700_000_000_000 },
|
||||
}, { timeoutMs: 40 });
|
||||
const elapsed = Date.now() - startedAt;
|
||||
|
||||
assert.equal(setStatStarted, true);
|
||||
assert.ok(elapsed < 1500, `expected bounded mtime stamp, elapsed=${elapsed}`);
|
||||
});
|
||||
|
||||
test("local-to-local transfer preserves source mtime on the destination", async (t) => {
|
||||
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-local-mtime-"));
|
||||
t.after(async () => fs.promises.rm(tempDir, { recursive: true, force: true }));
|
||||
|
||||
const sourcePath = path.join(tempDir, "source.bin");
|
||||
const targetPath = path.join(tempDir, "target.bin");
|
||||
await fs.promises.writeFile(sourcePath, Buffer.from("hello-mtime"));
|
||||
const sourceMtimeMs = 1_600_000_000_000; // 2020-09-13T12:26:40.000Z
|
||||
await fs.promises.utimes(sourcePath, new Date(sourceMtimeMs), new Date(sourceMtimeMs));
|
||||
|
||||
transferBridge.init({ sftpClients: new Map() });
|
||||
const result = await transferBridge.startTransfer({ sender: createSender() }, {
|
||||
transferId: "local-mtime-preserve",
|
||||
sourcePath,
|
||||
targetPath,
|
||||
sourceType: "local",
|
||||
targetType: "local",
|
||||
totalBytes: 11,
|
||||
resumable: false,
|
||||
});
|
||||
|
||||
assert.equal(result.error, undefined);
|
||||
const targetStat = await fs.promises.stat(targetPath);
|
||||
assert.equal(Math.floor(targetStat.mtimeMs / 1000), Math.floor(sourceMtimeMs / 1000));
|
||||
assert.deepEqual(await fs.promises.readFile(targetPath), Buffer.from("hello-mtime"));
|
||||
});
|
||||
|
||||
test("waitForPendingWriteOpenPathGate timeout fails closed without clearing poison", async () => {
|
||||
let resolveGate;
|
||||
const gate = new Promise((resolve) => { resolveGate = resolve; });
|
||||
let failCalls = 0;
|
||||
const transfer = {
|
||||
pendingWriteOpenPathGate: gate,
|
||||
_resolvePendingWriteOpenPathGate() {
|
||||
resolveGate();
|
||||
transfer.pendingWriteOpenPathGate = null;
|
||||
transfer._resolvePendingWriteOpenPathGate = null;
|
||||
},
|
||||
_failPendingWriteOpenPathGate() {
|
||||
failCalls += 1;
|
||||
},
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
() => transferBridge._waitForPendingWriteOpenPathGateForTests(transfer, { timeoutMs: 30 }),
|
||||
/Timed out waiting for prior write OPEN to settle/i,
|
||||
);
|
||||
assert.equal(transfer.pendingWriteOpenPathGate, gate, "timeout must not clear the published gate poison");
|
||||
assert.equal(failCalls, 1, "timeout must poison the shared path gate for later waiters");
|
||||
resolveGate();
|
||||
});
|
||||
|
||||
test("in-place isolated OPEN channel error fails closed without waiting on shared fallback", async (t) => {
|
||||
// Codex P1 on 3d4cecfa: keeping in-place poison without force-release must not
|
||||
// fall through into concurrent-shared, which waits forever on the prior gate.
|
||||
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-inplace-open-terminal-poison-"));
|
||||
t.after(async () => fs.promises.rm(tempDir, { recursive: true, force: true }));
|
||||
|
||||
const payload = Buffer.alloc(TRANSFER_CHUNK_SIZE, 73);
|
||||
const localPath = path.join(tempDir, "upload.bin");
|
||||
await fs.promises.writeFile(localPath, payload);
|
||||
const targetPath = "/tmp/inplace-terminal-poison.bin";
|
||||
const remoteFiles = new Map([[targetPath, Buffer.from("keep-original")]]);
|
||||
let openCalls = 0;
|
||||
let sharedOpenCalls = 0;
|
||||
|
||||
const isolatedSftp = createFastSftp({
|
||||
open(_remotePath, flags, _callback) {
|
||||
assert.equal(flags, "w");
|
||||
openCalls += 1;
|
||||
// Never invoke OPEN callback (dead channel after error).
|
||||
},
|
||||
write() {
|
||||
throw new Error("WRITE must not run while OPEN is pending");
|
||||
},
|
||||
end() {},
|
||||
fastPut() {
|
||||
throw new Error("fastPut must not run after in-place OPEN poison");
|
||||
},
|
||||
});
|
||||
|
||||
const sharedSftp = createFastSftp({
|
||||
open() {
|
||||
sharedOpenCalls += 1;
|
||||
throw new Error("shared OPEN must not run after terminal in-place poison");
|
||||
},
|
||||
createWriteStream() {
|
||||
throw new Error("serial WriteStream must not run");
|
||||
},
|
||||
lstat(remotePath, callback) {
|
||||
const key = String(remotePath);
|
||||
if (!remoteFiles.has(key)) {
|
||||
const error = new Error("ENOENT");
|
||||
error.code = 2;
|
||||
callback(error);
|
||||
return;
|
||||
}
|
||||
callback(null, {
|
||||
size: remoteFiles.get(key).length,
|
||||
mode: 0o120777,
|
||||
isDirectory: () => false,
|
||||
isSymbolicLink: () => true,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const client = {
|
||||
sftp: sharedSftp,
|
||||
async lstat(remotePath) {
|
||||
const key = String(remotePath);
|
||||
if (!remoteFiles.has(key)) {
|
||||
const error = new Error("ENOENT");
|
||||
error.code = 2;
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
size: remoteFiles.get(key).length,
|
||||
mode: 0o120777,
|
||||
isDirectory: () => false,
|
||||
isSymbolicLink: () => true,
|
||||
};
|
||||
},
|
||||
async stat(remotePath) {
|
||||
const key = String(remotePath);
|
||||
if (!remoteFiles.has(key)) {
|
||||
const error = new Error("ENOENT");
|
||||
error.code = 2;
|
||||
throw error;
|
||||
}
|
||||
return { size: remoteFiles.get(key).length };
|
||||
},
|
||||
rename() {
|
||||
return Promise.resolve();
|
||||
},
|
||||
async delete() {},
|
||||
client: {
|
||||
sftp(callback) {
|
||||
callback(null, isolatedSftp);
|
||||
},
|
||||
},
|
||||
};
|
||||
transferBridge.init({ sftpClients: new Map([["target", client]]) });
|
||||
|
||||
const sender = createSender();
|
||||
const running = transferBridge.startTransfer({ sender }, {
|
||||
transferId: "inplace-open-terminal-poison",
|
||||
sourcePath: localPath,
|
||||
targetPath,
|
||||
sourceType: "local",
|
||||
targetType: "sftp",
|
||||
targetSftpId: "target",
|
||||
totalBytes: payload.length,
|
||||
resumable: false,
|
||||
skipAdmission: true,
|
||||
});
|
||||
|
||||
const opened = await waitUntil(() => openCalls >= 1, 2000);
|
||||
assert.ok(opened, "expected isolated in-place OPEN to stall");
|
||||
isolatedSftp.emit("error", new Error("isolated channel died during in-place OPEN"));
|
||||
|
||||
const result = await Promise.race([
|
||||
running,
|
||||
new Promise((_, reject) => setTimeout(
|
||||
() => reject(new Error("transfer hung behind permanent in-place OPEN poison")),
|
||||
4000,
|
||||
)),
|
||||
]);
|
||||
|
||||
assert.ok(result.error, "expected fail-closed transfer");
|
||||
assert.match(result.error, /isolated channel died|pipelined upload failed/i);
|
||||
assert.equal(sharedOpenCalls, 0, "concurrent-shared must not wait on poisoned in-place gate");
|
||||
assert.equal(sender.sent.some((entry) => entry.channel === "netcatty:transfer:complete"), false);
|
||||
});
|
||||
|
||||
test("in-place isolated OPEN keeps path gate until late callback after channel error", async (t) => {
|
||||
// Codex P1 on e2cc8241: force-releasing an in-place truncating OPEN lets
|
||||
// fastPut complete, then a late OPEN truncates the reported-success file.
|
||||
const tempDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "netcatty-inplace-open-no-force-release-"));
|
||||
t.after(async () => fs.promises.rm(tempDir, { recursive: true, force: true }));
|
||||
|
||||
const payload = Buffer.alloc(TRANSFER_CHUNK_SIZE, 71);
|
||||
const localPath = path.join(tempDir, "upload.bin");
|
||||
await fs.promises.writeFile(localPath, payload);
|
||||
const targetPath = "/tmp/inplace-no-force.bin";
|
||||
const existingPayload = Buffer.from("keep-original");
|
||||
|
||||
const remoteFiles = new Map([[targetPath, Buffer.from(existingPayload)]]);
|
||||
const eventLog = [];
|
||||
let releaseOpen = null;
|
||||
let openCalls = 0;
|
||||
let fastPutCalls = 0;
|
||||
|
||||
const isolatedSftp = createFastSftp({
|
||||
open(remotePath, flags, callback) {
|
||||
assert.equal(flags, "w");
|
||||
openCalls += 1;
|
||||
const key = String(remotePath);
|
||||
releaseOpen = () => {
|
||||
remoteFiles.set(key, Buffer.alloc(0));
|
||||
eventLog.push(`late-open-truncate:${key}`);
|
||||
callback(null, Buffer.from(`handle:${key}`));
|
||||
};
|
||||
},
|
||||
write() {
|
||||
throw new Error("WRITE must not run while OPEN is pending");
|
||||
},
|
||||
close(_handle, callback) {
|
||||
eventLog.push("close");
|
||||
callback(null);
|
||||
},
|
||||
end() {
|
||||
eventLog.push("isolated-end");
|
||||
},
|
||||
fastPut() {
|
||||
fastPutCalls += 1;
|
||||
throw new Error("fastPut must not run while in-place OPEN poison is held");
|
||||
},
|
||||
});
|
||||
|
||||
const sharedSftp = createFastSftp({
|
||||
open() {
|
||||
throw new Error("shared OPEN must not run while isolated in-place gate is held");
|
||||
},
|
||||
createWriteStream() {
|
||||
throw new Error("serial WriteStream must not run");
|
||||
},
|
||||
lstat(remotePath, callback) {
|
||||
const key = String(remotePath);
|
||||
if (!remoteFiles.has(key)) {
|
||||
const error = new Error("ENOENT");
|
||||
error.code = 2;
|
||||
callback(error);
|
||||
return;
|
||||
}
|
||||
callback(null, {
|
||||
size: remoteFiles.get(key).length,
|
||||
mode: 0o120777,
|
||||
isDirectory: () => false,
|
||||
isSymbolicLink: () => true,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
const client = {
|
||||
sftp: sharedSftp,
|
||||
async lstat(remotePath) {
|
||||
const key = String(remotePath);
|
||||
if (!remoteFiles.has(key)) {
|
||||
const error = new Error("ENOENT");
|
||||
error.code = 2;
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
size: remoteFiles.get(key).length,
|
||||
mode: 0o120777,
|
||||
isDirectory: () => false,
|
||||
isSymbolicLink: () => true,
|
||||
};
|
||||
},
|
||||
async stat(remotePath) {
|
||||
const key = String(remotePath);
|
||||
if (!remoteFiles.has(key)) {
|
||||
const error = new Error("ENOENT");
|
||||
error.code = 2;
|
||||
throw error;
|
||||
}
|
||||
return { size: remoteFiles.get(key).length };
|
||||
},
|
||||
rename() {
|
||||
return Promise.resolve();
|
||||
},
|
||||
async delete() {},
|
||||
client: {
|
||||
sftp(callback) {
|
||||
callback(null, isolatedSftp);
|
||||
},
|
||||
},
|
||||
};
|
||||
transferBridge.init({ sftpClients: new Map([["target", client]]) });
|
||||
|
||||
const sender = createSender();
|
||||
const transferId = "inplace-open-no-force-release";
|
||||
const running = transferBridge.startTransfer({ sender }, {
|
||||
transferId,
|
||||
sourcePath: localPath,
|
||||
targetPath,
|
||||
sourceType: "local",
|
||||
targetType: "sftp",
|
||||
targetSftpId: "target",
|
||||
totalBytes: payload.length,
|
||||
resumable: false,
|
||||
skipAdmission: true,
|
||||
});
|
||||
|
||||
const opened = await waitUntil(() => openCalls >= 1, 2000);
|
||||
assert.ok(opened, "expected isolated in-place OPEN to stall");
|
||||
|
||||
isolatedSftp.emit("error", new Error("isolated channel died during in-place OPEN"));
|
||||
|
||||
// Former force-release window (2s). Path gate must still block fallbacks.
|
||||
await new Promise((resolve) => setTimeout(resolve, 2500));
|
||||
assert.equal(fastPutCalls, 0, "fastPut must not run while in-place OPEN is unsettled");
|
||||
assert.deepEqual(remoteFiles.get(targetPath), existingPayload, "destination must stay intact before late OPEN");
|
||||
|
||||
assert.equal(typeof releaseOpen, "function");
|
||||
releaseOpen();
|
||||
|
||||
const result = await Promise.race([
|
||||
running,
|
||||
new Promise((_, reject) => setTimeout(() => reject(new Error("transfer hung after late in-place OPEN")), 8000)),
|
||||
]);
|
||||
|
||||
assert.ok(result.error, "expected transfer to fail closed rather than report success after late truncate");
|
||||
assert.equal(sender.sent.some((entry) => entry.channel === "netcatty:transfer:complete"), false);
|
||||
assert.ok(eventLog.includes(`late-open-truncate:${targetPath}`) || eventLog.includes("close"));
|
||||
});
|
||||
@@ -19,8 +19,9 @@ const DOWNLOAD_TRANSFER_CONCURRENCY = 64;
|
||||
// (isolated channel or shared/sudo browse path). Additional downloads wait for
|
||||
// a free slot rather than degrading to serial createReadStream (#2719 / #2449
|
||||
// fail-closed). Folder fan-out is capped in the renderer by
|
||||
// runSftpTransferWorkers (settings transfer concurrency); multi-select
|
||||
// top-level files are not throttled by that setting.
|
||||
// runSftpTransferWorkers (settings transfer concurrency). Multi-select
|
||||
// top-level downloads enqueue in parallel and share the same host scheduler /
|
||||
// channel pool caps - they must never fall back to serial body streams.
|
||||
const FAST_DOWNLOAD_CHANNELS_PER_SESSION = 1;
|
||||
|
||||
module.exports = {
|
||||
|
||||
@@ -204,6 +204,10 @@ export const STORAGE_KEY_AI_EXTERNAL_MCP_SILENT_SESSIONS = 'netcatty_ai_external
|
||||
|
||||
// SFTP Transfer Concurrency
|
||||
export const STORAGE_KEY_SFTP_TRANSFER_CONCURRENCY = 'netcatty_sftp_transfer_concurrency_v1';
|
||||
/** Folder transfers: scan the full tree before moving bytes (stable file totals). */
|
||||
export const STORAGE_KEY_SFTP_FOLDER_PRESCAN = 'netcatty_sftp_folder_prescan_v1';
|
||||
/** Skip files when target size + mtime already match the source (rsync-like). */
|
||||
export const STORAGE_KEY_SFTP_SKIP_UNCHANGED = 'netcatty_sftp_skip_unchanged_v1';
|
||||
/**
|
||||
* @deprecated Legacy transfer-pool idle TTL. No longer read; SSH keep-alive uses
|
||||
* STORAGE_KEY_SSH_TRANSPORT_IDLE_TTL_MS. Kept so old localStorage entries are ignored safely.
|
||||
|
||||
Reference in new issue
Block a user