mirror of
https://github.com/binaricat/Netcatty.git
synced 2026-10-29 18:11:40 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
55f4e9cc56 | ||
|
|
80479e84d3 |
No files matched your search
@@ -109,6 +109,8 @@ export const enTerminalMessages: Messages = {
|
||||
'terminal.serverStats.cpu': 'CPU Usage',
|
||||
'terminal.serverStats.cpuCores': 'CPU Core Usage',
|
||||
'terminal.serverStats.gpu': 'GPU Usage',
|
||||
'terminal.serverStats.gpuDevices': 'GPU Device Usage',
|
||||
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
|
||||
'terminal.serverStats.memory': 'Memory Usage',
|
||||
'terminal.serverStats.memoryDetails': 'Memory Details',
|
||||
'terminal.serverStats.memUsed': 'Used',
|
||||
|
||||
@@ -109,6 +109,8 @@ export const esTerminalMessages: Messages = {
|
||||
'terminal.serverStats.cpu': 'Uso de CPU',
|
||||
'terminal.serverStats.cpuCores': 'Uso de núcleos de CPU',
|
||||
'terminal.serverStats.gpu': 'Uso de GPU',
|
||||
'terminal.serverStats.gpuDevices': 'Uso de cada GPU',
|
||||
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
|
||||
'terminal.serverStats.memory': 'Uso de memoria',
|
||||
'terminal.serverStats.memoryDetails': 'Detalles de memoria',
|
||||
'terminal.serverStats.memUsed': 'Usada',
|
||||
|
||||
@@ -130,6 +130,8 @@ export const ruTerminalMessages: Messages = {
|
||||
'terminal.serverStats.cpu': 'Использование CPU',
|
||||
'terminal.serverStats.cpuCores': 'Использование ядер CPU',
|
||||
'terminal.serverStats.gpu': 'Использование GPU',
|
||||
'terminal.serverStats.gpuDevices': 'Использование видеокарт',
|
||||
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
|
||||
'terminal.serverStats.memory': 'Использование памяти',
|
||||
'terminal.serverStats.memoryDetails': 'Сведения о памяти',
|
||||
'terminal.serverStats.memUsed': 'Использовано',
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import { enTerminalMessages } from "./en/terminal";
|
||||
import { esTerminalMessages } from "./es/terminal";
|
||||
import { ruTerminalMessages } from "./ru/terminal";
|
||||
import { zhCNVaultMessages } from "./zh-CN/vault";
|
||||
import { zhTWVaultMessages } from "./zh-TW/vault";
|
||||
|
||||
test("per-GPU stats strings exist in every locale", () => {
|
||||
const labels = [
|
||||
[enTerminalMessages, "GPU Device Usage", "GPU {{index}}"],
|
||||
[esTerminalMessages, "Uso de cada GPU", "GPU {{index}}"],
|
||||
[ruTerminalMessages, "Использование видеокарт", "GPU {{index}}"],
|
||||
[zhCNVaultMessages, "GPU 设备使用率", "GPU {{index}}"],
|
||||
[zhTWVaultMessages, "GPU 裝置使用率", "GPU {{index}}"],
|
||||
] as const;
|
||||
|
||||
for (const [messages, devices, indexLabel] of labels) {
|
||||
assert.equal(messages["terminal.serverStats.gpuDevices"], devices);
|
||||
assert.equal(messages["terminal.serverStats.gpuIndex"], indexLabel);
|
||||
}
|
||||
});
|
||||
@@ -532,6 +532,8 @@ export const zhCNVaultMessages: Messages = {
|
||||
'terminal.serverStats.cpu': 'CPU 使用率',
|
||||
'terminal.serverStats.cpuCores': 'CPU 核心使用率',
|
||||
'terminal.serverStats.gpu': 'GPU 使用率',
|
||||
'terminal.serverStats.gpuDevices': 'GPU 设备使用率',
|
||||
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
|
||||
'terminal.serverStats.memory': '内存使用',
|
||||
'terminal.serverStats.memoryDetails': '内存详情',
|
||||
'terminal.serverStats.memUsed': '已用',
|
||||
|
||||
@@ -532,6 +532,8 @@ export const zhTWVaultMessages: Messages = {
|
||||
'terminal.serverStats.cpu': 'CPU 使用率',
|
||||
'terminal.serverStats.cpuCores': 'CPU 核心使用率',
|
||||
'terminal.serverStats.gpu': 'GPU 使用率',
|
||||
'terminal.serverStats.gpuDevices': 'GPU 裝置使用率',
|
||||
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
|
||||
'terminal.serverStats.memory': '記憶體使用',
|
||||
'terminal.serverStats.memoryDetails': '記憶體詳情',
|
||||
'terminal.serverStats.memUsed': '已用',
|
||||
|
||||
@@ -24,6 +24,14 @@ export interface ProcessInfo {
|
||||
command: string;
|
||||
}
|
||||
|
||||
export interface GpuDeviceStats {
|
||||
index: number;
|
||||
name: string;
|
||||
utilization: number | null; // percent, null when nvidia-smi reports N/A
|
||||
memoryUsed: number | null; // MiB
|
||||
memoryTotal: number | null; // MiB
|
||||
}
|
||||
|
||||
export interface ServerStats {
|
||||
hostname?: string; // Remote hostname
|
||||
osName?: string; // Remote operating system name
|
||||
@@ -37,6 +45,7 @@ export interface ServerStats {
|
||||
gpuName: string | null; // First GPU name (tooltip)
|
||||
gpuMemUsed: number | null; // Summed VRAM used in MB
|
||||
gpuMemTotal: number | null; // Summed VRAM total in MB
|
||||
gpus: GpuDeviceStats[]; // Per-GPU utilization and VRAM
|
||||
memTotal: number | null; // Total memory in MB
|
||||
memUsed: number | null; // Used memory in MB (excluding buffers/cache)
|
||||
memFree: number | null; // Free memory in MB
|
||||
@@ -116,6 +125,7 @@ function createEmptyServerStats(): ServerStats {
|
||||
gpuName: null,
|
||||
gpuMemUsed: null,
|
||||
gpuMemTotal: null,
|
||||
gpus: [],
|
||||
memTotal: null,
|
||||
memUsed: null,
|
||||
memFree: null,
|
||||
@@ -218,6 +228,7 @@ function normalizeServerStats(stats: Partial<ServerStats>): ServerStats {
|
||||
gpuName: stats.gpuName ?? null,
|
||||
gpuMemUsed: stats.gpuMemUsed ?? null,
|
||||
gpuMemTotal: stats.gpuMemTotal ?? null,
|
||||
gpus: Array.isArray(stats.gpus) ? stats.gpus : [],
|
||||
memTotal: stats.memTotal ?? null,
|
||||
memUsed: stats.memUsed ?? null,
|
||||
memFree: stats.memFree ?? null,
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import React from "react";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
|
||||
import { I18nProvider } from "../../application/i18n/I18nProvider.tsx";
|
||||
import type { GpuDeviceStats } from "../../application/state/useServerStats.ts";
|
||||
import { TerminalGpuCards } from "./TerminalGpuCards.tsx";
|
||||
|
||||
function renderCards(gpus: GpuDeviceStats[], gpu = 40) {
|
||||
return renderToStaticMarkup(
|
||||
<I18nProvider locale="en">
|
||||
<TerminalGpuCards
|
||||
gpus={gpus}
|
||||
gpu={gpu}
|
||||
gpuName="NVIDIA GeForce RTX 4090"
|
||||
gpuMemUsed={2048}
|
||||
gpuMemTotal={24576}
|
||||
/>
|
||||
</I18nProvider>,
|
||||
);
|
||||
}
|
||||
|
||||
const card = (index: number, utilization: number | null, name = `NVIDIA A${index}`): GpuDeviceStats => ({
|
||||
index,
|
||||
name,
|
||||
utilization,
|
||||
memoryUsed: 1024 * (index + 1),
|
||||
memoryTotal: 8192,
|
||||
});
|
||||
|
||||
test("one GPU uses the same per-core bar, colors, and spacing", () => {
|
||||
const markup = renderCards([card(0, 73, "NVIDIA GeForce RTX 4090")], 73);
|
||||
|
||||
assert.match(markup, /GPU Device Usage/);
|
||||
assert.match(markup, /GPU 0/);
|
||||
assert.match(markup, /NVIDIA GeForce RTX 4090/);
|
||||
assert.match(markup, /73%/);
|
||||
assert.match(markup, /1\.0\/8\.0G/);
|
||||
assert.match(markup, /grid gap-1\.5 max-h-\[260px\] overflow-y-auto/);
|
||||
assert.match(markup, /min-w-\[160px\]/);
|
||||
assert.match(markup, /line-clamp-2/);
|
||||
assert.match(markup, /title="NVIDIA GeForce RTX 4090"/);
|
||||
assert.doesNotMatch(markup, /max-w-\[88px\]/);
|
||||
assert.match(markup, /h-1\.5 bg-muted rounded-full overflow-hidden/);
|
||||
assert.match(markup, /bg-amber-500/);
|
||||
assert.match(markup, /text-amber-400/);
|
||||
assert.match(markup, /repeat\(1, 1fr\)/);
|
||||
});
|
||||
|
||||
test("eight GPUs stay in four scrolling columns", () => {
|
||||
const markup = renderCards(Array.from({ length: 8 }, (_, index) => card(index, 10 + index)));
|
||||
|
||||
assert.match(markup, /repeat\(4, 1fr\)/);
|
||||
assert.match(markup, /max-h-\[260px\] overflow-y-auto/);
|
||||
assert.match(markup, /max-w-\[26rem\]/);
|
||||
assert.match(markup, /line-clamp-2/);
|
||||
assert.match(markup, /title="NVIDIA A0"/);
|
||||
assert.doesNotMatch(markup, /max-w-\[88px\]/);
|
||||
for (let index = 0; index < 8; index += 1) {
|
||||
assert.match(markup, new RegExp(`GPU ${index}`));
|
||||
}
|
||||
assert.match(markup, /text-emerald-400/);
|
||||
assert.doesNotMatch(markup, /repeat\(8,/);
|
||||
});
|
||||
|
||||
test("high utilization uses the CPU red threshold and unknown cards stay muted", () => {
|
||||
const markup = renderCards([
|
||||
card(0, 95, "NVIDIA H100"),
|
||||
{ index: 3, name: "NVIDIA A100", utilization: null, memoryUsed: null, memoryTotal: null },
|
||||
], 95);
|
||||
|
||||
assert.match(markup, /bg-red-500/);
|
||||
assert.match(markup, /text-red-400/);
|
||||
assert.match(markup, /95%/);
|
||||
assert.match(markup, /GPU 3/);
|
||||
assert.match(markup, />--</);
|
||||
assert.match(markup, /repeat\(2, 1fr\)/);
|
||||
});
|
||||
|
||||
test("a summary without per-GPU rows falls back to one aggregate bar", () => {
|
||||
const markup = renderCards([], 12);
|
||||
|
||||
assert.match(markup, /min-w-\[160px\]/);
|
||||
assert.match(markup, /h-2 bg-muted rounded-full/);
|
||||
assert.match(markup, /12%/);
|
||||
assert.match(markup, /2\.0\/24\.0G/);
|
||||
assert.match(markup, /text-emerald-400/);
|
||||
assert.doesNotMatch(markup, /grid gap-1\.5/);
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
import React from "react";
|
||||
|
||||
import { useI18n } from "../../application/i18n/I18nProvider";
|
||||
import type { GpuDeviceStats } from "../../application/state/useServerStats";
|
||||
import { cn } from "../../lib/utils";
|
||||
|
||||
export function formatGpuMemoryGib(usedMb: number | null, totalMb: number | null): string | null {
|
||||
if (usedMb === null || totalMb === null || !(totalMb > 0)) return null;
|
||||
return `${(usedMb / 1024).toFixed(1)}/${(totalMb / 1024).toFixed(1)}G`;
|
||||
}
|
||||
|
||||
function usageBarClass(usage: number): string {
|
||||
return usage >= 90 ? "bg-red-500" : usage >= 70 ? "bg-amber-500" : "bg-emerald-500";
|
||||
}
|
||||
|
||||
function usageTextClass(usage: number): string {
|
||||
return usage >= 90 ? "text-red-400" : usage >= 70 ? "text-amber-400" : "text-emerald-400";
|
||||
}
|
||||
|
||||
interface TerminalGpuCardsProps {
|
||||
gpus: GpuDeviceStats[];
|
||||
gpu: number;
|
||||
gpuName: string | null;
|
||||
gpuMemUsed: number | null;
|
||||
gpuMemTotal: number | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-GPU details for the terminal stats hover card.
|
||||
* Mirrors the CPU per-core grid: four columns, the same bar heights,
|
||||
* threshold colors, and a scroll cap so many cards stay inside the popover.
|
||||
*/
|
||||
export const TerminalGpuCards: React.FC<TerminalGpuCardsProps> = ({
|
||||
gpus,
|
||||
gpu,
|
||||
gpuName,
|
||||
gpuMemUsed,
|
||||
gpuMemTotal,
|
||||
}) => {
|
||||
const { t } = useI18n();
|
||||
const aggregateMemory = formatGpuMemoryGib(gpuMemUsed, gpuMemTotal);
|
||||
|
||||
const columns = Math.min(4, gpus.length);
|
||||
|
||||
return (
|
||||
<div className={cn("text-xs space-y-2 w-max max-w-[26rem]", gpus.length === 1 && "min-w-[160px]")}>
|
||||
<div className="font-medium text-sm mb-2">{t("terminal.serverStats.gpuDevices")}</div>
|
||||
{gpus.length > 0 ? (
|
||||
<div
|
||||
className="grid gap-1.5 max-h-[260px] overflow-y-auto pr-1 overscroll-contain"
|
||||
style={{ gridTemplateColumns: `repeat(${columns}, 1fr)` }}
|
||||
>
|
||||
{gpus.map((device) => {
|
||||
const usage = device.utilization;
|
||||
const memory = formatGpuMemoryGib(device.memoryUsed, device.memoryTotal);
|
||||
return (
|
||||
<div key={device.index} className="flex w-full min-w-0 flex-col items-center gap-1">
|
||||
<div className="text-[10px] text-muted-foreground">
|
||||
{t("terminal.serverStats.gpuIndex", { index: device.index })}
|
||||
</div>
|
||||
{device.name ? (
|
||||
<div
|
||||
className="line-clamp-2 w-full break-words text-center text-[10px] leading-tight text-muted-foreground"
|
||||
title={device.name}
|
||||
>
|
||||
{device.name}
|
||||
</div>
|
||||
) : null}
|
||||
<div className="w-full h-1.5 bg-muted rounded-full overflow-hidden">
|
||||
<div
|
||||
className={cn(
|
||||
"h-full rounded-full transition-all",
|
||||
usage === null ? "bg-muted" : usageBarClass(usage),
|
||||
)}
|
||||
style={{ width: `${usage ?? 0}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className={cn(
|
||||
"text-[11px] font-medium",
|
||||
usage === null ? "text-muted-foreground" : usageTextClass(usage),
|
||||
)}>
|
||||
{usage === null ? "--" : `${usage}%`}
|
||||
</div>
|
||||
{memory ? (
|
||||
<div className="text-[10px] text-muted-foreground whitespace-nowrap">{memory}</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex flex-col gap-1.5 min-w-[160px]">
|
||||
{gpuName ? (
|
||||
<div className="text-[10px] text-muted-foreground truncate" title={gpuName}>{gpuName}</div>
|
||||
) : null}
|
||||
<div className="w-full h-2 bg-muted rounded-full overflow-hidden">
|
||||
<div
|
||||
className={cn("h-full rounded-full transition-all", usageBarClass(gpu))}
|
||||
style={{ width: `${gpu}%` }}
|
||||
/>
|
||||
</div>
|
||||
<div className={cn("text-center text-[11px] font-medium", usageTextClass(gpu))}>
|
||||
{gpu}%{aggregateMemory ? ` · ${aggregateMemory}` : ""}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
@@ -28,4 +28,7 @@ test("GPU chip only renders when the remote reports GPU utilization", () => {
|
||||
source,
|
||||
/terminal\.serverStats\.gpu[\s\S]{0,2000}terminal\.serverStats\.memory/,
|
||||
);
|
||||
// Same hover card as CPU: compact aggregate trigger, per-device body.
|
||||
assert.match(source, /HoverCardTrigger asChild[\s\S]*terminal\.serverStats\.gpu[\s\S]*<TerminalGpuCards/);
|
||||
assert.match(source, /className="w-auto p-3"[\s\S]*side="bottom"[\s\S]*align="start"[\s\S]*sideOffset=\{8\}[\s\S]*<TerminalGpuCards/);
|
||||
});
|
||||
@@ -8,6 +8,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '../ui/tooltip';
|
||||
import { formatNetSpeed } from './terminalHelpers';
|
||||
import { useServerStats } from '../../application/state/useServerStats';
|
||||
import { formatDiskCapacityRange, resolveTerminalDiskSummary } from './serverStatsFormat';
|
||||
import { formatGpuMemoryGib, TerminalGpuCards } from './TerminalGpuCards';
|
||||
|
||||
interface TerminalServerStatsProps {
|
||||
sessionId: string;
|
||||
@@ -48,6 +49,7 @@ export const TerminalServerStats: React.FC<TerminalServerStatsProps> = ({
|
||||
const hasNetworkDetails = serverStats.netInterfaces.length > 0;
|
||||
const hasLatency = serverStats.latencyMs !== null;
|
||||
const diskSummary = resolveTerminalDiskSummary(serverStats);
|
||||
const gpuMemory = formatGpuMemoryGib(serverStats.gpuMemUsed, serverStats.gpuMemTotal);
|
||||
|
||||
if (!enabled || !isConnected || !serverStats.lastUpdated) return null;
|
||||
|
||||
@@ -122,28 +124,37 @@ export const TerminalServerStats: React.FC<TerminalServerStatsProps> = ({
|
||||
</div>
|
||||
</HoverCardContent>
|
||||
</HoverCard>
|
||||
{/* GPU — only rendered when the remote reports NVIDIA GPU data (Linux nvidia-smi) */}
|
||||
{/* GPU — only rendered when the remote reports NVIDIA GPU data (Linux nvidia-smi).
|
||||
Compact chip stays aggregate, matching CPU; per-card detail is the hover card. */}
|
||||
{serverStats.gpu !== null && (
|
||||
<div
|
||||
className="flex items-center gap-0.5 min-w-0 shrink"
|
||||
aria-label={t("terminal.serverStats.gpu")}
|
||||
title={
|
||||
[
|
||||
serverStats.gpuName,
|
||||
serverStats.gpuMemUsed !== null && serverStats.gpuMemTotal !== null && serverStats.gpuMemTotal > 0
|
||||
? `VRAM ${(serverStats.gpuMemUsed / 1024).toFixed(1)}/${(serverStats.gpuMemTotal / 1024).toFixed(1)}G`
|
||||
: null,
|
||||
].filter(Boolean).join(' · ') || undefined
|
||||
}
|
||||
>
|
||||
<Gpu size={10} className="flex-shrink-0" />
|
||||
<span className="truncate">
|
||||
{serverStats.gpu}%
|
||||
{serverStats.gpuMemUsed !== null && serverStats.gpuMemTotal !== null && serverStats.gpuMemTotal > 0
|
||||
? ` (${(serverStats.gpuMemUsed / 1024).toFixed(1)}/${(serverStats.gpuMemTotal / 1024).toFixed(1)}G)`
|
||||
: ''}
|
||||
</span>
|
||||
</div>
|
||||
<HoverCard openDelay={200} closeDelay={100}>
|
||||
<HoverCardTrigger asChild>
|
||||
<button
|
||||
className="flex items-center gap-0.5 hover:opacity-100 opacity-80 transition-opacity cursor-pointer min-w-0 shrink"
|
||||
aria-label={t("terminal.serverStats.gpu")}
|
||||
>
|
||||
<Gpu size={10} className="flex-shrink-0" />
|
||||
<span className="truncate">
|
||||
{serverStats.gpu}%
|
||||
{gpuMemory ? ` (${gpuMemory})` : ''}
|
||||
</span>
|
||||
</button>
|
||||
</HoverCardTrigger>
|
||||
<HoverCardContent
|
||||
className="w-auto p-3"
|
||||
side="bottom"
|
||||
align="start"
|
||||
sideOffset={8}
|
||||
>
|
||||
<TerminalGpuCards
|
||||
gpus={serverStats.gpus}
|
||||
gpu={serverStats.gpu}
|
||||
gpuName={serverStats.gpuName}
|
||||
gpuMemUsed={serverStats.gpuMemUsed}
|
||||
gpuMemTotal={serverStats.gpuMemTotal}
|
||||
/>
|
||||
</HoverCardContent>
|
||||
</HoverCard>
|
||||
)}
|
||||
{/* Memory with HoverCard for htop-style bar and top processes */}
|
||||
<HoverCard openDelay={200} closeDelay={100}>
|
||||
|
||||
@@ -1,13 +1,20 @@
|
||||
"use strict";
|
||||
const { executeBoundedSshCommand } = require("../boundedSshExec.cjs");
|
||||
|
||||
const GPU_ASSIGNMENT = `gpustat=$(gpucsv=$(timeout -s KILL 2 nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total,name --format=csv,noheader,nounits 2>/dev/null) && printf '%s\\n' "$gpucsv" | awk -F', *' 'NF >= 4 {g++; if($1 ~ /^[0-9]+$/) {n++; u+=$1} if($2 ~ /^[0-9]+$/) {mu+=$2; uc++} if($3 ~ /^[0-9]+$/) {mt+=$3; tc++} if(name=="") name=$4} END{if(n>0) printf "%.0f %s %s %s", u/n, (uc==g ? sprintf("%.0f",mu) : "N/A"), (tc==g ? sprintf("%.0f",mt) : "N/A"), name}' 2>/dev/null || echo "")`;
|
||||
// One bounded nvidia-smi query per poll. The awk summary stays
|
||||
// "<avgUtil> <sumUsed|N/A> <sumTotal|N/A> <firstName>" and, when cards parse,
|
||||
// appends " ;; index<TAB>name<TAB>util|N/A<TAB>used|N/A<TAB>total|N/A" records.
|
||||
// Names are taken from the middle columns so a comma inside the product name
|
||||
// does not shift utilization or memory. "|" cannot survive into the stats line.
|
||||
const GPU_ASSIGNMENT = `gpustat=$(gpucsv=$(timeout -s KILL 2 nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits 2>/dev/null) && printf '%s\\n' "$gpucsv" | awk -F', *' 'NF >= 5 {idx=$1; util=$(NF-2); used=$(NF-1); total=$NF; gsub(/^[ \\t"]+|[ \\t"]+$/, "", idx); gsub(/^[ \\t"]+|[ \\t"]+$/, "", util); gsub(/^[ \\t"]+|[ \\t"]+$/, "", used); gsub(/^[ \\t"]+|[ \\t"]+$/, "", total); if (idx !~ /^[0-9]+$/) next; name=$2; for (i=3; i<=NF-3; i++) name=name ", " $i; gsub(/^[ \\t"]+|[ \\t"]+$/, "", name); gsub(/\\t/, " ", name); gsub(/\\|/, "/", name); gsub(/ ;; /, " ", name); g++; if (util ~ /^[0-9]+$/) {n++; u+=util} if (used ~ /^[0-9]+$/) {mu+=used; uc++} if (total ~ /^[0-9]+$/) {mt+=total; tc++} if (name0=="") name0=name; utilOut=(util ~ /^[0-9]+$/) ? util : "N/A"; usedOut=(used ~ /^[0-9]+$/) ? used : "N/A"; totalOut=(total ~ /^[0-9]+$/) ? total : "N/A"; rec=idx "\\t" name "\\t" utilOut "\\t" usedOut "\\t" totalOut; list=(list=="") ? rec : list " ;; " rec} END{if (n>0) {printf "%.0f %s %s %s", u/n, (uc==g ? sprintf("%.0f", mu) : "N/A"), (tc==g ? sprintf("%.0f", mt) : "N/A"), name0; if (list != "") printf " ;; %s", list}}' 2>/dev/null || echo "")`;
|
||||
const EMPTY_GPU = {
|
||||
gpu: null,
|
||||
gpuName: null,
|
||||
gpuMemUsed: null,
|
||||
gpuMemTotal: null,
|
||||
gpus: [],
|
||||
};
|
||||
const GPU_LIST_MARK = " ;; ";
|
||||
const states = new WeakMap();
|
||||
function stateFor(conn) {
|
||||
let state = states.get(conn);
|
||||
@@ -24,8 +31,23 @@ function stateFor(conn) {
|
||||
}
|
||||
return state;
|
||||
}
|
||||
function sameGpuList(left, right) {
|
||||
const a = left || [];
|
||||
const b = right || [];
|
||||
return a.length === b.length && a.every((gpu, index) => {
|
||||
const other = b[index];
|
||||
return gpu.index === other.index
|
||||
&& gpu.name === other.name
|
||||
&& gpu.utilization === other.utilization
|
||||
&& gpu.memoryUsed === other.memoryUsed
|
||||
&& gpu.memoryTotal === other.memoryTotal;
|
||||
});
|
||||
}
|
||||
function updateGpuStats(state, next) {
|
||||
if (Object.keys(EMPTY_GPU).some((key) => state.stats[key] !== next[key])) {
|
||||
const changed = Object.keys(EMPTY_GPU).some((key) => (
|
||||
key === "gpus" ? !sameGpuList(state.stats[key], next[key]) : state.stats[key] !== next[key]
|
||||
));
|
||||
if (changed) {
|
||||
// Publication time is separate from CPU history and strictly advances even
|
||||
// when a fast GPU reply shares the CPU sample's millisecond.
|
||||
state.updatedAt = Math.max(Date.now() + 1, state.updatedAt + 1);
|
||||
@@ -35,23 +57,51 @@ function updateGpuStats(state, next) {
|
||||
function monitorGpuUpdatedAt(conn) {
|
||||
return states.get(conn)?.updatedAt ?? 0;
|
||||
}
|
||||
function emptyGpuStats() {
|
||||
return { ...EMPTY_GPU, gpus: [] };
|
||||
}
|
||||
function parseGpuNumber(value) {
|
||||
if (value == null || !/^[0-9]+$/.test(String(value).trim())) return null;
|
||||
return parseInt(value, 10);
|
||||
}
|
||||
function parseGpuCard(record) {
|
||||
const fields = String(record || "").split("\t");
|
||||
if (fields.length < 5) return null;
|
||||
const index = parseGpuNumber(fields[0]);
|
||||
if (index == null) return null;
|
||||
const memoryTotal = parseGpuNumber(fields[fields.length - 1]);
|
||||
const memoryUsed = parseGpuNumber(fields[fields.length - 2]);
|
||||
const utilization = parseGpuNumber(fields[fields.length - 3]);
|
||||
return {
|
||||
index,
|
||||
name: fields.slice(1, fields.length - 3).join("\t").trim(),
|
||||
utilization: utilization == null ? null : Math.max(0, Math.min(100, utilization)),
|
||||
memoryUsed,
|
||||
memoryTotal,
|
||||
};
|
||||
}
|
||||
function parseGpuStats(output) {
|
||||
const raw =
|
||||
String(output || "")
|
||||
.split("|")
|
||||
.find((part) => part.startsWith("GPU:"))
|
||||
?.slice(4)
|
||||
.trim()
|
||||
.split(/\s+/) ?? [];
|
||||
const utilization = parseInt(raw[0], 10);
|
||||
if (!Number.isFinite(utilization)) return EMPTY_GPU;
|
||||
const number = (value) =>
|
||||
Number.isFinite(parseInt(value, 10)) ? parseInt(value, 10) : null;
|
||||
const payload = String(output || "")
|
||||
.split("|")
|
||||
.find((part) => part.startsWith("GPU:"))
|
||||
?.slice(4)
|
||||
.trim() ?? "";
|
||||
if (!payload) return emptyGpuStats();
|
||||
const markAt = payload.indexOf(GPU_LIST_MARK);
|
||||
const summary = (markAt === -1 ? payload : payload.slice(0, markAt)).trim();
|
||||
const list = markAt === -1 ? "" : payload.slice(markAt + GPU_LIST_MARK.length);
|
||||
const raw = summary.split(/\s+/).filter(Boolean);
|
||||
const utilization = parseGpuNumber(raw[0]);
|
||||
if (utilization == null) return emptyGpuStats();
|
||||
const gpus = list
|
||||
? list.split(GPU_LIST_MARK).map(parseGpuCard).filter(Boolean)
|
||||
: [];
|
||||
return {
|
||||
gpu: Math.max(0, Math.min(100, utilization)),
|
||||
gpuName: raw.slice(3).join(" ") || null,
|
||||
gpuMemUsed: number(raw[1]),
|
||||
gpuMemTotal: number(raw[2]),
|
||||
gpuMemUsed: parseGpuNumber(raw[1]),
|
||||
gpuMemTotal: parseGpuNumber(raw[2]),
|
||||
gpus,
|
||||
};
|
||||
}
|
||||
function withCachedMonitorGpu(conn, result) {
|
||||
@@ -66,7 +116,7 @@ function cacheFullGpuStats(conn, stats) {
|
||||
updateGpuStats(
|
||||
state,
|
||||
Object.fromEntries(
|
||||
Object.keys(EMPTY_GPU).map((key) => [key, stats[key] ?? null]),
|
||||
Object.keys(EMPTY_GPU).map((key) => [key, stats[key] ?? (key === "gpus" ? [] : null)]),
|
||||
),
|
||||
);
|
||||
state.nextAt = Date.now() + 10000;
|
||||
|
||||
@@ -3,6 +3,8 @@ const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const { EventEmitter } = require("node:events");
|
||||
const {
|
||||
GPU_ASSIGNMENT,
|
||||
parseGpuStats,
|
||||
startMonitorGpuProbe,
|
||||
withCachedMonitorGpu,
|
||||
cacheFullGpuStats,
|
||||
@@ -17,6 +19,75 @@ function stream() {
|
||||
return s;
|
||||
}
|
||||
const sample = { success: true, stats: { cpu: 12, memTotal: 4096, gpu: null } };
|
||||
|
||||
test("GPU collection stays one nvidia-smi query and keeps per-GPU fields", () => {
|
||||
assert.equal(
|
||||
GPU_ASSIGNMENT.match(/nvidia-smi/g).length,
|
||||
1,
|
||||
"each poll must run a single nvidia-smi command",
|
||||
);
|
||||
assert.match(
|
||||
GPU_ASSIGNMENT,
|
||||
/nvidia-smi --query-gpu=index,name,utilization\.gpu,memory\.used,memory\.total --format=csv,noheader,nounits/,
|
||||
);
|
||||
});
|
||||
|
||||
test("parseGpuStats keeps a single aggregate GPU and an empty device list", () => {
|
||||
assert.deepEqual(parseGpuStats("GPU:73 2048 24576 NVIDIA GeForce RTX 4090"), {
|
||||
gpu: 73,
|
||||
gpuName: "NVIDIA GeForce RTX 4090",
|
||||
gpuMemUsed: 2048,
|
||||
gpuMemTotal: 24576,
|
||||
gpus: [],
|
||||
});
|
||||
});
|
||||
|
||||
test("parseGpuStats reads each GPU index, name, utilization, and memory", () => {
|
||||
const parsed = parseGpuStats(
|
||||
"GPU:72 3072 49152 NVIDIA RTX 4090 ;; 0\tNVIDIA RTX 4090\t73\t2048\t24576 ;; 1\tNVIDIA A100 80GB\t71\t1024\t24576",
|
||||
);
|
||||
assert.equal(parsed.gpu, 72);
|
||||
assert.equal(parsed.gpuMemUsed, 3072);
|
||||
assert.equal(parsed.gpuMemTotal, 49152);
|
||||
assert.deepEqual(parsed.gpus, [
|
||||
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: 2048, memoryTotal: 24576 },
|
||||
{ index: 1, name: "NVIDIA A100 80GB", utilization: 71, memoryUsed: 1024, memoryTotal: 24576 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("parseGpuStats keeps eight GPUs and clamps utilization", () => {
|
||||
const cards = Array.from({ length: 8 }, (_, index) =>
|
||||
`${index}\tNVIDIA A100 ${index}\t${index === 3 ? 140 : 10 + index}\t${1024 * (index + 1)}\t81920`,
|
||||
).join(" ;; ");
|
||||
const parsed = parseGpuStats(`GPU:40 36864 655360 NVIDIA A100 0 ;; ${cards}`);
|
||||
assert.equal(parsed.gpus.length, 8);
|
||||
assert.equal(parsed.gpus[3].utilization, 100);
|
||||
assert.equal(parsed.gpus[7].index, 7);
|
||||
assert.equal(parsed.gpus[7].memoryTotal, 81920);
|
||||
});
|
||||
|
||||
test("parseGpuStats skips a broken card and unknown memory without hiding the rest", () => {
|
||||
const parsed = parseGpuStats(
|
||||
"GPU:73 2048 24576 NVIDIA RTX 4090 ;; not-a-card ;; 0\tNVIDIA RTX 4090\t73\tN/A\t24576 ;; 2\tNVIDIA A100\tN/A\t1024\t8192",
|
||||
);
|
||||
assert.equal(parsed.gpu, 73);
|
||||
assert.deepEqual(parsed.gpus, [
|
||||
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: null, memoryTotal: 24576 },
|
||||
{ index: 2, name: "NVIDIA A100", utilization: null, memoryUsed: 1024, memoryTotal: 8192 },
|
||||
]);
|
||||
});
|
||||
|
||||
test("parseGpuStats hides GPU data when nvidia-smi output cannot be parsed", () => {
|
||||
for (const output of ["", "GPU:", "GPU:[Not Supported]", "GPU:N/A N/A N/A", "CPURAW:1 1|MEMINFO:1"]) {
|
||||
assert.deepEqual(parseGpuStats(output), {
|
||||
gpu: null,
|
||||
gpuName: null,
|
||||
gpuMemUsed: null,
|
||||
gpuMemTotal: null,
|
||||
gpus: [],
|
||||
});
|
||||
}
|
||||
});
|
||||
test("GPU work is coalesced, cached independently, and never changes CPU readings", async (t) => {
|
||||
let now = 100000,
|
||||
calls = 0,
|
||||
|
||||
@@ -1312,11 +1312,10 @@ function createSessionOpsApi(ctx) {
|
||||
`kernel=$(uname -r 2>/dev/null || echo "")`,
|
||||
`uptime=$(awk '{printf "%.0f",$1}' /proc/uptime 2>/dev/null || echo "")`,
|
||||
`loadavg=$(awk '{print $1" "$2" "$3}' /proc/loadavg 2>/dev/null || echo "")`,
|
||||
// GPU: NVIDIA GPUs expose utilization + VRAM through the driver-bundled
|
||||
// nvidia-smi. Best-effort — empty on hosts without the tool, so the UI
|
||||
// simply hides the GPU chip. Utilization is averaged across GPUs and
|
||||
// VRAM is summed independently; incomplete memory totals remain unknown.
|
||||
// The first GPU's name is kept for the tooltip.
|
||||
// GPU: NVIDIA GPUs expose utilization + VRAM through one bounded
|
||||
// nvidia-smi query. Best-effort — empty on hosts without the tool, so
|
||||
// the UI hides the GPU chip. Utilization is averaged across GPUs and
|
||||
// VRAM is summed independently; the same reply also lists each card.
|
||||
// Bound GPU queries independently so a stuck driver cannot stop CPU/memory
|
||||
// polling. Hosts without timeout also omit this optional metric.
|
||||
// macOS is skipped: powermetrics needs root, which stats must not.
|
||||
@@ -1461,6 +1460,7 @@ function createSessionOpsApi(ctx) {
|
||||
let gpuName = null;
|
||||
let gpuMemUsed = null;
|
||||
let gpuMemTotal = null;
|
||||
let gpus = [];
|
||||
for (const part of parts) {
|
||||
if (part.startsWith('CPU:')) {
|
||||
// macOS: command reports normalized CPU% directly (no delta needed)
|
||||
@@ -1588,7 +1588,7 @@ function createSessionOpsApi(ctx) {
|
||||
}
|
||||
}
|
||||
} else if (part.startsWith('GPU:')) {
|
||||
({gpu,gpuName,gpuMemUsed,gpuMemTotal}=parseGpuStats(part));
|
||||
({gpu,gpuName,gpuMemUsed,gpuMemTotal,gpus}=parseGpuStats(part));
|
||||
} else if (part.startsWith('HOST:')) {
|
||||
hostname = part.substring(5).trim();
|
||||
} else if (part.startsWith('OS:')) {
|
||||
@@ -1741,6 +1741,7 @@ function createSessionOpsApi(ctx) {
|
||||
gpuName, // First GPU name (tooltip), null when unavailable
|
||||
gpuMemUsed, // Summed VRAM used in MB, null when unavailable
|
||||
gpuMemTotal, // Summed VRAM total in MB, null when unavailable
|
||||
gpus, // Per-GPU index, name, utilization, and VRAM in MB
|
||||
memTotal, // Total memory in MB
|
||||
memUsed, // Used memory in MB (excluding buffers/cache)
|
||||
memFree, // Free memory in MB
|
||||
|
||||
@@ -634,6 +634,7 @@ test("getServerStats reports null GPU fields when the host has no nvidia-smi dat
|
||||
assert.equal(result.stats.gpuName, null);
|
||||
assert.equal(result.stats.gpuMemUsed, null);
|
||||
assert.equal(result.stats.gpuMemTotal, null);
|
||||
assert.deepEqual(result.stats.gpus, []);
|
||||
});
|
||||
|
||||
test("getServerStats parses NVIDIA GPU utilization and VRAM from the stats line", async () => {
|
||||
@@ -653,6 +654,7 @@ test("getServerStats parses NVIDIA GPU utilization and VRAM from the stats line"
|
||||
assert.equal(result.stats.gpuName, "NVIDIA GeForce RTX 4090");
|
||||
assert.equal(result.stats.gpuMemUsed, 2048);
|
||||
assert.equal(result.stats.gpuMemTotal, 24576);
|
||||
assert.deepEqual(result.stats.gpus, []);
|
||||
});
|
||||
|
||||
test("getServerStats tolerates a malformed GPU section", async () => {
|
||||
@@ -668,16 +670,40 @@ test("getServerStats tolerates a malformed GPU section", async () => {
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(result.stats.gpu, null);
|
||||
assert.equal(result.stats.gpuName, null);
|
||||
assert.deepEqual(result.stats.gpus, []);
|
||||
});
|
||||
|
||||
for (const scenario of [
|
||||
{ name: "multiple GPUs", script: "printf '%s\\n' '73, 2048, 24576, NVIDIA RTX 4090' '71, 1024, 24576, NVIDIA RTX 4090'", gpu: 72, used: 3072, total: 49152 },
|
||||
{ name: "idle GPU", script: "printf '%s\\n' '0, 0, 24576, NVIDIA RTX 4090'", gpu: 0, used: 0, total: 24576 },
|
||||
{ name: "mixed utilization availability", script: "printf '%s\\n' '73, 2048, 24576, NVIDIA RTX 4090' '[N/A], 1024, 8192, NVIDIA A100 MIG'", gpu: 73, used: 3072, total: 32768 },
|
||||
{ name: "unavailable memory usage", script: "printf '%s\\n' '73, [N/A], 24576, NVIDIA RTX 4090'", gpu: 73, total: 24576 },
|
||||
{ name: "partial memory totals", script: "printf '%s\\n' '73, 2048, 24576, NVIDIA RTX 4090' '71, 1024, [N/A], NVIDIA RTX 4090'", gpu: 72, used: 3072 },
|
||||
{ name: "unsupported utilization", script: "printf '%s\\n' '[N/A], 2048, 24576, NVIDIA RTX 4090'", gpu: null },
|
||||
{ name: "failed query with partial output", script: "printf '%s\\n' '73, 2048, 24576, NVIDIA RTX 4090'; exit 1", gpu: null },
|
||||
{ name: "multiple GPUs", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, 2048, 24576' '1, NVIDIA RTX 4090, 71, 1024, 24576'", gpu: 72, used: 3072, total: 49152, gpus: [
|
||||
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: 2048, memoryTotal: 24576 },
|
||||
{ index: 1, name: "NVIDIA RTX 4090", utilization: 71, memoryUsed: 1024, memoryTotal: 24576 },
|
||||
] },
|
||||
{ name: "idle GPU", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 0, 0, 24576'", gpu: 0, used: 0, total: 24576, gpus: [
|
||||
{ index: 0, name: "NVIDIA RTX 4090", utilization: 0, memoryUsed: 0, memoryTotal: 24576 },
|
||||
] },
|
||||
{ name: "mixed utilization availability", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, 2048, 24576' '1, NVIDIA A100 MIG, [N/A], 1024, 8192'", gpu: 73, used: 3072, total: 32768, gpus: [
|
||||
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: 2048, memoryTotal: 24576 },
|
||||
{ index: 1, name: "NVIDIA A100 MIG", utilization: null, memoryUsed: 1024, memoryTotal: 8192 },
|
||||
] },
|
||||
{ name: "unavailable memory usage", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, [N/A], 24576'", gpu: 73, total: 24576, gpus: [
|
||||
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: null, memoryTotal: 24576 },
|
||||
] },
|
||||
{ name: "partial memory totals", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, 2048, 24576' '1, NVIDIA RTX 4090, 71, 1024, [N/A]'", gpu: 72, used: 3072, gpus: [
|
||||
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: 2048, memoryTotal: 24576 },
|
||||
{ index: 1, name: "NVIDIA RTX 4090", utilization: 71, memoryUsed: 1024, memoryTotal: null },
|
||||
] },
|
||||
{ name: "comma in GPU name", script: "printf '%s\\n' '0, NVIDIA Foo, 80GB, 41, 2048, 24576'", gpu: 41, used: 2048, total: 24576, gpus: [
|
||||
{ index: 0, name: "NVIDIA Foo, 80GB", utilization: 41, memoryUsed: 2048, memoryTotal: 24576 },
|
||||
] },
|
||||
{ name: "eight GPUs", script: `printf '%s\\n' ${Array.from({ length: 8 }, (_, index) => `'${index}, NVIDIA A100 ${index}, ${10 + index * 2}, ${1024 * (index + 1)}, 81920'`).join(" ")}`, gpu: 17, used: 36864, total: 655360, gpus: Array.from({ length: 8 }, (_, index) => ({
|
||||
index,
|
||||
name: `NVIDIA A100 ${index}`,
|
||||
utilization: 10 + index * 2,
|
||||
memoryUsed: 1024 * (index + 1),
|
||||
memoryTotal: 81920,
|
||||
})) },
|
||||
{ name: "unsupported utilization", script: "printf '%s\\n' '0, NVIDIA RTX 4090, [N/A], 2048, 24576'", gpu: null },
|
||||
{ name: "failed query with partial output", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, 2048, 24576'; exit 1", gpu: null },
|
||||
{ name: "missing query tool", script: "exit 127", gpu: null },
|
||||
{ name: "missing timeout tool", script: "exec sleep 20", gpu: null, noTimeout: true },
|
||||
{ name: "blocked GPU query", script: "trap '' TERM; exec sleep 20", gpu: null },
|
||||
@@ -741,6 +767,7 @@ for (const scenario of [
|
||||
assert.equal(result.stats.gpu, scenario.gpu);
|
||||
assert.equal(result.stats.gpuMemUsed, scenario.used ?? null);
|
||||
assert.equal(result.stats.gpuMemTotal, scenario.total ?? null);
|
||||
assert.deepEqual(result.stats.gpus, scenario.gpus ?? []);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
+7
@@ -301,6 +301,13 @@ declare global {
|
||||
gpuName?: string | null; // First GPU name
|
||||
gpuMemUsed?: number | null; // Summed VRAM used in MB
|
||||
gpuMemTotal?: number | null; // Summed VRAM total in MB
|
||||
gpus?: Array<{ // Per-GPU utilization and VRAM (MiB)
|
||||
index: number;
|
||||
name: string;
|
||||
utilization: number | null;
|
||||
memoryUsed: number | null;
|
||||
memoryTotal: number | null;
|
||||
}>;
|
||||
memTotal: number | null; // Total memory in MB
|
||||
memUsed: number | null; // Used memory in MB (excluding buffers/cache)
|
||||
memFree: number | null; // Free memory in MB
|
||||
|
||||
Reference in new issue
Block a user