Compare commits

...
Author SHA1 Message Date
Cursor Agentand陈大猫 55f4e9cc56 fix(terminal): wrap GPU names and stretch a single card
Names use two lines plus the native title instead of an 88px clip, and one
GPU fills the hover card so the bar sits with the title.

Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>
2026-10-20 04:43:18 +00:00
Cursor Agentand陈大猫 80479e84d3 feat(terminal): show per-GPU usage in the server stats bar
Keep the existing single nvidia-smi poll and carry each card's index, name,
utilization, and VRAM into the same hover-card layout used for CPU cores.

Closes #3745

Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>
2026-10-20 04:26:23 +00:00
16 changed files with 463 additions and 50 deletions

No files matched your search

+2
View File
@@ -109,6 +109,8 @@ export const enTerminalMessages: Messages = {
'terminal.serverStats.cpu': 'CPU Usage',
'terminal.serverStats.cpuCores': 'CPU Core Usage',
'terminal.serverStats.gpu': 'GPU Usage',
'terminal.serverStats.gpuDevices': 'GPU Device Usage',
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
'terminal.serverStats.memory': 'Memory Usage',
'terminal.serverStats.memoryDetails': 'Memory Details',
'terminal.serverStats.memUsed': 'Used',
+2
View File
@@ -109,6 +109,8 @@ export const esTerminalMessages: Messages = {
'terminal.serverStats.cpu': 'Uso de CPU',
'terminal.serverStats.cpuCores': 'Uso de núcleos de CPU',
'terminal.serverStats.gpu': 'Uso de GPU',
'terminal.serverStats.gpuDevices': 'Uso de cada GPU',
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
'terminal.serverStats.memory': 'Uso de memoria',
'terminal.serverStats.memoryDetails': 'Detalles de memoria',
'terminal.serverStats.memUsed': 'Usada',
+2
View File
@@ -130,6 +130,8 @@ export const ruTerminalMessages: Messages = {
'terminal.serverStats.cpu': 'Использование CPU',
'terminal.serverStats.cpuCores': 'Использование ядер CPU',
'terminal.serverStats.gpu': 'Использование GPU',
'terminal.serverStats.gpuDevices': 'Использование видеокарт',
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
'terminal.serverStats.memory': 'Использование памяти',
'terminal.serverStats.memoryDetails': 'Сведения о памяти',
'terminal.serverStats.memUsed': 'Использовано',
@@ -0,0 +1,23 @@
import test from "node:test";
import assert from "node:assert/strict";
import { enTerminalMessages } from "./en/terminal";
import { esTerminalMessages } from "./es/terminal";
import { ruTerminalMessages } from "./ru/terminal";
import { zhCNVaultMessages } from "./zh-CN/vault";
import { zhTWVaultMessages } from "./zh-TW/vault";
test("per-GPU stats strings exist in every locale", () => {
const labels = [
[enTerminalMessages, "GPU Device Usage", "GPU {{index}}"],
[esTerminalMessages, "Uso de cada GPU", "GPU {{index}}"],
[ruTerminalMessages, "Использование видеокарт", "GPU {{index}}"],
[zhCNVaultMessages, "GPU 设备使用率", "GPU {{index}}"],
[zhTWVaultMessages, "GPU 裝置使用率", "GPU {{index}}"],
] as const;
for (const [messages, devices, indexLabel] of labels) {
assert.equal(messages["terminal.serverStats.gpuDevices"], devices);
assert.equal(messages["terminal.serverStats.gpuIndex"], indexLabel);
}
});
+2
View File
@@ -532,6 +532,8 @@ export const zhCNVaultMessages: Messages = {
'terminal.serverStats.cpu': 'CPU 使用率',
'terminal.serverStats.cpuCores': 'CPU 核心使用率',
'terminal.serverStats.gpu': 'GPU 使用率',
'terminal.serverStats.gpuDevices': 'GPU 设备使用率',
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
'terminal.serverStats.memory': '内存使用',
'terminal.serverStats.memoryDetails': '内存详情',
'terminal.serverStats.memUsed': '已用',
+2
View File
@@ -532,6 +532,8 @@ export const zhTWVaultMessages: Messages = {
'terminal.serverStats.cpu': 'CPU 使用率',
'terminal.serverStats.cpuCores': 'CPU 核心使用率',
'terminal.serverStats.gpu': 'GPU 使用率',
'terminal.serverStats.gpuDevices': 'GPU 裝置使用率',
'terminal.serverStats.gpuIndex': 'GPU {{index}}',
'terminal.serverStats.memory': '記憶體使用',
'terminal.serverStats.memoryDetails': '記憶體詳情',
'terminal.serverStats.memUsed': '已用',
+11
View File
@@ -24,6 +24,14 @@ export interface ProcessInfo {
command: string;
}
export interface GpuDeviceStats {
index: number;
name: string;
utilization: number | null; // percent, null when nvidia-smi reports N/A
memoryUsed: number | null; // MiB
memoryTotal: number | null; // MiB
}
export interface ServerStats {
hostname?: string; // Remote hostname
osName?: string; // Remote operating system name
@@ -37,6 +45,7 @@ export interface ServerStats {
gpuName: string | null; // First GPU name (tooltip)
gpuMemUsed: number | null; // Summed VRAM used in MB
gpuMemTotal: number | null; // Summed VRAM total in MB
gpus: GpuDeviceStats[]; // Per-GPU utilization and VRAM
memTotal: number | null; // Total memory in MB
memUsed: number | null; // Used memory in MB (excluding buffers/cache)
memFree: number | null; // Free memory in MB
@@ -116,6 +125,7 @@ function createEmptyServerStats(): ServerStats {
gpuName: null,
gpuMemUsed: null,
gpuMemTotal: null,
gpus: [],
memTotal: null,
memUsed: null,
memFree: null,
@@ -218,6 +228,7 @@ function normalizeServerStats(stats: Partial<ServerStats>): ServerStats {
gpuName: stats.gpuName ?? null,
gpuMemUsed: stats.gpuMemUsed ?? null,
gpuMemTotal: stats.gpuMemTotal ?? null,
gpus: Array.isArray(stats.gpus) ? stats.gpus : [],
memTotal: stats.memTotal ?? null,
memUsed: stats.memUsed ?? null,
memFree: stats.memFree ?? null,
@@ -0,0 +1,90 @@
import test from "node:test";
import assert from "node:assert/strict";
import React from "react";
import { renderToStaticMarkup } from "react-dom/server";
import { I18nProvider } from "../../application/i18n/I18nProvider.tsx";
import type { GpuDeviceStats } from "../../application/state/useServerStats.ts";
import { TerminalGpuCards } from "./TerminalGpuCards.tsx";
function renderCards(gpus: GpuDeviceStats[], gpu = 40) {
return renderToStaticMarkup(
<I18nProvider locale="en">
<TerminalGpuCards
gpus={gpus}
gpu={gpu}
gpuName="NVIDIA GeForce RTX 4090"
gpuMemUsed={2048}
gpuMemTotal={24576}
/>
</I18nProvider>,
);
}
const card = (index: number, utilization: number | null, name = `NVIDIA A${index}`): GpuDeviceStats => ({
index,
name,
utilization,
memoryUsed: 1024 * (index + 1),
memoryTotal: 8192,
});
test("one GPU uses the same per-core bar, colors, and spacing", () => {
const markup = renderCards([card(0, 73, "NVIDIA GeForce RTX 4090")], 73);
assert.match(markup, /GPU Device Usage/);
assert.match(markup, /GPU 0/);
assert.match(markup, /NVIDIA GeForce RTX 4090/);
assert.match(markup, /73%/);
assert.match(markup, /1\.0\/8\.0G/);
assert.match(markup, /grid gap-1\.5 max-h-\[260px\] overflow-y-auto/);
assert.match(markup, /min-w-\[160px\]/);
assert.match(markup, /line-clamp-2/);
assert.match(markup, /title="NVIDIA GeForce RTX 4090"/);
assert.doesNotMatch(markup, /max-w-\[88px\]/);
assert.match(markup, /h-1\.5 bg-muted rounded-full overflow-hidden/);
assert.match(markup, /bg-amber-500/);
assert.match(markup, /text-amber-400/);
assert.match(markup, /repeat\(1, 1fr\)/);
});
test("eight GPUs stay in four scrolling columns", () => {
const markup = renderCards(Array.from({ length: 8 }, (_, index) => card(index, 10 + index)));
assert.match(markup, /repeat\(4, 1fr\)/);
assert.match(markup, /max-h-\[260px\] overflow-y-auto/);
assert.match(markup, /max-w-\[26rem\]/);
assert.match(markup, /line-clamp-2/);
assert.match(markup, /title="NVIDIA A0"/);
assert.doesNotMatch(markup, /max-w-\[88px\]/);
for (let index = 0; index < 8; index += 1) {
assert.match(markup, new RegExp(`GPU ${index}`));
}
assert.match(markup, /text-emerald-400/);
assert.doesNotMatch(markup, /repeat\(8,/);
});
test("high utilization uses the CPU red threshold and unknown cards stay muted", () => {
const markup = renderCards([
card(0, 95, "NVIDIA H100"),
{ index: 3, name: "NVIDIA A100", utilization: null, memoryUsed: null, memoryTotal: null },
], 95);
assert.match(markup, /bg-red-500/);
assert.match(markup, /text-red-400/);
assert.match(markup, /95%/);
assert.match(markup, /GPU 3/);
assert.match(markup, />--</);
assert.match(markup, /repeat\(2, 1fr\)/);
});
test("a summary without per-GPU rows falls back to one aggregate bar", () => {
const markup = renderCards([], 12);
assert.match(markup, /min-w-\[160px\]/);
assert.match(markup, /h-2 bg-muted rounded-full/);
assert.match(markup, /12%/);
assert.match(markup, /2\.0\/24\.0G/);
assert.match(markup, /text-emerald-400/);
assert.doesNotMatch(markup, /grid gap-1\.5/);
});
+109
View File
@@ -0,0 +1,109 @@
import React from "react";
import { useI18n } from "../../application/i18n/I18nProvider";
import type { GpuDeviceStats } from "../../application/state/useServerStats";
import { cn } from "../../lib/utils";
export function formatGpuMemoryGib(usedMb: number | null, totalMb: number | null): string | null {
if (usedMb === null || totalMb === null || !(totalMb > 0)) return null;
return `${(usedMb / 1024).toFixed(1)}/${(totalMb / 1024).toFixed(1)}G`;
}
function usageBarClass(usage: number): string {
return usage >= 90 ? "bg-red-500" : usage >= 70 ? "bg-amber-500" : "bg-emerald-500";
}
function usageTextClass(usage: number): string {
return usage >= 90 ? "text-red-400" : usage >= 70 ? "text-amber-400" : "text-emerald-400";
}
interface TerminalGpuCardsProps {
gpus: GpuDeviceStats[];
gpu: number;
gpuName: string | null;
gpuMemUsed: number | null;
gpuMemTotal: number | null;
}
/**
* Per-GPU details for the terminal stats hover card.
* Mirrors the CPU per-core grid: four columns, the same bar heights,
* threshold colors, and a scroll cap so many cards stay inside the popover.
*/
export const TerminalGpuCards: React.FC<TerminalGpuCardsProps> = ({
gpus,
gpu,
gpuName,
gpuMemUsed,
gpuMemTotal,
}) => {
const { t } = useI18n();
const aggregateMemory = formatGpuMemoryGib(gpuMemUsed, gpuMemTotal);
const columns = Math.min(4, gpus.length);
return (
<div className={cn("text-xs space-y-2 w-max max-w-[26rem]", gpus.length === 1 && "min-w-[160px]")}>
<div className="font-medium text-sm mb-2">{t("terminal.serverStats.gpuDevices")}</div>
{gpus.length > 0 ? (
<div
className="grid gap-1.5 max-h-[260px] overflow-y-auto pr-1 overscroll-contain"
style={{ gridTemplateColumns: `repeat(${columns}, 1fr)` }}
>
{gpus.map((device) => {
const usage = device.utilization;
const memory = formatGpuMemoryGib(device.memoryUsed, device.memoryTotal);
return (
<div key={device.index} className="flex w-full min-w-0 flex-col items-center gap-1">
<div className="text-[10px] text-muted-foreground">
{t("terminal.serverStats.gpuIndex", { index: device.index })}
</div>
{device.name ? (
<div
className="line-clamp-2 w-full break-words text-center text-[10px] leading-tight text-muted-foreground"
title={device.name}
>
{device.name}
</div>
) : null}
<div className="w-full h-1.5 bg-muted rounded-full overflow-hidden">
<div
className={cn(
"h-full rounded-full transition-all",
usage === null ? "bg-muted" : usageBarClass(usage),
)}
style={{ width: `${usage ?? 0}%` }}
/>
</div>
<div className={cn(
"text-[11px] font-medium",
usage === null ? "text-muted-foreground" : usageTextClass(usage),
)}>
{usage === null ? "--" : `${usage}%`}
</div>
{memory ? (
<div className="text-[10px] text-muted-foreground whitespace-nowrap">{memory}</div>
) : null}
</div>
);
})}
</div>
) : (
<div className="flex flex-col gap-1.5 min-w-[160px]">
{gpuName ? (
<div className="text-[10px] text-muted-foreground truncate" title={gpuName}>{gpuName}</div>
) : null}
<div className="w-full h-2 bg-muted rounded-full overflow-hidden">
<div
className={cn("h-full rounded-full transition-all", usageBarClass(gpu))}
style={{ width: `${gpu}%` }}
/>
</div>
<div className={cn("text-center text-[11px] font-medium", usageTextClass(gpu))}>
{gpu}%{aggregateMemory ? ` · ${aggregateMemory}` : ""}
</div>
</div>
)}
</div>
);
};
@@ -28,4 +28,7 @@ test("GPU chip only renders when the remote reports GPU utilization", () => {
source,
/terminal\.serverStats\.gpu[\s\S]{0,2000}terminal\.serverStats\.memory/,
);
// Same hover card as CPU: compact aggregate trigger, per-device body.
assert.match(source, /HoverCardTrigger asChild[\s\S]*terminal\.serverStats\.gpu[\s\S]*<TerminalGpuCards/);
assert.match(source, /className="w-auto p-3"[\s\S]*side="bottom"[\s\S]*align="start"[\s\S]*sideOffset=\{8\}[\s\S]*<TerminalGpuCards/);
});
+32 -21
View File
@@ -8,6 +8,7 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '../ui/tooltip';
import { formatNetSpeed } from './terminalHelpers';
import { useServerStats } from '../../application/state/useServerStats';
import { formatDiskCapacityRange, resolveTerminalDiskSummary } from './serverStatsFormat';
import { formatGpuMemoryGib, TerminalGpuCards } from './TerminalGpuCards';
interface TerminalServerStatsProps {
sessionId: string;
@@ -48,6 +49,7 @@ export const TerminalServerStats: React.FC<TerminalServerStatsProps> = ({
const hasNetworkDetails = serverStats.netInterfaces.length > 0;
const hasLatency = serverStats.latencyMs !== null;
const diskSummary = resolveTerminalDiskSummary(serverStats);
const gpuMemory = formatGpuMemoryGib(serverStats.gpuMemUsed, serverStats.gpuMemTotal);
if (!enabled || !isConnected || !serverStats.lastUpdated) return null;
@@ -122,28 +124,37 @@ export const TerminalServerStats: React.FC<TerminalServerStatsProps> = ({
</div>
</HoverCardContent>
</HoverCard>
{/* GPU — only rendered when the remote reports NVIDIA GPU data (Linux nvidia-smi) */}
{/* GPU — only rendered when the remote reports NVIDIA GPU data (Linux nvidia-smi).
Compact chip stays aggregate, matching CPU; per-card detail is the hover card. */}
{serverStats.gpu !== null && (
<div
className="flex items-center gap-0.5 min-w-0 shrink"
aria-label={t("terminal.serverStats.gpu")}
title={
[
serverStats.gpuName,
serverStats.gpuMemUsed !== null && serverStats.gpuMemTotal !== null && serverStats.gpuMemTotal > 0
? `VRAM ${(serverStats.gpuMemUsed / 1024).toFixed(1)}/${(serverStats.gpuMemTotal / 1024).toFixed(1)}G`
: null,
].filter(Boolean).join(' · ') || undefined
}
>
<Gpu size={10} className="flex-shrink-0" />
<span className="truncate">
{serverStats.gpu}%
{serverStats.gpuMemUsed !== null && serverStats.gpuMemTotal !== null && serverStats.gpuMemTotal > 0
? ` (${(serverStats.gpuMemUsed / 1024).toFixed(1)}/${(serverStats.gpuMemTotal / 1024).toFixed(1)}G)`
: ''}
</span>
</div>
<HoverCard openDelay={200} closeDelay={100}>
<HoverCardTrigger asChild>
<button
className="flex items-center gap-0.5 hover:opacity-100 opacity-80 transition-opacity cursor-pointer min-w-0 shrink"
aria-label={t("terminal.serverStats.gpu")}
>
<Gpu size={10} className="flex-shrink-0" />
<span className="truncate">
{serverStats.gpu}%
{gpuMemory ? ` (${gpuMemory})` : ''}
</span>
</button>
</HoverCardTrigger>
<HoverCardContent
className="w-auto p-3"
side="bottom"
align="start"
sideOffset={8}
>
<TerminalGpuCards
gpus={serverStats.gpus}
gpu={serverStats.gpu}
gpuName={serverStats.gpuName}
gpuMemUsed={serverStats.gpuMemUsed}
gpuMemTotal={serverStats.gpuMemTotal}
/>
</HoverCardContent>
</HoverCard>
)}
{/* Memory with HoverCard for htop-style bar and top processes */}
<HoverCard openDelay={200} closeDelay={100}>
+66 -16
View File
@@ -1,13 +1,20 @@
"use strict";
const { executeBoundedSshCommand } = require("../boundedSshExec.cjs");
const GPU_ASSIGNMENT = `gpustat=$(gpucsv=$(timeout -s KILL 2 nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total,name --format=csv,noheader,nounits 2>/dev/null) && printf '%s\\n' "$gpucsv" | awk -F', *' 'NF >= 4 {g++; if($1 ~ /^[0-9]+$/) {n++; u+=$1} if($2 ~ /^[0-9]+$/) {mu+=$2; uc++} if($3 ~ /^[0-9]+$/) {mt+=$3; tc++} if(name=="") name=$4} END{if(n>0) printf "%.0f %s %s %s", u/n, (uc==g ? sprintf("%.0f",mu) : "N/A"), (tc==g ? sprintf("%.0f",mt) : "N/A"), name}' 2>/dev/null || echo "")`;
// One bounded nvidia-smi query per poll. The awk summary stays
// "<avgUtil> <sumUsed|N/A> <sumTotal|N/A> <firstName>" and, when cards parse,
// appends " ;; index<TAB>name<TAB>util|N/A<TAB>used|N/A<TAB>total|N/A" records.
// Names are taken from the middle columns so a comma inside the product name
// does not shift utilization or memory. "|" cannot survive into the stats line.
const GPU_ASSIGNMENT = `gpustat=$(gpucsv=$(timeout -s KILL 2 nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,memory.total --format=csv,noheader,nounits 2>/dev/null) && printf '%s\\n' "$gpucsv" | awk -F', *' 'NF >= 5 {idx=$1; util=$(NF-2); used=$(NF-1); total=$NF; gsub(/^[ \\t"]+|[ \\t"]+$/, "", idx); gsub(/^[ \\t"]+|[ \\t"]+$/, "", util); gsub(/^[ \\t"]+|[ \\t"]+$/, "", used); gsub(/^[ \\t"]+|[ \\t"]+$/, "", total); if (idx !~ /^[0-9]+$/) next; name=$2; for (i=3; i<=NF-3; i++) name=name ", " $i; gsub(/^[ \\t"]+|[ \\t"]+$/, "", name); gsub(/\\t/, " ", name); gsub(/\\|/, "/", name); gsub(/ ;; /, " ", name); g++; if (util ~ /^[0-9]+$/) {n++; u+=util} if (used ~ /^[0-9]+$/) {mu+=used; uc++} if (total ~ /^[0-9]+$/) {mt+=total; tc++} if (name0=="") name0=name; utilOut=(util ~ /^[0-9]+$/) ? util : "N/A"; usedOut=(used ~ /^[0-9]+$/) ? used : "N/A"; totalOut=(total ~ /^[0-9]+$/) ? total : "N/A"; rec=idx "\\t" name "\\t" utilOut "\\t" usedOut "\\t" totalOut; list=(list=="") ? rec : list " ;; " rec} END{if (n>0) {printf "%.0f %s %s %s", u/n, (uc==g ? sprintf("%.0f", mu) : "N/A"), (tc==g ? sprintf("%.0f", mt) : "N/A"), name0; if (list != "") printf " ;; %s", list}}' 2>/dev/null || echo "")`;
const EMPTY_GPU = {
gpu: null,
gpuName: null,
gpuMemUsed: null,
gpuMemTotal: null,
gpus: [],
};
const GPU_LIST_MARK = " ;; ";
const states = new WeakMap();
function stateFor(conn) {
let state = states.get(conn);
@@ -24,8 +31,23 @@ function stateFor(conn) {
}
return state;
}
function sameGpuList(left, right) {
const a = left || [];
const b = right || [];
return a.length === b.length && a.every((gpu, index) => {
const other = b[index];
return gpu.index === other.index
&& gpu.name === other.name
&& gpu.utilization === other.utilization
&& gpu.memoryUsed === other.memoryUsed
&& gpu.memoryTotal === other.memoryTotal;
});
}
function updateGpuStats(state, next) {
if (Object.keys(EMPTY_GPU).some((key) => state.stats[key] !== next[key])) {
const changed = Object.keys(EMPTY_GPU).some((key) => (
key === "gpus" ? !sameGpuList(state.stats[key], next[key]) : state.stats[key] !== next[key]
));
if (changed) {
// Publication time is separate from CPU history and strictly advances even
// when a fast GPU reply shares the CPU sample's millisecond.
state.updatedAt = Math.max(Date.now() + 1, state.updatedAt + 1);
@@ -35,23 +57,51 @@ function updateGpuStats(state, next) {
function monitorGpuUpdatedAt(conn) {
return states.get(conn)?.updatedAt ?? 0;
}
function emptyGpuStats() {
return { ...EMPTY_GPU, gpus: [] };
}
function parseGpuNumber(value) {
if (value == null || !/^[0-9]+$/.test(String(value).trim())) return null;
return parseInt(value, 10);
}
function parseGpuCard(record) {
const fields = String(record || "").split("\t");
if (fields.length < 5) return null;
const index = parseGpuNumber(fields[0]);
if (index == null) return null;
const memoryTotal = parseGpuNumber(fields[fields.length - 1]);
const memoryUsed = parseGpuNumber(fields[fields.length - 2]);
const utilization = parseGpuNumber(fields[fields.length - 3]);
return {
index,
name: fields.slice(1, fields.length - 3).join("\t").trim(),
utilization: utilization == null ? null : Math.max(0, Math.min(100, utilization)),
memoryUsed,
memoryTotal,
};
}
function parseGpuStats(output) {
const raw =
String(output || "")
.split("|")
.find((part) => part.startsWith("GPU:"))
?.slice(4)
.trim()
.split(/\s+/) ?? [];
const utilization = parseInt(raw[0], 10);
if (!Number.isFinite(utilization)) return EMPTY_GPU;
const number = (value) =>
Number.isFinite(parseInt(value, 10)) ? parseInt(value, 10) : null;
const payload = String(output || "")
.split("|")
.find((part) => part.startsWith("GPU:"))
?.slice(4)
.trim() ?? "";
if (!payload) return emptyGpuStats();
const markAt = payload.indexOf(GPU_LIST_MARK);
const summary = (markAt === -1 ? payload : payload.slice(0, markAt)).trim();
const list = markAt === -1 ? "" : payload.slice(markAt + GPU_LIST_MARK.length);
const raw = summary.split(/\s+/).filter(Boolean);
const utilization = parseGpuNumber(raw[0]);
if (utilization == null) return emptyGpuStats();
const gpus = list
? list.split(GPU_LIST_MARK).map(parseGpuCard).filter(Boolean)
: [];
return {
gpu: Math.max(0, Math.min(100, utilization)),
gpuName: raw.slice(3).join(" ") || null,
gpuMemUsed: number(raw[1]),
gpuMemTotal: number(raw[2]),
gpuMemUsed: parseGpuNumber(raw[1]),
gpuMemTotal: parseGpuNumber(raw[2]),
gpus,
};
}
function withCachedMonitorGpu(conn, result) {
@@ -66,7 +116,7 @@ function cacheFullGpuStats(conn, stats) {
updateGpuStats(
state,
Object.fromEntries(
Object.keys(EMPTY_GPU).map((key) => [key, stats[key] ?? null]),
Object.keys(EMPTY_GPU).map((key) => [key, stats[key] ?? (key === "gpus" ? [] : null)]),
),
);
state.nextAt = Date.now() + 10000;
@@ -3,6 +3,8 @@ const test = require("node:test");
const assert = require("node:assert/strict");
const { EventEmitter } = require("node:events");
const {
GPU_ASSIGNMENT,
parseGpuStats,
startMonitorGpuProbe,
withCachedMonitorGpu,
cacheFullGpuStats,
@@ -17,6 +19,75 @@ function stream() {
return s;
}
const sample = { success: true, stats: { cpu: 12, memTotal: 4096, gpu: null } };
test("GPU collection stays one nvidia-smi query and keeps per-GPU fields", () => {
assert.equal(
GPU_ASSIGNMENT.match(/nvidia-smi/g).length,
1,
"each poll must run a single nvidia-smi command",
);
assert.match(
GPU_ASSIGNMENT,
/nvidia-smi --query-gpu=index,name,utilization\.gpu,memory\.used,memory\.total --format=csv,noheader,nounits/,
);
});
test("parseGpuStats keeps a single aggregate GPU and an empty device list", () => {
assert.deepEqual(parseGpuStats("GPU:73 2048 24576 NVIDIA GeForce RTX 4090"), {
gpu: 73,
gpuName: "NVIDIA GeForce RTX 4090",
gpuMemUsed: 2048,
gpuMemTotal: 24576,
gpus: [],
});
});
test("parseGpuStats reads each GPU index, name, utilization, and memory", () => {
const parsed = parseGpuStats(
"GPU:72 3072 49152 NVIDIA RTX 4090 ;; 0\tNVIDIA RTX 4090\t73\t2048\t24576 ;; 1\tNVIDIA A100 80GB\t71\t1024\t24576",
);
assert.equal(parsed.gpu, 72);
assert.equal(parsed.gpuMemUsed, 3072);
assert.equal(parsed.gpuMemTotal, 49152);
assert.deepEqual(parsed.gpus, [
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: 2048, memoryTotal: 24576 },
{ index: 1, name: "NVIDIA A100 80GB", utilization: 71, memoryUsed: 1024, memoryTotal: 24576 },
]);
});
test("parseGpuStats keeps eight GPUs and clamps utilization", () => {
const cards = Array.from({ length: 8 }, (_, index) =>
`${index}\tNVIDIA A100 ${index}\t${index === 3 ? 140 : 10 + index}\t${1024 * (index + 1)}\t81920`,
).join(" ;; ");
const parsed = parseGpuStats(`GPU:40 36864 655360 NVIDIA A100 0 ;; ${cards}`);
assert.equal(parsed.gpus.length, 8);
assert.equal(parsed.gpus[3].utilization, 100);
assert.equal(parsed.gpus[7].index, 7);
assert.equal(parsed.gpus[7].memoryTotal, 81920);
});
test("parseGpuStats skips a broken card and unknown memory without hiding the rest", () => {
const parsed = parseGpuStats(
"GPU:73 2048 24576 NVIDIA RTX 4090 ;; not-a-card ;; 0\tNVIDIA RTX 4090\t73\tN/A\t24576 ;; 2\tNVIDIA A100\tN/A\t1024\t8192",
);
assert.equal(parsed.gpu, 73);
assert.deepEqual(parsed.gpus, [
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: null, memoryTotal: 24576 },
{ index: 2, name: "NVIDIA A100", utilization: null, memoryUsed: 1024, memoryTotal: 8192 },
]);
});
test("parseGpuStats hides GPU data when nvidia-smi output cannot be parsed", () => {
for (const output of ["", "GPU:", "GPU:[Not Supported]", "GPU:N/A N/A N/A", "CPURAW:1 1|MEMINFO:1"]) {
assert.deepEqual(parseGpuStats(output), {
gpu: null,
gpuName: null,
gpuMemUsed: null,
gpuMemTotal: null,
gpus: [],
});
}
});
test("GPU work is coalesced, cached independently, and never changes CPU readings", async (t) => {
let now = 100000,
calls = 0,
+7 -6
View File
@@ -1312,11 +1312,10 @@ function createSessionOpsApi(ctx) {
`kernel=$(uname -r 2>/dev/null || echo "")`,
`uptime=$(awk '{printf "%.0f",$1}' /proc/uptime 2>/dev/null || echo "")`,
`loadavg=$(awk '{print $1" "$2" "$3}' /proc/loadavg 2>/dev/null || echo "")`,
// GPU: NVIDIA GPUs expose utilization + VRAM through the driver-bundled
// nvidia-smi. Best-effort — empty on hosts without the tool, so the UI
// simply hides the GPU chip. Utilization is averaged across GPUs and
// VRAM is summed independently; incomplete memory totals remain unknown.
// The first GPU's name is kept for the tooltip.
// GPU: NVIDIA GPUs expose utilization + VRAM through one bounded
// nvidia-smi query. Best-effort — empty on hosts without the tool, so
// the UI hides the GPU chip. Utilization is averaged across GPUs and
// VRAM is summed independently; the same reply also lists each card.
// Bound GPU queries independently so a stuck driver cannot stop CPU/memory
// polling. Hosts without timeout also omit this optional metric.
// macOS is skipped: powermetrics needs root, which stats must not.
@@ -1461,6 +1460,7 @@ function createSessionOpsApi(ctx) {
let gpuName = null;
let gpuMemUsed = null;
let gpuMemTotal = null;
let gpus = [];
for (const part of parts) {
if (part.startsWith('CPU:')) {
// macOS: command reports normalized CPU% directly (no delta needed)
@@ -1588,7 +1588,7 @@ function createSessionOpsApi(ctx) {
}
}
} else if (part.startsWith('GPU:')) {
({gpu,gpuName,gpuMemUsed,gpuMemTotal}=parseGpuStats(part));
({gpu,gpuName,gpuMemUsed,gpuMemTotal,gpus}=parseGpuStats(part));
} else if (part.startsWith('HOST:')) {
hostname = part.substring(5).trim();
} else if (part.startsWith('OS:')) {
@@ -1741,6 +1741,7 @@ function createSessionOpsApi(ctx) {
gpuName, // First GPU name (tooltip), null when unavailable
gpuMemUsed, // Summed VRAM used in MB, null when unavailable
gpuMemTotal, // Summed VRAM total in MB, null when unavailable
gpus, // Per-GPU index, name, utilization, and VRAM in MB
memTotal, // Total memory in MB
memUsed, // Used memory in MB (excluding buffers/cache)
memFree, // Free memory in MB
@@ -634,6 +634,7 @@ test("getServerStats reports null GPU fields when the host has no nvidia-smi dat
assert.equal(result.stats.gpuName, null);
assert.equal(result.stats.gpuMemUsed, null);
assert.equal(result.stats.gpuMemTotal, null);
assert.deepEqual(result.stats.gpus, []);
});
test("getServerStats parses NVIDIA GPU utilization and VRAM from the stats line", async () => {
@@ -653,6 +654,7 @@ test("getServerStats parses NVIDIA GPU utilization and VRAM from the stats line"
assert.equal(result.stats.gpuName, "NVIDIA GeForce RTX 4090");
assert.equal(result.stats.gpuMemUsed, 2048);
assert.equal(result.stats.gpuMemTotal, 24576);
assert.deepEqual(result.stats.gpus, []);
});
test("getServerStats tolerates a malformed GPU section", async () => {
@@ -668,16 +670,40 @@ test("getServerStats tolerates a malformed GPU section", async () => {
assert.equal(result.success, true);
assert.equal(result.stats.gpu, null);
assert.equal(result.stats.gpuName, null);
assert.deepEqual(result.stats.gpus, []);
});
for (const scenario of [
{ name: "multiple GPUs", script: "printf '%s\\n' '73, 2048, 24576, NVIDIA RTX 4090' '71, 1024, 24576, NVIDIA RTX 4090'", gpu: 72, used: 3072, total: 49152 },
{ name: "idle GPU", script: "printf '%s\\n' '0, 0, 24576, NVIDIA RTX 4090'", gpu: 0, used: 0, total: 24576 },
{ name: "mixed utilization availability", script: "printf '%s\\n' '73, 2048, 24576, NVIDIA RTX 4090' '[N/A], 1024, 8192, NVIDIA A100 MIG'", gpu: 73, used: 3072, total: 32768 },
{ name: "unavailable memory usage", script: "printf '%s\\n' '73, [N/A], 24576, NVIDIA RTX 4090'", gpu: 73, total: 24576 },
{ name: "partial memory totals", script: "printf '%s\\n' '73, 2048, 24576, NVIDIA RTX 4090' '71, 1024, [N/A], NVIDIA RTX 4090'", gpu: 72, used: 3072 },
{ name: "unsupported utilization", script: "printf '%s\\n' '[N/A], 2048, 24576, NVIDIA RTX 4090'", gpu: null },
{ name: "failed query with partial output", script: "printf '%s\\n' '73, 2048, 24576, NVIDIA RTX 4090'; exit 1", gpu: null },
{ name: "multiple GPUs", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, 2048, 24576' '1, NVIDIA RTX 4090, 71, 1024, 24576'", gpu: 72, used: 3072, total: 49152, gpus: [
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: 2048, memoryTotal: 24576 },
{ index: 1, name: "NVIDIA RTX 4090", utilization: 71, memoryUsed: 1024, memoryTotal: 24576 },
] },
{ name: "idle GPU", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 0, 0, 24576'", gpu: 0, used: 0, total: 24576, gpus: [
{ index: 0, name: "NVIDIA RTX 4090", utilization: 0, memoryUsed: 0, memoryTotal: 24576 },
] },
{ name: "mixed utilization availability", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, 2048, 24576' '1, NVIDIA A100 MIG, [N/A], 1024, 8192'", gpu: 73, used: 3072, total: 32768, gpus: [
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: 2048, memoryTotal: 24576 },
{ index: 1, name: "NVIDIA A100 MIG", utilization: null, memoryUsed: 1024, memoryTotal: 8192 },
] },
{ name: "unavailable memory usage", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, [N/A], 24576'", gpu: 73, total: 24576, gpus: [
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: null, memoryTotal: 24576 },
] },
{ name: "partial memory totals", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, 2048, 24576' '1, NVIDIA RTX 4090, 71, 1024, [N/A]'", gpu: 72, used: 3072, gpus: [
{ index: 0, name: "NVIDIA RTX 4090", utilization: 73, memoryUsed: 2048, memoryTotal: 24576 },
{ index: 1, name: "NVIDIA RTX 4090", utilization: 71, memoryUsed: 1024, memoryTotal: null },
] },
{ name: "comma in GPU name", script: "printf '%s\\n' '0, NVIDIA Foo, 80GB, 41, 2048, 24576'", gpu: 41, used: 2048, total: 24576, gpus: [
{ index: 0, name: "NVIDIA Foo, 80GB", utilization: 41, memoryUsed: 2048, memoryTotal: 24576 },
] },
{ name: "eight GPUs", script: `printf '%s\\n' ${Array.from({ length: 8 }, (_, index) => `'${index}, NVIDIA A100 ${index}, ${10 + index * 2}, ${1024 * (index + 1)}, 81920'`).join(" ")}`, gpu: 17, used: 36864, total: 655360, gpus: Array.from({ length: 8 }, (_, index) => ({
index,
name: `NVIDIA A100 ${index}`,
utilization: 10 + index * 2,
memoryUsed: 1024 * (index + 1),
memoryTotal: 81920,
})) },
{ name: "unsupported utilization", script: "printf '%s\\n' '0, NVIDIA RTX 4090, [N/A], 2048, 24576'", gpu: null },
{ name: "failed query with partial output", script: "printf '%s\\n' '0, NVIDIA RTX 4090, 73, 2048, 24576'; exit 1", gpu: null },
{ name: "missing query tool", script: "exit 127", gpu: null },
{ name: "missing timeout tool", script: "exec sleep 20", gpu: null, noTimeout: true },
{ name: "blocked GPU query", script: "trap '' TERM; exec sleep 20", gpu: null },
@@ -741,6 +767,7 @@ for (const scenario of [
assert.equal(result.stats.gpu, scenario.gpu);
assert.equal(result.stats.gpuMemUsed, scenario.used ?? null);
assert.equal(result.stats.gpuMemTotal, scenario.total ?? null);
assert.deepEqual(result.stats.gpus, scenario.gpus ?? []);
});
}
}
+7
View File
@@ -301,6 +301,13 @@ declare global {
gpuName?: string | null; // First GPU name
gpuMemUsed?: number | null; // Summed VRAM used in MB
gpuMemTotal?: number | null; // Summed VRAM total in MB
gpus?: Array<{ // Per-GPU utilization and VRAM (MiB)
index: number;
name: string;
utilization: number | null;
memoryUsed: number | null;
memoryTotal: number | null;
}>;
memTotal: number | null; // Total memory in MB
memUsed: number | null; // Used memory in MB (excluding buffers/cache)
memFree: number | null; // Free memory in MB