Files
Netcatty/scripts/compare-ci-test-baseline.cjs
T
e40aace189 fix(ci): stop green Codex fixes failing baseline name matching (#2747)
* fix(ci): stop green Codex fixes failing baseline name matching

Accept exit-0 candidates when quantitative coverage does not shrink,
even if individual TAP success titles were renamed. Keep rejecting
true coverage loss, upload baseline test logs, and surface comparison
details on rejected verification runs.

Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>

* fix(ci): require baseline failures to be fixed by name

Address Codex P1: a red→green candidate must keep each failing test
title as a success, so deleting the failing test and adding an unrelated
passer cannot pass the quantitative clean gate.

Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>

* fix(ci): keep comparator comments ASCII-only

Replace the non-ASCII arrow in the red-to-green comment so the file
stays consistent with the repo ASCII convention.

Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>

* fix(ci): fail closed on incomplete baseline TAP summaries

Require a complete exact-base summary before accepting a quantitative
clean candidate, so a truncated red baseline cannot be bypassed by
deleting crashing coverage.

Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>

* test(ci): clarify incomplete-baseline fail-closed coverage

Keep the comparator fail-closed on truncated exact-base TAP summaries
and re-request Codex review on the current head.

Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>
2026-08-05 19:00:32 +08:00

486 lines
15 KiB
JavaScript

'use strict';
const fs = require('node:fs');
const ANSI_RE = /\x1b\[[0-?]*[ -/]*[@-~]/g;
function normalizeRuntimeDetail(detail) {
return detail
.replace(
/\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\b/g,
'<timestamp>',
)
.replace(/(?:\/private)?\/var\/folders\/\S+|\/tmp\/\S+/g, '<tmp-path>')
.replace(/:\d+:\d+\b/g, ':<line>:<column>');
}
function normalizeAssertionDetail(detail) {
return normalizeRuntimeDetail(detail).replace(
/\b(now|timestamp|pid|nonce|random(?:Value)?)\b(['"]?\s*[:=]\s*)[^,\s}\]]+/gi,
'$1$2<volatile>',
);
}
function normalizeStableDiagnosticDetail(detail) {
return normalizeAssertionDetail(detail);
}
function collectNonTapOutput(lines) {
const output = [];
let inDiagnostic = false;
let diagnosticOwnerIndent = -1;
let awaitingDiagnosticIndent = null;
for (const rawLine of lines) {
const line = rawLine.trim();
if (inDiagnostic) {
const indent = rawLine.match(/^\s*/)?.[0].length || 0;
if (/^\s+\.\.\.\s*$/.test(rawLine) && indent > diagnosticOwnerIndent) {
inDiagnostic = false;
diagnosticOwnerIndent = -1;
continue;
}
const boundary = line.match(
/^(?:(?:ok|not ok) \d+ - |# (?:tests|suites|pass|fail|cancelled|skipped|todo|duration_ms) |1\.\.\d+$)/,
);
if (!boundary || indent > diagnosticOwnerIndent) continue;
output.push('<unterminated-tap-diagnostic>');
inDiagnostic = false;
diagnosticOwnerIndent = -1;
}
if (awaitingDiagnosticIndent !== null) {
if (!line) continue;
const indent = rawLine.match(/^\s*/)?.[0].length || 0;
if (/^\s+---\s*$/.test(rawLine) && indent > awaitingDiagnosticIndent) {
inDiagnostic = true;
diagnosticOwnerIndent = awaitingDiagnosticIndent;
awaitingDiagnosticIndent = null;
continue;
}
awaitingDiagnosticIndent = null;
}
const failed = rawLine.match(/^(\s*)not ok \d+ - /);
if (failed) {
awaitingDiagnosticIndent = failed[1].length;
continue;
}
if (
!line ||
/^TAP version \d+$/.test(line) ||
/^ok \d+ - /.test(line) ||
/^# (?:Subtest:|tests |suites |pass |fail |cancelled |skipped |todo |duration_ms )/.test(line) ||
/^1\.\.\d+$/.test(line)
) {
continue;
}
output.push(
line
.replace(/\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\b/g, '<timestamp>')
.replace(/(?:\/private)?\/var\/folders\/\S+|\/tmp\/\S+/g, '<tmp-path>')
.replace(/:\d+:\d+\b/g, ':<line>:<column>'),
);
}
if (inDiagnostic) output.push('<unterminated-tap-diagnostic>');
return output;
}
function parseTapResult(text, exitCode) {
const normalized = String(text || '').replace(ANSI_RE, '').replace(/\r\n?/g, '\n');
const lines = normalized.split('\n');
const failures = [];
const failureRecords = [];
const successes = [];
let failCount = null;
let cancelledCount = null;
let skippedCount = null;
let todoCount = null;
let testCount = null;
for (let index = 0; index < lines.length; index += 1) {
const line = lines[index];
const succeeded = line.match(/^\s*ok \d+ - (.+)$/);
const tapDirective = /\s+#\s*(?:SKIP|TODO)\b/i;
if (succeeded && !tapDirective.test(line)) {
successes.push(succeeded[1].trim());
}
const failed = line.match(/^\s*not ok \d+ - (.+)$/);
if (failed && !tapDirective.test(line)) {
const name = failed[1].trim();
const diagnostic = [];
const errorDetails = [];
let inDiagnostic = false;
let errorBlockIndent = -1;
let skippingStack = false;
let stackIndent = -1;
for (let detailIndex = index + 1; detailIndex < lines.length; detailIndex += 1) {
const detail = lines[detailIndex];
if (/^\s*(?:ok|not ok) \d+ - /.test(detail) || /^\s*# (?:tests|fail|cancelled|skipped|todo) \d+\s*$/.test(detail)) {
break;
}
if (/^\s*---\s*$/.test(detail)) {
inDiagnostic = true;
continue;
}
if (inDiagnostic && /^\s*\.\.\.\s*$/.test(detail)) break;
if (!inDiagnostic) continue;
const indent = detail.match(/^\s*/)?.[0].length || 0;
if (errorBlockIndent >= 0) {
if (!detail.trim()) continue;
if (indent > errorBlockIndent) {
errorDetails.push(detail.trim());
continue;
}
errorBlockIndent = -1;
}
if (skippingStack) {
if (!detail.trim() || indent > stackIndent) continue;
skippingStack = false;
}
if (/^\s*stack:\s*(?:\|-)?\s*$/.test(detail)) {
skippingStack = true;
stackIndent = indent;
continue;
}
if (/^\s*duration_ms:/.test(detail)) continue;
if (/^\s*(?:actual|expected):/.test(detail)) continue;
if (/^\s*error:\s*(?:\|-|>)\s*$/.test(detail)) {
diagnostic.push('error:');
errorBlockIndent = indent;
continue;
}
if (detail.trim()) {
diagnostic.push(normalizeStableDiagnosticDetail(detail.trimEnd()));
}
}
const assertionFailure = diagnostic.some((detail) =>
/^\s*code:\s*['"]?ERR_ASSERTION['"]?\s*$/.test(detail),
);
const stableErrorDetails = assertionFailure
? errorDetails.map(normalizeStableDiagnosticDetail)
: errorDetails.map(normalizeRuntimeDetail);
diagnostic.push(...stableErrorDetails.map((detail) => `error-detail: ${detail}`));
failures.push(name);
failureRecords.push({
name,
identity: diagnostic.length ? `${name}\n${diagnostic.join('\n')}` : name,
});
}
const failSummary = line.match(/^\s*# fail (\d+)\s*$/);
if (failSummary) {
failCount = Number(failSummary[1]);
}
const cancelledSummary = line.match(/^\s*# cancelled (\d+)\s*$/);
if (cancelledSummary) {
cancelledCount = Number(cancelledSummary[1]);
}
const skippedSummary = line.match(/^\s*# skipped (\d+)\s*$/);
if (skippedSummary) {
skippedCount = Number(skippedSummary[1]);
}
const todoSummary = line.match(/^\s*# todo (\d+)\s*$/);
if (todoSummary) {
todoCount = Number(todoSummary[1]);
}
const testSummary = line.match(/^\s*# tests (\d+)\s*$/);
if (testSummary) {
testCount = Number(testSummary[1]);
}
}
return {
exitCode: Number(exitCode),
failures,
failureRecords,
successes,
failCount,
cancelledCount,
skippedCount,
todoCount,
testCount,
nonTapOutput: collectNonTapOutput(lines),
complete:
failCount !== null &&
cancelledCount !== null &&
skippedCount !== null &&
todoCount !== null &&
testCount !== null,
};
}
function countFailures(failures) {
const counts = new Map();
for (const failure of failures) {
counts.set(failure, (counts.get(failure) || 0) + 1);
}
return counts;
}
function compareTapResults(baseline, candidate) {
const baselineSuccessCounts = countFailures(baseline.successes);
const candidateSuccessCounts = countFailures(candidate.successes);
const candidateSuccessPool = new Map(candidateSuccessCounts);
const missingBaselineSuccesses = [];
for (const [success, count] of baselineSuccessCounts) {
const available = candidateSuccessPool.get(success) || 0;
const missing = count - available;
candidateSuccessPool.set(success, Math.max(0, available - count));
for (let i = 0; i < missing; i += 1) missingBaselineSuccesses.push(success);
}
const candidateFailurePool = countFailures(
candidate.failureRecords.map((failure) => failure.identity),
);
const missingBaselineFailures = [];
for (const failure of baseline.failureRecords) {
const matchingFailures = candidateFailurePool.get(failure.identity) || 0;
if (matchingFailures > 0) {
candidateFailurePool.set(failure.identity, matchingFailures - 1);
continue;
}
const matchingSuccesses = candidateSuccessPool.get(failure.name) || 0;
if (matchingSuccesses > 0) {
candidateSuccessPool.set(failure.name, matchingSuccesses - 1);
continue;
}
missingBaselineFailures.push(failure.name);
}
const result = ({
passed,
kind,
newFailures = [],
candidateFailures = candidate.exitCode === 0 ? [] : candidate.failures,
}) => ({
passed,
kind,
baselineFailures: baseline.failures,
candidateFailures,
newFailures,
missingBaselineSuccesses,
missingBaselineFailures,
});
if (candidate.exitCode === 0) {
// Require every baseline failure title to reappear as a same-named success
// before accepting a red-to-green transition. Aggregate counts alone would allow
// deleting the failing test and adding an unrelated passer.
const baselineFailCounts = countFailures(baseline.failures);
const baselineSuccessCountsForFix = countFailures(baseline.successes);
const candidateSuccessCountsForFix = countFailures(candidate.successes);
const unresolvedBaselineFailures = [];
for (const [name, failCount] of baselineFailCounts) {
const required =
failCount + (baselineSuccessCountsForFix.get(name) || 0);
const available = candidateSuccessCountsForFix.get(name) || 0;
for (let i = 0; i < required - available; i += 1) {
unresolvedBaselineFailures.push(name);
}
}
// A fully green candidate may rename success titles when quantitative
// coverage does not shrink, but only against a complete baseline summary.
// An incomplete baseline cannot prove coverage was preserved, so fail closed.
const quantitativeClean =
baseline.complete &&
candidate.complete &&
candidate.failCount === 0 &&
candidate.cancelledCount === 0 &&
unresolvedBaselineFailures.length === 0 &&
candidate.skippedCount <= baseline.skippedCount &&
candidate.todoCount <= baseline.todoCount &&
candidate.testCount >= baseline.testCount;
if (quantitativeClean) {
return result({ passed: true, kind: 'clean' });
}
const coverageShrunk =
baseline.complete &&
candidate.complete &&
candidate.failCount === 0 &&
candidate.cancelledCount === 0 &&
candidate.skippedCount <= baseline.skippedCount &&
candidate.todoCount <= baseline.todoCount &&
candidate.testCount < baseline.testCount &&
missingBaselineSuccesses.length > 0;
if (coverageShrunk) {
return result({
passed: false,
kind: 'missing_baseline_successes',
newFailures: missingBaselineSuccesses,
});
}
return result({
passed: false,
kind: 'unclassified_failure',
newFailures: unresolvedBaselineFailures.length
? unresolvedBaselineFailures
: missingBaselineSuccesses,
});
}
if (baseline.exitCode === 0) {
return result({
passed: false,
kind: 'new_failures',
candidateFailures: candidate.failures,
newFailures: candidate.failures,
});
}
if (!baseline.complete || !candidate.complete) {
return result({
passed: false,
kind: 'unclassified_failure',
candidateFailures: candidate.failures,
newFailures: candidate.failures,
});
}
if (missingBaselineSuccesses.length) {
return result({
passed: false,
kind: 'missing_baseline_successes',
candidateFailures: candidate.failures,
newFailures: missingBaselineSuccesses,
});
}
if (candidate.exitCode !== baseline.exitCode) {
return result({
passed: false,
kind: 'unclassified_failure',
candidateFailures: candidate.failures,
newFailures: candidate.failures,
});
}
const baselineNonTapCounts = countFailures(baseline.nonTapOutput);
const candidateNonTapCounts = countFailures(candidate.nonTapOutput);
const addedNonTapOutput = [...candidateNonTapCounts].some(
([line, count]) => count > (baselineNonTapCounts.get(line) || 0),
);
if (addedNonTapOutput) {
return result({
passed: false,
kind: 'unclassified_failure',
candidateFailures: candidate.failures,
newFailures: candidate.failures,
});
}
if (candidate.failCount === 0 || candidate.testCount < baseline.testCount) {
return result({
passed: false,
kind: 'unclassified_failure',
candidateFailures: candidate.failures,
newFailures: candidate.failures,
});
}
if (
candidate.cancelledCount > 0 ||
candidate.skippedCount > baseline.skippedCount ||
candidate.todoCount > baseline.todoCount
) {
return result({
passed: false,
kind: 'cancelled_tests',
candidateFailures: candidate.failures,
newFailures: candidate.failures,
});
}
const baselineCounts = countFailures(
baseline.failureRecords.map((failure) => failure.identity),
);
const candidateCounts = countFailures(
candidate.failureRecords.map((failure) => failure.identity),
);
const newFailures = [];
for (const [identity, count] of candidateCounts) {
const extra = count - (baselineCounts.get(identity) || 0);
const name = candidate.failureRecords.find(
(failure) => failure.identity === identity,
)?.name || identity;
for (let i = 0; i < extra; i += 1) newFailures.push(name);
}
const parsedCountsMatch =
baseline.failureRecords.length >= Number(baseline.failCount) &&
candidate.failureRecords.length >= Number(candidate.failCount);
const passed =
newFailures.length === 0 &&
missingBaselineFailures.length === 0 &&
parsedCountsMatch;
const reportedFailures = newFailures.length
? newFailures
: missingBaselineFailures;
return result({
passed,
kind: passed
? 'baseline_only'
: newFailures.length
? 'new_failures'
: 'unclassified_failure',
candidateFailures: candidate.failures,
newFailures: reportedFailures,
});
}
function parseArgs(argv) {
const values = {};
for (let i = 0; i < argv.length; i += 2) {
const key = argv[i];
if (!key?.startsWith('--') || argv[i + 1] === undefined) {
throw new Error('Expected --name value arguments.');
}
values[key.slice(2)] = argv[i + 1];
}
return values;
}
function main(argv = process.argv.slice(2)) {
const args = parseArgs(argv);
for (const required of [
'baseline-log',
'baseline-exit',
'candidate-log',
'candidate-exit',
'output',
]) {
if (!(required in args)) throw new Error(`Missing --${required}.`);
}
const baseline = parseTapResult(
fs.readFileSync(args['baseline-log'], 'utf8'),
args['baseline-exit'],
);
const candidate = parseTapResult(
fs.readFileSync(args['candidate-log'], 'utf8'),
args['candidate-exit'],
);
const result = compareTapResults(baseline, candidate);
fs.writeFileSync(args.output, `${JSON.stringify(result, null, 2)}\n`, 'utf8');
console.log(
result.passed
? `Test comparison accepted: ${result.kind}.`
: `Test comparison failed: ${result.kind}.`,
);
if (result.newFailures.length) {
console.error(`New failures:\n- ${result.newFailures.join('\n- ')}`);
}
if (!result.passed && result.missingBaselineSuccesses?.length) {
console.error(
`Missing baseline successes:\n- ${result.missingBaselineSuccesses.join('\n- ')}`,
);
}
return result.passed ? 0 : 1;
}
if (require.main === module) {
try {
process.exitCode = main();
} catch (error) {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 2;
}
}
module.exports = {
compareTapResults,
main,
parseTapResult,
};