mirror of
https://github.com/binaricat/Netcatty.git
synced 2026-11-20 23:20:46 +00:00
* fix(ci): stop green Codex fixes failing baseline name matching Accept exit-0 candidates when quantitative coverage does not shrink, even if individual TAP success titles were renamed. Keep rejecting true coverage loss, upload baseline test logs, and surface comparison details on rejected verification runs. Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com> * fix(ci): require baseline failures to be fixed by name Address Codex P1: a red→green candidate must keep each failing test title as a success, so deleting the failing test and adding an unrelated passer cannot pass the quantitative clean gate. Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com> * fix(ci): keep comparator comments ASCII-only Replace the non-ASCII arrow in the red-to-green comment so the file stays consistent with the repo ASCII convention. Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com> * fix(ci): fail closed on incomplete baseline TAP summaries Require a complete exact-base summary before accepting a quantitative clean candidate, so a truncated red baseline cannot be bypassed by deleting crashing coverage. Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com> * test(ci): clarify incomplete-baseline fail-closed coverage Keep the comparator fail-closed on truncated exact-base TAP summaries and re-request Codex review on the current head. Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: 陈大猫 <binaricat@users.noreply.github.com>
432 lines
15 KiB
JavaScript
432 lines
15 KiB
JavaScript
'use strict';
|
|
|
|
const test = require('node:test');
|
|
const assert = require('node:assert/strict');
|
|
const {
|
|
compareTapResults,
|
|
parseTapResult,
|
|
} = require('./compare-ci-test-baseline.cjs');
|
|
|
|
const tap = ({ failures = [], successes = [], fail = failures.length, cancelled = 0, skipped = 0, todo = 0, tests = 10 } = {}) => [
|
|
'TAP version 13',
|
|
...failures.map((name, index) => `not ok ${index + 1} - ${name}`),
|
|
...successes.map((name, index) => `ok ${failures.length + index + 1} - ${name}`),
|
|
`# fail ${fail}`,
|
|
`# cancelled ${cancelled}`,
|
|
`# skipped ${skipped}`,
|
|
`# todo ${todo}`,
|
|
`# tests ${tests}`,
|
|
].join('\n');
|
|
|
|
test('accepts a clean candidate even when the base was red', () => {
|
|
const result = compareTapResults(
|
|
parseTapResult(tap({ failures: ['base failure'] }), 1),
|
|
parseTapResult(tap({ successes: ['base failure'] }), 0),
|
|
);
|
|
assert.equal(result.passed, true);
|
|
assert.equal(result.kind, 'clean');
|
|
});
|
|
|
|
test('rejects a zero-exit candidate without a complete clean TAP summary', () => {
|
|
const result = compareTapResults(
|
|
parseTapResult(tap(), 0),
|
|
parseTapResult('custom test command completed', 0),
|
|
);
|
|
assert.equal(result.passed, false);
|
|
assert.equal(result.kind, 'unclassified_failure');
|
|
});
|
|
|
|
test('accepts renaming a successful test when quantitative coverage does not shrink', () => {
|
|
const result = compareTapResults(
|
|
parseTapResult(tap({ successes: ['kept test', 'removed test'] }), 0),
|
|
parseTapResult(tap({ successes: ['kept test', 'unrelated replacement'] }), 0),
|
|
);
|
|
assert.equal(result.passed, true);
|
|
assert.equal(result.kind, 'clean');
|
|
assert.deepEqual(result.missingBaselineSuccesses, ['removed test']);
|
|
|
|
const issueNumberName = compareTapResults(
|
|
parseTapResult(tap({
|
|
successes: ['accepts packages that ship upstream #6055/#5987/#6043'],
|
|
}), 0),
|
|
parseTapResult(tap({
|
|
successes: ['accepts packages that ship upstream #9999'],
|
|
}), 0),
|
|
);
|
|
assert.equal(issueNumberName.passed, true);
|
|
assert.equal(issueNumberName.kind, 'clean');
|
|
|
|
// Clean candidate that fixes a prior red must keep the failing test title.
|
|
const fixedFailure = compareTapResults(
|
|
parseTapResult(tap({ failures: ['broken test'], successes: ['other test'] }), 1),
|
|
parseTapResult(tap({ successes: ['broken test', 'other test'] }), 0),
|
|
);
|
|
assert.equal(fixedFailure.passed, true);
|
|
assert.equal(fixedFailure.kind, 'clean');
|
|
|
|
// Deleting the failing test and adding an unrelated passer is not a clean fix.
|
|
const deletedFailure = compareTapResults(
|
|
parseTapResult(tap({ failures: ['broken test'], successes: ['other test'] }), 1),
|
|
parseTapResult(tap({ successes: ['other test', 'unrelated replacement'] }), 0),
|
|
);
|
|
assert.equal(deletedFailure.passed, false);
|
|
assert.equal(deletedFailure.kind, 'unclassified_failure');
|
|
assert.deepEqual(deletedFailure.missingBaselineFailures, ['broken test']);
|
|
});
|
|
|
|
test('accepts a green candidate that renames one success while adding another', () => {
|
|
const result = compareTapResults(
|
|
parseTapResult(tap({
|
|
successes: ['alpha', 'beta', 'gamma'],
|
|
tests: 8886,
|
|
}), 0),
|
|
parseTapResult(tap({
|
|
successes: ['alpha', 'beta', 'delta', 'epsilon'],
|
|
tests: 8887,
|
|
}), 0),
|
|
);
|
|
assert.equal(result.passed, true);
|
|
assert.equal(result.kind, 'clean');
|
|
assert.deepEqual(result.missingBaselineSuccesses, ['gamma']);
|
|
});
|
|
|
|
test('rejects deleting successful coverage even when the run stays green', () => {
|
|
const result = compareTapResults(
|
|
parseTapResult(tap({ successes: ['kept test', 'removed test'], tests: 12 }), 0),
|
|
parseTapResult(tap({ successes: ['kept test'], tests: 11 }), 0),
|
|
);
|
|
assert.equal(result.passed, false);
|
|
assert.equal(result.kind, 'missing_baseline_successes');
|
|
assert.deepEqual(result.missingBaselineSuccesses, ['removed test']);
|
|
assert.deepEqual(result.newFailures, ['removed test']);
|
|
});
|
|
|
|
test('rejects a clean candidate when the exact-base TAP summary is incomplete (fail closed)', () => {
|
|
const result = compareTapResults(
|
|
parseTapResult('base runner stopped early', 1),
|
|
parseTapResult(tap(), 0),
|
|
);
|
|
assert.equal(result.passed, false);
|
|
assert.equal(result.kind, 'unclassified_failure');
|
|
});
|
|
|
|
test('accepts only failures already present on the exact base', () => {
|
|
const result = compareTapResults(
|
|
parseTapResult(tap({ failures: ['base A', 'base B'] }), 1),
|
|
parseTapResult(tap({ failures: ['base B'], successes: ['base A'] }), 1),
|
|
);
|
|
assert.equal(result.passed, true);
|
|
assert.equal(result.kind, 'baseline_only');
|
|
assert.deepEqual(result.newFailures, []);
|
|
|
|
const removedFailure = compareTapResults(
|
|
parseTapResult(tap({ failures: ['base A', 'base B'] }), 1),
|
|
parseTapResult(tap({
|
|
failures: ['base B'],
|
|
successes: ['unrelated replacement'],
|
|
}), 1),
|
|
);
|
|
assert.equal(removedFailure.passed, false);
|
|
assert.equal(removedFailure.kind, 'unclassified_failure');
|
|
});
|
|
|
|
test('rejects a different candidate failure even when both runs are red', () => {
|
|
const result = compareTapResults(
|
|
parseTapResult(tap({ failures: ['base failure'] }), 1),
|
|
parseTapResult(tap({ failures: ['candidate regression'] }), 1),
|
|
);
|
|
assert.equal(result.passed, false);
|
|
assert.equal(result.kind, 'new_failures');
|
|
assert.deepEqual(result.newFailures, ['candidate regression']);
|
|
});
|
|
|
|
test('rejects an additional runner failure after comparable TAP output', () => {
|
|
const sameTap = tap({ failures: ['base failure'] });
|
|
const differentExit = compareTapResults(
|
|
parseTapResult(sameTap, 1),
|
|
parseTapResult(`${sameTap}\nrunner crashed after tests`, 2),
|
|
);
|
|
assert.equal(differentExit.passed, false);
|
|
assert.equal(differentExit.kind, 'unclassified_failure');
|
|
|
|
const sameExit = compareTapResults(
|
|
parseTapResult(sameTap, 1),
|
|
parseTapResult(`${sameTap}\nrunner crashed after tests`, 1),
|
|
);
|
|
assert.equal(sameExit.passed, false);
|
|
assert.equal(sameExit.kind, 'unclassified_failure');
|
|
|
|
const preSummaryCrash = compareTapResults(
|
|
parseTapResult(sameTap, 1),
|
|
parseTapResult(`runner crashed before summary\n${sameTap}`, 1),
|
|
);
|
|
assert.equal(preSummaryCrash.passed, false);
|
|
assert.equal(preSummaryCrash.kind, 'unclassified_failure');
|
|
|
|
const crashWithoutFailureKeyword = compareTapResults(
|
|
parseTapResult(sameTap, 1),
|
|
parseTapResult(`${sameTap}\nSegmentation fault (core dumped)`, 1),
|
|
);
|
|
assert.equal(crashWithoutFailureKeyword.passed, false);
|
|
assert.equal(crashWithoutFailureKeyword.kind, 'unclassified_failure');
|
|
|
|
const crashInsideUnattachedYamlMarkers = compareTapResults(
|
|
parseTapResult(sameTap, 1),
|
|
parseTapResult(`${sameTap}\n---\nSegmentation fault (core dumped)\n...`, 1),
|
|
);
|
|
assert.equal(crashInsideUnattachedYamlMarkers.passed, false);
|
|
assert.equal(crashInsideUnattachedYamlMarkers.kind, 'unclassified_failure');
|
|
|
|
const diagnosticTap = (closed) => [
|
|
'TAP version 13',
|
|
'not ok 1 - base failure',
|
|
' ---',
|
|
" error: 'existing failure'",
|
|
" code: 'ERR_TEST_FAILURE'",
|
|
...(closed ? [' ...'] : []),
|
|
'# fail 1',
|
|
'# cancelled 0',
|
|
'# skipped 0',
|
|
'# todo 0',
|
|
'# tests 10',
|
|
].join('\n');
|
|
const crashAfterUnterminatedDiagnostic = compareTapResults(
|
|
parseTapResult(diagnosticTap(true), 1),
|
|
parseTapResult(`${diagnosticTap(false)}\nSegmentation fault (core dumped)`, 1),
|
|
);
|
|
assert.equal(crashAfterUnterminatedDiagnostic.passed, false);
|
|
assert.equal(crashAfterUnterminatedDiagnostic.kind, 'unclassified_failure');
|
|
|
|
const diagnosticWithExtra = (extra) => [
|
|
'TAP version 13',
|
|
'not ok 1 - base failure',
|
|
' ---',
|
|
" error: 'existing failure'",
|
|
" code: 'ERR_TEST_FAILURE'",
|
|
...(extra ? [` ${extra}`] : []),
|
|
' ...',
|
|
'# fail 1',
|
|
'# cancelled 0',
|
|
'# skipped 0',
|
|
'# todo 0',
|
|
'# tests 10',
|
|
].join('\n');
|
|
for (const extra of ['Segmentation fault (core dumped)', 'signal: SIGSEGV']) {
|
|
const crashInsideClosedDiagnostic = compareTapResults(
|
|
parseTapResult(diagnosticWithExtra(''), 1),
|
|
parseTapResult(diagnosticWithExtra(extra), 1),
|
|
);
|
|
assert.equal(crashInsideClosedDiagnostic.passed, false);
|
|
assert.equal(crashInsideClosedDiagnostic.kind, 'new_failures');
|
|
}
|
|
|
|
const unchangedArbitraryOutput = compareTapResults(
|
|
parseTapResult(`starting custom runner\n${sameTap}`, 1),
|
|
parseTapResult(`starting custom runner\n${sameTap}`, 1),
|
|
);
|
|
assert.equal(unchangedArbitraryOutput.passed, true);
|
|
assert.equal(unchangedArbitraryOutput.kind, 'baseline_only');
|
|
});
|
|
|
|
test('distinguishes same-title failures by stable TAP diagnostics', () => {
|
|
const withDiagnostic = (error, location = '/workspace/example.test.js:10:1') => [
|
|
'TAP version 13',
|
|
'not ok 1 - duplicate title',
|
|
' ---',
|
|
` location: '${location}'`,
|
|
" failureType: 'testCodeFailure'",
|
|
` error: '${error}'`,
|
|
" code: 'ERR_ASSERTION'",
|
|
' stack: |- ',
|
|
' volatile stack line',
|
|
' ...',
|
|
'# fail 1',
|
|
'# cancelled 0',
|
|
'# skipped 0',
|
|
'# todo 0',
|
|
'# tests 10',
|
|
].join('\n');
|
|
const same = compareTapResults(
|
|
parseTapResult(withDiagnostic('old failure'), 1),
|
|
parseTapResult(withDiagnostic('old failure'), 1),
|
|
);
|
|
assert.equal(same.passed, true);
|
|
|
|
const moved = compareTapResults(
|
|
parseTapResult(withDiagnostic('old failure'), 1),
|
|
parseTapResult(withDiagnostic('old failure', '/workspace/example.test.js:11:1'), 1),
|
|
);
|
|
assert.equal(moved.passed, true);
|
|
|
|
const changed = compareTapResults(
|
|
parseTapResult(withDiagnostic('old failure'), 1),
|
|
parseTapResult(withDiagnostic('new regression'), 1),
|
|
);
|
|
assert.equal(changed.passed, false);
|
|
assert.deepEqual(changed.newFailures, ['duplicate title']);
|
|
|
|
const dynamicValues = (actual, nextTitle) => [
|
|
'TAP version 13',
|
|
'not ok 1 - duplicate title',
|
|
' ---',
|
|
" location: '/workspace/example.test.js:10:1'",
|
|
" failureType: 'testCodeFailure'",
|
|
" error: 'old failure'",
|
|
` actual: ${actual}`,
|
|
' expected: 1',
|
|
" code: 'ERR_ASSERTION'",
|
|
' stack: |- ',
|
|
' volatile stack line',
|
|
' ...',
|
|
`# Subtest: ${nextTitle}`,
|
|
`ok 2 - ${nextTitle}`,
|
|
'1..2',
|
|
'# fail 1',
|
|
'# cancelled 0',
|
|
'# skipped 0',
|
|
'# todo 0',
|
|
'# tests 2',
|
|
].join('\n');
|
|
const dynamic = compareTapResults(
|
|
parseTapResult(dynamicValues(41831, 'later test'), 1),
|
|
parseTapResult(dynamicValues(52942, 'later test'), 1),
|
|
);
|
|
assert.equal(dynamic.passed, true);
|
|
|
|
const multilineError = (reason) => [
|
|
'TAP version 13',
|
|
'not ok 1 - duplicate title',
|
|
' ---',
|
|
" location: '/workspace/example.test.js:10:1'",
|
|
" failureType: 'testCodeFailure'",
|
|
' error: |-',
|
|
` ${reason}`,
|
|
" code: 'ERR_ASSERTION'",
|
|
" operator: 'strictEqual'",
|
|
' ...',
|
|
'# fail 1',
|
|
'# cancelled 0',
|
|
'# skipped 0',
|
|
'# todo 0',
|
|
'# tests 10',
|
|
].join('\n');
|
|
const changedMultilineError = compareTapResults(
|
|
parseTapResult(multilineError('old failure'), 1),
|
|
parseTapResult(multilineError('new regression'), 1),
|
|
);
|
|
assert.equal(changedMultilineError.passed, false);
|
|
assert.deepEqual(changedMultilineError.newFailures, ['duplicate title']);
|
|
|
|
const assertionDiff = (field, actual) => [
|
|
'TAP version 13',
|
|
'not ok 1 - duplicate title',
|
|
' ---',
|
|
" failureType: 'testCodeFailure'",
|
|
' error: |-',
|
|
' Expected values to be strictly equal:',
|
|
'',
|
|
' {',
|
|
` ${field}: ${actual}`,
|
|
' }',
|
|
" code: 'ERR_ASSERTION'",
|
|
" operator: 'strictEqual'",
|
|
' ...',
|
|
'# fail 1',
|
|
'# cancelled 0',
|
|
'# skipped 0',
|
|
'# todo 0',
|
|
'# tests 10',
|
|
].join('\n');
|
|
const changedAssertionValue = compareTapResults(
|
|
parseTapResult(assertionDiff('now', 111), 1),
|
|
parseTapResult(assertionDiff('now', 222), 1),
|
|
);
|
|
assert.equal(changedAssertionValue.passed, true);
|
|
|
|
const changedStableAssertionValue = compareTapResults(
|
|
parseTapResult(assertionDiff('x', 1), 1),
|
|
parseTapResult(assertionDiff('x', 3), 1),
|
|
);
|
|
assert.equal(changedStableAssertionValue.passed, false);
|
|
|
|
const changedPortAssertion = compareTapResults(
|
|
parseTapResult(assertionDiff('port', 2222), 1),
|
|
parseTapResult(assertionDiff('port', 443), 1),
|
|
);
|
|
assert.equal(changedPortAssertion.passed, false);
|
|
|
|
const customMultilineError = (reason) => multilineError(reason).replace(
|
|
"code: 'ERR_ASSERTION'",
|
|
"code: 'ERR_CUSTOM'",
|
|
);
|
|
const changedCustomError = compareTapResults(
|
|
parseTapResult(customMultilineError('service rejected: old reason'), 1),
|
|
parseTapResult(customMultilineError('service rejected: new reason'), 1),
|
|
);
|
|
assert.equal(changedCustomError.passed, false);
|
|
|
|
const sameCustomErrorAcrossRuns = compareTapResults(
|
|
parseTapResult(customMultilineError(
|
|
'service failed at /tmp/run-A/result on 2026-07-31T10:00:00Z in worker.js:10:2',
|
|
), 1),
|
|
parseTapResult(customMultilineError(
|
|
'service failed at /tmp/run-B/result on 2026-07-31T10:01:00Z in worker.js:11:3',
|
|
), 1),
|
|
);
|
|
assert.equal(sameCustomErrorAcrossRuns.passed, true);
|
|
});
|
|
|
|
test('rejects added duplicate failures, cancellations, skipped tests, and TODOs', () => {
|
|
const duplicate = compareTapResults(
|
|
parseTapResult(tap({ failures: ['same'] }), 1),
|
|
parseTapResult(tap({ failures: ['same', 'same'] }), 1),
|
|
);
|
|
assert.equal(duplicate.passed, false);
|
|
|
|
const cancelled = compareTapResults(
|
|
parseTapResult(tap({ failures: ['same'] }), 1),
|
|
parseTapResult(tap({ failures: ['same'], cancelled: 2 }), 1),
|
|
);
|
|
assert.equal(cancelled.passed, false);
|
|
assert.equal(cancelled.kind, 'cancelled_tests');
|
|
|
|
const skipped = compareTapResults(
|
|
parseTapResult(tap({ skipped: 1 }), 0),
|
|
parseTapResult(tap({ skipped: 10 }), 0),
|
|
);
|
|
assert.equal(skipped.passed, false);
|
|
assert.equal(skipped.kind, 'unclassified_failure');
|
|
|
|
const todo = compareTapResults(
|
|
parseTapResult(tap({ todo: 1 }), 0),
|
|
parseTapResult(tap({ todo: 10 }), 0),
|
|
);
|
|
assert.equal(todo.passed, false);
|
|
assert.equal(todo.kind, 'unclassified_failure');
|
|
});
|
|
|
|
test('fails closed when a red run has no complete TAP summary', () => {
|
|
const result = compareTapResults(
|
|
parseTapResult('runner stopped early', 1),
|
|
parseTapResult('runner stopped early', 1),
|
|
);
|
|
assert.equal(result.passed, false);
|
|
assert.equal(result.kind, 'unclassified_failure');
|
|
});
|
|
|
|
test('rejects non-test failures and a candidate that runs fewer tests', () => {
|
|
const posttestFailure = compareTapResults(
|
|
parseTapResult(tap({ failures: ['existing'] }), 1),
|
|
parseTapResult(tap({ fail: 0 }), 1),
|
|
);
|
|
assert.equal(posttestFailure.passed, false);
|
|
assert.equal(posttestFailure.kind, 'unclassified_failure');
|
|
|
|
const fewerTests = compareTapResults(
|
|
parseTapResult(tap({ failures: ['existing'], tests: 20 }), 1),
|
|
parseTapResult(tap({ failures: ['existing'], tests: 19 }), 1),
|
|
);
|
|
assert.equal(fewerTests.passed, false);
|
|
assert.equal(fewerTests.kind, 'unclassified_failure');
|
|
});
|