diff --git a/scripts/compare-ci-test-baseline.cjs b/scripts/compare-ci-test-baseline.cjs index 453c2c795..a93aaea06 100644 --- a/scripts/compare-ci-test-baseline.cjs +++ b/scripts/compare-ci-test-baseline.cjs @@ -170,6 +170,16 @@ function compareTapResults(baseline, candidate) { }; } + if (candidate.exitCode !== baseline.exitCode) { + return { + passed: false, + kind: 'unclassified_failure', + baselineFailures: baseline.failures, + candidateFailures: candidate.failures, + newFailures: candidate.failures, + }; + } + if (candidate.failCount === 0 || candidate.testCount < baseline.testCount) { return { passed: false, diff --git a/scripts/compare-ci-test-baseline.test.cjs b/scripts/compare-ci-test-baseline.test.cjs index 94d72266e..9ffef636a 100644 --- a/scripts/compare-ci-test-baseline.test.cjs +++ b/scripts/compare-ci-test-baseline.test.cjs @@ -64,6 +64,16 @@ test('rejects a different candidate failure even when both runs are red', () => assert.deepEqual(result.newFailures, ['candidate regression']); }); +test('rejects an additional runner failure after comparable TAP output', () => { + const sameTap = tap({ failures: ['base failure'] }); + const result = compareTapResults( + parseTapResult(sameTap, 1), + parseTapResult(`${sameTap}\nrunner crashed after tests`, 2), + ); + assert.equal(result.passed, false); + assert.equal(result.kind, 'unclassified_failure'); +}); + test('distinguishes same-title failures by stable TAP diagnostics', () => { const withDiagnostic = (error, location = '/workspace/example.test.js:10:1') => [ 'TAP version 13',