Failure signatures: comparing failures, not logs

/**
 * Extracts a structural fingerprint from an executed command's output.
 *
 * Comparing raw output across a patch is useless: timestamps, absolute paths,
 * durations, object addresses and test ordering all churn between runs. The
 * signature normalises those away so that "did this specific failure go away"
 * becomes a real question with a real answer.
 *
 * The distinction that matters most is between a failure that is GONE and one
 * that merely CHANGED SHAPE. A patch that turns a TypeError into an
 * AssertionError has not fixed anything, but its output differs from the
 * original, so any comparison based on "output changed" would call it fixed.
 */

export interface FailureSignature {
  readonly command: string;
  readonly exitCode: number | null;
  /** e.g. "TypeError", "AssertionError", "ENOENT". Null when not classifiable. */
  readonly errorClass: string | null;
  /** Normalized message: absolute paths, timestamps, hex addresses stripped. */
  readonly normalizedMessage: string | null;
  /** Innermost stack frame within the repository under investigation, if any. */
  readonly originFile: string | null;
  readonly originLine: number | null;
  readonly failingTestIds: readonly string[];
  readonly hash: string;
}

.replace(/[A-Za-z]:[\\/][^\s:)'"]+/g, '<path>')
.replace(/(?<![\w<])\/(?:[\w.@-]+\/)+[\w.@-]+/g, '<path>')
.replace(/\d{4}-\d{2}-\d{2}T[\d:.]+Z?/g, '<ts>')
.replace(/\b\d{2}:\d{2}:\d{2}(?:\.\d+)?\b/g, '<time>')
.replace(/\b\d+(?:\.\d+)?\s?(?:ms|s|sec|seconds)\b/gi, '<dur>')
.replace(/\b0x[0-9a-fA-F]+\b/g, '<addr>')
.replace(/\b[0-9a-f]{32,}\b/g, '<hash>')

/**
 * Deliberately conservative: it removes things that provably vary between runs
 * and leaves everything else alone. Over-normalising would collapse genuinely
 * different failures into one signature, which is the more dangerous error.
 */

const NON_REPOSITORY_PATH = /(?:^|[\\/])(?:node_modules|internal|node:)[\\/]?/;

/**
 * V8 synthetic frames: `[eval]`, `[eval]-wrapper`, `<anonymous>`, `[stdin]`.
 *
 * These look like ordinary paths to a naive check but name no real file. Left
 * unfiltered they can be selected as a failure's origin, producing a signature
 * that points at a file nobody can open.
 */
const SYNTHETIC_FRAME = /^[[<]|^[^\\/.]+$/;

/**
 * The column number is deliberately excluded: a fix on the same line commonly
 * shifts columns without changing the failure, and including it would report
 * spurious FAILURE_MUTATED results.
 */
const canonical = JSON.stringify({
  command: signature.command,
  errorClass: signature.errorClass,
  normalizedMessage: signature.normalizedMessage,
  originFile: signature.originFile,
  originLine: signature.originLine,
  failingTestIds: signature.failingTestIds,
});

export function compareSignatures(
  before: FailureSignature | null,
  after: FailureSignature | null,
): SignatureComparison {
  if (before === null && after === null) return 'INCOMPARABLE';
  if (before === null) return 'NEW_FAILURE';
  if (after === null) return 'FAILURE_RESOLVED';
  if (before.command !== after.command) return 'INCOMPARABLE';
  if (before.hash === after.hash) return 'FAILURE_UNCHANGED';
  return 'FAILURE_MUTATED';
}

// A mutated failure is never a verified fix, whatever the other signals say.
if (comparison === 'FAILURE_MUTATED' && verdict === 'VERIFIED') {
  verdict = 'PARTIALLY_VERIFIED';
  notes.push('Downgraded from VERIFIED because the failure signature changed shape rather than resolving.');
}