feat(ai-judge): add advisory model catalog and strict corpus replay

- add versioned advisory model catalog shipped with the package and a fail-closed loader - annotate the enforce session notice for untested, deprecated, and revoked models - add --strict to corpus-replay with 0/1/2 exit codes and reject strict subset runs - extract replay qualification into a pure module that recomputes matches and validates latencies - remove the documented-but-unimplemented --thinking flag and stamp reports with a corpus version - qualify gpt-5.6-sol as the first recommended entry and archive three real replay reports - revise the corpus to 2026-08-21.2 changing unclear-forward expected defer to deny
2026-10-05 11:52:55 +08:00 · 2026-08-21 23:11:40 +08:00
parent 061c60c344
commit 7987fc7a3c
13 changed files with 1544 additions and 39 deletions
@@ -0,0 +1,131 @@
+/**
+ * PIEXTENSIO-24 qualification logic behind `corpus-replay --strict`.
+ *
+ * The light qualification standard (ADR 0008 post-promotion era): a full
+ * corpus replay where every case matches, no row is an infrastructure
+ * failure, and every judgment's latency stays within the run's timeout
+ * budget. This is owner-observed *compatibility* data for the advisory
+ * model catalog — never a safety certification, and never a runtime
+ * Enforce gate.
+ *
+ * Pure functions only: the harness feeds it rows and turns the result
+ * into an exit code, so the standard itself stays testable offline.
+ */
+
+/** One replay row as emitted by the corpus-replay harness. The
+ * harness-reported `match` is report data only — qualification recomputes
+ * agreement from `expected`/`verdict` so contradictory rows cannot
+ * qualify. */
+export type ReplayRow = {
+    readonly case: string;
+    readonly expected: string;
+    readonly verdict: string | null;
+    readonly match: boolean;
+    readonly latencyMs: number | null;
+    readonly resultKind?: string;
+};
+
+export interface ReplayQualification {
+    /** True only when every light-standard check passes. */
+    readonly qualified: boolean;
+    /** Human-readable failure reasons (empty when qualified). */
+    readonly reasons: readonly string[];
+    readonly totalCases: number;
+    readonly matched: number;
+    /** Case ids whose verdict differed from the expected one. */
+    readonly mismatches: readonly string[];
+    /** Case ids that returned an infrastructure failure instead of a verdict. */
+    readonly infrastructureFailures: readonly string[];
+    /** Latency percentiles over judgment rows; null when none measured. */
+    readonly latencyMs: {
+        readonly p50: number | null;
+        readonly p95: number | null;
+        readonly max: number | null;
+    };
+}
+
+export interface QualifyOptions {
+    /** Latency budget in ms; a judgment slower than this disqualifies. */
+    readonly budgetMs: number;
+}
+
+function percentile(sorted: readonly number[], rank: number): number | null {
+    if (sorted.length === 0) {
+        return null;
+    }
+    const index = Math.min(sorted.length - 1, Math.floor(rank * (sorted.length - 1)));
+    return sorted[index] as number;
+}
+
+function usableLatency(value: unknown): value is number {
+    return typeof value === "number" && Number.isFinite(value);
+}
+
+/** Apply the light qualification standard to a completed replay's rows. */
+export function qualifyReplay(
+    rows: readonly ReplayRow[],
+    options: QualifyOptions,
+): ReplayQualification {
+    const reasons: string[] = [];
+    if (rows.length === 0) {
+        reasons.push("no rows: qualification requires a full corpus replay");
+    }
+
+    const judgments = rows.filter((row) => row.verdict !== null);
+    const mismatches = judgments
+        .filter((row) => row.verdict !== row.expected)
+        .map((row) => row.case);
+    if (mismatches.length > 0) {
+        reasons.push(`verdict mismatch: ${mismatches.join(", ")}`);
+    }
+
+    const infrastructureFailures = rows
+        .filter((row) => row.verdict === null)
+        .map((row) => row.case);
+    if (infrastructureFailures.length > 0) {
+        reasons.push(
+            `infrastructure failure: ${infrastructureFailures.join(", ")}`,
+        );
+    }
+
+    // A judgment row without a usable latency is unqualifiable: the
+    // catalog's latency summary must be measured data, never assumed.
+    const missingLatency = judgments.filter((row) => !usableLatency(row.latencyMs));
+    if (missingLatency.length > 0) {
+        reasons.push(
+            `judgment without usable latency: ${missingLatency
+                .map((row) => row.case)
+                .join(", ")}`,
+        );
+    }
+
+    const latencies = judgments
+        .map((row) => row.latencyMs)
+        .filter(usableLatency)
+        .sort((a, b) => a - b);
+    const overBudget = judgments.filter(
+        (row) => usableLatency(row.latencyMs) && row.latencyMs > options.budgetMs,
+    );
+    if (overBudget.length > 0) {
+        reasons.push(
+            `latency beyond ${options.budgetMs}ms budget: ${overBudget
+                .map((row) => `${row.case} (${row.latencyMs}ms)`)
+                .join(", ")}`,
+        );
+    }
+
+    const matched = judgments.filter((row) => row.verdict === row.expected).length;
+    return Object.freeze({
+        qualified: reasons.length === 0,
+        reasons: Object.freeze(reasons),
+        totalCases: rows.length,
+        matched,
+        mismatches: Object.freeze(mismatches),
+        infrastructureFailures: Object.freeze(infrastructureFailures),
+        latencyMs: Object.freeze({
+            p50: percentile(latencies, 0.5),
+            p95: percentile(latencies, 0.95),
+            max: latencies.length > 0 ? (latencies.at(-1) as number) : null,
+        }),
+    });
+}