feat(ai-judge): add advisory model catalog and strict corpus replay

- add versioned advisory model catalog shipped with the package and a fail-closed loader
- annotate the enforce session notice for untested, deprecated, and revoked models
- add --strict to corpus-replay with 0/1/2 exit codes and reject strict subset runs
- extract replay qualification into a pure module that recomputes matches and validates latencies
- remove the documented-but-unimplemented --thinking flag and stamp reports with a corpus version
- qualify gpt-5.6-sol as the first recommended entry and archive three real replay reports
- revise the corpus to 2026-08-21.2 changing unclear-forward expected defer to deny
This commit is contained in:
2026-08-21 23:11:40 +08:00
parent 061c60c344
commit 7987fc7a3c
13 changed files with 1544 additions and 39 deletions
@@ -0,0 +1,41 @@
import { describe, expect, it } from "vitest";
import { parseArgs } from "../tools/corpus-replay";
/**
* PIEXTENSIO-24 CLI contract: qualification (--strict) is defined over the
* full corpus only — a --case subset is observation-only and must never be
* able to report qualified. Exit-code mapping itself lives in main()
* (0 qualified / 1 harness failure / 2 qualification failure) and needs a
* live model; the parse contract is what keeps the standard honest.
*/
const BASE = ["node", "corpus-replay.ts", "--provider", "p", "--model", "m"];
describe("corpus-replay parseArgs", () => {
it("accepts --strict with the full corpus", () => {
const parsed = parseArgs([...BASE, "--strict"]);
expect("error" in parsed && parsed.error).toBeFalsy();
expect("strict" in parsed && parsed.strict).toBe(true);
expect("cases" in parsed && parsed.cases).toBeNull();
});
it("rejects --strict together with --case", () => {
const parsed = parseArgs([...BASE, "--strict", "--case", "a,b"]);
expect("error" in parsed && /full corpus/.test(parsed.error)).toBe(true);
});
it("keeps --case usable without --strict", () => {
const parsed = parseArgs([...BASE, "--case", "a,b"]);
expect("error" in parsed && parsed.error).toBeFalsy();
expect("cases" in parsed && parsed.cases).toEqual(new Set(["a", "b"]));
expect("strict" in parsed && parsed.strict).toBe(false);
});
it("rejects unknown options and missing values", () => {
expect(
"error" in parseArgs([...BASE, "--thinking", "high"]),
).toBe(true);
expect("error" in parseArgs([...BASE, "--out"])).toBe(true);
expect("error" in parseArgs(["node", "corpus-replay.ts"])).toBe(true);
});
});