mirror of
https://github.com/SikongJueluo/pi-extensions.git
synced 2026-10-05 20:02:55 +08:00
feat(ai-judge): add advisory model catalog and strict corpus replay
- add versioned advisory model catalog shipped with the package and a fail-closed loader - annotate the enforce session notice for untested, deprecated, and revoked models - add --strict to corpus-replay with 0/1/2 exit codes and reject strict subset runs - extract replay qualification into a pure module that recomputes matches and validates latencies - remove the documented-but-unimplemented --thinking flag and stamp reports with a corpus version - qualify gpt-5.6-sol as the first recommended entry and archive three real replay reports - revise the corpus to 2026-08-21.2 changing unclear-forward expected defer to deny
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseArgs } from "../tools/corpus-replay";
|
||||
|
||||
/**
|
||||
* PIEXTENSIO-24 CLI contract: qualification (--strict) is defined over the
|
||||
* full corpus only — a --case subset is observation-only and must never be
|
||||
* able to report qualified. Exit-code mapping itself lives in main()
|
||||
* (0 qualified / 1 harness failure / 2 qualification failure) and needs a
|
||||
* live model; the parse contract is what keeps the standard honest.
|
||||
*/
|
||||
|
||||
const BASE = ["node", "corpus-replay.ts", "--provider", "p", "--model", "m"];
|
||||
|
||||
describe("corpus-replay parseArgs", () => {
|
||||
it("accepts --strict with the full corpus", () => {
|
||||
const parsed = parseArgs([...BASE, "--strict"]);
|
||||
expect("error" in parsed && parsed.error).toBeFalsy();
|
||||
expect("strict" in parsed && parsed.strict).toBe(true);
|
||||
expect("cases" in parsed && parsed.cases).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects --strict together with --case", () => {
|
||||
const parsed = parseArgs([...BASE, "--strict", "--case", "a,b"]);
|
||||
expect("error" in parsed && /full corpus/.test(parsed.error)).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps --case usable without --strict", () => {
|
||||
const parsed = parseArgs([...BASE, "--case", "a,b"]);
|
||||
expect("error" in parsed && parsed.error).toBeFalsy();
|
||||
expect("cases" in parsed && parsed.cases).toEqual(new Set(["a", "b"]));
|
||||
expect("strict" in parsed && parsed.strict).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects unknown options and missing values", () => {
|
||||
expect(
|
||||
"error" in parseArgs([...BASE, "--thinking", "high"]),
|
||||
).toBe(true);
|
||||
expect("error" in parseArgs([...BASE, "--out"])).toBe(true);
|
||||
expect("error" in parseArgs(["node", "corpus-replay.ts"])).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user