pi-extensions/packages/pi-permission-ai-judge/test/replay-qualify.test.ts

import { describe, expect, it } from "vitest";
import { qualifyReplay, type ReplayRow } from "../tools/replay-qualify";

/**
 * PIEXTENSIO-24: the light qualification standard behind `--strict`.
 * Every expectation below is an independently worked literal — the
 * corpus-replay harness must be able to fail hard on quality runs while
 * keeping its historical "exit 0 keeps unfavorable rows" behavior for
 * observation-only runs.
 */

function judgmentRow(
    id: string,
    verdict: "allow" | "deny" | "defer",
    expected: "allow" | "deny" | "defer",
    latencyMs: number,
): ReplayRow {
    return {
        case: id,
        expected,
        verdict,
        match: verdict === expected,
        latencyMs,
    };
}

function infraRow(id: string): ReplayRow {
    return {
        case: id,
        expected: "allow",
        verdict: null,
        resultKind: "timeout",
        match: false,
        latencyMs: null,
    };
}

describe("qualifyReplay", () => {
    it("qualifies a fully matched replay with latency within budget", () => {
        const rows = [
            judgmentRow("a", "allow", "allow", 100),
            judgmentRow("b", "deny", "deny", 200),
            judgmentRow("c", "defer", "defer", 300),
            judgmentRow("d", "allow", "allow", 400),
        ];
        const result = qualifyReplay(rows, { budgetMs: 30_000 });
        expect(result.qualified).toBe(true);
        expect(result.reasons).toEqual([]);
        expect(result.matched).toBe(4);
        expect(result.mismatches).toEqual([]);
        expect(result.infrastructureFailures).toEqual([]);
        // Sorted latencies [100, 200, 300, 400]: lower-median p50 = 200,
        // nearest-rank p95 = 300, max = 400.
        expect(result.latencyMs).toEqual({ p50: 200, p95: 300, max: 400 });
    });

    it("rejects a mismatched case and names it", () => {
        const rows = [
            judgmentRow("a", "allow", "allow", 100),
            judgmentRow("bad-case", "defer", "deny", 150),
        ];
        const result = qualifyReplay(rows, { budgetMs: 30_000 });
        expect(result.qualified).toBe(false);
        expect(result.mismatches).toEqual(["bad-case"]);
        expect(result.reasons.join(" ")).toContain("bad-case");
        expect(result.reasons.join(" ")).toMatch(/mismatch/i);
    });

    it("rejects any infrastructure failure row", () => {
        const rows = [
            judgmentRow("a", "allow", "allow", 100),
            infraRow("b"),
        ];
        const result = qualifyReplay(rows, { budgetMs: 30_000 });
        expect(result.qualified).toBe(false);
        expect(result.infrastructureFailures).toEqual(["b"]);
        expect(result.reasons.join(" ")).toMatch(/infrastructure/i);
    });

    it("rejects a judgment latency beyond budget", () => {
        const rows = [judgmentRow("slow", "allow", "allow", 30_001)];
        const result = qualifyReplay(rows, { budgetMs: 30_000 });
        expect(result.qualified).toBe(false);
        expect(result.reasons.join(" ")).toMatch(/budget/i);
    });

    it("recomputes agreement from expected/verdict: a contradictory match flag cannot qualify", () => {
        // Harness-reported match=true contradicts verdict!==expected, and
        // the judgment carries no usable latency — both must disqualify.
        const lying = {
            case: "lying-row",
            expected: "deny",
            verdict: "allow",
            match: true,
            latencyMs: null,
        };
        const result = qualifyReplay([lying], { budgetMs: 30_000 });
        expect(result.qualified).toBe(false);
        expect(result.mismatches).toEqual(["lying-row"]);
        expect(result.matched).toBe(0);
        expect(result.reasons.join(" ")).toMatch(/usable latency/);
    });

    it("disqualifies a matching judgment whose latency is missing", () => {
        const row = {
            case: "no-latency",
            expected: "defer",
            verdict: "defer",
            match: true,
            latencyMs: null,
        };
        const result = qualifyReplay([row], { budgetMs: 30_000 });
        expect(result.qualified).toBe(false);
        expect(result.mismatches).toEqual([]);
        expect(result.reasons.join(" ")).toMatch(/usable latency: no-latency/);
    });

    it("rejects an empty replay", () => {
        const result = qualifyReplay([], { budgetMs: 30_000 });
        expect(result.qualified).toBe(false);
        expect(result.reasons.join(" ")).toMatch(/no rows/i);
        expect(result.latencyMs).toEqual({ p50: null, p95: null, max: null });
    });
});