pi-extensions/packages/pi-permission-ai-judge/tools/replay-qualify.ts

/**
 * PIEXTENSIO-24 qualification logic behind `corpus-replay --strict`.
 *
 * The light qualification standard (ADR 0008 post-promotion era): a full
 * corpus replay where every case matches, no row is an infrastructure
 * failure, and every judgment's latency stays within the run's timeout
 * budget. This is owner-observed *compatibility* data for the advisory
 * model catalog — never a safety certification, and never a runtime
 * Enforce gate.
 *
 * Pure functions only: the harness feeds it rows and turns the result
 * into an exit code, so the standard itself stays testable offline.
 */

/** One replay row as emitted by the corpus-replay harness. The
 * harness-reported `match` is report data only — qualification recomputes
 * agreement from `expected`/`verdict` so contradictory rows cannot
 * qualify. */
export type ReplayRow = {
    readonly case: string;
    readonly expected: string;
    readonly verdict: string | null;
    readonly match: boolean;
    readonly latencyMs: number | null;
    readonly resultKind?: string;
};

export interface ReplayQualification {
    /** True only when every light-standard check passes. */
    readonly qualified: boolean;
    /** Human-readable failure reasons (empty when qualified). */
    readonly reasons: readonly string[];
    readonly totalCases: number;
    readonly matched: number;
    /** Case ids whose verdict differed from the expected one. */
    readonly mismatches: readonly string[];
    /** Case ids that returned an infrastructure failure instead of a verdict. */
    readonly infrastructureFailures: readonly string[];
    /** Latency percentiles over judgment rows; null when none measured. */
    readonly latencyMs: {
        readonly p50: number | null;
        readonly p95: number | null;
        readonly max: number | null;
    };
}

export interface QualifyOptions {
    /** Latency budget in ms; a judgment slower than this disqualifies. */
    readonly budgetMs: number;
}

function percentile(sorted: readonly number[], rank: number): number | null {
    if (sorted.length === 0) {
        return null;
    }
    const index = Math.min(sorted.length - 1, Math.floor(rank * (sorted.length - 1)));
    return sorted[index] as number;
}

function usableLatency(value: unknown): value is number {
    return typeof value === "number" && Number.isFinite(value);
}

/** Apply the light qualification standard to a completed replay's rows. */
export function qualifyReplay(
    rows: readonly ReplayRow[],
    options: QualifyOptions,
): ReplayQualification {
    const reasons: string[] = [];
    if (rows.length === 0) {
        reasons.push("no rows: qualification requires a full corpus replay");
    }

    const judgments = rows.filter((row) => row.verdict !== null);
    const mismatches = judgments
        .filter((row) => row.verdict !== row.expected)
        .map((row) => row.case);
    if (mismatches.length > 0) {
        reasons.push(`verdict mismatch: ${mismatches.join(", ")}`);
    }

    const infrastructureFailures = rows
        .filter((row) => row.verdict === null)
        .map((row) => row.case);
    if (infrastructureFailures.length > 0) {
        reasons.push(
            `infrastructure failure: ${infrastructureFailures.join(", ")}`,
        );
    }

    // A judgment row without a usable latency is unqualifiable: the
    // catalog's latency summary must be measured data, never assumed.
    const missingLatency = judgments.filter((row) => !usableLatency(row.latencyMs));
    if (missingLatency.length > 0) {
        reasons.push(
            `judgment without usable latency: ${missingLatency
                .map((row) => row.case)
                .join(", ")}`,
        );
    }

    const latencies = judgments
        .map((row) => row.latencyMs)
        .filter(usableLatency)
        .sort((a, b) => a - b);
    const overBudget = judgments.filter(
        (row) => usableLatency(row.latencyMs) && row.latencyMs > options.budgetMs,
    );
    if (overBudget.length > 0) {
        reasons.push(
            `latency beyond ${options.budgetMs}ms budget: ${overBudget
                .map((row) => `${row.case} (${row.latencyMs}ms)`)
                .join(", ")}`,
        );
    }

    const matched = judgments.filter((row) => row.verdict === row.expected).length;
    return Object.freeze({
        qualified: reasons.length === 0,
        reasons: Object.freeze(reasons),
        totalCases: rows.length,
        matched,
        mismatches: Object.freeze(mismatches),
        infrastructureFailures: Object.freeze(infrastructureFailures),
        latencyMs: Object.freeze({
            p50: percentile(latencies, 0.5),
            p95: percentile(latencies, 0.95),
            max: latencies.length > 0 ? (latencies.at(-1) as number) : null,
        }),
    });
}