feat(ai-judge): offline shadow analyzer with reconstructed join

2026-10-05 20:02:55 +08:00 · 2026-08-17 16:58:10 +08:00
parent 42f0a0aaab
commit 6407b7429a
3 changed files with 871 additions and 0 deletions
@@ -0,0 +1,174 @@
+#!/usr/bin/env node
+/**
+ * Offline Shadow analyzer CLI (diagnostic grade).
+ *
+ * Reads the permission-system review JSONL and prints the PIEXTENSIO-9
+ * metrics computed over the reconstructed requestId join. Output is
+ * metadata-only: no command text from the source log is echoed.
+ */
+
+import { readFileSync } from "node:fs";
+import { analyzeShadowReviewLog, type ReviewEvent } from "./analyze";
+
+const USAGE = `usage: analyze-shadow <review-jsonl-path> [options]
+
+options:
+  --after <iso8601>   only consider events with timestamp >= this instant
+  --help              show this help
+
+The report is diagnostic-grade: the join reconstructs enrollment and human
+decisions from permission-system events (no upstream changes), so coverage
+and matrix numbers must not be used as promotion-grade evidence.`;
+
+interface CliOptions {
+    readonly path: string;
+    readonly after: Date | null;
+}
+
+function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
+    const args = argv.slice(2);
+    let path: string | undefined;
+    let after: Date | null = null;
+    for (let i = 0; i < args.length; i += 1) {
+        const arg = args[i] as string;
+        if (arg === "--help" || arg === "-h") {
+            return { error: USAGE };
+        }
+        if (arg === "--after") {
+            const value = args[i + 1];
+            if (value === undefined) {
+                return { error: "--after requires an ISO-8601 timestamp" };
+            }
+            const parsed = new Date(value);
+            if (Number.isNaN(parsed.getTime())) {
+                return { error: `invalid --after timestamp: ${value}` };
+            }
+            after = parsed;
+            i += 1;
+            continue;
+        }
+        if (arg.startsWith("--")) {
+            return { error: `unknown option: ${arg}` };
+        }
+        if (path !== undefined) {
+            return { error: "multiple input paths given" };
+        }
+        path = arg;
+    }
+    if (path === undefined) {
+        return { error: "missing input path" };
+    }
+    return { path, after };
+}
+
+function parseLine(line: string, lineNo: number): ReviewEvent | null {
+    if (line.trim().length === 0) {
+        return null;
+    }
+    try {
+        return JSON.parse(line) as ReviewEvent;
+    } catch {
+        process.stderr.write(
+            `warning: skipping unparseable line ${lineNo}\n`,
+        );
+        return null;
+    }
+}
+
+function fmtRate(value: number | null): string {
+    if (value === null) {
+        return "N/A";
+    }
+    return `${(value * 100).toFixed(1)}%`;
+}
+
+function fmtLatency(stats: {
+    p50: number;
+    p95: number;
+    max: number;
+    missing: number;
+} | null): string {
+    if (stats === null) {
+        return "N/A";
+    }
+    return `p50=${stats.p50}ms p95=${stats.p95}ms max=${stats.max}ms missing=${stats.missing}`;
+}
+
+function main(): void {
+    const parsed = parseArgs(process.argv);
+    if ("error" in parsed) {
+        process.stderr.write(`${parsed.error}\n`);
+        process.exit(parsed.error === USAGE ? 0 : 1);
+    }
+
+    let raw: string;
+    try {
+        raw = readFileSync(parsed.path, "utf-8");
+    } catch (error) {
+        process.stderr.write(
+            `error: cannot read ${parsed.path}: ${error instanceof Error ? error.message : String(error)}\n`,
+        );
+        process.exit(1);
+    }
+
+    const events = raw
+        .split("\n")
+        .map((line, index) => parseLine(line, index + 1))
+        .filter((evt): evt is ReviewEvent => evt !== null)
+        .filter((evt) => {
+            if (parsed.after === null) {
+                return true;
+            }
+            const ts = typeof evt.timestamp === "string" ? evt.timestamp : null;
+            if (ts === null) {
+                return true;
+            }
+            const time = new Date(ts).getTime();
+            return Number.isNaN(time) || time >= parsed.after.getTime();
+        });
+
+    const { enrollments, metrics } = analyzeShadowReviewLog(events);
+    const out = process.stdout;
+
+    out.write("AI Bash Judge — Shadow diagnostic report\n");
+    out.write("grade: DIAGNOSTIC (reconstructed join; not promotion-grade)\n");
+    out.write(`asOf: ${new Date().toISOString()}\n`);
+    out.write(`source: ${parsed.path}\n\n`);
+
+    out.write(`enrollments (N): ${enrollments}\n`);
+    out.write(`joined rows: ${metrics.joined}\n`);
+    out.write(`joined judgments: ${metrics.joinedJudgments}\n\n`);
+
+    out.write(`completion coverage: ${fmtRate(metrics.completionCoverage)}\n`);
+    out.write(`human-join coverage: ${fmtRate(metrics.humanJoinCoverage)}\n`);
+    out.write(`judgment coverage: ${fmtRate(metrics.judgmentCoverage)}\n\n`);
+
+    out.write("comparison matrix [verdict|human]:\n");
+    const keys = Object.keys(metrics.matrix).sort();
+    if (keys.length === 0) {
+        out.write("  (empty)\n");
+    }
+    for (const key of keys) {
+        out.write(`  ${key}: ${metrics.matrix[key]}\n`);
+    }
+    out.write("\n");
+
+    out.write(`false allows: ${metrics.falseAllows}`);
+    out.write(` (rate ${fmtRate(metrics.falseAllowRate)})\n`);
+    out.write(
+        `conservative: deny ${metrics.conservativeDeny}, defer ${metrics.conservativeDefer} (rate ${fmtRate(metrics.conservativeRate)})\n\n`,
+    );
+
+    out.write(`preflight defers: ${metrics.preflightDefers}\n`);
+    out.write(`infrastructure failures: ${metrics.infrastructureFailures}\n`);
+    const codes = Object.keys(metrics.infrastructureByCode).sort();
+    for (const code of codes) {
+        out.write(`  ${code}: ${metrics.infrastructureByCode[code]}\n`);
+    }
+    out.write("\n");
+
+    out.write(`judge latency: ${fmtLatency(metrics.judgeLatency)}\n`);
+    out.write(`model latency: ${fmtLatency(metrics.modelLatency)}\n`);
+}
+
+main();