docs(research): archive shadow replay rounds and analyzer round-1 fixes

- rejoin round-1 rows hidden by terminal-event handling: normalize denied_with_reason, collapse forwarded double terminal rows, print quarantine counts, add --before window bound
- archive rounds 1-3 reports with blind-deny protocol, cross-round totals, and PIEXTENSIO-11 latency evidence
This commit is contained in:
2026-08-17 18:59:19 +08:00
parent ea7d93d63f
commit 850f36c7a4
8 changed files with 211 additions and 22 deletions
@@ -178,6 +178,8 @@ function humanFromResolution(
case "approved_for_serving_session":
return { decision: "allow", state: resolution, denialReason: reason };
case "denied":
// Upstream's provide-reason deny writes `denied_with_reason`.
case "denied_with_reason":
return { decision: "deny", state: resolution, denialReason: reason };
default:
return { error: "terminal_event_unreadable" };
@@ -275,8 +277,21 @@ export function analyzeShadowReviewLog(events: readonly ReviewEvent[]): AnalyzeR
continue;
}
if (terminalEvents.length > 1) {
quarantine("multiple_human_decisions");
continue;
// Upstream's forwarded decision path double-writes the terminal
// event with the same requestId and resolution in adjacent file
// order (round 1: two `approved` or two `denied_with_reason`
// rows in the same second). Identical-resolution duplicates are
// that pattern, not an integrity fault: collapse to the first
// row. Conflicting resolutions stay quarantined — the analyzer
// must never pick a convenient outcome among alternatives
// (PIEXTENSIO-9).
const distinct = new Set(
terminalEvents.map((t) => asString(t.resolution) ?? ""),
);
if (distinct.size > 1) {
quarantine("multiple_human_decisions");
continue;
}
}
const terminalEvent = terminalEvents[0] as ReviewEvent;
const human = humanFromResolution(
@@ -14,6 +14,7 @@ const USAGE = `usage: analyze-shadow <review-jsonl-path> [options]
options:
--after <iso8601> only consider events with timestamp >= this instant
--before <iso8601> only consider events with timestamp <= this instant
--help show this help
The report is diagnostic-grade: the join reconstructs enrollment and human
@@ -23,27 +24,33 @@ and matrix numbers must not be used as promotion-grade evidence.`;
interface CliOptions {
readonly path: string;
readonly after: Date | null;
readonly before: Date | null;
}
function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
const args = argv.slice(2);
let path: string | undefined;
let after: Date | null = null;
let before: Date | null = null;
for (let i = 0; i < args.length; i += 1) {
const arg = args[i] as string;
if (arg === "--help" || arg === "-h") {
return { error: USAGE };
}
if (arg === "--after") {
if (arg === "--after" || arg === "--before") {
const value = args[i + 1];
if (value === undefined) {
return { error: "--after requires an ISO-8601 timestamp" };
return { error: `${arg} requires an ISO-8601 timestamp` };
}
const parsed = new Date(value);
if (Number.isNaN(parsed.getTime())) {
return { error: `invalid --after timestamp: ${value}` };
return { error: `invalid ${arg} timestamp: ${value}` };
}
if (arg === "--after") {
after = parsed;
} else {
before = parsed;
}
after = parsed;
i += 1;
continue;
}
@@ -58,7 +65,7 @@ function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
if (path === undefined) {
return { error: "missing input path" };
}
return { path, after };
return { path, after, before };
}
function parseLine(line: string, lineNo: number): ReviewEvent | null {
@@ -116,15 +123,18 @@ function main(): void {
.map((line, index) => parseLine(line, index + 1))
.filter((evt): evt is ReviewEvent => evt !== null)
.filter((evt) => {
if (parsed.after === null) {
return true;
}
const ts = typeof evt.timestamp === "string" ? evt.timestamp : null;
if (ts === null) {
return true;
}
const time = new Date(ts).getTime();
return Number.isNaN(time) || time >= parsed.after.getTime();
if (parsed.after !== null && !Number.isNaN(time) && time < parsed.after.getTime()) {
return false;
}
if (parsed.before !== null && !Number.isNaN(time) && time > parsed.before.getTime()) {
return false;
}
return true;
});
const { enrollments, metrics } = analyzeShadowReviewLog(events);
@@ -143,6 +153,17 @@ function main(): void {
out.write(`human-join coverage: ${fmtRate(metrics.humanJoinCoverage)}\n`);
out.write(`judgment coverage: ${fmtRate(metrics.judgmentCoverage)}\n\n`);
const quarantineEntries = Object.entries(metrics.quarantined).sort(
([a], [b]) => a.localeCompare(b),
);
if (quarantineEntries.length > 0) {
out.write("quarantined rows:\n");
for (const [category, count] of quarantineEntries) {
out.write(` ${category}: ${count}\n`);
}
out.write("\n");
}
out.write("comparison matrix [verdict|human]:\n");
const keys = Object.keys(metrics.matrix).sort();
if (keys.length === 0) {
@@ -157,6 +157,53 @@ describe("analyzeShadowReviewLog — attribution", () => {
});
describe("analyzeShadowReviewLog — integrity", () => {
it("normalizes denied_with_reason to a human deny (false-allow rows stay visible)", () => {
const { metrics } = analyzeShadowReviewLog(
lifecycle({ verdict: "allow", resolution: "denied_with_reason" }),
);
expect(metrics.quarantined).toEqual({});
expect(metrics.matrix).toEqual({ "allow|deny": 1 });
expect(metrics.falseAllows).toBe(1);
expect(metrics.falseAllowRate).toBe(1);
});
it("collapses the forwarded path's identical double terminal rows", () => {
const events: ReviewEvent[] = [
{ event: "authorizer_chain_resolved", requestId: "f", links: ["ai-bash-judge"] },
{
event: "ai_bash_judge.result",
requestId: "f",
resultKind: "preflight_defer",
verdict: null,
code: "missing_structured_input",
origin: "forwarded",
},
{ event: "permission_request.approved", requestId: "f", resolution: "approved" },
{ event: "permission_request.approved", requestId: "f", resolution: "approved" },
];
const { metrics } = analyzeShadowReviewLog(events);
expect(metrics.quarantined).toEqual({});
expect(metrics.joined).toBe(1);
expect(metrics.preflightDefers).toBe(1);
expect(metrics.matrix).toEqual({});
});
it("still quarantines conflicting duplicate human decisions", () => {
const events: ReviewEvent[] = [
{ event: "authorizer_chain_resolved", requestId: "c", links: ["ai-bash-judge"] },
{
event: "ai_bash_judge.result",
requestId: "c",
resultKind: "judgment",
verdict: "allow",
},
{ event: "permission_request.approved", requestId: "c", resolution: "approved" },
{ event: "permission_request.denied", requestId: "c", resolution: "denied" },
];
const { metrics } = analyzeShadowReviewLog(events);
expect(metrics.quarantined).toEqual({ multiple_human_decisions: 1 });
expect(metrics.joined).toBe(0);
});
it("quarantines a duplicate judge result", () => {
const base = lifecycle({ verdict: "allow" });
const dup = base.map((e) => e).concat([