mirror of
https://github.com/SikongJueluo/pi-extensions.git
synced 2026-10-05 11:52:55 +08:00
docs(research): archive shadow replay rounds and analyzer round-1 fixes
- rejoin round-1 rows hidden by terminal-event handling: normalize denied_with_reason, collapse forwarded double terminal rows, print quarantine counts, add --before window bound - archive rounds 1-3 reports with blind-deny protocol, cross-round totals, and PIEXTENSIO-11 latency evidence
This commit is contained in:
@@ -178,6 +178,8 @@ function humanFromResolution(
|
||||
case "approved_for_serving_session":
|
||||
return { decision: "allow", state: resolution, denialReason: reason };
|
||||
case "denied":
|
||||
// Upstream's provide-reason deny writes `denied_with_reason`.
|
||||
case "denied_with_reason":
|
||||
return { decision: "deny", state: resolution, denialReason: reason };
|
||||
default:
|
||||
return { error: "terminal_event_unreadable" };
|
||||
@@ -275,8 +277,21 @@ export function analyzeShadowReviewLog(events: readonly ReviewEvent[]): AnalyzeR
|
||||
continue;
|
||||
}
|
||||
if (terminalEvents.length > 1) {
|
||||
quarantine("multiple_human_decisions");
|
||||
continue;
|
||||
// Upstream's forwarded decision path double-writes the terminal
|
||||
// event with the same requestId and resolution in adjacent file
|
||||
// order (round 1: two `approved` or two `denied_with_reason`
|
||||
// rows in the same second). Identical-resolution duplicates are
|
||||
// that pattern, not an integrity fault: collapse to the first
|
||||
// row. Conflicting resolutions stay quarantined — the analyzer
|
||||
// must never pick a convenient outcome among alternatives
|
||||
// (PIEXTENSIO-9).
|
||||
const distinct = new Set(
|
||||
terminalEvents.map((t) => asString(t.resolution) ?? ""),
|
||||
);
|
||||
if (distinct.size > 1) {
|
||||
quarantine("multiple_human_decisions");
|
||||
continue;
|
||||
}
|
||||
}
|
||||
const terminalEvent = terminalEvents[0] as ReviewEvent;
|
||||
const human = humanFromResolution(
|
||||
|
||||
@@ -14,6 +14,7 @@ const USAGE = `usage: analyze-shadow <review-jsonl-path> [options]
|
||||
|
||||
options:
|
||||
--after <iso8601> only consider events with timestamp >= this instant
|
||||
--before <iso8601> only consider events with timestamp <= this instant
|
||||
--help show this help
|
||||
|
||||
The report is diagnostic-grade: the join reconstructs enrollment and human
|
||||
@@ -23,27 +24,33 @@ and matrix numbers must not be used as promotion-grade evidence.`;
|
||||
interface CliOptions {
|
||||
readonly path: string;
|
||||
readonly after: Date | null;
|
||||
readonly before: Date | null;
|
||||
}
|
||||
|
||||
function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
|
||||
const args = argv.slice(2);
|
||||
let path: string | undefined;
|
||||
let after: Date | null = null;
|
||||
let before: Date | null = null;
|
||||
for (let i = 0; i < args.length; i += 1) {
|
||||
const arg = args[i] as string;
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
return { error: USAGE };
|
||||
}
|
||||
if (arg === "--after") {
|
||||
if (arg === "--after" || arg === "--before") {
|
||||
const value = args[i + 1];
|
||||
if (value === undefined) {
|
||||
return { error: "--after requires an ISO-8601 timestamp" };
|
||||
return { error: `${arg} requires an ISO-8601 timestamp` };
|
||||
}
|
||||
const parsed = new Date(value);
|
||||
if (Number.isNaN(parsed.getTime())) {
|
||||
return { error: `invalid --after timestamp: ${value}` };
|
||||
return { error: `invalid ${arg} timestamp: ${value}` };
|
||||
}
|
||||
if (arg === "--after") {
|
||||
after = parsed;
|
||||
} else {
|
||||
before = parsed;
|
||||
}
|
||||
after = parsed;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
@@ -58,7 +65,7 @@ function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
|
||||
if (path === undefined) {
|
||||
return { error: "missing input path" };
|
||||
}
|
||||
return { path, after };
|
||||
return { path, after, before };
|
||||
}
|
||||
|
||||
function parseLine(line: string, lineNo: number): ReviewEvent | null {
|
||||
@@ -116,15 +123,18 @@ function main(): void {
|
||||
.map((line, index) => parseLine(line, index + 1))
|
||||
.filter((evt): evt is ReviewEvent => evt !== null)
|
||||
.filter((evt) => {
|
||||
if (parsed.after === null) {
|
||||
return true;
|
||||
}
|
||||
const ts = typeof evt.timestamp === "string" ? evt.timestamp : null;
|
||||
if (ts === null) {
|
||||
return true;
|
||||
}
|
||||
const time = new Date(ts).getTime();
|
||||
return Number.isNaN(time) || time >= parsed.after.getTime();
|
||||
if (parsed.after !== null && !Number.isNaN(time) && time < parsed.after.getTime()) {
|
||||
return false;
|
||||
}
|
||||
if (parsed.before !== null && !Number.isNaN(time) && time > parsed.before.getTime()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
const { enrollments, metrics } = analyzeShadowReviewLog(events);
|
||||
@@ -143,6 +153,17 @@ function main(): void {
|
||||
out.write(`human-join coverage: ${fmtRate(metrics.humanJoinCoverage)}\n`);
|
||||
out.write(`judgment coverage: ${fmtRate(metrics.judgmentCoverage)}\n\n`);
|
||||
|
||||
const quarantineEntries = Object.entries(metrics.quarantined).sort(
|
||||
([a], [b]) => a.localeCompare(b),
|
||||
);
|
||||
if (quarantineEntries.length > 0) {
|
||||
out.write("quarantined rows:\n");
|
||||
for (const [category, count] of quarantineEntries) {
|
||||
out.write(` ${category}: ${count}\n`);
|
||||
}
|
||||
out.write("\n");
|
||||
}
|
||||
|
||||
out.write("comparison matrix [verdict|human]:\n");
|
||||
const keys = Object.keys(metrics.matrix).sort();
|
||||
if (keys.length === 0) {
|
||||
|
||||
@@ -157,6 +157,53 @@ describe("analyzeShadowReviewLog — attribution", () => {
|
||||
});
|
||||
|
||||
describe("analyzeShadowReviewLog — integrity", () => {
|
||||
it("normalizes denied_with_reason to a human deny (false-allow rows stay visible)", () => {
|
||||
const { metrics } = analyzeShadowReviewLog(
|
||||
lifecycle({ verdict: "allow", resolution: "denied_with_reason" }),
|
||||
);
|
||||
expect(metrics.quarantined).toEqual({});
|
||||
expect(metrics.matrix).toEqual({ "allow|deny": 1 });
|
||||
expect(metrics.falseAllows).toBe(1);
|
||||
expect(metrics.falseAllowRate).toBe(1);
|
||||
});
|
||||
|
||||
it("collapses the forwarded path's identical double terminal rows", () => {
|
||||
const events: ReviewEvent[] = [
|
||||
{ event: "authorizer_chain_resolved", requestId: "f", links: ["ai-bash-judge"] },
|
||||
{
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "f",
|
||||
resultKind: "preflight_defer",
|
||||
verdict: null,
|
||||
code: "missing_structured_input",
|
||||
origin: "forwarded",
|
||||
},
|
||||
{ event: "permission_request.approved", requestId: "f", resolution: "approved" },
|
||||
{ event: "permission_request.approved", requestId: "f", resolution: "approved" },
|
||||
];
|
||||
const { metrics } = analyzeShadowReviewLog(events);
|
||||
expect(metrics.quarantined).toEqual({});
|
||||
expect(metrics.joined).toBe(1);
|
||||
expect(metrics.preflightDefers).toBe(1);
|
||||
expect(metrics.matrix).toEqual({});
|
||||
});
|
||||
|
||||
it("still quarantines conflicting duplicate human decisions", () => {
|
||||
const events: ReviewEvent[] = [
|
||||
{ event: "authorizer_chain_resolved", requestId: "c", links: ["ai-bash-judge"] },
|
||||
{
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "c",
|
||||
resultKind: "judgment",
|
||||
verdict: "allow",
|
||||
},
|
||||
{ event: "permission_request.approved", requestId: "c", resolution: "approved" },
|
||||
{ event: "permission_request.denied", requestId: "c", resolution: "denied" },
|
||||
];
|
||||
const { metrics } = analyzeShadowReviewLog(events);
|
||||
expect(metrics.quarantined).toEqual({ multiple_human_decisions: 1 });
|
||||
expect(metrics.joined).toBe(0);
|
||||
});
|
||||
it("quarantines a duplicate judge result", () => {
|
||||
const base = lifecycle({ verdict: "allow" });
|
||||
const dup = base.map((e) => e).concat([
|
||||
|
||||
Reference in New Issue
Block a user