feat(ai-judge): judge-owned audit log with local health gate (ADR 0006)

This commit is contained in:
2026-08-19 23:45:43 +08:00
parent 13cb19c349
commit 232bad549e
9 changed files with 498 additions and 25 deletions
+15 -1
View File
@@ -122,7 +122,21 @@ report.
## 5. Log contract (pairing keys)
Event order per request, keyed by `requestId`:
**Two logs exist since ADR 0006** (Judge-owned audit + permission review):
1. **Judge audit log** `~/.pi/agent/extensions/pi-permission-ai-judge/logs/audit.jsonl`
— judge-owned events only: `ai_bash_judge.enrolled` (ask received:
requestId, origin, surface, command — the cohort denominator) and a
mirror of every `ai_bash_judge.result` row. Append + fsync per record;
write failure is sticky-unhealthy and refuses Enforce authority.
2. **Permission review log** (below) — still the source for human-decision
attribution and upstream chain events.
The analyzer takes enrollments from the audit log via `--audit <path>`
(dual-log mode); without the flag it reconstructs enrollment from the
review log as before.
Event order per request in the review log, keyed by `requestId`:
```
permission_request.waiting
@@ -15,6 +15,10 @@ const USAGE = `usage: analyze-shadow <review-jsonl-path> [options]
options:
--after <iso8601> only consider events with timestamp >= this instant
--before <iso8601> only consider events with timestamp <= this instant
--audit <path> Judge-owned audit JSONL (ADR 0006); enrollment (the
denominator N) is then taken from its
ai_bash_judge.enrolled rows instead of the reconstructed
permission-system chain events
--help show this help
The report is diagnostic-grade: the join reconstructs enrollment and human
@@ -25,6 +29,7 @@ interface CliOptions {
readonly path: string;
readonly after: Date | null;
readonly before: Date | null;
readonly audit: string | null;
}
function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
@@ -32,15 +37,21 @@ function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
let path: string | undefined;
let after: Date | null = null;
let before: Date | null = null;
let audit: string | null = null;
for (let i = 0; i < args.length; i += 1) {
const arg = args[i] as string;
if (arg === "--help" || arg === "-h") {
return { error: USAGE };
}
if (arg === "--after" || arg === "--before") {
if (arg === "--after" || arg === "--before" || arg === "--audit") {
const value = args[i + 1];
if (value === undefined) {
return { error: `${arg} requires an ISO-8601 timestamp` };
return { error: `${arg} requires a value` };
}
if (arg === "--audit") {
audit = value;
i += 1;
continue;
}
const parsed = new Date(value);
if (Number.isNaN(parsed.getTime())) {
@@ -65,7 +76,7 @@ function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
if (path === undefined) {
return { error: "missing input path" };
}
return { path, after, before };
return { path, after, before, audit };
}
function parseLine(line: string, lineNo: number): ReviewEvent | null {
@@ -137,13 +148,63 @@ function main(): void {
return true;
});
const { enrollments, metrics } = analyzeShadowReviewLog(events);
// ADR 0006 dual-log mode: with --audit, the denominator comes from the
// Judge's own enrolled rows (asks it received), and the reconstructed
// permission-system enrollment proxy is dropped to avoid a second
// denominator source. Human decisions still come from the permission
// log (attribution join).
let joined = events;
if (parsed.audit !== null) {
let auditRaw: string;
try {
auditRaw = readFileSync(parsed.audit, "utf-8");
} catch (error) {
process.stderr.write(
`error: cannot read audit log ${parsed.audit}: ${error instanceof Error ? error.message : String(error)}\n`,
);
process.exit(1);
}
const auditEnrolled = auditRaw
.split("\n")
.map((line, index) => parseLine(line, index + 1))
.filter((evt): evt is ReviewEvent => evt !== null)
.filter((evt) => evt.event === "ai_bash_judge.enrolled")
.filter((evt) => {
const ts = typeof evt.timestamp === "string" ? evt.timestamp : null;
if (ts === null) {
return true;
}
const time = new Date(ts).getTime();
if (parsed.after !== null && !Number.isNaN(time) && time < parsed.after.getTime()) {
return false;
}
if (parsed.before !== null && !Number.isNaN(time) && time > parsed.before.getTime()) {
return false;
}
return true;
})
.map((evt) => ({
...evt,
event: "authorizer_chain_resolved",
links: ["ai-bash-judge"],
}) as ReviewEvent);
joined = [
...events.filter((evt) => evt.event !== "authorizer_chain_resolved"),
...auditEnrolled,
];
}
const { enrollments, metrics } = analyzeShadowReviewLog(joined);
const out = process.stdout;
out.write("AI Bash Judge — Shadow diagnostic report\n");
out.write("grade: DIAGNOSTIC (reconstructed join; not promotion-grade)\n");
out.write(`asOf: ${new Date().toISOString()}\n`);
out.write(`source: ${parsed.path}\n\n`);
out.write(`source: ${parsed.path}\n`);
if (parsed.audit !== null) {
out.write(`audit: ${parsed.audit} (enrollment source, ADR 0006)\n`);
}
out.write("\n");
out.write(`enrollments (N): ${enrollments}\n`);
out.write(`joined rows: ${metrics.joined}\n`);
@@ -0,0 +1,91 @@
import {
closeSync,
existsSync,
mkdirSync,
openSync,
writeSync,
fsyncSync,
} from "node:fs";
import { join } from "node:path";
import { stripForbiddenKeys } from "./review";
/**
* Judge-owned audit log (ADR 0006: enforce audit self-sufficiency).
*
* The accountability record Enforce needs lives in the Judge package, not
* in an upstream host contract: a separate JSONL file under the agent dir,
* append + fsync per record. A failed write marks this runtime
* **permanently unhealthy** (sticky — a flip-flopping audit trail is worse
* than a dead one), and the Enforce truth table's `auditHealthy` gate
* refuses authority while unhealthy.
*
* The privacy denylist matches the review sink: event keys matching the
* forbidden patterns are stripped before the record is serialized.
*/
export interface AuditLog {
/** Append one metadata-only record; marks the runtime unhealthy on failure. */
readonly audit: (event: string, details: Record<string, unknown>) => void;
/** False after any write or setup failure (sticky for the runtime). */
readonly healthy: () => boolean;
/** Absolute path of the audit file. */
readonly path: () => string;
}
export interface AuditLogDeps {
/** User-global agent dir (`~/.pi/agent`); the audit file lives under it. */
readonly agentDir: string;
/** Runtime identity stamped on every record. */
readonly runtimeId: string;
/** Injectable clock for tests; defaults to ISO-now. */
readonly now?: () => string;
}
const AUDIT_DIR_SEGMENTS = ["extensions", "pi-permission-ai-judge", "logs"];
const AUDIT_FILENAME = "audit.jsonl";
export function createAuditLog(deps: AuditLogDeps): AuditLog {
const now = deps.now ?? (() => new Date().toISOString());
const dir = join(deps.agentDir, ...AUDIT_DIR_SEGMENTS);
const file = join(dir, AUDIT_FILENAME);
let healthy = true;
try {
mkdirSync(dir, { recursive: true });
} catch {
healthy = false;
}
return {
audit: (event, details) => {
if (!healthy) {
// Sticky-fail: keep the API total but write nothing further.
return;
}
const record = JSON.stringify({
timestamp: now(),
judgeRuntimeId: deps.runtimeId,
event,
...stripForbiddenKeys(details),
});
let fd: number | undefined;
try {
fd = openSync(file, "a");
writeSync(fd, `${record}\n`);
fsyncSync(fd);
} catch {
healthy = false;
} finally {
if (fd !== undefined) {
try {
closeSync(fd);
} catch {
// Close failure does not un-fail the write.
}
}
}
},
healthy: () => healthy,
path: () => file,
};
}
+43 -6
View File
@@ -18,6 +18,7 @@ import {
import { PROMPT_VERSION, TOOL_SCHEMA_VERSION } from "./prompt";
import { loadJudgeConfig, type EffectiveJudgeConfig } from "./config";
import { createReviewSink, type ReviewSink } from "./review";
import { createAuditLog, type AuditLog } from "./audit";
import {
buildConversationEvidence,
conversationProbeFromSession,
@@ -46,6 +47,8 @@ interface RootSession {
readonly conversation: ReturnType<typeof conversationProbeFromSession>;
/** Requesting-session cwd for relative-path meaning. */
readonly getCwd: () => string;
/** Judge-owned audit log (ADR 0006); unhealthy refuses Enforce authority. */
readonly auditLog: AuditLog;
}
const EMPTY_CONVERSATION: ConversationEvidence = {
@@ -161,6 +164,34 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
reviewLogEnabled: captured.reviewLogEnabled,
});
try {
// Judge-owned enrollment record (ADR 0006 denominator:
// asks the Judge received, per its own audit log).
// Non-bash surfaces are ignored below without a shadow
// row; they also do not enroll (v0.1 cohort is bash-only).
if (
details.forwarding !== undefined ||
details.payload.kind === "forwarded" ||
details.payload.kind === "bash"
) {
captured.auditLog.audit("ai_bash_judge.enrolled", {
requestId: details.requestId,
origin:
details.forwarding !== undefined ||
details.payload.kind === "forwarded"
? "forwarded"
: "local",
surface: "bash",
command:
details.payload.kind === "bash" &&
details.payload.request?.value
? details.payload.request.value
: (details.command ?? null),
});
}
const emitResult = (record: Record<string, unknown>): void => {
sink.review("ai_bash_judge.result", record);
captured.auditLog.audit("ai_bash_judge.result", record);
};
// Forwarded asks do not carry a structured child full
// command in permission-system 25.3/25.4. Never parse the
// legacy prose. The deferral is recorded so the request
@@ -170,7 +201,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
details.forwarding !== undefined ||
details.payload.kind === "forwarded"
) {
sink.review("ai_bash_judge.result", {
emitResult({
...resultBase(
captured.judgeRuntimeId,
details,
@@ -197,7 +228,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
if (
captured.getSessionId() !== captured.expectedSessionId
) {
sink.review("ai_bash_judge.result", {
emitResult({
...resultBase(
captured.judgeRuntimeId,
details,
@@ -216,7 +247,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
const evidence = buildBashJudgmentEvidence(details);
if (evidence === undefined) {
sink.review("ai_bash_judge.result", {
emitResult({
...resultBase(
captured.judgeRuntimeId,
details,
@@ -255,7 +286,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
);
if (result.kind === "judgment") {
sink.review("ai_bash_judge.result", {
emitResult({
...resultBase(
captured.judgeRuntimeId,
details,
@@ -281,7 +312,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
evidenceQuality: evidenceQuality(true, conversation, captured.getCwd()),
});
} else {
sink.review("ai_bash_judge.result", {
emitResult({
...resultBase(
captured.judgeRuntimeId,
details,
@@ -313,6 +344,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
v01ProductionGateState(
captured.config.mode,
sink.health(),
captured.auditLog.healthy(),
),
);
return authority.kind === "allow"
@@ -339,15 +371,20 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
return;
}
const runtimeId = crypto.randomUUID();
root = {
getSessionId: () => ctx.sessionManager.getSessionId(),
expectedSessionId: sessionId,
getModel: () => ctx.model,
modelRegistry: ctx.modelRegistry,
shutdown: new AbortController(),
judgeRuntimeId: crypto.randomUUID(),
judgeRuntimeId: runtimeId,
config: loadJudgeConfig({ agentDir: getAgentDir() }),
reviewLogEnabled: readPermissionReviewLogEnabled(),
auditLog: createAuditLog({
agentDir: getAgentDir(),
runtimeId,
}),
conversation: conversationProbeFromSession(ctx.sessionManager),
getCwd: () => ctx.sessionManager.getCwd(),
};
+11 -8
View File
@@ -12,8 +12,8 @@ import type { TelemetryHealth } from "./review";
*/
export type EnforceGateState = {
/** Host contract present (permission-system host version/seams). */
readonly hostContractPresent: boolean;
/** The Judge-owned audit log is healthy (ADR 0006 self-check gate). */
readonly auditHealthy: boolean;
/** Runtime telemetry healthy at decision time. */
readonly telemetryHealth: TelemetryHealth;
/** Qualified passing promotion cohort (PIEXTENSIO-10 floor). */
@@ -47,8 +47,8 @@ export function evaluateEnforceAuthority(
if (state.mode !== "enforce") {
return { kind: "defer", blockedBy: "mode_shadow" };
}
if (!state.hostContractPresent) {
return { kind: "defer", blockedBy: "host_contract_absent" };
if (!state.auditHealthy) {
return { kind: "defer", blockedBy: "audit_unhealthy" };
}
if (state.telemetryHealth !== "healthy") {
return { kind: "defer", blockedBy: `telemetry_${state.telemetryHealth}` };
@@ -79,16 +79,19 @@ export function evaluateEnforceAuthority(
/**
* The v0.1 production gate state: the promotion gates are structurally
* unreachable until the upstream seams exist, so `enforce` defers here
* regardless of configuration. The full truth table above is exercised
* through injected fake gate states in tests only.
* unreachable until the owner records a qualifying cohort, approval, and
* activation (ADR 0006: recorded Judge-side, not via a host contract),
* so `enforce` defers here regardless of configuration. The full truth
* table above is exercised through injected fake gate states in tests
* only.
*/
export function v01ProductionGateState(
mode: "shadow" | "enforce",
telemetryHealth: TelemetryHealth,
auditHealthy = true,
): EnforceGateState {
return {
hostContractPresent: true,
auditHealthy,
telemetryHealth,
cohortQualified: false,
ownerApprovalRecorded: false,
@@ -58,7 +58,7 @@ export interface ReviewSinkDeps {
}
/** Strip forbidden keys defensively; a metadata-only event never carries them. */
function stripForbiddenKeys(
export function stripForbiddenKeys(
details: Record<string, unknown>,
): Record<string, unknown> {
const clean: Record<string, unknown> = {};
@@ -0,0 +1,136 @@
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { execFileSync } from "node:child_process";
import { afterEach, describe, expect, it } from "vitest";
/**
* CLI-level test for the ADR 0006 dual-log mode: `--audit` switches the
* enrollment denominator to the Judge-owned audit log's enrolled rows.
*/
const dirs: string[] = [];
function tmp(): string {
const dir = mkdtempSync(join(tmpdir(), "ai-judge-cli-"));
dirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of dirs.splice(0)) {
rmSync(dir, { recursive: true, force: true });
}
});
function run(args: readonly string[]): string {
const cli = join(import.meta.dirname, "..", "..", "src", "analyzer", "cli.ts");
return execFileSync("npx", ["tsx", cli, ...args], {
encoding: "utf-8",
// The review log path is the first positional arg.
});
}
describe("analyze-shadow CLI — --audit dual-log mode", () => {
it("takes enrollment from the audit log and keeps human decisions from the review log", () => {
const dir = tmp();
const reviewLog = join(dir, "review.jsonl");
const auditLog = join(dir, "audit.jsonl");
// Review log: chain_resolved for req-A (should be IGNORED as
// enrollment when --audit is given) and req-B has no chain row at
// all (chain event lost) but IS in the audit log — it must enroll.
writeFileSync(
reviewLog,
[
JSON.stringify({
timestamp: "2026-08-18T00:00:01Z",
event: "authorizer_chain_resolved",
requestId: "req-A",
links: ["ai-bash-judge"],
}),
JSON.stringify({
timestamp: "2026-08-18T00:00:02Z",
event: "ai_bash_judge.result",
requestId: "req-A",
resultKind: "judgment",
verdict: "allow",
}),
JSON.stringify({
timestamp: "2026-08-18T00:00:03Z",
event: "permission_request.approved",
requestId: "req-A",
resolution: "approved",
}),
JSON.stringify({
timestamp: "2026-08-18T00:00:04Z",
event: "ai_bash_judge.result",
requestId: "req-B",
resultKind: "judgment",
verdict: "defer",
}),
JSON.stringify({
timestamp: "2026-08-18T00:00:05Z",
event: "permission_request.denied",
requestId: "req-B",
resolution: "denied_with_reason",
}),
].join("\n") + "\n",
);
// Audit log: only req-B enrolled (judge received req-B; req-A never
// reached the judge callback even though the chain resolved).
writeFileSync(
auditLog,
[
JSON.stringify({
timestamp: "2026-08-18T00:00:04Z",
judgeRuntimeId: "rt-1",
event: "ai_bash_judge.enrolled",
requestId: "req-B",
origin: "local",
surface: "bash",
command: "git push origin main",
}),
].join("\n") + "\n",
);
const out = run([reviewLog, "--audit", auditLog]);
expect(out).toContain("audit: " + auditLog);
// Denominator is the audit enrollment only: req-A does not enroll.
expect(out).toContain("enrollments (N): 1");
expect(out).toContain("joined rows: 1");
// req-B: defer|deny is a conservative row, no false allow.
expect(out).not.toContain("false allows: 1");
// req-A's result+decision without enrollment stays out of the join.
expect(out).toContain("joined judgments: 1");
});
it("applies the --after window to audit enrolled rows too", () => {
const dir = tmp();
const reviewLog = join(dir, "review.jsonl");
const auditLog = join(dir, "audit.jsonl");
writeFileSync(reviewLog, "\n");
writeFileSync(
auditLog,
[
JSON.stringify({
timestamp: "2026-08-17T00:00:00Z",
event: "ai_bash_judge.enrolled",
requestId: "req-old",
}),
JSON.stringify({
timestamp: "2026-08-18T00:00:00Z",
event: "ai_bash_judge.enrolled",
requestId: "req-new",
}),
].join("\n") + "\n",
);
const out = run([
reviewLog,
"--audit", auditLog,
"--after", "2026-08-17T12:00:00Z",
]);
expect(out).toContain("enrollments (N): 1");
});
});
@@ -0,0 +1,124 @@
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { createAuditLog } from "../src/audit";
const dirs: string[] = [];
function tmp(): string {
const dir = mkdtempSync(join(tmpdir(), "ai-judge-audit-"));
dirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of dirs.splice(0)) {
rmSync(dir, { recursive: true, force: true });
}
});
function readAudit(agentDir: string): Array<Record<string, unknown>> {
return readFileSync(
join(agentDir, "extensions", "pi-permission-ai-judge", "logs", "audit.jsonl"),
"utf-8",
)
.trim()
.split("\n")
.filter((line) => line.length > 0)
.map((line) => JSON.parse(line) as Record<string, unknown>);
}
describe("createAuditLog — healthy path", () => {
it("appends one JSONL record per audit call with timestamp and runtime id", () => {
const agentDir = tmp();
const log = createAuditLog({
agentDir,
runtimeId: "rt-1",
now: () => "2026-08-18T00:00:00.000Z",
});
expect(log.healthy()).toBe(true);
log.audit("ai_bash_judge.enrolled", { requestId: "perm-a", origin: "local" });
log.audit("ai_bash_judge.result", { requestId: "perm-a", resultKind: "judgment" });
const rows = readAudit(agentDir);
expect(rows).toHaveLength(2);
expect(rows[0]).toEqual({
timestamp: "2026-08-18T00:00:00.000Z",
judgeRuntimeId: "rt-1",
event: "ai_bash_judge.enrolled",
requestId: "perm-a",
origin: "local",
});
expect(rows[1]).toEqual({
timestamp: "2026-08-18T00:00:00.000Z",
judgeRuntimeId: "rt-1",
event: "ai_bash_judge.result",
requestId: "perm-a",
resultKind: "judgment",
});
});
it("strips privacy-forbidden keys from audit records", () => {
const agentDir = tmp();
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
log.audit("ai_bash_judge.result", {
requestId: "perm-a",
apiToken: "leak",
secretPath: "/x",
verdict: "allow",
});
const rows = readAudit(agentDir);
expect(rows[0]).toEqual({
timestamp: rows[0].timestamp,
judgeRuntimeId: "rt-1",
event: "ai_bash_judge.result",
requestId: "perm-a",
verdict: "allow",
});
expect(rows[0].apiToken).toBeUndefined();
expect(rows[0].secretPath).toBeUndefined();
});
it("creates nested log directories on first use", () => {
const agentDir = join(tmp(), "deep", "agent");
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
log.audit("e", {});
expect(readAudit(agentDir)).toHaveLength(1);
});
});
describe("createAuditLog — fail-closed health", () => {
it("marks unhealthy permanently when the write fails once", () => {
const agentDir = tmp();
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
// Corrupt the logs dir into a file: open("a") on a path whose parent
// is a regular file throws ENSUREDIR/ENOTDIR.
const logsDir = join(
agentDir,
"extensions",
"pi-permission-ai-judge",
"logs",
);
rmSync(logsDir, { recursive: true, force: true });
writeFileSync(logsDir, "not a directory");
log.audit("e1", {});
expect(log.healthy()).toBe(false);
// Sticky: health never recovers within this runtime (ADR 0006).
log.audit("e2", {});
expect(log.healthy()).toBe(false);
});
it("is unhealthy from creation when the directory cannot be made", () => {
const agentDir = tmp();
const poisoned = join(agentDir, "extensions");
writeFileSync(poisoned, "file blocks mkdir");
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
expect(log.healthy()).toBe(false);
log.audit("e", {});
expect(log.healthy()).toBe(false);
});
});
@@ -6,7 +6,7 @@ import {
} from "../src/judge";
const ALL_OPEN: EnforceGateState = {
hostContractPresent: true,
auditHealthy: true,
telemetryHealth: "healthy",
cohortQualified: true,
ownerApprovalRecorded: true,
@@ -29,7 +29,7 @@ describe("evaluateEnforceAuthority — every gate independently forces defer", (
expectedReason: string;
}> = [
{ name: "shadow mode", patch: { mode: "shadow" }, expectedReason: "mode_shadow" },
{ name: "host contract absent", patch: { hostContractPresent: false }, expectedReason: "host_contract_absent" },
{ name: "audit log unhealthy", patch: { auditHealthy: false }, expectedReason: "audit_unhealthy" },
{ name: "telemetry disabled", patch: { telemetryHealth: "disabled" }, expectedReason: "telemetry_disabled" },
{ name: "telemetry write failed", patch: { telemetryHealth: "write_failed" }, expectedReason: "telemetry_write_failed" },
{ name: "telemetry integrity anomaly", patch: { telemetryHealth: "integrity_anomaly" }, expectedReason: "telemetry_integrity_anomaly" },
@@ -69,10 +69,17 @@ describe("evaluateEnforceAuthority — v0.1 production state", () => {
}
});
it("blocks v0.1 enforce on the cohort gate", () => {
it("blocks v0.1 enforce on the cohort gate even with healthy audit", () => {
const outcome = evaluateEnforceAuthority(
v01ProductionGateState("enforce", "healthy"),
v01ProductionGateState("enforce", "healthy", true),
);
expect(outcome).toEqual({ kind: "defer", blockedBy: "cohort_not_qualified" });
});
it("blocks v0.1 enforce on the audit gate when the audit log is unhealthy", () => {
const outcome = evaluateEnforceAuthority(
v01ProductionGateState("enforce", "healthy", false),
);
expect(outcome).toEqual({ kind: "defer", blockedBy: "audit_unhealthy" });
});
});