mirror of
https://github.com/SikongJueluo/pi-extensions.git
synced 2026-10-05 11:52:55 +08:00
feat(ai-judge): judge-owned audit log with local health gate (ADR 0006)
This commit is contained in:
@@ -0,0 +1,136 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
/**
|
||||
* CLI-level test for the ADR 0006 dual-log mode: `--audit` switches the
|
||||
* enrollment denominator to the Judge-owned audit log's enrolled rows.
|
||||
*/
|
||||
|
||||
const dirs: string[] = [];
|
||||
|
||||
function tmp(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "ai-judge-cli-"));
|
||||
dirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function run(args: readonly string[]): string {
|
||||
const cli = join(import.meta.dirname, "..", "..", "src", "analyzer", "cli.ts");
|
||||
return execFileSync("npx", ["tsx", cli, ...args], {
|
||||
encoding: "utf-8",
|
||||
// The review log path is the first positional arg.
|
||||
});
|
||||
}
|
||||
|
||||
describe("analyze-shadow CLI — --audit dual-log mode", () => {
|
||||
it("takes enrollment from the audit log and keeps human decisions from the review log", () => {
|
||||
const dir = tmp();
|
||||
const reviewLog = join(dir, "review.jsonl");
|
||||
const auditLog = join(dir, "audit.jsonl");
|
||||
|
||||
// Review log: chain_resolved for req-A (should be IGNORED as
|
||||
// enrollment when --audit is given) and req-B has no chain row at
|
||||
// all (chain event lost) but IS in the audit log — it must enroll.
|
||||
writeFileSync(
|
||||
reviewLog,
|
||||
[
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:01Z",
|
||||
event: "authorizer_chain_resolved",
|
||||
requestId: "req-A",
|
||||
links: ["ai-bash-judge"],
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:02Z",
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "req-A",
|
||||
resultKind: "judgment",
|
||||
verdict: "allow",
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:03Z",
|
||||
event: "permission_request.approved",
|
||||
requestId: "req-A",
|
||||
resolution: "approved",
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:04Z",
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "req-B",
|
||||
resultKind: "judgment",
|
||||
verdict: "defer",
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:05Z",
|
||||
event: "permission_request.denied",
|
||||
requestId: "req-B",
|
||||
resolution: "denied_with_reason",
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
|
||||
// Audit log: only req-B enrolled (judge received req-B; req-A never
|
||||
// reached the judge callback even though the chain resolved).
|
||||
writeFileSync(
|
||||
auditLog,
|
||||
[
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:04Z",
|
||||
judgeRuntimeId: "rt-1",
|
||||
event: "ai_bash_judge.enrolled",
|
||||
requestId: "req-B",
|
||||
origin: "local",
|
||||
surface: "bash",
|
||||
command: "git push origin main",
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
|
||||
const out = run([reviewLog, "--audit", auditLog]);
|
||||
expect(out).toContain("audit: " + auditLog);
|
||||
// Denominator is the audit enrollment only: req-A does not enroll.
|
||||
expect(out).toContain("enrollments (N): 1");
|
||||
expect(out).toContain("joined rows: 1");
|
||||
// req-B: defer|deny is a conservative row, no false allow.
|
||||
expect(out).not.toContain("false allows: 1");
|
||||
// req-A's result+decision without enrollment stays out of the join.
|
||||
expect(out).toContain("joined judgments: 1");
|
||||
});
|
||||
|
||||
it("applies the --after window to audit enrolled rows too", () => {
|
||||
const dir = tmp();
|
||||
const reviewLog = join(dir, "review.jsonl");
|
||||
const auditLog = join(dir, "audit.jsonl");
|
||||
writeFileSync(reviewLog, "\n");
|
||||
writeFileSync(
|
||||
auditLog,
|
||||
[
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-17T00:00:00Z",
|
||||
event: "ai_bash_judge.enrolled",
|
||||
requestId: "req-old",
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:00Z",
|
||||
event: "ai_bash_judge.enrolled",
|
||||
requestId: "req-new",
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
const out = run([
|
||||
reviewLog,
|
||||
"--audit", auditLog,
|
||||
"--after", "2026-08-17T12:00:00Z",
|
||||
]);
|
||||
expect(out).toContain("enrollments (N): 1");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createAuditLog } from "../src/audit";
|
||||
|
||||
const dirs: string[] = [];
|
||||
|
||||
function tmp(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "ai-judge-audit-"));
|
||||
dirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function readAudit(agentDir: string): Array<Record<string, unknown>> {
|
||||
return readFileSync(
|
||||
join(agentDir, "extensions", "pi-permission-ai-judge", "logs", "audit.jsonl"),
|
||||
"utf-8",
|
||||
)
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter((line) => line.length > 0)
|
||||
.map((line) => JSON.parse(line) as Record<string, unknown>);
|
||||
}
|
||||
|
||||
describe("createAuditLog — healthy path", () => {
|
||||
it("appends one JSONL record per audit call with timestamp and runtime id", () => {
|
||||
const agentDir = tmp();
|
||||
const log = createAuditLog({
|
||||
agentDir,
|
||||
runtimeId: "rt-1",
|
||||
now: () => "2026-08-18T00:00:00.000Z",
|
||||
});
|
||||
expect(log.healthy()).toBe(true);
|
||||
|
||||
log.audit("ai_bash_judge.enrolled", { requestId: "perm-a", origin: "local" });
|
||||
log.audit("ai_bash_judge.result", { requestId: "perm-a", resultKind: "judgment" });
|
||||
|
||||
const rows = readAudit(agentDir);
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(rows[0]).toEqual({
|
||||
timestamp: "2026-08-18T00:00:00.000Z",
|
||||
judgeRuntimeId: "rt-1",
|
||||
event: "ai_bash_judge.enrolled",
|
||||
requestId: "perm-a",
|
||||
origin: "local",
|
||||
});
|
||||
expect(rows[1]).toEqual({
|
||||
timestamp: "2026-08-18T00:00:00.000Z",
|
||||
judgeRuntimeId: "rt-1",
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "perm-a",
|
||||
resultKind: "judgment",
|
||||
});
|
||||
});
|
||||
|
||||
it("strips privacy-forbidden keys from audit records", () => {
|
||||
const agentDir = tmp();
|
||||
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
|
||||
log.audit("ai_bash_judge.result", {
|
||||
requestId: "perm-a",
|
||||
apiToken: "leak",
|
||||
secretPath: "/x",
|
||||
verdict: "allow",
|
||||
});
|
||||
const rows = readAudit(agentDir);
|
||||
expect(rows[0]).toEqual({
|
||||
timestamp: rows[0].timestamp,
|
||||
judgeRuntimeId: "rt-1",
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "perm-a",
|
||||
verdict: "allow",
|
||||
});
|
||||
expect(rows[0].apiToken).toBeUndefined();
|
||||
expect(rows[0].secretPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("creates nested log directories on first use", () => {
|
||||
const agentDir = join(tmp(), "deep", "agent");
|
||||
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
|
||||
log.audit("e", {});
|
||||
expect(readAudit(agentDir)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createAuditLog — fail-closed health", () => {
|
||||
it("marks unhealthy permanently when the write fails once", () => {
|
||||
const agentDir = tmp();
|
||||
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
|
||||
// Corrupt the logs dir into a file: open("a") on a path whose parent
|
||||
// is a regular file throws ENSUREDIR/ENOTDIR.
|
||||
const logsDir = join(
|
||||
agentDir,
|
||||
"extensions",
|
||||
"pi-permission-ai-judge",
|
||||
"logs",
|
||||
);
|
||||
rmSync(logsDir, { recursive: true, force: true });
|
||||
writeFileSync(logsDir, "not a directory");
|
||||
|
||||
log.audit("e1", {});
|
||||
expect(log.healthy()).toBe(false);
|
||||
|
||||
// Sticky: health never recovers within this runtime (ADR 0006).
|
||||
log.audit("e2", {});
|
||||
expect(log.healthy()).toBe(false);
|
||||
});
|
||||
|
||||
it("is unhealthy from creation when the directory cannot be made", () => {
|
||||
const agentDir = tmp();
|
||||
const poisoned = join(agentDir, "extensions");
|
||||
writeFileSync(poisoned, "file blocks mkdir");
|
||||
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
|
||||
expect(log.healthy()).toBe(false);
|
||||
log.audit("e", {});
|
||||
expect(log.healthy()).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
} from "../src/judge";
|
||||
|
||||
const ALL_OPEN: EnforceGateState = {
|
||||
hostContractPresent: true,
|
||||
auditHealthy: true,
|
||||
telemetryHealth: "healthy",
|
||||
cohortQualified: true,
|
||||
ownerApprovalRecorded: true,
|
||||
@@ -29,7 +29,7 @@ describe("evaluateEnforceAuthority — every gate independently forces defer", (
|
||||
expectedReason: string;
|
||||
}> = [
|
||||
{ name: "shadow mode", patch: { mode: "shadow" }, expectedReason: "mode_shadow" },
|
||||
{ name: "host contract absent", patch: { hostContractPresent: false }, expectedReason: "host_contract_absent" },
|
||||
{ name: "audit log unhealthy", patch: { auditHealthy: false }, expectedReason: "audit_unhealthy" },
|
||||
{ name: "telemetry disabled", patch: { telemetryHealth: "disabled" }, expectedReason: "telemetry_disabled" },
|
||||
{ name: "telemetry write failed", patch: { telemetryHealth: "write_failed" }, expectedReason: "telemetry_write_failed" },
|
||||
{ name: "telemetry integrity anomaly", patch: { telemetryHealth: "integrity_anomaly" }, expectedReason: "telemetry_integrity_anomaly" },
|
||||
@@ -69,10 +69,17 @@ describe("evaluateEnforceAuthority — v0.1 production state", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("blocks v0.1 enforce on the cohort gate", () => {
|
||||
it("blocks v0.1 enforce on the cohort gate even with healthy audit", () => {
|
||||
const outcome = evaluateEnforceAuthority(
|
||||
v01ProductionGateState("enforce", "healthy"),
|
||||
v01ProductionGateState("enforce", "healthy", true),
|
||||
);
|
||||
expect(outcome).toEqual({ kind: "defer", blockedBy: "cohort_not_qualified" });
|
||||
});
|
||||
|
||||
it("blocks v0.1 enforce on the audit gate when the audit log is unhealthy", () => {
|
||||
const outcome = evaluateEnforceAuthority(
|
||||
v01ProductionGateState("enforce", "healthy", false),
|
||||
);
|
||||
expect(outcome).toEqual({ kind: "defer", blockedBy: "audit_unhealthy" });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user