mirror of
https://github.com/SikongJueluo/pi-extensions.git
synced 2026-10-05 11:52:55 +08:00
feat(ai-judge): judge-owned audit log with local health gate (ADR 0006)
This commit is contained in:
@@ -122,7 +122,21 @@ report.
|
||||
|
||||
## 5. Log contract (pairing keys)
|
||||
|
||||
Event order per request, keyed by `requestId`:
|
||||
**Two logs exist since ADR 0006** (Judge-owned audit + permission review):
|
||||
|
||||
1. **Judge audit log** `~/.pi/agent/extensions/pi-permission-ai-judge/logs/audit.jsonl`
|
||||
— judge-owned events only: `ai_bash_judge.enrolled` (ask received:
|
||||
requestId, origin, surface, command — the cohort denominator) and a
|
||||
mirror of every `ai_bash_judge.result` row. Append + fsync per record;
|
||||
write failure is sticky-unhealthy and refuses Enforce authority.
|
||||
2. **Permission review log** (below) — still the source for human-decision
|
||||
attribution and upstream chain events.
|
||||
|
||||
The analyzer takes enrollments from the audit log via `--audit <path>`
|
||||
(dual-log mode); without the flag it reconstructs enrollment from the
|
||||
review log as before.
|
||||
|
||||
Event order per request in the review log, keyed by `requestId`:
|
||||
|
||||
```
|
||||
permission_request.waiting
|
||||
|
||||
@@ -15,6 +15,10 @@ const USAGE = `usage: analyze-shadow <review-jsonl-path> [options]
|
||||
options:
|
||||
--after <iso8601> only consider events with timestamp >= this instant
|
||||
--before <iso8601> only consider events with timestamp <= this instant
|
||||
--audit <path> Judge-owned audit JSONL (ADR 0006); enrollment (the
|
||||
denominator N) is then taken from its
|
||||
ai_bash_judge.enrolled rows instead of the reconstructed
|
||||
permission-system chain events
|
||||
--help show this help
|
||||
|
||||
The report is diagnostic-grade: the join reconstructs enrollment and human
|
||||
@@ -25,6 +29,7 @@ interface CliOptions {
|
||||
readonly path: string;
|
||||
readonly after: Date | null;
|
||||
readonly before: Date | null;
|
||||
readonly audit: string | null;
|
||||
}
|
||||
|
||||
function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
|
||||
@@ -32,15 +37,21 @@ function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
|
||||
let path: string | undefined;
|
||||
let after: Date | null = null;
|
||||
let before: Date | null = null;
|
||||
let audit: string | null = null;
|
||||
for (let i = 0; i < args.length; i += 1) {
|
||||
const arg = args[i] as string;
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
return { error: USAGE };
|
||||
}
|
||||
if (arg === "--after" || arg === "--before") {
|
||||
if (arg === "--after" || arg === "--before" || arg === "--audit") {
|
||||
const value = args[i + 1];
|
||||
if (value === undefined) {
|
||||
return { error: `${arg} requires an ISO-8601 timestamp` };
|
||||
return { error: `${arg} requires a value` };
|
||||
}
|
||||
if (arg === "--audit") {
|
||||
audit = value;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
const parsed = new Date(value);
|
||||
if (Number.isNaN(parsed.getTime())) {
|
||||
@@ -65,7 +76,7 @@ function parseArgs(argv: readonly string[]): CliOptions | { error: string } {
|
||||
if (path === undefined) {
|
||||
return { error: "missing input path" };
|
||||
}
|
||||
return { path, after, before };
|
||||
return { path, after, before, audit };
|
||||
}
|
||||
|
||||
function parseLine(line: string, lineNo: number): ReviewEvent | null {
|
||||
@@ -137,13 +148,63 @@ function main(): void {
|
||||
return true;
|
||||
});
|
||||
|
||||
const { enrollments, metrics } = analyzeShadowReviewLog(events);
|
||||
// ADR 0006 dual-log mode: with --audit, the denominator comes from the
|
||||
// Judge's own enrolled rows (asks it received), and the reconstructed
|
||||
// permission-system enrollment proxy is dropped to avoid a second
|
||||
// denominator source. Human decisions still come from the permission
|
||||
// log (attribution join).
|
||||
let joined = events;
|
||||
if (parsed.audit !== null) {
|
||||
let auditRaw: string;
|
||||
try {
|
||||
auditRaw = readFileSync(parsed.audit, "utf-8");
|
||||
} catch (error) {
|
||||
process.stderr.write(
|
||||
`error: cannot read audit log ${parsed.audit}: ${error instanceof Error ? error.message : String(error)}\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const auditEnrolled = auditRaw
|
||||
.split("\n")
|
||||
.map((line, index) => parseLine(line, index + 1))
|
||||
.filter((evt): evt is ReviewEvent => evt !== null)
|
||||
.filter((evt) => evt.event === "ai_bash_judge.enrolled")
|
||||
.filter((evt) => {
|
||||
const ts = typeof evt.timestamp === "string" ? evt.timestamp : null;
|
||||
if (ts === null) {
|
||||
return true;
|
||||
}
|
||||
const time = new Date(ts).getTime();
|
||||
if (parsed.after !== null && !Number.isNaN(time) && time < parsed.after.getTime()) {
|
||||
return false;
|
||||
}
|
||||
if (parsed.before !== null && !Number.isNaN(time) && time > parsed.before.getTime()) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.map((evt) => ({
|
||||
...evt,
|
||||
event: "authorizer_chain_resolved",
|
||||
links: ["ai-bash-judge"],
|
||||
}) as ReviewEvent);
|
||||
joined = [
|
||||
...events.filter((evt) => evt.event !== "authorizer_chain_resolved"),
|
||||
...auditEnrolled,
|
||||
];
|
||||
}
|
||||
|
||||
const { enrollments, metrics } = analyzeShadowReviewLog(joined);
|
||||
const out = process.stdout;
|
||||
|
||||
out.write("AI Bash Judge — Shadow diagnostic report\n");
|
||||
out.write("grade: DIAGNOSTIC (reconstructed join; not promotion-grade)\n");
|
||||
out.write(`asOf: ${new Date().toISOString()}\n`);
|
||||
out.write(`source: ${parsed.path}\n\n`);
|
||||
out.write(`source: ${parsed.path}\n`);
|
||||
if (parsed.audit !== null) {
|
||||
out.write(`audit: ${parsed.audit} (enrollment source, ADR 0006)\n`);
|
||||
}
|
||||
out.write("\n");
|
||||
|
||||
out.write(`enrollments (N): ${enrollments}\n`);
|
||||
out.write(`joined rows: ${metrics.joined}\n`);
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import {
|
||||
closeSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
openSync,
|
||||
writeSync,
|
||||
fsyncSync,
|
||||
} from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { stripForbiddenKeys } from "./review";
|
||||
|
||||
/**
|
||||
* Judge-owned audit log (ADR 0006: enforce audit self-sufficiency).
|
||||
*
|
||||
* The accountability record Enforce needs lives in the Judge package, not
|
||||
* in an upstream host contract: a separate JSONL file under the agent dir,
|
||||
* append + fsync per record. A failed write marks this runtime
|
||||
* **permanently unhealthy** (sticky — a flip-flopping audit trail is worse
|
||||
* than a dead one), and the Enforce truth table's `auditHealthy` gate
|
||||
* refuses authority while unhealthy.
|
||||
*
|
||||
* The privacy denylist matches the review sink: event keys matching the
|
||||
* forbidden patterns are stripped before the record is serialized.
|
||||
*/
|
||||
|
||||
export interface AuditLog {
|
||||
/** Append one metadata-only record; marks the runtime unhealthy on failure. */
|
||||
readonly audit: (event: string, details: Record<string, unknown>) => void;
|
||||
/** False after any write or setup failure (sticky for the runtime). */
|
||||
readonly healthy: () => boolean;
|
||||
/** Absolute path of the audit file. */
|
||||
readonly path: () => string;
|
||||
}
|
||||
|
||||
export interface AuditLogDeps {
|
||||
/** User-global agent dir (`~/.pi/agent`); the audit file lives under it. */
|
||||
readonly agentDir: string;
|
||||
/** Runtime identity stamped on every record. */
|
||||
readonly runtimeId: string;
|
||||
/** Injectable clock for tests; defaults to ISO-now. */
|
||||
readonly now?: () => string;
|
||||
}
|
||||
|
||||
const AUDIT_DIR_SEGMENTS = ["extensions", "pi-permission-ai-judge", "logs"];
|
||||
const AUDIT_FILENAME = "audit.jsonl";
|
||||
|
||||
export function createAuditLog(deps: AuditLogDeps): AuditLog {
|
||||
const now = deps.now ?? (() => new Date().toISOString());
|
||||
const dir = join(deps.agentDir, ...AUDIT_DIR_SEGMENTS);
|
||||
const file = join(dir, AUDIT_FILENAME);
|
||||
let healthy = true;
|
||||
|
||||
try {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
} catch {
|
||||
healthy = false;
|
||||
}
|
||||
|
||||
return {
|
||||
audit: (event, details) => {
|
||||
if (!healthy) {
|
||||
// Sticky-fail: keep the API total but write nothing further.
|
||||
return;
|
||||
}
|
||||
const record = JSON.stringify({
|
||||
timestamp: now(),
|
||||
judgeRuntimeId: deps.runtimeId,
|
||||
event,
|
||||
...stripForbiddenKeys(details),
|
||||
});
|
||||
let fd: number | undefined;
|
||||
try {
|
||||
fd = openSync(file, "a");
|
||||
writeSync(fd, `${record}\n`);
|
||||
fsyncSync(fd);
|
||||
} catch {
|
||||
healthy = false;
|
||||
} finally {
|
||||
if (fd !== undefined) {
|
||||
try {
|
||||
closeSync(fd);
|
||||
} catch {
|
||||
// Close failure does not un-fail the write.
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
healthy: () => healthy,
|
||||
path: () => file,
|
||||
};
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
import { PROMPT_VERSION, TOOL_SCHEMA_VERSION } from "./prompt";
|
||||
import { loadJudgeConfig, type EffectiveJudgeConfig } from "./config";
|
||||
import { createReviewSink, type ReviewSink } from "./review";
|
||||
import { createAuditLog, type AuditLog } from "./audit";
|
||||
import {
|
||||
buildConversationEvidence,
|
||||
conversationProbeFromSession,
|
||||
@@ -46,6 +47,8 @@ interface RootSession {
|
||||
readonly conversation: ReturnType<typeof conversationProbeFromSession>;
|
||||
/** Requesting-session cwd for relative-path meaning. */
|
||||
readonly getCwd: () => string;
|
||||
/** Judge-owned audit log (ADR 0006); unhealthy refuses Enforce authority. */
|
||||
readonly auditLog: AuditLog;
|
||||
}
|
||||
|
||||
const EMPTY_CONVERSATION: ConversationEvidence = {
|
||||
@@ -161,6 +164,34 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
|
||||
reviewLogEnabled: captured.reviewLogEnabled,
|
||||
});
|
||||
try {
|
||||
// Judge-owned enrollment record (ADR 0006 denominator:
|
||||
// asks the Judge received, per its own audit log).
|
||||
// Non-bash surfaces are ignored below without a shadow
|
||||
// row; they also do not enroll (v0.1 cohort is bash-only).
|
||||
if (
|
||||
details.forwarding !== undefined ||
|
||||
details.payload.kind === "forwarded" ||
|
||||
details.payload.kind === "bash"
|
||||
) {
|
||||
captured.auditLog.audit("ai_bash_judge.enrolled", {
|
||||
requestId: details.requestId,
|
||||
origin:
|
||||
details.forwarding !== undefined ||
|
||||
details.payload.kind === "forwarded"
|
||||
? "forwarded"
|
||||
: "local",
|
||||
surface: "bash",
|
||||
command:
|
||||
details.payload.kind === "bash" &&
|
||||
details.payload.request?.value
|
||||
? details.payload.request.value
|
||||
: (details.command ?? null),
|
||||
});
|
||||
}
|
||||
const emitResult = (record: Record<string, unknown>): void => {
|
||||
sink.review("ai_bash_judge.result", record);
|
||||
captured.auditLog.audit("ai_bash_judge.result", record);
|
||||
};
|
||||
// Forwarded asks do not carry a structured child full
|
||||
// command in permission-system 25.3/25.4. Never parse the
|
||||
// legacy prose. The deferral is recorded so the request
|
||||
@@ -170,7 +201,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
|
||||
details.forwarding !== undefined ||
|
||||
details.payload.kind === "forwarded"
|
||||
) {
|
||||
sink.review("ai_bash_judge.result", {
|
||||
emitResult({
|
||||
...resultBase(
|
||||
captured.judgeRuntimeId,
|
||||
details,
|
||||
@@ -197,7 +228,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
|
||||
if (
|
||||
captured.getSessionId() !== captured.expectedSessionId
|
||||
) {
|
||||
sink.review("ai_bash_judge.result", {
|
||||
emitResult({
|
||||
...resultBase(
|
||||
captured.judgeRuntimeId,
|
||||
details,
|
||||
@@ -216,7 +247,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
|
||||
|
||||
const evidence = buildBashJudgmentEvidence(details);
|
||||
if (evidence === undefined) {
|
||||
sink.review("ai_bash_judge.result", {
|
||||
emitResult({
|
||||
...resultBase(
|
||||
captured.judgeRuntimeId,
|
||||
details,
|
||||
@@ -255,7 +286,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
|
||||
);
|
||||
|
||||
if (result.kind === "judgment") {
|
||||
sink.review("ai_bash_judge.result", {
|
||||
emitResult({
|
||||
...resultBase(
|
||||
captured.judgeRuntimeId,
|
||||
details,
|
||||
@@ -281,7 +312,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
|
||||
evidenceQuality: evidenceQuality(true, conversation, captured.getCwd()),
|
||||
});
|
||||
} else {
|
||||
sink.review("ai_bash_judge.result", {
|
||||
emitResult({
|
||||
...resultBase(
|
||||
captured.judgeRuntimeId,
|
||||
details,
|
||||
@@ -313,6 +344,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
|
||||
v01ProductionGateState(
|
||||
captured.config.mode,
|
||||
sink.health(),
|
||||
captured.auditLog.healthy(),
|
||||
),
|
||||
);
|
||||
return authority.kind === "allow"
|
||||
@@ -339,15 +371,20 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
|
||||
return;
|
||||
}
|
||||
|
||||
const runtimeId = crypto.randomUUID();
|
||||
root = {
|
||||
getSessionId: () => ctx.sessionManager.getSessionId(),
|
||||
expectedSessionId: sessionId,
|
||||
getModel: () => ctx.model,
|
||||
modelRegistry: ctx.modelRegistry,
|
||||
shutdown: new AbortController(),
|
||||
judgeRuntimeId: crypto.randomUUID(),
|
||||
judgeRuntimeId: runtimeId,
|
||||
config: loadJudgeConfig({ agentDir: getAgentDir() }),
|
||||
reviewLogEnabled: readPermissionReviewLogEnabled(),
|
||||
auditLog: createAuditLog({
|
||||
agentDir: getAgentDir(),
|
||||
runtimeId,
|
||||
}),
|
||||
conversation: conversationProbeFromSession(ctx.sessionManager),
|
||||
getCwd: () => ctx.sessionManager.getCwd(),
|
||||
};
|
||||
|
||||
@@ -12,8 +12,8 @@ import type { TelemetryHealth } from "./review";
|
||||
*/
|
||||
|
||||
export type EnforceGateState = {
|
||||
/** Host contract present (permission-system host version/seams). */
|
||||
readonly hostContractPresent: boolean;
|
||||
/** The Judge-owned audit log is healthy (ADR 0006 self-check gate). */
|
||||
readonly auditHealthy: boolean;
|
||||
/** Runtime telemetry healthy at decision time. */
|
||||
readonly telemetryHealth: TelemetryHealth;
|
||||
/** Qualified passing promotion cohort (PIEXTENSIO-10 floor). */
|
||||
@@ -47,8 +47,8 @@ export function evaluateEnforceAuthority(
|
||||
if (state.mode !== "enforce") {
|
||||
return { kind: "defer", blockedBy: "mode_shadow" };
|
||||
}
|
||||
if (!state.hostContractPresent) {
|
||||
return { kind: "defer", blockedBy: "host_contract_absent" };
|
||||
if (!state.auditHealthy) {
|
||||
return { kind: "defer", blockedBy: "audit_unhealthy" };
|
||||
}
|
||||
if (state.telemetryHealth !== "healthy") {
|
||||
return { kind: "defer", blockedBy: `telemetry_${state.telemetryHealth}` };
|
||||
@@ -79,16 +79,19 @@ export function evaluateEnforceAuthority(
|
||||
|
||||
/**
|
||||
* The v0.1 production gate state: the promotion gates are structurally
|
||||
* unreachable until the upstream seams exist, so `enforce` defers here
|
||||
* regardless of configuration. The full truth table above is exercised
|
||||
* through injected fake gate states in tests only.
|
||||
* unreachable until the owner records a qualifying cohort, approval, and
|
||||
* activation (ADR 0006: recorded Judge-side, not via a host contract),
|
||||
* so `enforce` defers here regardless of configuration. The full truth
|
||||
* table above is exercised through injected fake gate states in tests
|
||||
* only.
|
||||
*/
|
||||
export function v01ProductionGateState(
|
||||
mode: "shadow" | "enforce",
|
||||
telemetryHealth: TelemetryHealth,
|
||||
auditHealthy = true,
|
||||
): EnforceGateState {
|
||||
return {
|
||||
hostContractPresent: true,
|
||||
auditHealthy,
|
||||
telemetryHealth,
|
||||
cohortQualified: false,
|
||||
ownerApprovalRecorded: false,
|
||||
|
||||
@@ -58,7 +58,7 @@ export interface ReviewSinkDeps {
|
||||
}
|
||||
|
||||
/** Strip forbidden keys defensively; a metadata-only event never carries them. */
|
||||
function stripForbiddenKeys(
|
||||
export function stripForbiddenKeys(
|
||||
details: Record<string, unknown>,
|
||||
): Record<string, unknown> {
|
||||
const clean: Record<string, unknown> = {};
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
/**
|
||||
* CLI-level test for the ADR 0006 dual-log mode: `--audit` switches the
|
||||
* enrollment denominator to the Judge-owned audit log's enrolled rows.
|
||||
*/
|
||||
|
||||
const dirs: string[] = [];
|
||||
|
||||
function tmp(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "ai-judge-cli-"));
|
||||
dirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function run(args: readonly string[]): string {
|
||||
const cli = join(import.meta.dirname, "..", "..", "src", "analyzer", "cli.ts");
|
||||
return execFileSync("npx", ["tsx", cli, ...args], {
|
||||
encoding: "utf-8",
|
||||
// The review log path is the first positional arg.
|
||||
});
|
||||
}
|
||||
|
||||
describe("analyze-shadow CLI — --audit dual-log mode", () => {
|
||||
it("takes enrollment from the audit log and keeps human decisions from the review log", () => {
|
||||
const dir = tmp();
|
||||
const reviewLog = join(dir, "review.jsonl");
|
||||
const auditLog = join(dir, "audit.jsonl");
|
||||
|
||||
// Review log: chain_resolved for req-A (should be IGNORED as
|
||||
// enrollment when --audit is given) and req-B has no chain row at
|
||||
// all (chain event lost) but IS in the audit log — it must enroll.
|
||||
writeFileSync(
|
||||
reviewLog,
|
||||
[
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:01Z",
|
||||
event: "authorizer_chain_resolved",
|
||||
requestId: "req-A",
|
||||
links: ["ai-bash-judge"],
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:02Z",
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "req-A",
|
||||
resultKind: "judgment",
|
||||
verdict: "allow",
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:03Z",
|
||||
event: "permission_request.approved",
|
||||
requestId: "req-A",
|
||||
resolution: "approved",
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:04Z",
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "req-B",
|
||||
resultKind: "judgment",
|
||||
verdict: "defer",
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:05Z",
|
||||
event: "permission_request.denied",
|
||||
requestId: "req-B",
|
||||
resolution: "denied_with_reason",
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
|
||||
// Audit log: only req-B enrolled (judge received req-B; req-A never
|
||||
// reached the judge callback even though the chain resolved).
|
||||
writeFileSync(
|
||||
auditLog,
|
||||
[
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:04Z",
|
||||
judgeRuntimeId: "rt-1",
|
||||
event: "ai_bash_judge.enrolled",
|
||||
requestId: "req-B",
|
||||
origin: "local",
|
||||
surface: "bash",
|
||||
command: "git push origin main",
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
|
||||
const out = run([reviewLog, "--audit", auditLog]);
|
||||
expect(out).toContain("audit: " + auditLog);
|
||||
// Denominator is the audit enrollment only: req-A does not enroll.
|
||||
expect(out).toContain("enrollments (N): 1");
|
||||
expect(out).toContain("joined rows: 1");
|
||||
// req-B: defer|deny is a conservative row, no false allow.
|
||||
expect(out).not.toContain("false allows: 1");
|
||||
// req-A's result+decision without enrollment stays out of the join.
|
||||
expect(out).toContain("joined judgments: 1");
|
||||
});
|
||||
|
||||
it("applies the --after window to audit enrolled rows too", () => {
|
||||
const dir = tmp();
|
||||
const reviewLog = join(dir, "review.jsonl");
|
||||
const auditLog = join(dir, "audit.jsonl");
|
||||
writeFileSync(reviewLog, "\n");
|
||||
writeFileSync(
|
||||
auditLog,
|
||||
[
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-17T00:00:00Z",
|
||||
event: "ai_bash_judge.enrolled",
|
||||
requestId: "req-old",
|
||||
}),
|
||||
JSON.stringify({
|
||||
timestamp: "2026-08-18T00:00:00Z",
|
||||
event: "ai_bash_judge.enrolled",
|
||||
requestId: "req-new",
|
||||
}),
|
||||
].join("\n") + "\n",
|
||||
);
|
||||
const out = run([
|
||||
reviewLog,
|
||||
"--audit", auditLog,
|
||||
"--after", "2026-08-17T12:00:00Z",
|
||||
]);
|
||||
expect(out).toContain("enrollments (N): 1");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,124 @@
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createAuditLog } from "../src/audit";
|
||||
|
||||
const dirs: string[] = [];
|
||||
|
||||
function tmp(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "ai-judge-audit-"));
|
||||
dirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of dirs.splice(0)) {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function readAudit(agentDir: string): Array<Record<string, unknown>> {
|
||||
return readFileSync(
|
||||
join(agentDir, "extensions", "pi-permission-ai-judge", "logs", "audit.jsonl"),
|
||||
"utf-8",
|
||||
)
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter((line) => line.length > 0)
|
||||
.map((line) => JSON.parse(line) as Record<string, unknown>);
|
||||
}
|
||||
|
||||
describe("createAuditLog — healthy path", () => {
|
||||
it("appends one JSONL record per audit call with timestamp and runtime id", () => {
|
||||
const agentDir = tmp();
|
||||
const log = createAuditLog({
|
||||
agentDir,
|
||||
runtimeId: "rt-1",
|
||||
now: () => "2026-08-18T00:00:00.000Z",
|
||||
});
|
||||
expect(log.healthy()).toBe(true);
|
||||
|
||||
log.audit("ai_bash_judge.enrolled", { requestId: "perm-a", origin: "local" });
|
||||
log.audit("ai_bash_judge.result", { requestId: "perm-a", resultKind: "judgment" });
|
||||
|
||||
const rows = readAudit(agentDir);
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(rows[0]).toEqual({
|
||||
timestamp: "2026-08-18T00:00:00.000Z",
|
||||
judgeRuntimeId: "rt-1",
|
||||
event: "ai_bash_judge.enrolled",
|
||||
requestId: "perm-a",
|
||||
origin: "local",
|
||||
});
|
||||
expect(rows[1]).toEqual({
|
||||
timestamp: "2026-08-18T00:00:00.000Z",
|
||||
judgeRuntimeId: "rt-1",
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "perm-a",
|
||||
resultKind: "judgment",
|
||||
});
|
||||
});
|
||||
|
||||
it("strips privacy-forbidden keys from audit records", () => {
|
||||
const agentDir = tmp();
|
||||
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
|
||||
log.audit("ai_bash_judge.result", {
|
||||
requestId: "perm-a",
|
||||
apiToken: "leak",
|
||||
secretPath: "/x",
|
||||
verdict: "allow",
|
||||
});
|
||||
const rows = readAudit(agentDir);
|
||||
expect(rows[0]).toEqual({
|
||||
timestamp: rows[0].timestamp,
|
||||
judgeRuntimeId: "rt-1",
|
||||
event: "ai_bash_judge.result",
|
||||
requestId: "perm-a",
|
||||
verdict: "allow",
|
||||
});
|
||||
expect(rows[0].apiToken).toBeUndefined();
|
||||
expect(rows[0].secretPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("creates nested log directories on first use", () => {
|
||||
const agentDir = join(tmp(), "deep", "agent");
|
||||
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
|
||||
log.audit("e", {});
|
||||
expect(readAudit(agentDir)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createAuditLog — fail-closed health", () => {
|
||||
it("marks unhealthy permanently when the write fails once", () => {
|
||||
const agentDir = tmp();
|
||||
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
|
||||
// Corrupt the logs dir into a file: open("a") on a path whose parent
|
||||
// is a regular file throws ENSUREDIR/ENOTDIR.
|
||||
const logsDir = join(
|
||||
agentDir,
|
||||
"extensions",
|
||||
"pi-permission-ai-judge",
|
||||
"logs",
|
||||
);
|
||||
rmSync(logsDir, { recursive: true, force: true });
|
||||
writeFileSync(logsDir, "not a directory");
|
||||
|
||||
log.audit("e1", {});
|
||||
expect(log.healthy()).toBe(false);
|
||||
|
||||
// Sticky: health never recovers within this runtime (ADR 0006).
|
||||
log.audit("e2", {});
|
||||
expect(log.healthy()).toBe(false);
|
||||
});
|
||||
|
||||
it("is unhealthy from creation when the directory cannot be made", () => {
|
||||
const agentDir = tmp();
|
||||
const poisoned = join(agentDir, "extensions");
|
||||
writeFileSync(poisoned, "file blocks mkdir");
|
||||
const log = createAuditLog({ agentDir, runtimeId: "rt-1" });
|
||||
expect(log.healthy()).toBe(false);
|
||||
log.audit("e", {});
|
||||
expect(log.healthy()).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
} from "../src/judge";
|
||||
|
||||
const ALL_OPEN: EnforceGateState = {
|
||||
hostContractPresent: true,
|
||||
auditHealthy: true,
|
||||
telemetryHealth: "healthy",
|
||||
cohortQualified: true,
|
||||
ownerApprovalRecorded: true,
|
||||
@@ -29,7 +29,7 @@ describe("evaluateEnforceAuthority — every gate independently forces defer", (
|
||||
expectedReason: string;
|
||||
}> = [
|
||||
{ name: "shadow mode", patch: { mode: "shadow" }, expectedReason: "mode_shadow" },
|
||||
{ name: "host contract absent", patch: { hostContractPresent: false }, expectedReason: "host_contract_absent" },
|
||||
{ name: "audit log unhealthy", patch: { auditHealthy: false }, expectedReason: "audit_unhealthy" },
|
||||
{ name: "telemetry disabled", patch: { telemetryHealth: "disabled" }, expectedReason: "telemetry_disabled" },
|
||||
{ name: "telemetry write failed", patch: { telemetryHealth: "write_failed" }, expectedReason: "telemetry_write_failed" },
|
||||
{ name: "telemetry integrity anomaly", patch: { telemetryHealth: "integrity_anomaly" }, expectedReason: "telemetry_integrity_anomaly" },
|
||||
@@ -69,10 +69,17 @@ describe("evaluateEnforceAuthority — v0.1 production state", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("blocks v0.1 enforce on the cohort gate", () => {
|
||||
it("blocks v0.1 enforce on the cohort gate even with healthy audit", () => {
|
||||
const outcome = evaluateEnforceAuthority(
|
||||
v01ProductionGateState("enforce", "healthy"),
|
||||
v01ProductionGateState("enforce", "healthy", true),
|
||||
);
|
||||
expect(outcome).toEqual({ kind: "defer", blockedBy: "cohort_not_qualified" });
|
||||
});
|
||||
|
||||
it("blocks v0.1 enforce on the audit gate when the audit log is unhealthy", () => {
|
||||
const outcome = evaluateEnforceAuthority(
|
||||
v01ProductionGateState("enforce", "healthy", false),
|
||||
);
|
||||
expect(outcome).toEqual({ kind: "defer", blockedBy: "audit_unhealthy" });
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user