feat(ai-judge): promotion-gate records seam and truth-table wiring (PIEXTENSIO-21)

This commit is contained in:
2026-08-20 17:45:07 +08:00
parent 45e91a700d
commit d57892d5ab
7 changed files with 1116 additions and 68 deletions
@@ -1,9 +1,13 @@
import { describe, expect, it } from "vitest";
import {
evaluateEnforceAuthority,
v01ProductionGateState,
type EnforceGateState,
} from "../src/judge";
import {
loadPromotionRecords,
resolvePromotionGates,
type CandidateIdentity,
} from "../src/promotion";
const ALL_OPEN: EnforceGateState = {
auditHealthy: true,
@@ -55,31 +59,85 @@ describe("evaluateEnforceAuthority — every gate independently forces defer", (
}
});
describe("evaluateEnforceAuthority — v0.1 production state", () => {
it("never grants authority for any mode or telemetry state in v0.1", () => {
describe("evaluateEnforceAuthority — production state with no promotion records", () => {
// The real post-PIEXTENSIO-21 seam: an empty records file (the normal
// pre-promotion state) closes all promotion gates, so every mode and
// telemetry state defers — mechanically identical to v0.1's hardcoded
// closure, now derived from actual storage.
const emptySnapshot = loadPromotionRecords({
agentDir: "/nonexistent-agent-dir",
});
const identity: CandidateIdentity = {
judge: "@sikongjueluo/pi-permission-ai-judge@0.0.1",
permissionSystem: "25.4.0",
provider: "openai-codex",
model: "gpt-5.6-sol",
api: "openai-codex-responses",
promptVersion: "bash-shadow-v4",
toolSchemaVersion: "report-verdict-v1",
reviewSchemaVersion: "1",
timeoutCohort: 30000,
};
it("never grants authority for any mode or telemetry state without records", () => {
const modes = ["shadow", "enforce"] as const;
const healths = ["healthy", "disabled", "write_failed", "integrity_anomaly"] as const;
for (const mode of modes) {
for (const health of healths) {
const outcome = evaluateEnforceAuthority(
v01ProductionGateState(mode, health),
);
const gates = resolvePromotionGates(emptySnapshot, identity);
const outcome = evaluateEnforceAuthority({
...ALL_OPEN,
mode,
telemetryHealth: health,
...gates,
});
expect(outcome.kind).toBe("defer");
}
}
});
it("blocks v0.1 enforce on the cohort gate even with healthy audit", () => {
const outcome = evaluateEnforceAuthority(
v01ProductionGateState("enforce", "healthy", true),
);
expect(outcome).toEqual({ kind: "defer", blockedBy: "cohort_not_qualified" });
it("blocks enforce on the cohort gate first even with healthy audit", () => {
const gates = resolvePromotionGates(emptySnapshot, identity);
const outcome = evaluateEnforceAuthority({
...ALL_OPEN,
...gates,
});
expect(outcome).toEqual({
kind: "defer",
blockedBy: "cohort_not_qualified",
});
});
it("blocks v0.1 enforce on the audit gate when the audit log is unhealthy", () => {
const outcome = evaluateEnforceAuthority(
v01ProductionGateState("enforce", "healthy", false),
);
expect(outcome).toEqual({ kind: "defer", blockedBy: "audit_unhealthy" });
it("grants authority only when every record kind exists for the exact identity", () => {
const records = [
{
kind: "cohort_qualified",
candidateIdentity: identity,
recordedAt: "2026-08-20T12:00:00Z",
basis: "cohort test",
},
{
kind: "owner_approval",
candidateIdentity: identity,
recordedAt: "2026-08-20T12:01:00Z",
basis: "approved",
},
{
kind: "activation",
candidateIdentity: identity,
recordedAt: "2026-08-20T12:02:00Z",
basis: "activated",
},
] as const;
const snapshot = {
records,
healthy: true,
diagnostic: null,
path: "unused",
};
const gates = resolvePromotionGates(snapshot, identity);
expect(evaluateEnforceAuthority({ ...ALL_OPEN, ...gates })).toEqual({
kind: "allow",
});
});
});
@@ -1,4 +1,4 @@
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
@@ -50,6 +50,8 @@ import {
unpublishPermissionsService,
} from "@gotgenes/pi-permission-system";
import extension from "../src/index";
import { appendPromotionRecord, type CandidateIdentity } from "../src/promotion";
import { PROMPT_VERSION, TOOL_SCHEMA_VERSION } from "../src/prompt";
function createFakePi(): {
pi: ExtensionAPI;
@@ -539,3 +541,194 @@ describe("AI judge lifecycle", () => {
expect(service.registerAuthorizer).not.toHaveBeenCalled();
});
});
describe("AI judge Enforce authority seam (PIEXTENSIO-21)", () => {
beforeEach(() => {
createMockAgentDir();
writeFileSync(
join(mockAgentDir.dir, "pi-permission-ai-judge.config.json"),
JSON.stringify({ mode: "enforce" }),
);
});
/** Records identity matching the lifecycle fake model + static fields. */
function lifecycleIdentity(): CandidateIdentity {
return {
judge: "@sikongjueluo/pi-permission-ai-judge@0.0.1",
permissionSystem: "25.4.0",
provider: "test-provider",
model: "test-model",
api: "openai-codex-responses",
promptVersion: PROMPT_VERSION,
toolSchemaVersion: TOOL_SCHEMA_VERSION,
reviewSchemaVersion: "1",
timeoutCohort: "default",
};
}
async function runAsk(): Promise<{
verdict: { kind: string };
reviews: Array<{ event: string; details?: Record<string, unknown> }>;
}> {
let authorize: Authorizer["authorize"] | undefined;
const service = {
registerAuthorizer: vi.fn((_name, callback) => {
authorize = callback;
return vi.fn();
}),
checkPermission: vi.fn(),
getToolPermission: vi.fn(),
} as unknown as PermissionsService;
publishPermissionsService(service);
publishedService = service;
const complete = vi.fn(async () => modelResponse());
const ctx = {
hasUI: true,
sessionManager: fakeSessionManager(),
model: {
id: "test-model",
provider: "test-provider",
api: "openai-codex-responses",
} as Model<any>,
modelRegistry: { complete },
ui: { notify: vi.fn() },
} as unknown as ExtensionContext;
const harness = createFakePi();
extension(harness.pi);
harness.start(ctx);
harness.ready();
expect(authorize).toBeDefined();
const reviews: Array<{ event: string; details?: Record<string, unknown> }> = [];
const verdict = await authorize!(
ask(),
{
checkPermission: vi.fn(),
getToolPermission: vi.fn(),
},
{
review: (event, details) => reviews.push({ event, details }),
debug: vi.fn(),
},
);
harness.shutdown();
return { verdict, reviews };
}
it("defers in enforce mode when no promotion records exist", async () => {
const { verdict, reviews } = await runAsk();
expect(verdict).toEqual({ kind: "defer" });
expect(reviews).toMatchObject([
{
event: "ai_bash_judge.result",
details: expect.objectContaining({
mode: "enforce",
verdict: "allow",
effectiveVerdict: "defer",
authorityBlockedBy: "cohort_not_qualified",
}),
},
]);
});
it("grants authority in enforce mode only with all three exact-identity records", async () => {
const identity = lifecycleIdentity();
for (const [kind, basis] of [
["cohort_qualified", "cohort piextensio-test"],
["owner_approval", "approved for test"],
["activation", "activated for test"],
] as const) {
expect(
appendPromotionRecord({
agentDir: mockAgentDir.dir,
record: {
kind,
candidateIdentity: identity,
recordedAt: "2026-08-21T10:00:00Z",
basis,
},
}),
).toBeNull();
}
const { verdict, reviews } = await runAsk();
expect(verdict).toEqual({ kind: "allow" });
expect(reviews).toMatchObject([
{
event: "ai_bash_judge.result",
details: expect.objectContaining({
mode: "enforce",
verdict: "allow",
effectiveVerdict: "allow",
authorityBlockedBy: null,
}),
},
]);
});
it("defers in enforce mode when records exist for another identity", async () => {
const identity = { ...lifecycleIdentity(), model: "other-model" };
for (const kind of [
"cohort_qualified",
"owner_approval",
"activation",
] as const) {
appendPromotionRecord({
agentDir: mockAgentDir.dir,
record: {
kind,
candidateIdentity: identity,
recordedAt: "2026-08-21T10:00:00Z",
basis: "other identity",
},
});
}
const { verdict, reviews } = await runAsk();
expect(verdict).toEqual({ kind: "defer" });
expect(reviews).toMatchObject([
{
event: "ai_bash_judge.result",
details: expect.objectContaining({
effectiveVerdict: "defer",
authorityBlockedBy: "cohort_not_qualified",
}),
},
]);
});
it("never grants authority in shadow mode regardless of records", async () => {
writeFileSync(
join(mockAgentDir.dir, "pi-permission-ai-judge.config.json"),
JSON.stringify({ mode: "shadow" }),
);
const identity = lifecycleIdentity();
for (const kind of [
"cohort_qualified",
"owner_approval",
"activation",
] as const) {
appendPromotionRecord({
agentDir: mockAgentDir.dir,
record: {
kind,
candidateIdentity: identity,
recordedAt: "2026-08-21T10:00:00Z",
basis: "shadow still defers",
},
});
}
const { verdict, reviews } = await runAsk();
expect(verdict).toEqual({ kind: "defer" });
expect(reviews).toMatchObject([
{
event: "ai_bash_judge.result",
details: expect.objectContaining({
mode: "shadow",
effectiveVerdict: "defer",
authorityBlockedBy: "mode_shadow",
}),
},
]);
});
});
@@ -0,0 +1,260 @@
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import {
appendPromotionRecord,
loadPromotionRecords,
promotionRecordsPath,
resolvePromotionGates,
type CandidateIdentity,
type PromotionRecord,
} from "../src/promotion";
const IDENTITY: CandidateIdentity = {
judge: "@sikongjueluo/pi-permission-ai-judge@0.0.1",
permissionSystem: "25.4.0",
provider: "openai-codex",
model: "gpt-5.6-sol",
api: "openai-codex-responses",
promptVersion: "bash-shadow-v4",
toolSchemaVersion: "report-verdict-v1",
reviewSchemaVersion: "1",
timeoutCohort: 30000,
};
function record(
kind: PromotionRecord["kind"],
identity: CandidateIdentity = IDENTITY,
basis = "test basis",
): PromotionRecord {
return {
kind,
candidateIdentity: identity,
recordedAt: "2026-08-20T12:00:00Z",
basis,
};
}
function line(value: unknown): string {
return `${JSON.stringify(value)}\n`;
}
/** writeFileSync, but creating the records directory first. */
function writeRecords(dir: string, content: string): void {
const path = promotionRecordsPath(dir);
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, content);
}
describe("promotion records — loadPromotionRecords", () => {
let dir: string;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), "ai-judge-promotion-"));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
it("treats a missing file as the healthy pre-promotion state", () => {
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(snapshot).toEqual({
records: [],
healthy: true,
diagnostic: null,
path: promotionRecordsPath(dir),
});
});
it("parses well-formed records of every kind", () => {
writeRecords(
dir,
[record("cohort_qualified"), record("owner_approval"), record("activation")]
.map(line)
.join(""),
);
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(snapshot.healthy).toBe(true);
expect(snapshot.records).toHaveLength(3);
expect(snapshot.records.map((r) => r.kind)).toEqual([
"cohort_qualified",
"owner_approval",
"activation",
]);
});
it("fails closed on malformed JSON lines", () => {
writeRecords(
dir,
line(record("cohort_qualified")) + "{not json\n",
);
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(snapshot.healthy).toBe(false);
expect(snapshot.records).toEqual([]);
expect(snapshot.diagnostic).toContain("malformed");
});
it("fails closed on shape-invalid records", () => {
writeRecords(
dir,
line({ kind: "activation" }), // missing identity/basis/recordedAt
);
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(snapshot.healthy).toBe(false);
expect(snapshot.diagnostic).toContain("malformed");
});
it("skips blank lines without failing", () => {
writeRecords(
dir,
"\n" + line(record("activation")) + "\n\n",
);
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(snapshot.healthy).toBe(true);
expect(snapshot.records).toHaveLength(1);
});
});
describe("promotion records — resolvePromotionGates", () => {
let dir: string;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), "ai-judge-promotion-"));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
it("flips only its own gate per record kind", () => {
writeRecords(dir, line(record("owner_approval")));
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(resolvePromotionGates(snapshot, IDENTITY)).toEqual({
cohortQualified: false,
ownerApprovalRecorded: true,
activationRecorded: false,
});
});
it("all three gates open with all three records for the exact identity", () => {
writeRecords(
dir,
[
record("cohort_qualified", IDENTITY, "cohort id x"),
record("owner_approval", IDENTITY, "approved"),
record("activation", IDENTITY, "activated"),
]
.map(line)
.join(""),
);
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(resolvePromotionGates(snapshot, IDENTITY)).toEqual({
cohortQualified: true,
ownerApprovalRecorded: true,
activationRecorded: true,
});
});
const identityDrifts: ReadonlyArray<[string, Partial<CandidateIdentity>]> = [
["provider", { provider: "other-provider" }],
["model", { model: "gpt-5.7" }],
["api", { api: "openai-responses" }],
["promptVersion", { promptVersion: "bash-shadow-v5" }],
["toolSchemaVersion", { toolSchemaVersion: "report-verdict-v2" }],
["reviewSchemaVersion", { reviewSchemaVersion: "2" }],
["timeoutCohort", { timeoutCohort: "default" }],
["permissionSystem", { permissionSystem: "25.5.0" }],
["judge package", { judge: "@sikongjueluo/pi-permission-ai-judge@0.0.2" }],
];
for (const [name, patch] of identityDrifts) {
it(`keeps every gate closed when the live identity drifts on ${name}`, () => {
writeRecords(
dir,
[
record("cohort_qualified"),
record("owner_approval"),
record("activation"),
]
.map(line)
.join(""),
);
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(
resolvePromotionGates(snapshot, { ...IDENTITY, ...patch }),
).toEqual({
cohortQualified: false,
ownerApprovalRecorded: false,
activationRecorded: false,
});
});
}
it("closes every gate when the snapshot is unhealthy", () => {
writeRecords(dir, "garbage\n");
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(snapshot.healthy).toBe(false);
expect(resolvePromotionGates(snapshot, IDENTITY)).toEqual({
cohortQualified: false,
ownerApprovalRecorded: false,
activationRecorded: false,
});
});
it("leaves other-identity records inert, not malformed", () => {
const other: CandidateIdentity = {
...IDENTITY,
model: "glm-5.2",
};
writeRecords(
dir,
[
record("cohort_qualified", other),
record("activation", other),
]
.map(line)
.join(""),
);
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(snapshot.healthy).toBe(true);
expect(resolvePromotionGates(snapshot, IDENTITY)).toEqual({
cohortQualified: false,
ownerApprovalRecorded: false,
activationRecorded: false,
});
});
});
describe("promotion records — appendPromotionRecord", () => {
let dir: string;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), "ai-judge-promotion-"));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
it("appends a shape-valid record that round-trips through the loader", () => {
const error = appendPromotionRecord({
agentDir: dir,
record: record("owner_approval", IDENTITY, "approved v4"),
now: () => "2026-08-21T09:00:00Z",
});
expect(error).toBeNull();
const raw = readFileSync(promotionRecordsPath(dir), "utf-8");
expect(raw).toContain('"recordedAt":"2026-08-21T09:00:00Z"');
expect(raw).toContain('"basis":"approved v4"');
const snapshot = loadPromotionRecords({ agentDir: dir });
expect(snapshot.healthy).toBe(true);
expect(resolvePromotionGates(snapshot, IDENTITY).ownerApprovalRecorded).toBe(true);
});
it("rejects a shape-invalid record without touching the file", () => {
const bad = {
kind: "activation",
candidateIdentity: { judge: "x" },
recordedAt: "",
basis: "",
} as unknown as PromotionRecord;
const error = appendPromotionRecord({ agentDir: dir, record: bad });
expect(error).toBe("record is not shape-valid");
expect(loadPromotionRecords({ agentDir: dir }).records).toEqual([]);
});
});