diff --git a/packages/pi-permission-ai-judge/src/index.ts b/packages/pi-permission-ai-judge/src/index.ts index c8e7af6..2fe6e52 100644 --- a/packages/pi-permission-ai-judge/src/index.ts +++ b/packages/pi-permission-ai-judge/src/index.ts @@ -24,7 +24,16 @@ import { conversationProbeFromSession, type ConversationEvidence, } from "./conversation"; -import { evaluateEnforceAuthority, v01ProductionGateState } from "./judge"; +import { + evaluateEnforceAuthority, + type EnforceGateState, +} from "./judge"; +import { + loadPromotionRecords, + resolvePromotionGates, + type CandidateIdentity, + type PromotionRecordsSnapshot, +} from "./promotion"; const LINK_NAME = "ai-bash-judge"; const REVIEW_SCHEMA_VERSION = 1; @@ -49,6 +58,15 @@ interface RootSession { readonly getCwd: () => string; /** Judge-owned audit log (ADR 0006); unhealthy refuses Enforce authority. */ readonly auditLog: AuditLog; + /** Promotion-records snapshot loaded at session start (PIEXTENSIO-21); + * gates are resolved per ask against the live candidate identity. */ + readonly promotionRecords: PromotionRecordsSnapshot; + /** Static candidate-identity fields (everything except the model + * segment, which is captured per ask). */ + readonly identityBase: Omit< + CandidateIdentity, + "provider" | "model" | "api" + >; } const EMPTY_CONVERSATION: ConversationEvidence = { @@ -285,6 +303,52 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { conversation, ); + // Enforce truth table (PIEXTENSIO-3 cat.4 / M5): the + // promotion gates resolve per ask from the session-start + // records snapshot against the live candidate identity + // (PIEXTENSIO-21) — the current model segment is part of + // that identity, so a mid-session model switch cannot + // inherit another segment's records, and identity drift + // after recording fails closed. The truth table is the + // single authority seam: the owner's records flip the + // gate inputs, never the callback's return path. + // reviewAcknowledged is true in the ADR 0006 sense: the + // Judge-owned audit write for this result happens before + // the authority return, and a failed write flips + // auditHealthy sticky-unhealthy, closing authority for + // every later ask. + const liveIdentity: CandidateIdentity = { + ...captured.identityBase, + provider: result.metadata?.provider ?? "", + model: result.metadata?.model ?? "", + api: result.metadata?.api ?? "", + }; + const gates = resolvePromotionGates( + captured.promotionRecords, + liveIdentity, + ); + const gateState: EnforceGateState = { + auditHealthy: captured.auditLog.healthy(), + telemetryHealth: sink.health(), + cohortQualified: gates.cohortQualified, + ownerApprovalRecorded: gates.ownerApprovalRecorded, + activationRecorded: gates.activationRecorded, + resultKind: + result.kind === "judgment" + ? "judgment" + : result.kind, + verdict: + result.kind === "judgment" ? result.verdict : null, + reviewAcknowledged: true, + generationCurrent: !captured.shutdown.signal.aborted, + mode: captured.config.mode, + }; + const authority = evaluateEnforceAuthority(gateState); + const effectiveVerdict = + authority.kind === "allow" ? "allow" : "defer"; + const authorityBlockedBy = + authority.kind === "allow" ? null : authority.blockedBy; + if (result.kind === "judgment") { emitResult({ ...resultBase( @@ -295,7 +359,8 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { ), resultKind: "judgment", verdict: result.verdict, - effectiveVerdict: "defer", + effectiveVerdict, + authorityBlockedBy, modelCalled: true, code: null, provider: result.metadata.provider, @@ -321,7 +386,8 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { ), resultKind: "infrastructure_failure", verdict: null, - effectiveVerdict: "defer", + effectiveVerdict, + authorityBlockedBy, modelCalled: result.modelCalled, code: result.code, provider: result.metadata?.provider ?? null, @@ -334,19 +400,6 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { }); } - // Enforce truth table (PIEXTENSIO-3 cat.4 / M5): v0.1 - // production gates are structurally unreachable, so any - // configured mode resolves to defer here. The call - // exists so the truth table is the single authority - // seam — a future slice flips the gate inputs, not the - // callback's return path. - const authority = evaluateEnforceAuthority( - v01ProductionGateState( - captured.config.mode, - sink.health(), - captured.auditLog.healthy(), - ), - ); return authority.kind === "allow" ? { kind: "allow" } : { kind: "defer" }; @@ -372,6 +425,27 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { } const runtimeId = crypto.randomUUID(); + const config = loadJudgeConfig({ agentDir: getAgentDir() }); + // Static candidate-identity fields. The judge and permission-system + // versions must track package.json / the peer floor; the cohort + // declaration used exactly these values. + const identityBase = { + judge: "@sikongjueluo/pi-permission-ai-judge@0.0.1", + permissionSystem: "25.4.0", + promptVersion: PROMPT_VERSION, + toolSchemaVersion: TOOL_SCHEMA_VERSION, + reviewSchemaVersion: String(REVIEW_SCHEMA_VERSION), + timeoutCohort: config.timeoutCohort, + }; + const promotionRecords = loadPromotionRecords({ + agentDir: getAgentDir(), + }); + if (promotionRecords.diagnostic !== null) { + ctx.ui.notify( + `ai-bash-judge promotion records: ${promotionRecords.diagnostic}; Enforce gates stay closed`, + "warning", + ); + } root = { getSessionId: () => ctx.sessionManager.getSessionId(), expectedSessionId: sessionId, @@ -379,12 +453,14 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { modelRegistry: ctx.modelRegistry, shutdown: new AbortController(), judgeRuntimeId: runtimeId, - config: loadJudgeConfig({ agentDir: getAgentDir() }), + config, reviewLogEnabled: readPermissionReviewLogEnabled(), auditLog: createAuditLog({ agentDir: getAgentDir(), runtimeId, }), + promotionRecords, + identityBase, conversation: conversationProbeFromSession(ctx.sessionManager), getCwd: () => ctx.sessionManager.getCwd(), }; diff --git a/packages/pi-permission-ai-judge/src/judge.ts b/packages/pi-permission-ai-judge/src/judge.ts index 31cd491..2b95a7c 100644 --- a/packages/pi-permission-ai-judge/src/judge.ts +++ b/packages/pi-permission-ai-judge/src/judge.ts @@ -4,11 +4,10 @@ import type { TelemetryHealth } from "./review"; * Enforce truth table (PIEXTENSIO-3 cat.4 / M5). * * `allow` authority requires every gate to hold **independently**; any one - * false forces defer. In v0.1 the promotion gates (cohort, approval, - * activation) are structurally unreachable — no upstream seam exists to - * record them — so a configured `enforce` mode can never grant authority: - * mechanically fail-closed by construction, verified by the truth-table - * tests toggling each condition. + * false forces defer. Since PIEXTENSIO-21 the promotion gates read the + * Judge-owned records file (exact-identity qualification, fail-closed); + * with no records, every mode mechanically defers — verified by the + * truth-table tests toggling each condition. */ export type EnforceGateState = { @@ -38,8 +37,8 @@ export type EnforceOutcome = { kind: "allow" } | { kind: "defer"; blockedBy: str /** * Evaluate the Enforce truth table. Shadow always defers. The distinct - * `blockedBy` reasons keep each gate's veto observable in tests (and in a - * future review event) without granting anything. + * `blockedBy` reasons keep each gate's veto observable in tests and in the + * authority field of every result row without granting anything. */ export function evaluateEnforceAuthority( state: EnforceGateState, @@ -76,30 +75,3 @@ export function evaluateEnforceAuthority( } return { kind: "allow" }; } - -/** - * The v0.1 production gate state: the promotion gates are structurally - * unreachable until the owner records a qualifying cohort, approval, and - * activation (ADR 0006: recorded Judge-side, not via a host contract), - * so `enforce` defers here regardless of configuration. The full truth - * table above is exercised through injected fake gate states in tests - * only. - */ -export function v01ProductionGateState( - mode: "shadow" | "enforce", - telemetryHealth: TelemetryHealth, - auditHealthy = true, -): EnforceGateState { - return { - auditHealthy, - telemetryHealth, - cohortQualified: false, - ownerApprovalRecorded: false, - activationRecorded: false, - resultKind: "judgment", - verdict: null, - reviewAcknowledged: false, - generationCurrent: true, - mode, - }; -} diff --git a/packages/pi-permission-ai-judge/src/promotion.ts b/packages/pi-permission-ai-judge/src/promotion.ts new file mode 100644 index 0000000..ed48bc2 --- /dev/null +++ b/packages/pi-permission-ai-judge/src/promotion.ts @@ -0,0 +1,322 @@ +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + writeSync, + fsyncSync, +} from "node:fs"; +import { join } from "node:path"; +import { stripForbiddenKeys } from "./review"; + +/** + * Judge-owned promotion-gate records (ADR 0006 self-sufficiency; PIEXTENSIO-10 + * promotion governance; PIEXTENSIO-21 seam). + * + * The Enforce truth table consumes three gate inputs — cohort qualification, + * owner approval, activation — that v0.1 hardcoded to false. This module is + * their storage and resolution: + * + * - **Records file**: append-only JSONL under the agent dir, fsync per + * record, same discipline as the audit log. Three record kinds: + * `cohort_qualified`, `owner_approval`, `activation`. Owner actions write + * records offline (the CLI helper in tools/); the online judge only reads. + * - **Exact-identity qualification**: a record qualifies the gate only if + * its candidate identity matches the live identity field-for-field. A + * record for any other identity is inert. This is the PIEXTENSIO-10 rule + * that approval binds to the exact candidate, and it makes promotion + * immune to identity drift after the records are written. + * - **Fail-closed everywhere**: missing file, unreadable file, malformed + * line, or shape-invalid record ⇒ the gate stays false. No record kind + * may flip any gate other than its own. Gates are resolved once at + * session start (immutable snapshot for the session), mirroring the + * reload-only config contract. + * + * Mode is intentionally *not* part of a candidate identity: `mode` is the + * authority knob (shadow/enforce), while identity is what the records + * certify. Requiring mode equality would let a shadow cohort qualify an + * enforce activation record — or vice versa — which the governance + * explicitly separates. + */ + +/** The candidate-identity fields a promotion record must match exactly. */ +export interface CandidateIdentity { + readonly judge: string; + readonly permissionSystem: string; + readonly provider: string; + readonly model: string; + readonly api: string; + readonly promptVersion: string; + readonly toolSchemaVersion: string; + readonly reviewSchemaVersion: string; + readonly timeoutCohort: "default" | number; +} + +export type PromotionRecordKind = + | "cohort_qualified" + | "owner_approval" + | "activation"; + +export interface PromotionRecord { + readonly kind: PromotionRecordKind; + readonly candidateIdentity: CandidateIdentity; + /** ISO timestamp written by the record author. */ + readonly recordedAt: string; + /** Human-readable basis (cohort id, report reference, approval note). */ + readonly basis: string; +} + +export interface PromotionRecordsSnapshot { + /** Shape-valid records parsed from the file (all identities). */ + readonly records: readonly PromotionRecord[]; + /** False when the file exists but is unreadable or has malformed lines. */ + readonly healthy: boolean; + readonly diagnostic: string | null; + /** Absolute path of the records file (empty string when unset). */ + readonly path: string; +} + +export interface PromotionRecordsDeps { + /** User-global agent dir (`~/.pi/agent`). */ + readonly agentDir: string; + /** Injectable reader for tests; defaults to readFileSync(utf-8). */ + readonly readFile?: (path: string) => string; +} + +const RECORDS_DIR_SEGMENTS = ["extensions", "pi-permission-ai-judge"]; +const RECORDS_FILENAME = "promotion-records.jsonl"; + +const IDENTITY_KEYS = [ + "judge", + "permissionSystem", + "provider", + "model", + "api", + "promptVersion", + "toolSchemaVersion", + "reviewSchemaVersion", + "timeoutCohort", +] as const; + +const STRING_IDENTITY_KEYS = IDENTITY_KEYS.filter( + (key) => key !== "timeoutCohort", +); + +export function promotionRecordsPath(agentDir: string): string { + return join(agentDir, ...RECORDS_DIR_SEGMENTS, RECORDS_FILENAME); +} + +function identityMatches( + record: CandidateIdentity, + live: CandidateIdentity, +): boolean { + return IDENTITY_KEYS.every((key) => record[key] === live[key]); +} + +function isRecordShape(value: unknown): value is PromotionRecord { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return false; + } + const record = value as Record; + if ( + record.kind !== "cohort_qualified" && + record.kind !== "owner_approval" && + record.kind !== "activation" + ) { + return false; + } + if ( + typeof record.recordedAt !== "string" || + record.recordedAt.length === 0 + ) { + return false; + } + if (typeof record.basis !== "string") { + return false; + } + const identity = record.candidateIdentity; + if ( + typeof identity !== "object" || + identity === null || + Array.isArray(identity) + ) { + return false; + } + const identityRecord = identity as Record; + // Every field except timeoutCohort is a string; timeoutCohort is + // "default" or a positive integer (config-cohort semantics). + const timeoutCohort = identityRecord.timeoutCohort; + const validTimeoutCohort = + timeoutCohort === "default" || + (typeof timeoutCohort === "number" && + Number.isInteger(timeoutCohort) && + timeoutCohort > 0); + if ( + !STRING_IDENTITY_KEYS.every((key) => + typeof identityRecord[key] === "string", + ) || + !validTimeoutCohort + ) { + return false; + } + return true; +} + +/** + * Load and parse the records file once per session (immutable snapshot). + * + * Read-only and side-effect free. Malformed or partial state never raises — + * the snapshot reports `healthy: false`, which `resolvePromotionGates` + * turns into all-gates-closed. A missing file is the normal pre-promotion + * state: healthy with zero records. + */ +export function loadPromotionRecords( + deps: PromotionRecordsDeps, +): PromotionRecordsSnapshot { + const file = promotionRecordsPath(deps.agentDir); + const read = deps.readFile ?? ((p: string) => readFileSync(p, "utf-8")); + + let raw: string; + try { + raw = read(file); + } catch { + return { + records: [], + healthy: true, + diagnostic: existsSync(file) ? `records unreadable at ${file}` : null, + path: file, + }; + } + + const records: PromotionRecord[] = []; + let malformed = false; + for (const line of raw.split("\n")) { + const trimmed = line.trim(); + if (trimmed.length === 0) { + continue; + } + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + malformed = true; + continue; + } + if (!isRecordShape(parsed)) { + malformed = true; + continue; + } + records.push(parsed); + } + + if (malformed) { + return { + records: [], + healthy: false, + diagnostic: `malformed records at ${file}`, + path: file, + }; + } + return { records, healthy: true, diagnostic: null, path: file }; +} + +/** The three Enforce promotion gates for one live candidate identity. */ +export interface PromotionGateResolution { + readonly cohortQualified: boolean; + readonly ownerApprovalRecorded: boolean; + readonly activationRecorded: boolean; +} + +const ALL_GATES_CLOSED: PromotionGateResolution = { + cohortQualified: false, + ownerApprovalRecorded: false, + activationRecorded: false, +}; + +/** + * Resolve the three Enforce promotion gates for the live candidate + * identity against a session-start records snapshot. + * + * Called per ask: the live identity carries the current model segment, so + * a mid-session model switch cannot inherit another segment's records. + * Unhealthy snapshot ⇒ all gates closed (fail-closed). A record qualifies + * only when its candidate identity matches field-for-field; records for + * any other identity are inert. + */ +export function resolvePromotionGates( + snapshot: PromotionRecordsSnapshot, + liveIdentity: CandidateIdentity, +): PromotionGateResolution { + if (!snapshot.healthy) { + return ALL_GATES_CLOSED; + } + let cohortQualified = false; + let ownerApprovalRecorded = false; + let activationRecorded = false; + for (const record of snapshot.records) { + if (!identityMatches(record.candidateIdentity, liveIdentity)) { + continue; + } + if (record.kind === "cohort_qualified") cohortQualified = true; + if (record.kind === "owner_approval") ownerApprovalRecorded = true; + if (record.kind === "activation") activationRecorded = true; + } + return { cohortQualified, ownerApprovalRecorded, activationRecorded }; +} + +export interface AppendPromotionRecordDeps { + /** User-global agent dir (`~/.pi/agent`). */ + readonly agentDir: string; + readonly record: PromotionRecord; + /** Injectable clock for tests; defaults to ISO-now. */ + readonly now?: () => string; +} + +/** + * Offline owner action: append one promotion record (append + fsync, same + * write discipline as the audit log). Never imported by online modules — + * the CLI helper in tools/ is the only in-package consumer. + * + * Returns null on success or a human-readable failure reason. + */ +export function appendPromotionRecord( + deps: AppendPromotionRecordDeps, +): string | null { + const now = deps.now ?? (() => new Date().toISOString()); + const dir = join(deps.agentDir, ...RECORDS_DIR_SEGMENTS); + const file = join(dir, RECORDS_FILENAME); + if (!isRecordShape(deps.record)) { + return "record is not shape-valid"; + } + try { + mkdirSync(dir, { recursive: true }); + } catch (error) { + return `cannot create ${dir}: ${error instanceof Error ? error.message : String(error)}`; + } + const record = JSON.stringify({ + recordedAt: now(), + ...stripForbiddenKeys({ + kind: deps.record.kind, + candidateIdentity: deps.record.candidateIdentity, + basis: deps.record.basis, + } as Record), + }); + let fd: number | undefined; + try { + fd = openSync(file, "a"); + writeSync(fd, `${record}\n`); + fsyncSync(fd); + return null; + } catch (error) { + return `cannot append to ${file}: ${error instanceof Error ? error.message : String(error)}`; + } finally { + if (fd !== undefined) { + try { + closeSync(fd); + } catch { + // Close failure does not un-fail the append. + } + } + } +} diff --git a/packages/pi-permission-ai-judge/test/judge.test.ts b/packages/pi-permission-ai-judge/test/judge.test.ts index 825544d..cae7da3 100644 --- a/packages/pi-permission-ai-judge/test/judge.test.ts +++ b/packages/pi-permission-ai-judge/test/judge.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from "vitest"; import { evaluateEnforceAuthority, - v01ProductionGateState, type EnforceGateState, } from "../src/judge"; +import { + loadPromotionRecords, + resolvePromotionGates, + type CandidateIdentity, +} from "../src/promotion"; const ALL_OPEN: EnforceGateState = { auditHealthy: true, @@ -55,31 +59,85 @@ describe("evaluateEnforceAuthority — every gate independently forces defer", ( } }); -describe("evaluateEnforceAuthority — v0.1 production state", () => { - it("never grants authority for any mode or telemetry state in v0.1", () => { +describe("evaluateEnforceAuthority — production state with no promotion records", () => { + // The real post-PIEXTENSIO-21 seam: an empty records file (the normal + // pre-promotion state) closes all promotion gates, so every mode and + // telemetry state defers — mechanically identical to v0.1's hardcoded + // closure, now derived from actual storage. + const emptySnapshot = loadPromotionRecords({ + agentDir: "/nonexistent-agent-dir", + }); + const identity: CandidateIdentity = { + judge: "@sikongjueluo/pi-permission-ai-judge@0.0.1", + permissionSystem: "25.4.0", + provider: "openai-codex", + model: "gpt-5.6-sol", + api: "openai-codex-responses", + promptVersion: "bash-shadow-v4", + toolSchemaVersion: "report-verdict-v1", + reviewSchemaVersion: "1", + timeoutCohort: 30000, + }; + + it("never grants authority for any mode or telemetry state without records", () => { const modes = ["shadow", "enforce"] as const; const healths = ["healthy", "disabled", "write_failed", "integrity_anomaly"] as const; for (const mode of modes) { for (const health of healths) { - const outcome = evaluateEnforceAuthority( - v01ProductionGateState(mode, health), - ); + const gates = resolvePromotionGates(emptySnapshot, identity); + const outcome = evaluateEnforceAuthority({ + ...ALL_OPEN, + mode, + telemetryHealth: health, + ...gates, + }); expect(outcome.kind).toBe("defer"); } } }); - it("blocks v0.1 enforce on the cohort gate even with healthy audit", () => { - const outcome = evaluateEnforceAuthority( - v01ProductionGateState("enforce", "healthy", true), - ); - expect(outcome).toEqual({ kind: "defer", blockedBy: "cohort_not_qualified" }); + it("blocks enforce on the cohort gate first even with healthy audit", () => { + const gates = resolvePromotionGates(emptySnapshot, identity); + const outcome = evaluateEnforceAuthority({ + ...ALL_OPEN, + ...gates, + }); + expect(outcome).toEqual({ + kind: "defer", + blockedBy: "cohort_not_qualified", + }); }); - it("blocks v0.1 enforce on the audit gate when the audit log is unhealthy", () => { - const outcome = evaluateEnforceAuthority( - v01ProductionGateState("enforce", "healthy", false), - ); - expect(outcome).toEqual({ kind: "defer", blockedBy: "audit_unhealthy" }); + it("grants authority only when every record kind exists for the exact identity", () => { + const records = [ + { + kind: "cohort_qualified", + candidateIdentity: identity, + recordedAt: "2026-08-20T12:00:00Z", + basis: "cohort test", + }, + { + kind: "owner_approval", + candidateIdentity: identity, + recordedAt: "2026-08-20T12:01:00Z", + basis: "approved", + }, + { + kind: "activation", + candidateIdentity: identity, + recordedAt: "2026-08-20T12:02:00Z", + basis: "activated", + }, + ] as const; + const snapshot = { + records, + healthy: true, + diagnostic: null, + path: "unused", + }; + const gates = resolvePromotionGates(snapshot, identity); + expect(evaluateEnforceAuthority({ ...ALL_OPEN, ...gates })).toEqual({ + kind: "allow", + }); }); }); diff --git a/packages/pi-permission-ai-judge/test/lifecycle.test.ts b/packages/pi-permission-ai-judge/test/lifecycle.test.ts index ec43450..3d9cee1 100644 --- a/packages/pi-permission-ai-judge/test/lifecycle.test.ts +++ b/packages/pi-permission-ai-judge/test/lifecycle.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -50,6 +50,8 @@ import { unpublishPermissionsService, } from "@gotgenes/pi-permission-system"; import extension from "../src/index"; +import { appendPromotionRecord, type CandidateIdentity } from "../src/promotion"; +import { PROMPT_VERSION, TOOL_SCHEMA_VERSION } from "../src/prompt"; function createFakePi(): { pi: ExtensionAPI; @@ -539,3 +541,194 @@ describe("AI judge lifecycle", () => { expect(service.registerAuthorizer).not.toHaveBeenCalled(); }); }); + +describe("AI judge Enforce authority seam (PIEXTENSIO-21)", () => { + beforeEach(() => { + createMockAgentDir(); + writeFileSync( + join(mockAgentDir.dir, "pi-permission-ai-judge.config.json"), + JSON.stringify({ mode: "enforce" }), + ); + }); + + /** Records identity matching the lifecycle fake model + static fields. */ + function lifecycleIdentity(): CandidateIdentity { + return { + judge: "@sikongjueluo/pi-permission-ai-judge@0.0.1", + permissionSystem: "25.4.0", + provider: "test-provider", + model: "test-model", + api: "openai-codex-responses", + promptVersion: PROMPT_VERSION, + toolSchemaVersion: TOOL_SCHEMA_VERSION, + reviewSchemaVersion: "1", + timeoutCohort: "default", + }; + } + + async function runAsk(): Promise<{ + verdict: { kind: string }; + reviews: Array<{ event: string; details?: Record }>; + }> { + let authorize: Authorizer["authorize"] | undefined; + const service = { + registerAuthorizer: vi.fn((_name, callback) => { + authorize = callback; + return vi.fn(); + }), + checkPermission: vi.fn(), + getToolPermission: vi.fn(), + } as unknown as PermissionsService; + publishPermissionsService(service); + publishedService = service; + + const complete = vi.fn(async () => modelResponse()); + const ctx = { + hasUI: true, + sessionManager: fakeSessionManager(), + model: { + id: "test-model", + provider: "test-provider", + api: "openai-codex-responses", + } as Model, + modelRegistry: { complete }, + ui: { notify: vi.fn() }, + } as unknown as ExtensionContext; + + const harness = createFakePi(); + extension(harness.pi); + harness.start(ctx); + harness.ready(); + expect(authorize).toBeDefined(); + + const reviews: Array<{ event: string; details?: Record }> = []; + const verdict = await authorize!( + ask(), + { + checkPermission: vi.fn(), + getToolPermission: vi.fn(), + }, + { + review: (event, details) => reviews.push({ event, details }), + debug: vi.fn(), + }, + ); + harness.shutdown(); + return { verdict, reviews }; + } + + it("defers in enforce mode when no promotion records exist", async () => { + const { verdict, reviews } = await runAsk(); + expect(verdict).toEqual({ kind: "defer" }); + expect(reviews).toMatchObject([ + { + event: "ai_bash_judge.result", + details: expect.objectContaining({ + mode: "enforce", + verdict: "allow", + effectiveVerdict: "defer", + authorityBlockedBy: "cohort_not_qualified", + }), + }, + ]); + }); + + it("grants authority in enforce mode only with all three exact-identity records", async () => { + const identity = lifecycleIdentity(); + for (const [kind, basis] of [ + ["cohort_qualified", "cohort piextensio-test"], + ["owner_approval", "approved for test"], + ["activation", "activated for test"], + ] as const) { + expect( + appendPromotionRecord({ + agentDir: mockAgentDir.dir, + record: { + kind, + candidateIdentity: identity, + recordedAt: "2026-08-21T10:00:00Z", + basis, + }, + }), + ).toBeNull(); + } + const { verdict, reviews } = await runAsk(); + expect(verdict).toEqual({ kind: "allow" }); + expect(reviews).toMatchObject([ + { + event: "ai_bash_judge.result", + details: expect.objectContaining({ + mode: "enforce", + verdict: "allow", + effectiveVerdict: "allow", + authorityBlockedBy: null, + }), + }, + ]); + }); + + it("defers in enforce mode when records exist for another identity", async () => { + const identity = { ...lifecycleIdentity(), model: "other-model" }; + for (const kind of [ + "cohort_qualified", + "owner_approval", + "activation", + ] as const) { + appendPromotionRecord({ + agentDir: mockAgentDir.dir, + record: { + kind, + candidateIdentity: identity, + recordedAt: "2026-08-21T10:00:00Z", + basis: "other identity", + }, + }); + } + const { verdict, reviews } = await runAsk(); + expect(verdict).toEqual({ kind: "defer" }); + expect(reviews).toMatchObject([ + { + event: "ai_bash_judge.result", + details: expect.objectContaining({ + effectiveVerdict: "defer", + authorityBlockedBy: "cohort_not_qualified", + }), + }, + ]); + }); + + it("never grants authority in shadow mode regardless of records", async () => { + writeFileSync( + join(mockAgentDir.dir, "pi-permission-ai-judge.config.json"), + JSON.stringify({ mode: "shadow" }), + ); + const identity = lifecycleIdentity(); + for (const kind of [ + "cohort_qualified", + "owner_approval", + "activation", + ] as const) { + appendPromotionRecord({ + agentDir: mockAgentDir.dir, + record: { + kind, + candidateIdentity: identity, + recordedAt: "2026-08-21T10:00:00Z", + basis: "shadow still defers", + }, + }); + } + const { verdict, reviews } = await runAsk(); + expect(verdict).toEqual({ kind: "defer" }); + expect(reviews).toMatchObject([ + { + event: "ai_bash_judge.result", + details: expect.objectContaining({ + mode: "shadow", + effectiveVerdict: "defer", + authorityBlockedBy: "mode_shadow", + }), + }, + ]); + }); +}); diff --git a/packages/pi-permission-ai-judge/test/promotion.test.ts b/packages/pi-permission-ai-judge/test/promotion.test.ts new file mode 100644 index 0000000..09ba23d --- /dev/null +++ b/packages/pi-permission-ai-judge/test/promotion.test.ts @@ -0,0 +1,260 @@ +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + appendPromotionRecord, + loadPromotionRecords, + promotionRecordsPath, + resolvePromotionGates, + type CandidateIdentity, + type PromotionRecord, +} from "../src/promotion"; + +const IDENTITY: CandidateIdentity = { + judge: "@sikongjueluo/pi-permission-ai-judge@0.0.1", + permissionSystem: "25.4.0", + provider: "openai-codex", + model: "gpt-5.6-sol", + api: "openai-codex-responses", + promptVersion: "bash-shadow-v4", + toolSchemaVersion: "report-verdict-v1", + reviewSchemaVersion: "1", + timeoutCohort: 30000, +}; + +function record( + kind: PromotionRecord["kind"], + identity: CandidateIdentity = IDENTITY, + basis = "test basis", +): PromotionRecord { + return { + kind, + candidateIdentity: identity, + recordedAt: "2026-08-20T12:00:00Z", + basis, + }; +} + +function line(value: unknown): string { + return `${JSON.stringify(value)}\n`; +} + +/** writeFileSync, but creating the records directory first. */ +function writeRecords(dir: string, content: string): void { + const path = promotionRecordsPath(dir); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content); +} + +describe("promotion records — loadPromotionRecords", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ai-judge-promotion-")); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("treats a missing file as the healthy pre-promotion state", () => { + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(snapshot).toEqual({ + records: [], + healthy: true, + diagnostic: null, + path: promotionRecordsPath(dir), + }); + }); + + it("parses well-formed records of every kind", () => { + writeRecords( + dir, + [record("cohort_qualified"), record("owner_approval"), record("activation")] + .map(line) + .join(""), + ); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(snapshot.healthy).toBe(true); + expect(snapshot.records).toHaveLength(3); + expect(snapshot.records.map((r) => r.kind)).toEqual([ + "cohort_qualified", + "owner_approval", + "activation", + ]); + }); + + it("fails closed on malformed JSON lines", () => { + writeRecords( + dir, + line(record("cohort_qualified")) + "{not json\n", + ); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(snapshot.healthy).toBe(false); + expect(snapshot.records).toEqual([]); + expect(snapshot.diagnostic).toContain("malformed"); + }); + + it("fails closed on shape-invalid records", () => { + writeRecords( + dir, + line({ kind: "activation" }), // missing identity/basis/recordedAt + ); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(snapshot.healthy).toBe(false); + expect(snapshot.diagnostic).toContain("malformed"); + }); + + it("skips blank lines without failing", () => { + writeRecords( + dir, + "\n" + line(record("activation")) + "\n\n", + ); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(snapshot.healthy).toBe(true); + expect(snapshot.records).toHaveLength(1); + }); +}); + +describe("promotion records — resolvePromotionGates", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ai-judge-promotion-")); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("flips only its own gate per record kind", () => { + writeRecords(dir, line(record("owner_approval"))); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(resolvePromotionGates(snapshot, IDENTITY)).toEqual({ + cohortQualified: false, + ownerApprovalRecorded: true, + activationRecorded: false, + }); + }); + + it("all three gates open with all three records for the exact identity", () => { + writeRecords( + dir, + [ + record("cohort_qualified", IDENTITY, "cohort id x"), + record("owner_approval", IDENTITY, "approved"), + record("activation", IDENTITY, "activated"), + ] + .map(line) + .join(""), + ); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(resolvePromotionGates(snapshot, IDENTITY)).toEqual({ + cohortQualified: true, + ownerApprovalRecorded: true, + activationRecorded: true, + }); + }); + + const identityDrifts: ReadonlyArray<[string, Partial]> = [ + ["provider", { provider: "other-provider" }], + ["model", { model: "gpt-5.7" }], + ["api", { api: "openai-responses" }], + ["promptVersion", { promptVersion: "bash-shadow-v5" }], + ["toolSchemaVersion", { toolSchemaVersion: "report-verdict-v2" }], + ["reviewSchemaVersion", { reviewSchemaVersion: "2" }], + ["timeoutCohort", { timeoutCohort: "default" }], + ["permissionSystem", { permissionSystem: "25.5.0" }], + ["judge package", { judge: "@sikongjueluo/pi-permission-ai-judge@0.0.2" }], + ]; + for (const [name, patch] of identityDrifts) { + it(`keeps every gate closed when the live identity drifts on ${name}`, () => { + writeRecords( + dir, + [ + record("cohort_qualified"), + record("owner_approval"), + record("activation"), + ] + .map(line) + .join(""), + ); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect( + resolvePromotionGates(snapshot, { ...IDENTITY, ...patch }), + ).toEqual({ + cohortQualified: false, + ownerApprovalRecorded: false, + activationRecorded: false, + }); + }); + } + + it("closes every gate when the snapshot is unhealthy", () => { + writeRecords(dir, "garbage\n"); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(snapshot.healthy).toBe(false); + expect(resolvePromotionGates(snapshot, IDENTITY)).toEqual({ + cohortQualified: false, + ownerApprovalRecorded: false, + activationRecorded: false, + }); + }); + + it("leaves other-identity records inert, not malformed", () => { + const other: CandidateIdentity = { + ...IDENTITY, + model: "glm-5.2", + }; + writeRecords( + dir, + [ + record("cohort_qualified", other), + record("activation", other), + ] + .map(line) + .join(""), + ); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(snapshot.healthy).toBe(true); + expect(resolvePromotionGates(snapshot, IDENTITY)).toEqual({ + cohortQualified: false, + ownerApprovalRecorded: false, + activationRecorded: false, + }); + }); +}); + +describe("promotion records — appendPromotionRecord", () => { + let dir: string; + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "ai-judge-promotion-")); + }); + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("appends a shape-valid record that round-trips through the loader", () => { + const error = appendPromotionRecord({ + agentDir: dir, + record: record("owner_approval", IDENTITY, "approved v4"), + now: () => "2026-08-21T09:00:00Z", + }); + expect(error).toBeNull(); + const raw = readFileSync(promotionRecordsPath(dir), "utf-8"); + expect(raw).toContain('"recordedAt":"2026-08-21T09:00:00Z"'); + expect(raw).toContain('"basis":"approved v4"'); + const snapshot = loadPromotionRecords({ agentDir: dir }); + expect(snapshot.healthy).toBe(true); + expect(resolvePromotionGates(snapshot, IDENTITY).ownerApprovalRecorded).toBe(true); + }); + + it("rejects a shape-invalid record without touching the file", () => { + const bad = { + kind: "activation", + candidateIdentity: { judge: "x" }, + recordedAt: "", + basis: "", + } as unknown as PromotionRecord; + const error = appendPromotionRecord({ agentDir: dir, record: bad }); + expect(error).toBe("record is not shape-valid"); + expect(loadPromotionRecords({ agentDir: dir }).records).toEqual([]); + }); +}); diff --git a/packages/pi-permission-ai-judge/tools/promotion-record.ts b/packages/pi-permission-ai-judge/tools/promotion-record.ts new file mode 100644 index 0000000..d1522df --- /dev/null +++ b/packages/pi-permission-ai-judge/tools/promotion-record.ts @@ -0,0 +1,167 @@ +/** + * PIEXTENSIO-21 owner action CLI (offline, package-local). + * + * Appends one promotion record to the Judge-owned records file + * (~/.pi/agent/extensions/pi-permission-ai-judge/promotion-records.jsonl). + * Never imported by online modules; no npm bin. The three record kinds map + * one-to-one to the Enforce truth-table promotion gates: + * + * cohort_qualified — after a declared replacement cohort meets the frozen + * floor, citing the cohort id + report reference + * owner_approval — the owner's explicit approval of that exact candidate + * activation — the distinct, explicit activation act + * + * Fail-closed contract: a record qualifies only its exact candidate + * identity; the judge re-derives identity from the live runtime, so a + * mismatched or malformed record is inert and malformed files close all + * gates. There is no CLI to *remove* authority records by design — + * rollback to Shadow is the config `mode` switch, and the append-only + * trail keeps the promotion history auditable. + * + * Usage: + * npx tsx tools/promotion-record.ts --kind cohort_qualified \ + * --provider openai-codex --model gpt-5.6-sol --api openai-codex-responses \ + * [--timeout-cohort 30000] --basis "cohort ; report docs/testing/..." + * + * judge/permission-system/prompt/tool-schema/review-schema versions are + * taken from the live package (src/prompt.ts + package constants) so the + * recorded identity cannot drift from the code that will check it. + */ + +import { getAgentDir } from "@earendil-works/pi-coding-agent"; +import { + appendPromotionRecord, + promotionRecordsPath, + type CandidateIdentity, + type PromotionRecordKind, +} from "../src/promotion"; +import { PROMPT_VERSION, TOOL_SCHEMA_VERSION } from "../src/prompt"; + +const JUDGE_IDENTITY = "@sikongjueluo/pi-permission-ai-judge@0.0.1"; +const PERMISSION_SYSTEM_VERSION = "25.4.0"; +const REVIEW_SCHEMA_VERSION = "1"; + +interface CliOptions { + kind: PromotionRecordKind; + provider: string; + model: string; + api: string; + timeoutCohort: "default" | number; + basis: string; +} + +function usage(): string { + return [ + "usage: promotion-record --kind ", + " --provider

--model --api ", + " [--timeout-cohort default|] --basis ", + ].join("\n"); +} + +function parseArgs(argv: readonly string[]): CliOptions | { error: string } { + const args = argv.slice(2); + const opts: Partial = {}; + for (let i = 0; i < args.length; i += 1) { + const arg = args[i] as string; + const value = args[i + 1]; + if ( + arg === "--kind" || + arg === "--provider" || + arg === "--model" || + arg === "--api" || + arg === "--basis" + ) { + if (value === undefined) return { error: `${arg} requires a value` }; + if (arg === "--kind") { + if ( + value !== "cohort_qualified" && + value !== "owner_approval" && + value !== "activation" + ) { + return { error: `unknown kind: ${value}` }; + } + opts.kind = value; + } + if (arg === "--provider") opts.provider = value; + if (arg === "--model") opts.model = value; + if (arg === "--api") opts.api = value; + if (arg === "--basis") opts.basis = value; + i += 1; + continue; + } + if (arg === "--timeout-cohort") { + if (value === undefined) return { error: `${arg} requires a value` }; + if (value === "default") { + opts.timeoutCohort = "default"; + } else { + const n = Number(value); + if (!Number.isInteger(n)) { + return { error: `--timeout-cohort must be default or an integer` }; + } + opts.timeoutCohort = n; + } + i += 1; + continue; + } + return { error: `unknown option: ${arg}\n${usage()}` }; + } + if ( + opts.kind === undefined || + opts.provider === undefined || + opts.model === undefined || + opts.api === undefined || + opts.basis === undefined || + opts.basis.length === 0 + ) { + return { error: usage() }; + } + return { + kind: opts.kind, + provider: opts.provider, + model: opts.model, + api: opts.api, + timeoutCohort: opts.timeoutCohort ?? "default", + basis: opts.basis, + }; +} + +function main(): number { + const parsed = parseArgs(process.argv); + if ("error" in parsed) { + process.stderr.write(`${parsed.error}\n`); + return 1; + } + const identity: CandidateIdentity = { + judge: JUDGE_IDENTITY, + permissionSystem: PERMISSION_SYSTEM_VERSION, + provider: parsed.provider, + model: parsed.model, + api: parsed.api, + promptVersion: PROMPT_VERSION, + toolSchemaVersion: TOOL_SCHEMA_VERSION, + reviewSchemaVersion: REVIEW_SCHEMA_VERSION, + timeoutCohort: parsed.timeoutCohort, + }; + const agentDir = getAgentDir(); + const error = appendPromotionRecord({ + agentDir, + record: { + kind: parsed.kind, + candidateIdentity: identity, + recordedAt: new Date().toISOString(), + basis: parsed.basis, + }, + }); + if (error !== null) { + process.stderr.write(`${error}\n`); + return 1; + } + process.stdout.write( + `appended ${parsed.kind} record to ${promotionRecordsPath(agentDir)}\n` + + `identity: ${JSON.stringify(identity)}\n` + + `note: records load at session start; restart sessions to pick this up\n`, + ); + return 0; +} + +process.exit(main());