From 0546a8049712d899aef19496b7305ddbcc7fd018 Mon Sep 17 00:00:00 2001 From: SikongJueluo Date: Mon, 17 Aug 2026 19:20:49 +0800 Subject: [PATCH] feat(ai-judge): global config module with validation and cohort identity --- packages/pi-permission-ai-judge/src/config.ts | 153 ++++++++++++++++++ packages/pi-permission-ai-judge/src/index.ts | 25 ++- packages/pi-permission-ai-judge/src/model.ts | 2 +- .../test/config.test.ts | 130 +++++++++++++++ .../test/lifecycle.test.ts | 2 + 5 files changed, 310 insertions(+), 2 deletions(-) create mode 100644 packages/pi-permission-ai-judge/src/config.ts create mode 100644 packages/pi-permission-ai-judge/test/config.test.ts diff --git a/packages/pi-permission-ai-judge/src/config.ts b/packages/pi-permission-ai-judge/src/config.ts new file mode 100644 index 0000000..9dabf8f --- /dev/null +++ b/packages/pi-permission-ai-judge/src/config.ts @@ -0,0 +1,153 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS, MIN_TIMEOUT_MS } from "./model"; + +/** + * Judge configuration (PIEXTENSIO-3 Config acceptance, PIEXTENSIO-11 + * configuration contract). + * + * The single user-configurable field is `timeoutMs` because acceptable + * interactive wait varies by deployment. Mode is read but v0.1 authority is + * mechanically fail-closed: `enforce` loads but the judge truth table + * (PIEXTENSIO-12) can never produce real authority until the promotion + * gates exist upstream, so it always defers. + * + * Global config path only — no project/env override, matching the trusted + * user-global Judge configuration of PIEXTENSIO-10. + */ + +export type JudgeMode = "shadow" | "enforce"; + +export interface EffectiveJudgeConfig { + readonly mode: JudgeMode; + readonly timeoutMs: number; + /** + * Cohort identity: a non-default timeout is a distinct configuration + * cohort and never inherits the default cohort's calibration + * (PIEXTENSIO-11). `default` marks the calibrated default. + */ + readonly timeoutCohort: "default" | number; + /** Validation diagnostics for the loaded raw file, newest wins per key. */ + readonly diagnostics: readonly ConfigDiagnostic[]; +} + +export interface ConfigDiagnostic { + readonly key: string; + readonly problem: string; + readonly fallback: string; +} + +export interface ConfigLoadDeps { + /** User-global agent dir (`~/.pi/agent`). */ + readonly agentDir: string; + /** Injectable for tests; defaults to `readFileSync`. */ + readonly readFile?: (path: string) => string; +} + +const CONFIG_FILENAME = "pi-permission-ai-judge.config.json"; +const DEFAULT_CONFIG: EffectiveJudgeConfig = { + mode: "shadow", + timeoutMs: DEFAULT_TIMEOUT_MS, + timeoutCohort: "default", + diagnostics: [], +}; + +/** + * Load and validate the global config. Missing file, malformed JSON, + * unknown mode, or out-of-range timeout all fail closed to the documented + * defaults (PIEXTENSIO-11: "invalid values fail closed to the documented + * default rather than becoming unbounded") and record a diagnostic. + */ +export function loadJudgeConfig( + deps: ConfigLoadDeps, +): EffectiveJudgeConfig { + const read = deps.readFile ?? ((p: string) => readFileSync(p, "utf-8")); + const path = join(deps.agentDir, CONFIG_FILENAME); + let raw: string; + try { + raw = read(path); + } catch (error) { + return { + ...DEFAULT_CONFIG, + diagnostics: [ + { + key: "file", + problem: `config not readable at ${path}: ${error instanceof Error ? error.message : String(error)}`, + fallback: "all defaults", + }, + ], + }; + } + + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (error) { + return { + ...DEFAULT_CONFIG, + diagnostics: [ + { + key: "file", + problem: `malformed JSON: ${error instanceof Error ? error.message : String(error)}`, + fallback: "all defaults", + }, + ], + }; + } + + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return { + ...DEFAULT_CONFIG, + diagnostics: [ + { + key: "file", + problem: "top-level value is not an object", + fallback: "all defaults", + }, + ], + }; + } + + const record = parsed as Record; + const diagnostics: ConfigDiagnostic[] = []; + + // Mode: unknown or missing resolves to shadow (fail-closed). + let mode: JudgeMode = "shadow"; + if (record.mode !== undefined) { + if (record.mode === "shadow" || record.mode === "enforce") { + mode = record.mode; + } else { + diagnostics.push({ + key: "mode", + problem: `unknown mode ${JSON.stringify(record.mode)}`, + fallback: "shadow", + }); + } + } + + // Timeout: integers in [5_000, 30_000]; anything else falls back to the + // documented 15,000 ms default. Boundary semantics: 4,999 and 30,001 + // are invalid, 5,000 and 30,000 are valid (PIEXTENSIO-3 boundaries). + let timeoutMs = DEFAULT_TIMEOUT_MS; + let timeoutCohort: EffectiveJudgeConfig["timeoutCohort"] = "default"; + if (record.timeoutMs !== undefined) { + const value = record.timeoutMs; + if ( + typeof value === "number" && + Number.isInteger(value) && + value >= MIN_TIMEOUT_MS && + value <= MAX_TIMEOUT_MS + ) { + timeoutMs = value; + timeoutCohort = value === DEFAULT_TIMEOUT_MS ? "default" : value; + } else { + diagnostics.push({ + key: "timeoutMs", + problem: `invalid timeoutMs ${JSON.stringify(value)}`, + fallback: `${DEFAULT_TIMEOUT_MS} (default)`, + }); + } + } + + return Object.freeze({ mode, timeoutMs, timeoutCohort, diagnostics }); +} \ No newline at end of file diff --git a/packages/pi-permission-ai-judge/src/index.ts b/packages/pi-permission-ai-judge/src/index.ts index 7cc4ed0..84c2ec5 100644 --- a/packages/pi-permission-ai-judge/src/index.ts +++ b/packages/pi-permission-ai-judge/src/index.ts @@ -1,4 +1,5 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import { getAgentDir } from "@earendil-works/pi-coding-agent"; import { getPermissionsService, PERMISSIONS_READY_CHANNEL, @@ -11,6 +12,7 @@ import { type ModelAvailability, } from "./model"; import { PROMPT_VERSION, TOOL_SCHEMA_VERSION } from "./prompt"; +import { loadJudgeConfig, type EffectiveJudgeConfig } from "./config"; const LINK_NAME = "ai-bash-judge"; const REVIEW_SCHEMA_VERSION = 1; @@ -22,6 +24,9 @@ interface RootSession { readonly shutdown: AbortController; /** Opaque per-runtime identity for cohort segmentation. */ readonly judgeRuntimeId: string; + /** Immutable effective-config snapshot captured at session start + * (reload-only application: a config edit lands on the next session). */ + readonly config: EffectiveJudgeConfig; } function reasonLength(reason: string): number { @@ -61,12 +66,17 @@ function resultBase( judgeRuntimeId: string, details: PromptPermissionDetails, startedAt: number, + config: EffectiveJudgeConfig, ): Record { return { schemaVersion: REVIEW_SCHEMA_VERSION, requestId: details.requestId, judgeRuntimeId, - mode: "shadow", + mode: config.mode, + // v0.1 fail-closed: `enforce` loads here but the truth table can + // never grant real authority, so the effective verdict below stays + // defer; the configured mode is recorded for audit. + timeoutCohort: config.timeoutCohort, origin: details.forwarding !== undefined || details.payload.kind === "forwarded" @@ -113,6 +123,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { captured.judgeRuntimeId, details, startedAt, + captured.config, ), resultKind: "preflight_defer", verdict: null, @@ -139,6 +150,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { captured.judgeRuntimeId, details, startedAt, + captured.config, ), resultKind: "preflight_defer", verdict: null, @@ -157,6 +169,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { captured.judgeRuntimeId, details, startedAt, + captured.config, ), resultKind: "preflight_defer", verdict: null, @@ -174,6 +187,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { captured.model, evidence, captured.shutdown.signal, + captured.config.timeoutMs, ); if (result.kind === "judgment") { @@ -182,6 +196,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { captured.judgeRuntimeId, details, startedAt, + captured.config, ), resultKind: "judgment", verdict: result.verdict, @@ -207,6 +222,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { captured.judgeRuntimeId, details, startedAt, + captured.config, ), resultKind: "infrastructure_failure", verdict: null, @@ -253,7 +269,14 @@ export default function permissionAiJudge(pi: ExtensionAPI): void { model: createModelAvailability(ctx.model, ctx.modelRegistry), shutdown: new AbortController(), judgeRuntimeId: crypto.randomUUID(), + config: loadJudgeConfig({ agentDir: getAgentDir() }), }; + for (const diagnostic of root.config.diagnostics) { + ctx.ui.notify( + `ai-bash-judge config: ${diagnostic.key} — ${diagnostic.problem}; using ${diagnostic.fallback}`, + "warning", + ); + } tryRegister(); }); diff --git a/packages/pi-permission-ai-judge/src/model.ts b/packages/pi-permission-ai-judge/src/model.ts index d5913d0..c97e354 100644 --- a/packages/pi-permission-ai-judge/src/model.ts +++ b/packages/pi-permission-ai-judge/src/model.ts @@ -14,7 +14,7 @@ import type { BashJudgmentEvidence } from "./evidence"; // provider segments (e.g. zai glm-5.2, observed racing the deadline at // `thinking: high`) should configure a non-default timeoutMs within the range // and be treated as a distinct configuration cohort. -const DEFAULT_TIMEOUT_MS = 15_000; +export const DEFAULT_TIMEOUT_MS = 15_000; export const MIN_TIMEOUT_MS = 5_000; export const MAX_TIMEOUT_MS = 30_000; // Reasoning-token aware cap. Providers that bill chain-of-thought inside diff --git a/packages/pi-permission-ai-judge/test/config.test.ts b/packages/pi-permission-ai-judge/test/config.test.ts new file mode 100644 index 0000000..a5cb2e0 --- /dev/null +++ b/packages/pi-permission-ai-judge/test/config.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; +import { loadJudgeConfig, type ConfigLoadDeps } from "../src/config"; + +function deps(files: Record = {}): ConfigLoadDeps { + return { + agentDir: "/agent", + readFile: (path: string) => { + const content = files[path]; + if (content === undefined) { + throw new Error("ENOENT"); + } + return content; + }, + }; +} + +const CONFIG_PATH = "/agent/pi-permission-ai-judge.config.json"; + +describe("loadJudgeConfig — missing and malformed", () => { + it("resolves a missing file to all defaults with one diagnostic", () => { + const config = loadJudgeConfig(deps()); + expect(config).toEqual({ + mode: "shadow", + timeoutMs: 15_000, + timeoutCohort: "default", + diagnostics: [ + expect.objectContaining({ key: "file", fallback: "all defaults" }), + ], + }); + }); + + it("resolves malformed JSON to defaults", () => { + const config = loadJudgeConfig( + deps({ [CONFIG_PATH]: "{ not json" }), + ); + expect(config.mode).toBe("shadow"); + expect(config.timeoutMs).toBe(15_000); + expect(config.diagnostics[0]?.key).toBe("file"); + }); + + it("resolves a non-object top level to defaults", () => { + const config = loadJudgeConfig(deps({ [CONFIG_PATH]: "[1,2,3]" })); + expect(config.mode).toBe("shadow"); + expect(config.diagnostics[0]?.key).toBe("file"); + }); +}); + +describe("loadJudgeConfig — mode", () => { + it("accepts shadow and enforce", () => { + expect( + loadJudgeConfig(deps({ [CONFIG_PATH]: '{"mode":"shadow"}' })).mode, + ).toBe("shadow"); + expect( + loadJudgeConfig(deps({ [CONFIG_PATH]: '{"mode":"enforce"}' })).mode, + ).toBe("enforce"); + }); + + it("resolves an unknown mode to shadow with a diagnostic", () => { + const config = loadJudgeConfig( + deps({ [CONFIG_PATH]: '{"mode":"yolo"}' }), + ); + expect(config.mode).toBe("shadow"); + expect(config.diagnostics).toEqual([ + { + key: "mode", + problem: 'unknown mode "yolo"', + fallback: "shadow", + }, + ]); + }); + + it("resolves a missing mode to shadow without diagnostics", () => { + const config = loadJudgeConfig(deps({ [CONFIG_PATH]: "{}" })); + expect(config.mode).toBe("shadow"); + expect(config.diagnostics).toEqual([]); + }); +}); + +describe("loadJudgeConfig — timeout boundaries", () => { + it.each([4_999, 30_001, 0, -5_000, 15.5, NaN, Infinity, "20000"])( + "rejects invalid timeoutMs %p with fallback to 15,000", + (value) => { + const config = loadJudgeConfig( + deps({ [CONFIG_PATH]: JSON.stringify({ timeoutMs: value }) }), + ); + expect(config.timeoutMs).toBe(15_000); + expect(config.timeoutCohort).toBe("default"); + expect(config.diagnostics[0]?.key).toBe("timeoutMs"); + }, + ); + + it("accepts the inclusive boundaries 5,000 and 30,000", () => { + expect( + loadJudgeConfig(deps({ [CONFIG_PATH]: '{"timeoutMs":5000}' })), + ).toMatchObject({ timeoutMs: 5_000, timeoutCohort: 5_000 }); + expect( + loadJudgeConfig(deps({ [CONFIG_PATH]: '{"timeoutMs":30000}' })), + ).toMatchObject({ timeoutMs: 30_000, timeoutCohort: 30_000 }); + }); + + it("marks an explicit default timeout as the default cohort", () => { + const config = loadJudgeConfig( + deps({ [CONFIG_PATH]: '{"timeoutMs":15000}' }), + ); + expect(config.timeoutCohort).toBe("default"); + expect(config.diagnostics).toEqual([]); + }); + + it("marks a non-default timeout as a distinct cohort", () => { + const config = loadJudgeConfig( + deps({ [CONFIG_PATH]: '{"timeoutMs":30000}' }), + ); + expect(config.timeoutCohort).toBe(30_000); + }); +}); + +describe("loadJudgeConfig — snapshot immutability", () => { + it("returns an immutable effective-config snapshot", () => { + const config = loadJudgeConfig( + deps({ [CONFIG_PATH]: '{"mode":"enforce","timeoutMs":20000}' }), + ); + expect(Object.isFrozen(config)).toBe(true); + expect(config).toEqual({ + mode: "enforce", + timeoutMs: 20_000, + timeoutCohort: 20_000, + diagnostics: [], + }); + }); +}); diff --git a/packages/pi-permission-ai-judge/test/lifecycle.test.ts b/packages/pi-permission-ai-judge/test/lifecycle.test.ts index 9d4da3a..ec782d7 100644 --- a/packages/pi-permission-ai-judge/test/lifecycle.test.ts +++ b/packages/pi-permission-ai-judge/test/lifecycle.test.ts @@ -179,6 +179,7 @@ describe("AI judge lifecycle", () => { sessionManager, model, modelRegistry: { complete }, + ui: { notify: vi.fn() }, } as unknown as ExtensionContext; const harness = createFakePi(); @@ -264,6 +265,7 @@ describe("AI judge lifecycle", () => { api: "openai-codex-responses", } as Model, modelRegistry: { complete }, + ui: { notify: vi.fn() }, } as unknown as ExtensionContext; const harness = createFakePi();