SikongJueluo/pi-extensions
1
0
Fork 0
mirror of https://github.com/SikongJueluo/pi-extensions.git synced 2026-10-05 11:52:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(ai-judge): global config module with validation and cohort identity

Browse Source
This commit is contained in:
SikongJueluo
2026-08-17 19:53:48 +08:00
parent 0a1b9f259d
commit 0546a80497
5 changed files with 310 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/pi-permission-ai-judge/src/config.ts
+153
View File
@@ -0,0 +1,153 @@
import { readFileSync } from "node:fs";
import { join } from "node:path";
import { DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS, MIN_TIMEOUT_MS } from "./model";
/**
* Judge configuration (PIEXTENSIO-3 Config acceptance, PIEXTENSIO-11
* configuration contract).
*
* The single user-configurable field is `timeoutMs` because acceptable
* interactive wait varies by deployment. Mode is read but v0.1 authority is
* mechanically fail-closed: `enforce` loads but the judge truth table
* (PIEXTENSIO-12) can never produce real authority until the promotion
* gates exist upstream, so it always defers.
*
* Global config path only — no project/env override, matching the trusted
* user-global Judge configuration of PIEXTENSIO-10.
*/
export type JudgeMode = "shadow" | "enforce";
export interface EffectiveJudgeConfig {
readonly mode: JudgeMode;
readonly timeoutMs: number;
/**
* Cohort identity: a non-default timeout is a distinct configuration
* cohort and never inherits the default cohort's calibration
* (PIEXTENSIO-11). `default` marks the calibrated default.
*/
readonly timeoutCohort: "default" | number;
/** Validation diagnostics for the loaded raw file, newest wins per key. */
readonly diagnostics: readonly ConfigDiagnostic[];
}
export interface ConfigDiagnostic {
readonly key: string;
readonly problem: string;
readonly fallback: string;
}
export interface ConfigLoadDeps {
/** User-global agent dir (`~/.pi/agent`). */
readonly agentDir: string;
/** Injectable for tests; defaults to `readFileSync`. */
readonly readFile?: (path: string) => string;
}
const CONFIG_FILENAME = "pi-permission-ai-judge.config.json";
const DEFAULT_CONFIG: EffectiveJudgeConfig = {
mode: "shadow",
timeoutMs: DEFAULT_TIMEOUT_MS,
timeoutCohort: "default",
diagnostics: [],
};
/**
* Load and validate the global config. Missing file, malformed JSON,
* unknown mode, or out-of-range timeout all fail closed to the documented
* defaults (PIEXTENSIO-11: "invalid values fail closed to the documented
* default rather than becoming unbounded") and record a diagnostic.
*/
export function loadJudgeConfig(
deps: ConfigLoadDeps,
): EffectiveJudgeConfig {
const read = deps.readFile ?? ((p: string) => readFileSync(p, "utf-8"));
const path = join(deps.agentDir, CONFIG_FILENAME);
let raw: string;
try {
raw = read(path);
} catch (error) {
return {
...DEFAULT_CONFIG,
diagnostics: [
{
key: "file",
problem: `config not readable at ${path}: ${error instanceof Error ? error.message : String(error)}`,
fallback: "all defaults",
},
],
};
}
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch (error) {
return {
...DEFAULT_CONFIG,
diagnostics: [
{
key: "file",
problem: `malformed JSON: ${error instanceof Error ? error.message : String(error)}`,
fallback: "all defaults",
},
],
};
}
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
return {
...DEFAULT_CONFIG,
diagnostics: [
{
key: "file",
problem: "top-level value is not an object",
fallback: "all defaults",
},
],
};
}
const record = parsed as Record<string, unknown>;
const diagnostics: ConfigDiagnostic[] = [];
// Mode: unknown or missing resolves to shadow (fail-closed).
let mode: JudgeMode = "shadow";
if (record.mode !== undefined) {
if (record.mode === "shadow" || record.mode === "enforce") {
mode = record.mode;
} else {
diagnostics.push({
key: "mode",
problem: `unknown mode ${JSON.stringify(record.mode)}`,
fallback: "shadow",
});
}
}
// Timeout: integers in [5_000, 30_000]; anything else falls back to the
// documented 15,000 ms default. Boundary semantics: 4,999 and 30,001
// are invalid, 5,000 and 30,000 are valid (PIEXTENSIO-3 boundaries).
let timeoutMs = DEFAULT_TIMEOUT_MS;
let timeoutCohort: EffectiveJudgeConfig["timeoutCohort"] = "default";
if (record.timeoutMs !== undefined) {
const value = record.timeoutMs;
if (
typeof value === "number" &&
Number.isInteger(value) &&
value >= MIN_TIMEOUT_MS &&
value <= MAX_TIMEOUT_MS
) {
timeoutMs = value;
timeoutCohort = value === DEFAULT_TIMEOUT_MS ? "default" : value;
} else {
diagnostics.push({
key: "timeoutMs",
problem: `invalid timeoutMs ${JSON.stringify(value)}`,
fallback: `${DEFAULT_TIMEOUT_MS} (default)`,
});
}
}
return Object.freeze({ mode, timeoutMs, timeoutCohort, diagnostics });
}
packages/pi-permission-ai-judge/src/index.ts
+24 -1
View File
@@ -1,4 +1,5 @@
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
import { getAgentDir } from "@earendil-works/pi-coding-agent";
import {
getPermissionsService,
PERMISSIONS_READY_CHANNEL,
@@ -11,6 +12,7 @@ import {
type ModelAvailability,
} from "./model";
import { PROMPT_VERSION, TOOL_SCHEMA_VERSION } from "./prompt";
import { loadJudgeConfig, type EffectiveJudgeConfig } from "./config";
const LINK_NAME = "ai-bash-judge";
const REVIEW_SCHEMA_VERSION = 1;
@@ -22,6 +24,9 @@ interface RootSession {
readonly shutdown: AbortController;
/** Opaque per-runtime identity for cohort segmentation. */
readonly judgeRuntimeId: string;
/** Immutable effective-config snapshot captured at session start
* (reload-only application: a config edit lands on the next session). */
readonly config: EffectiveJudgeConfig;
}
function reasonLength(reason: string): number {
@@ -61,12 +66,17 @@ function resultBase(
judgeRuntimeId: string,
details: PromptPermissionDetails,
startedAt: number,
config: EffectiveJudgeConfig,
): Record<string, unknown> {
return {
schemaVersion: REVIEW_SCHEMA_VERSION,
requestId: details.requestId,
judgeRuntimeId,
mode: "shadow",
mode: config.mode,
// v0.1 fail-closed: `enforce` loads here but the truth table can
// never grant real authority, so the effective verdict below stays
// defer; the configured mode is recorded for audit.
timeoutCohort: config.timeoutCohort,
origin:
details.forwarding !== undefined ||
details.payload.kind === "forwarded"
@@ -113,6 +123,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
captured.judgeRuntimeId,
details,
startedAt,
captured.config,
),
resultKind: "preflight_defer",
verdict: null,
@@ -139,6 +150,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
captured.judgeRuntimeId,
details,
startedAt,
captured.config,
),
resultKind: "preflight_defer",
verdict: null,
@@ -157,6 +169,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
captured.judgeRuntimeId,
details,
startedAt,
captured.config,
),
resultKind: "preflight_defer",
verdict: null,
@@ -174,6 +187,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
captured.model,
evidence,
captured.shutdown.signal,
captured.config.timeoutMs,
);
if (result.kind === "judgment") {
@@ -182,6 +196,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
captured.judgeRuntimeId,
details,
startedAt,
captured.config,
),
resultKind: "judgment",
verdict: result.verdict,
@@ -207,6 +222,7 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
captured.judgeRuntimeId,
details,
startedAt,
captured.config,
),
resultKind: "infrastructure_failure",
verdict: null,
@@ -253,7 +269,14 @@ export default function permissionAiJudge(pi: ExtensionAPI): void {
model: createModelAvailability(ctx.model, ctx.modelRegistry),
shutdown: new AbortController(),
judgeRuntimeId: crypto.randomUUID(),
config: loadJudgeConfig({ agentDir: getAgentDir() }),
};
for (const diagnostic of root.config.diagnostics) {
ctx.ui.notify(
`ai-bash-judge config: ${diagnostic.key} — ${diagnostic.problem}; using ${diagnostic.fallback}`,
"warning",
);
}
tryRegister();
});
packages/pi-permission-ai-judge/src/model.ts
+1 -1
View File
@@ -14,7 +14,7 @@ import type { BashJudgmentEvidence } from "./evidence";
// provider segments (e.g. zai glm-5.2, observed racing the deadline at
// `thinking: high`) should configure a non-default timeoutMs within the range
// and be treated as a distinct configuration cohort.
const DEFAULT_TIMEOUT_MS = 15_000;
export const DEFAULT_TIMEOUT_MS = 15_000;
export const MIN_TIMEOUT_MS = 5_000;
export const MAX_TIMEOUT_MS = 30_000;
// Reasoning-token aware cap. Providers that bill chain-of-thought inside
packages/pi-permission-ai-judge/test/config.test.ts
+130
View File
@@ -0,0 +1,130 @@
import { describe, expect, it } from "vitest";
import { loadJudgeConfig, type ConfigLoadDeps } from "../src/config";
function deps(files: Record<string, string> = {}): ConfigLoadDeps {
return {
agentDir: "/agent",
readFile: (path: string) => {
const content = files[path];
if (content === undefined) {
throw new Error("ENOENT");
}
return content;
},
};
}
const CONFIG_PATH = "/agent/pi-permission-ai-judge.config.json";
describe("loadJudgeConfig — missing and malformed", () => {
it("resolves a missing file to all defaults with one diagnostic", () => {
const config = loadJudgeConfig(deps());
expect(config).toEqual({
mode: "shadow",
timeoutMs: 15_000,
timeoutCohort: "default",
diagnostics: [
expect.objectContaining({ key: "file", fallback: "all defaults" }),
],
});
});
it("resolves malformed JSON to defaults", () => {
const config = loadJudgeConfig(
deps({ [CONFIG_PATH]: "{ not json" }),
);
expect(config.mode).toBe("shadow");
expect(config.timeoutMs).toBe(15_000);
expect(config.diagnostics[0]?.key).toBe("file");
});
it("resolves a non-object top level to defaults", () => {
const config = loadJudgeConfig(deps({ [CONFIG_PATH]: "[1,2,3]" }));
expect(config.mode).toBe("shadow");
expect(config.diagnostics[0]?.key).toBe("file");
});
});
describe("loadJudgeConfig — mode", () => {
it("accepts shadow and enforce", () => {
expect(
loadJudgeConfig(deps({ [CONFIG_PATH]: '{"mode":"shadow"}' })).mode,
).toBe("shadow");
expect(
loadJudgeConfig(deps({ [CONFIG_PATH]: '{"mode":"enforce"}' })).mode,
).toBe("enforce");
});
it("resolves an unknown mode to shadow with a diagnostic", () => {
const config = loadJudgeConfig(
deps({ [CONFIG_PATH]: '{"mode":"yolo"}' }),
);
expect(config.mode).toBe("shadow");
expect(config.diagnostics).toEqual([
{
key: "mode",
problem: 'unknown mode "yolo"',
fallback: "shadow",
},
]);
});
it("resolves a missing mode to shadow without diagnostics", () => {
const config = loadJudgeConfig(deps({ [CONFIG_PATH]: "{}" }));
expect(config.mode).toBe("shadow");
expect(config.diagnostics).toEqual([]);
});
});
describe("loadJudgeConfig — timeout boundaries", () => {
it.each([4_999, 30_001, 0, -5_000, 15.5, NaN, Infinity, "20000"])(
"rejects invalid timeoutMs %p with fallback to 15,000",
(value) => {
const config = loadJudgeConfig(
deps({ [CONFIG_PATH]: JSON.stringify({ timeoutMs: value }) }),
);
expect(config.timeoutMs).toBe(15_000);
expect(config.timeoutCohort).toBe("default");
expect(config.diagnostics[0]?.key).toBe("timeoutMs");
},
);
it("accepts the inclusive boundaries 5,000 and 30,000", () => {
expect(
loadJudgeConfig(deps({ [CONFIG_PATH]: '{"timeoutMs":5000}' })),
).toMatchObject({ timeoutMs: 5_000, timeoutCohort: 5_000 });
expect(
loadJudgeConfig(deps({ [CONFIG_PATH]: '{"timeoutMs":30000}' })),
).toMatchObject({ timeoutMs: 30_000, timeoutCohort: 30_000 });
});
it("marks an explicit default timeout as the default cohort", () => {
const config = loadJudgeConfig(
deps({ [CONFIG_PATH]: '{"timeoutMs":15000}' }),
);
expect(config.timeoutCohort).toBe("default");
expect(config.diagnostics).toEqual([]);
});
it("marks a non-default timeout as a distinct cohort", () => {
const config = loadJudgeConfig(
deps({ [CONFIG_PATH]: '{"timeoutMs":30000}' }),
);
expect(config.timeoutCohort).toBe(30_000);
});
});
describe("loadJudgeConfig — snapshot immutability", () => {
it("returns an immutable effective-config snapshot", () => {
const config = loadJudgeConfig(
deps({ [CONFIG_PATH]: '{"mode":"enforce","timeoutMs":20000}' }),
);
expect(Object.isFrozen(config)).toBe(true);
expect(config).toEqual({
mode: "enforce",
timeoutMs: 20_000,
timeoutCohort: 20_000,
diagnostics: [],
});
});
});
packages/pi-permission-ai-judge/test/lifecycle.test.ts
+2
View File
@@ -179,6 +179,7 @@ describe("AI judge lifecycle", () => {
sessionManager,
model,
modelRegistry: { complete },
ui: { notify: vi.fn() },
} as unknown as ExtensionContext;
const harness = createFakePi();
@@ -264,6 +265,7 @@ describe("AI judge lifecycle", () => {
api: "openai-codex-responses",
} as Model<any>,
modelRegistry: { complete },
ui: { notify: vi.fn() },
} as unknown as ExtensionContext;
const harness = createFakePi();