feat(permission): complete shadow review events for offline analysis

- key inner-cmd decisive review events by requestId so link decisions join offline
- record judge runtime id, prompt and tool schema versions, end-to-end and model latency, input and output usage, and evidence-quality flags on every judge result row
- record forwarded and session-mismatch preflight defers so they stay visible in the offline denominator
This commit is contained in:
2026-08-17 16:58:10 +08:00
parent 81f1da4100
commit 42f0a0aaab
6 changed files with 289 additions and 77 deletions
@@ -215,10 +215,22 @@ describe("AI judge lifecycle", () => {
details: expect.objectContaining({
requestId: "req-1",
mode: "shadow",
origin: "local",
judgeRuntimeId: expect.any(String),
promptVersion: "bash-shadow-v1",
toolSchemaVersion: "report-verdict-v1",
judgeLatencyMs: expect.any(Number),
modelLatencyMs: expect.any(Number),
inputUsage: 20,
outputUsage: 10,
resultKind: "judgment",
verdict: "allow",
effectiveVerdict: "defer",
modelCalled: true,
evidenceQuality: expect.objectContaining({
structuredFullInput: true,
explicitUserText: false,
}),
}),
},
]);
@@ -229,6 +241,80 @@ describe("AI judge lifecycle", () => {
expect(dispose).toHaveBeenCalledTimes(1);
});
it("records a forwarded ask as a preflight defer without calling the model", async () => {
let authorize: Authorizer["authorize"] | undefined;
const service = {
registerAuthorizer: vi.fn((_name, callback) => {
authorize = callback;
return vi.fn();
}),
checkPermission: vi.fn(),
getToolPermission: vi.fn(),
} as unknown as PermissionsService;
publishPermissionsService(service);
publishedService = service;
const complete = vi.fn();
const ctx = {
hasUI: true,
sessionManager: { getSessionId: () => "session-root" },
model: {
id: "test-model",
provider: "test-provider",
api: "openai-codex-responses",
} as Model<any>,
modelRegistry: { complete },
} as unknown as ExtensionContext;
const harness = createFakePi();
extension(harness.pi);
harness.start(ctx);
harness.ready();
const reviews: Array<{
event: string;
details?: Record<string, unknown>;
}> = [];
const forwarded = {
...ask(),
forwarding: { requestId: "fwd-1" } as never,
} as PromptPermissionDetails;
const verdict = await authorize!(
forwarded,
{
checkPermission: vi.fn(),
getToolPermission: vi.fn(),
},
{
review: (event, details) => reviews.push({ event, details }),
debug: vi.fn(),
},
);
expect(verdict).toEqual({ kind: "defer" });
expect(complete).not.toHaveBeenCalled();
expect(reviews).toEqual([
{
event: "ai_bash_judge.result",
details: expect.objectContaining({
requestId: "req-1",
origin: "forwarded",
resultKind: "preflight_defer",
verdict: null,
effectiveVerdict: "defer",
modelCalled: false,
code: "missing_structured_input",
evidenceQuality: expect.objectContaining({
structuredFullInput: false,
forwardedProvenance: false,
}),
}),
},
]);
harness.shutdown();
});
it("does not register from a headless child", () => {
const service = {
registerAuthorizer: vi.fn(),
@@ -84,7 +84,9 @@ describe("requestStructuredVerdict", () => {
verdict: "defer",
reason: "User intent is unavailable.",
metadata,
inputTokens: 10,
outputTokens: 12,
modelLatencyMs: expect.any(Number),
});
});
@@ -264,6 +266,7 @@ describe("requestStructuredVerdict", () => {
code: "no_model",
metadata: undefined,
modelCalled: false,
modelLatencyMs: null,
});
await expect(
@@ -277,6 +280,7 @@ describe("requestStructuredVerdict", () => {
code: "unsupported_api",
metadata,
modelCalled: false,
modelLatencyMs: null,
});
const shutdown = new AbortController();