From e03f2f72e4a34b11b95daff6738e862696fe3cc9 Mon Sep 17 00:00:00 2001 From: maaz519 Date: Wed, 1 Jul 2026 16:29:51 +0530 Subject: [PATCH] feat(p7): FakeInference deterministic provider + golden eval registry Task 7.2: FakeInference implements the InferencePort with keyword vocabulary mirroring the P6 RuleEngine; deterministic (no clock/random) so AI artifacts are replayable. Golden fixtures (AI_GOLDEN) + inference.spec.ts assert byte-identical replay, abstention, and Scenario-7 (EVENT never exceeds DISCUSSION certainty). Controls: abstain()/deny()/setCost(). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../iios-service/src/ai/inference.spec.ts | 45 ++++++ packages/iios-testkit/src/fakes/index.ts | 2 + packages/iios-testkit/src/fakes/inference.ts | 152 ++++++++++++++++++ .../iios-testkit/src/fixtures/ai-golden.ts | 52 ++++++ packages/iios-testkit/src/index.ts | 1 + 5 files changed, 252 insertions(+) create mode 100644 packages/iios-service/src/ai/inference.spec.ts create mode 100644 packages/iios-testkit/src/fakes/inference.ts create mode 100644 packages/iios-testkit/src/fixtures/ai-golden.ts diff --git a/packages/iios-service/src/ai/inference.spec.ts b/packages/iios-service/src/ai/inference.spec.ts new file mode 100644 index 0000000..4336b21 --- /dev/null +++ b/packages/iios-service/src/ai/inference.spec.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from 'vitest'; +import { makeFakeInference, AI_GOLDEN } from '@insignia/iios-testkit'; +import type { AiJobType } from '@insignia/iios-contracts'; + +const run = (jobType: AiJobType, text: string) => + makeFakeInference().infer({ jobType, text, purpose: `ai_${jobType.toLowerCase()}`, scopeId: 'scope1', interactionId: 'int1' }); + +describe('FakeInference — deterministic golden provider (P7)', () => { + it('is replayable: every golden case yields byte-identical output twice', async () => { + for (const c of AI_GOLDEN) { + const a = await run(c.jobType, c.text); + const b = await run(c.jobType, c.text); + expect(JSON.stringify(a)).toEqual(JSON.stringify(b)); + } + }); + + it('matches golden expectations (artifact type, claims, abstention)', async () => { + for (const c of AI_GOLDEN) { + const r = await run(c.jobType, c.text); + expect(r.artifactType).toBe(c.expect.artifactType); + expect(r.claims.map((x) => x.claimType).sort()).toEqual([...c.expect.claimTypes].sort()); + expect(!!r.abstentionReason).toBe(c.expect.abstained); + } + }); + + it('never asserts an event beyond DISCUSSION certainty (Scenario 7)', async () => { + for (const c of AI_GOLDEN.filter((g) => g.expect.maxCertainty)) { + const r = await run(c.jobType, c.text); + const event = r.claims.find((x) => x.claimType === 'EVENT'); + expect(event?.claimJson.certainty).toBe('DISCUSSION'); + } + }); + + it('abstain control forces no claims + a reason', async () => { + const r = await makeFakeInference().abstain().infer({ jobType: 'EXTRACT', text: 'party at 9 pm', purpose: 'x', scopeId: 's' }); + expect(r.claims).toHaveLength(0); + expect(r.abstentionReason).toBeTruthy(); + }); + + it('emits a positive integer cost (budget governor input)', async () => { + const r = await run('SUMMARIZE', 'hello world this is a message'); + expect(Number.isInteger(r.modelRun.costUnits)).toBe(true); + expect(r.modelRun.costUnits).toBeGreaterThan(0); + }); +}); diff --git a/packages/iios-testkit/src/fakes/index.ts b/packages/iios-testkit/src/fakes/index.ts index 33465e4..f45eb9f 100644 --- a/packages/iios-testkit/src/fakes/index.ts +++ b/packages/iios-testkit/src/fakes/index.ts @@ -6,8 +6,10 @@ import { FakeMdm } from './mdm'; import { FakeCrre } from './crre'; import { FakeSas } from './sas'; import { FakeCapability } from './capability'; +import { FakeInference, makeFakeInference } from './inference'; export { FakeSession, FakeOpa, FakeCmp, FakeMdm, FakeCrre, FakeSas, FakeCapability }; +export { FakeInference, makeFakeInference }; /** The full set of fakes, typed so tests can reach each fake's control methods. */ export interface FakePorts extends IiosPlatformPorts { diff --git a/packages/iios-testkit/src/fakes/inference.ts b/packages/iios-testkit/src/fakes/inference.ts new file mode 100644 index 0000000..2a125d8 --- /dev/null +++ b/packages/iios-testkit/src/fakes/inference.ts @@ -0,0 +1,152 @@ +import type { + InferencePort, + InferenceRequest, + InferenceResult, + InferenceClaim, + InferenceEvidence, + InferenceModelRun, +} from '@insignia/iios-contracts'; + +/** + * Deterministic, golden fake of the AI inference port (P7). No network, no clock, + * no randomness — the same input always yields byte-identical output, which is what + * makes AI artifacts replayable. Keyword vocabulary intentionally mirrors the P6 + * RuleEngine so AI-proposed flags line up with deterministic ones. + * + * Controls: `.abstain()` (always decline a claim), `.deny()` (simulate a provider + * error → the service must fail closed), `.setCost(n)` (force costUnits for budget + * tests). + */ +export class FakeInference implements InferencePort { + private mode: 'normal' | 'abstain' | 'deny' = 'normal'; + private costOverride?: number; + + abstain(): this { + this.mode = 'abstain'; + return this; + } + + deny(): this { + this.mode = 'deny'; + return this; + } + + setCost(n: number): this { + this.costOverride = n; + return this; + } + + async infer(req: InferenceRequest): Promise { + if (this.mode === 'deny') throw new Error('fake inference provider error'); + + const text = req.text ?? ''; + const evidence: InferenceEvidence[] = [ + { sourceType: 'INTERACTION', sourceId: req.interactionId ?? 'unknown', relevanceScore: 1 }, + ]; + const modelRun = this.run(req, text); + + if (this.mode === 'abstain') { + return { + artifactType: req.jobType === 'SUMMARIZE' ? 'SUMMARY' : req.jobType === 'EXTRACT' ? 'EXTRACTION' : 'CLASSIFICATION', + content: {}, + confidence: 0, + abstentionReason: 'ABSTAINED', + claims: [], + evidence, + modelRun, + }; + } + + if (req.jobType === 'CLASSIFY') { + const flagTypes = this.flags(text); + const claims: InferenceClaim[] = flagTypes.map((flagType) => ({ + claimType: 'MODERATION_FLAG', + claimJson: { flagType, explanation: `keyword rule matched for ${flagType}` }, + confidence: 0.9, + })); + return { + artifactType: 'CLASSIFICATION', + content: { flags: flagTypes }, + confidence: flagTypes.length > 0 ? 0.9 : 0.95, + explanation: flagTypes.length > 0 ? `flagged: ${flagTypes.join(', ')}` : 'no moderation concerns', + claims, + evidence, + modelRun, + }; + } + + if (req.jobType === 'SUMMARIZE') { + const first = (text.split(/[.!?]/)[0] ?? '').trim(); + return { + artifactType: 'SUMMARY', + content: { text: `[ai] ${first}`, sourceInteractionId: req.interactionId ?? null }, + confidence: 0.8, + explanation: 'extractive first-sentence summary (golden)', + claims: [], + evidence, + modelRun, + }; + } + + // EXTRACT + const event = this.extractEvent(text); + if (!event) { + return { + artifactType: 'EXTRACTION', + content: { claims: [] }, + confidence: 0, + abstentionReason: 'NO_CLAIM', + claims: [], + evidence, + modelRun, + }; + } + return { + artifactType: 'EXTRACTION', + content: { claims: [event] }, + confidence: 0.7, + explanation: 'candidate event extracted (unconfirmed)', + // Scenario 7: never CONFIRMED/OFFICIAL without a deterministic confirmation. + claims: [{ claimType: 'EVENT', claimJson: event, confidence: 0.7 }], + evidence, + modelRun, + }; + } + + private flags(text: string): string[] { + const t = text.toLowerCase(); + const out: string[] = []; + if (t.includes('party') || /\b(9|10|11)\s*pm\b/.test(t)) out.push('AFTER_HOURS_EVENT'); + if (t.includes('sale') || t.includes('promo') || t.includes('discount')) out.push('PROMOTION'); + return out; + } + + private extractEvent(text: string): { title: string; when: string; certainty: string } | null { + const t = text.toLowerCase(); + const timeMatch = t.match(/\b(9|10|11|12|[1-8])\s*(am|pm)\b/); + if (t.includes('party') || t.includes('event') || t.includes('meeting') || timeMatch) { + const title = t.includes('party') ? 'party' : t.includes('meeting') ? 'meeting' : 'event'; + return { title, when: timeMatch ? timeMatch[0] : 'unspecified', certainty: 'DISCUSSION' }; + } + return null; + } + + private run(req: InferenceRequest, text: string): InferenceModelRun { + const tokensIn = text.length === 0 ? 0 : text.trim().split(/\s+/).length; + const model = + req.jobType === 'CLASSIFY' ? 'golden-classifier' : req.jobType === 'SUMMARIZE' ? 'golden-summarizer' : 'golden-extractor'; + return { + model, + modelVersion: '1', + promptVersion: 'p1', + tokensIn, + tokensOut: Math.max(1, Math.ceil(tokensIn / 4)), + costUnits: this.costOverride ?? Math.max(1, Math.ceil(text.length / 10)), + latencyMs: 1, + }; + } +} + +export function makeFakeInference(): FakeInference { + return new FakeInference(); +} diff --git a/packages/iios-testkit/src/fixtures/ai-golden.ts b/packages/iios-testkit/src/fixtures/ai-golden.ts new file mode 100644 index 0000000..7cc66d1 --- /dev/null +++ b/packages/iios-testkit/src/fixtures/ai-golden.ts @@ -0,0 +1,52 @@ +import type { AiJobType } from '@insignia/iios-contracts'; + +/** + * The golden prompt/eval registry for the deterministic AI provider (P7). Each row + * is an input + the invariant the eval asserts. This is the offline eval set the + * atlas requires; it also pins the abstention / Scenario-7 behaviour. + */ +export interface AiGoldenCase { + name: string; + jobType: AiJobType; + text: string; + expect: { + artifactType: string; + claimTypes: string[]; + abstained: boolean; + /** For EXTRACT event cases: the certainty must never exceed DISCUSSION (Scenario 7). */ + maxCertainty?: string; + }; +} + +export const AI_GOLDEN: AiGoldenCase[] = [ + { + name: 'classify after-hours party', + jobType: 'CLASSIFY', + text: 'There is a party at 9 PM tonight!', + expect: { artifactType: 'CLASSIFICATION', claimTypes: ['MODERATION_FLAG'], abstained: false }, + }, + { + name: 'classify safe message', + jobType: 'CLASSIFY', + text: 'Community cleanup on Saturday morning.', + expect: { artifactType: 'CLASSIFICATION', claimTypes: [], abstained: false }, + }, + { + name: 'summarize', + jobType: 'SUMMARIZE', + text: 'The committee met today. We discussed the budget. Next steps to follow.', + expect: { artifactType: 'SUMMARY', claimTypes: [], abstained: false }, + }, + { + name: 'extract candidate event stays DISCUSSION', + jobType: 'EXTRACT', + text: 'Maybe we do a party at 9 PM?', + expect: { artifactType: 'EXTRACTION', claimTypes: ['EVENT'], abstained: false, maxCertainty: 'DISCUSSION' }, + }, + { + name: 'extract abstains when no claim', + jobType: 'EXTRACT', + text: 'Thanks everyone, appreciate it.', + expect: { artifactType: 'EXTRACTION', claimTypes: [], abstained: true }, + }, +]; diff --git a/packages/iios-testkit/src/index.ts b/packages/iios-testkit/src/index.ts index f303a51..bcdbc28 100644 --- a/packages/iios-testkit/src/index.ts +++ b/packages/iios-testkit/src/index.ts @@ -2,3 +2,4 @@ export * from './fakes'; export * from './replay'; export { portalMessageBasic } from './fixtures/portal-message-basic'; export { emailFromUnknown } from './fixtures/email-from-unknown'; +export { AI_GOLDEN, type AiGoldenCase } from './fixtures/ai-golden';