diff --git a/.env.example b/.env.example index e4929e0..b99a182 100644 --- a/.env.example +++ b/.env.example @@ -6,3 +6,18 @@ DATABASE_URL="postgresql://iios:iios@localhost:5434/iios?schema=public" JWT_SECRET="dev-only-change-me" # Per-app HS256 secrets, JSON map keyed by appId (the `session` platform port) APP_SECRETS={"portal-demo":"dev-secret"} + +# Full-text message search (Meilisearch). Unset MEILI_URL → search is disabled (no-op). +# MEILI_KEY must equal the meilisearch server's MEILI_MASTER_KEY (docker-compose default below). +MEILI_URL="http://localhost:7700" +MEILI_KEY="dev-meili-master-key" +# Consumed by docker-compose's meilisearch service; keep in sync with MEILI_KEY. +MEILI_MASTER_KEY="dev-meili-master-key" + +# BYO integration credentials at rest (Twilio SMS, SMTP): 32-byte base64 AES-256-GCM key. +# Generate with: openssl rand -base64 32 +IIOS_CRED_KEY="" + +# Orphaned-media garbage collection (uploaded-but-never-sent attachments). +# Interval 0 = off; set e.g. 3600000 (hourly) in prod. Grace defaults to 24h. +IIOS_MEDIA_GC_INTERVAL_MS=0 diff --git a/docker-compose.yml b/docker-compose.yml index 275a98c..b5eea66 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,5 +29,24 @@ services: timeout: 5s retries: 10 + # Full-text message search. The service uses it when MEILI_URL is set (else search no-ops). + # Point the service at it with MEILI_URL=http://localhost:7700 + MEILI_KEY=. + meilisearch: + image: getmeili/meilisearch:v1.11 + container_name: iios-meili + ports: + - "7700:7700" + environment: + MEILI_NO_ANALYTICS: "true" + MEILI_MASTER_KEY: ${MEILI_MASTER_KEY:-dev-meili-master-key} + volumes: + - iios_meili_data:/meili_data + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:7700/health || exit 1"] + interval: 5s + timeout: 5s + retries: 10 + volumes: iios_postgres_data: + iios_meili_data: diff --git a/packages/iios-messaging-ui/package.json b/packages/iios-messaging-ui/package.json index e785610..3274af2 100644 --- a/packages/iios-messaging-ui/package.json +++ b/packages/iios-messaging-ui/package.json @@ -1,6 +1,6 @@ { "name": "@insignia/iios-messaging-ui", - "version": "0.1.5", + "version": "0.1.6", "type": "module", "main": "dist/index.js", "module": "dist/index.js", diff --git a/packages/iios-messaging-ui/src/components/messenger.tsx b/packages/iios-messaging-ui/src/components/messenger.tsx index e28c2b7..dd2effc 100644 --- a/packages/iios-messaging-ui/src/components/messenger.tsx +++ b/packages/iios-messaging-ui/src/components/messenger.tsx @@ -12,7 +12,7 @@ import { ThreadPane } from './thread-pane'; * with a channel browser when the adapter supports channels. Owns only selection + browse state; * all data flows through the injected adapter via the hooks. */ -export function Messenger() { +export function Messenger({ focusThreadId }: { focusThreadId?: string | null } = {}) { const { conversations, loading, error, refetch } = useConversations(); const adapter = useAdapter(); const channelsSupported = typeof adapter.browseChannels === 'function'; @@ -32,6 +32,14 @@ export function Messenger() { // Switching conversations (or into browse/compose) closes any open thread pane. useEffect(() => setActiveRoot(null), [selected, browsing, composing]); + // Deep link (e.g. from global search): focus the requested conversation. + useEffect(() => { + if (!focusThreadId) return; + setBrowsing(false); + setComposing(false); + setSelected(focusThreadId); + }, [focusThreadId]); + const channels = useMemo(() => conversations.filter((c) => c.membership === 'channel'), [conversations]); const dms = useMemo(() => conversations.filter((c) => c.membership !== 'channel'), [conversations]); const selectedConversation = useMemo(() => conversations.find((c) => c.threadId === selected) ?? null, [conversations, selected]); diff --git a/packages/iios-messaging-ui/src/inbox/inbox.tsx b/packages/iios-messaging-ui/src/inbox/inbox.tsx index a4e42b2..e3b5c8c 100644 --- a/packages/iios-messaging-ui/src/inbox/inbox.tsx +++ b/packages/iios-messaging-ui/src/inbox/inbox.tsx @@ -32,17 +32,30 @@ const timeOf = (iso?: string): string => { }; /** The unified inbox: work items + mail in one list; click a threaded row to read + reply. */ -export function Inbox() { +export function Inbox({ focusThreadId }: { focusThreadId?: string | null } = {}) { const [filter, setFilter] = useState('OPEN'); const { items, loading, error, transition, refetch } = useInbox(filter); const compose = useCompose(); const [selectedId, setSelectedId] = useState(null); const [composing, setComposing] = useState(false); + // Deep link (e.g. from global search): mail lives in the Open view, so switch there and let the + // selection effect pick the matching thread once the list loads. useEffect(() => { + if (focusThreadId) setFilter('OPEN'); + }, [focusThreadId]); + + useEffect(() => { + if (focusThreadId) { + const hit = items.find((i) => i.threadId === focusThreadId); + if (hit) { + setSelectedId(hit.id); + return; + } + } if (selectedId && items.some((i) => i.id === selectedId)) return; setSelectedId(items[0]?.id ?? null); - }, [items, selectedId]); + }, [items, selectedId, focusThreadId]); const selected = items.find((i) => i.id === selectedId) ?? null; diff --git a/packages/iios-service/deploy/meilisearch.yaml b/packages/iios-service/deploy/meilisearch.yaml new file mode 100644 index 0000000..d8be578 --- /dev/null +++ b/packages/iios-service/deploy/meilisearch.yaml @@ -0,0 +1,98 @@ +# Meilisearch for IIOS message search (prod). +# +# IIOS deploys to Kubernetes via ArgoCD from the platform-engineering/k8s-pods repo +# (services/iios/). This file is the template for the prod Meilisearch instance — copy it into +# k8s-pods/services/iios/meilisearch.yaml and let ArgoCD sync it. Then wire the service: +# +# 1. Create the master key once (32+ random chars) and set it in the Secret below (or via a +# sealed-secret / your secret manager — do NOT commit a real key in plaintext): +# kubectl -n create secret generic iios-meili \ +# --from-literal=MEILI_MASTER_KEY="$(openssl rand -base64 32)" +# 2. Add these env vars to the iios-service Deployment (services/iios/deployment.yaml): +# - name: MEILI_URL +# value: http://meilisearch:7700 +# - name: MEILI_KEY +# valueFrom: { secretKeyRef: { name: iios-meili, key: MEILI_MASTER_KEY } } +# 3. After the first deploy, backfill existing messages once: POST /v1/search/reindex per tenant +# (authenticated as a scope member) — new messages index automatically on message.sent. +# +# Set the namespace on each object (or via kustomize) to match the iios-service namespace so the +# `meilisearch` Service DNS resolves as http://meilisearch:7700 from the pod. +--- +apiVersion: v1 +kind: Secret +metadata: + name: iios-meili +type: Opaque +stringData: + # Replace with a real key (or manage out-of-band per note #1 and delete this stringData block). + MEILI_MASTER_KEY: "CHANGE_ME_meili_master_key" +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: iios-meili-data +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: meilisearch + labels: { app: meilisearch } +spec: + replicas: 1 + strategy: { type: Recreate } # single RWO volume — never run two writers + selector: + matchLabels: { app: meilisearch } + template: + metadata: + labels: { app: meilisearch } + spec: + containers: + - name: meilisearch + image: getmeili/meilisearch:v1.11 + ports: + - containerPort: 7700 + env: + - name: MEILI_ENV + value: "production" + - name: MEILI_NO_ANALYTICS + value: "true" + - name: MEILI_MASTER_KEY + valueFrom: + secretKeyRef: { name: iios-meili, key: MEILI_MASTER_KEY } + volumeMounts: + - name: data + mountPath: /meili_data + resources: + requests: { cpu: "100m", memory: "256Mi" } + limits: { cpu: "1", memory: "1Gi" } + readinessProbe: + httpGet: { path: /health, port: 7700 } + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: { path: /health, port: 7700 } + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - name: data + persistentVolumeClaim: + claimName: iios-meili-data +--- +apiVersion: v1 +kind: Service +metadata: + name: meilisearch + labels: { app: meilisearch } +spec: + selector: { app: meilisearch } + ports: + - port: 7700 + targetPort: 7700 + # ClusterIP (internal only) — reached by iios-service as http://meilisearch:7700. + type: ClusterIP diff --git a/packages/iios-service/package.json b/packages/iios-service/package.json index 07b68b7..d4c1f4e 100644 --- a/packages/iios-service/package.json +++ b/packages/iios-service/package.json @@ -29,6 +29,7 @@ "ioredis": "^5.11.1", "jsonwebtoken": "^9.0.3", "jwks-rsa": "^4.1.0", + "meilisearch": "^0.45.0", "nodemailer": "^9.0.3", "prisma": "^6.2.1", "reflect-metadata": "^0.2.2", diff --git a/packages/iios-service/src/app.module.ts b/packages/iios-service/src/app.module.ts index 1afdae9..a7ca8e6 100644 --- a/packages/iios-service/src/app.module.ts +++ b/packages/iios-service/src/app.module.ts @@ -10,6 +10,7 @@ import { OutboxModule } from './outbox/outbox.module'; import { ThreadsModule } from './threads/threads.module'; import { MessageModule } from './messaging/message.module'; import { InboxModule } from './inbox/inbox.module'; +import { SearchModule } from './search/search.module'; import { TemplateModule } from './templates/template.module'; import { MailModule } from './mail/mail.module'; import { MediaModule } from './media/media.module'; @@ -39,6 +40,7 @@ import { DevController } from './dev/dev.controller'; ThreadsModule, MessageModule, InboxModule, + SearchModule, TemplateModule, MailModule, MediaModule, diff --git a/packages/iios-service/src/capability/capability.registry.ts b/packages/iios-service/src/capability/capability.registry.ts index 76b32a0..dd896b3 100644 --- a/packages/iios-service/src/capability/capability.registry.ts +++ b/packages/iios-service/src/capability/capability.registry.ts @@ -3,7 +3,7 @@ import type { CapabilityProvider } from '@insignia/iios-contracts'; import { SandboxProvider } from './sandbox.provider'; import { HttpProvider } from './http.provider'; import { EmailProvider } from './email.provider'; -import { SmtpProvider, smtpFallbackFromEnv, smtpIdentityFromEnv, storageResolver } from './smtp.provider'; +import { SmtpProvider, smtpFallbackFromEnv, smtpIdentityFromConfig, smtpIdentityFromEnv, storageResolver } from './smtp.provider'; import { TwilioSmsProvider, type TwilioCreds } from './twilio-sms.provider'; import { credKeyFromEnv } from './secret-crypto'; import { ProviderCredentialService } from './provider-credential.service'; @@ -36,11 +36,15 @@ export class CapabilityProviderRegistry { this.register(ch === 'EMAIL' ? new EmailProvider(url) : new HttpProvider(ch, url)); } // Real SMTP for EMAIL wins over the HTTP relay when configured (registered last). Attachments - // resolve through the media StoragePort when one is bound (else attachments FAIL closed). + // resolve through the media StoragePort when one is bound (else attachments FAIL closed). A + // tenant's own SMTP (BYO) is resolved per scope at send time and takes precedence over the env + // identity; register the provider whenever EITHER the env SMTP or the credential store exists. const smtp = smtpIdentityFromEnv(); - if (smtp) { + const smtpCreds = credKeyFromEnv() && this.credentials ? this.credentials : undefined; + if (smtp || smtpCreds) { const resolver = this.storage ? storageResolver(this.storage) : undefined; - this.register(new SmtpProvider(smtp, smtpFallbackFromEnv() ?? undefined, undefined, resolver)); + const credResolver = smtpCreds ? (scopeId: string) => smtpCreds.resolve(scopeId, 'SMTP').then(smtpIdentityFromConfig) : undefined; + this.register(new SmtpProvider(smtp ?? undefined, smtpFallbackFromEnv() ?? undefined, undefined, resolver, credResolver)); } // BYO SMS via each tenant's own Twilio creds (resolved per scope at send time). Registered // only when the platform key + credential store are present — else SMS stays on the sandbox. diff --git a/packages/iios-service/src/capability/provider-credential.controller.ts b/packages/iios-service/src/capability/provider-credential.controller.ts index 8b69fb1..4dd5fa3 100644 --- a/packages/iios-service/src/capability/provider-credential.controller.ts +++ b/packages/iios-service/src/capability/provider-credential.controller.ts @@ -2,9 +2,34 @@ import { BadRequestException, Body, Controller, Get, Headers, Param, Put } from import { SessionVerifier } from '../platform/session.verifier'; import { ActorResolver, type MessagePrincipal } from '../identity/actor.resolver'; import { ProviderCredentialService } from './provider-credential.service'; -import { TwilioCredentialsDto } from './provider-credential.dto'; -const SUPPORTED = new Set(['TWILIO_SMS']); +const SUPPORTED = new Set(['TWILIO_SMS', 'SMTP']); + +const str = (v: unknown): string => (typeof v === 'string' ? v.trim() : ''); +const isEmail = (v: string): boolean => /^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(v); + +/** Validate + shape the credential config for a provider type. Throws 400 on bad input. + * be-crm does the strict client-facing validation; this is IIOS's own guard. */ +function buildConfig(providerType: string, body: Record): Record { + if (providerType === 'TWILIO_SMS') { + const accountSid = str(body.accountSid); + const authToken = str(body.authToken); + const fromNumber = str(body.fromNumber); + if (!accountSid || !authToken || !fromNumber) throw new BadRequestException('accountSid, authToken and fromNumber are required'); + return { accountSid, authToken, fromNumber }; + } + if (providerType === 'SMTP') { + const host = str(body.host); + const user = str(body.user); + const pass = str(body.pass); + const fromEmail = str(body.fromEmail); + const fromName = str(body.fromName); + if (!host || !user || !pass || !fromEmail) throw new BadRequestException('host, user, pass and fromEmail are required'); + if (!isEmail(fromEmail)) throw new BadRequestException('fromEmail must be a valid email'); + return { host, port: Number(body.port) || 587, secure: body.secure === true || body.secure === 'true', user, pass, fromEmail, ...(fromName ? { fromName } : {}) }; + } + throw new BadRequestException(`unsupported providerType: ${providerType}`); +} /** * Per-scope BYO integration credentials. The caller's attested session decides the scope, so a @@ -22,16 +47,13 @@ export class ProviderCredentialController { @Put(':providerType/credentials') async put( @Param('providerType') providerType: string, - @Body() body: TwilioCredentialsDto, + @Body() body: Record, @Headers('authorization') authorization?: string, ) { this.assertSupported(providerType); + const config = buildConfig(providerType, body ?? {}); const scope = await this.actors.resolveScope(this.principal(authorization)); - await this.credentials.upsert(scope.id, providerType, { - accountSid: body.accountSid, - authToken: body.authToken, - fromNumber: body.fromNumber, - }); + await this.credentials.upsert(scope.id, providerType, config); return this.credentials.status(scope.id, providerType); } diff --git a/packages/iios-service/src/capability/provider-credential.dto.ts b/packages/iios-service/src/capability/provider-credential.dto.ts deleted file mode 100644 index 75c88e9..0000000 --- a/packages/iios-service/src/capability/provider-credential.dto.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { IsNotEmpty, IsString } from 'class-validator'; - -/** - * PUT /v1/providers/TWILIO_SMS/credentials — a tenant's own Twilio credentials. Step 1 supports - * TWILIO_SMS only; when a second provider (SMTP, …) is added, switch to a per-type validated body. - */ -export class TwilioCredentialsDto { - @IsString() @IsNotEmpty() accountSid!: string; - @IsString() @IsNotEmpty() authToken!: string; - @IsString() @IsNotEmpty() fromNumber!: string; -} diff --git a/packages/iios-service/src/capability/provider-credential.service.ts b/packages/iios-service/src/capability/provider-credential.service.ts index 574913b..3963bd4 100644 --- a/packages/iios-service/src/capability/provider-credential.service.ts +++ b/packages/iios-service/src/capability/provider-credential.service.ts @@ -18,6 +18,9 @@ function hintsFor(providerType: string, config: ProviderConfig): Record { expect(await r('nope')).toBeNull(); }); }); + +describe('BYO SMTP (scope-aware)', () => { + const TENANT: SmtpIdentity = { host: 'smtp.acme.com', port: 465, secure: true, user: 'apikey', pass: 't', from: 'Acme ' }; + const scoped = (payload: Record, scopeId?: string): CapabilityRequest => ({ + capability: 'channel.send', channelType: 'EMAIL', target: 'dana@acme.com', payload, idempotencyKey: 'k1', ...(scopeId ? { scopeId } : {}), + }); + + it('sends from the tenant identity when a scope has its own SMTP', async () => { + const s = stub(); + const p = new SmtpProvider(ID, FB, s.make, undefined, async (sid) => (sid === 'scope_1' ? TENANT : null)); + const res = await p.send(scoped({ subject: 'Hi', text: 'x' }, 'scope_1')); + expect(res.outcome).toBe('SENT'); + expect(s.calls[0]!.id).toEqual(TENANT); + expect(s.calls[0]!.mail.from).toBe('Acme '); + }); + + it('falls back to the platform identity when the scope has no SMTP', async () => { + const s = stub(); + const p = new SmtpProvider(ID, FB, s.make, undefined, async () => null); + await p.send(scoped({ subject: 'Hi', text: 'x' }, 'scope_2')); + expect(s.calls[0]!.id).toEqual(ID); + }); + + it('fails closed (NOT_CONFIGURED) when there is neither a tenant nor a platform identity', async () => { + const s = stub(); + const p = new SmtpProvider(undefined, undefined, s.make, undefined, async () => null); + const res = await p.send(scoped({ subject: 'Hi', text: 'x' }, 'scope_3')); + expect(res.outcome).toBe('FAILED'); + expect(res.errorCode).toBe('NOT_CONFIGURED'); + expect(s.calls).toHaveLength(0); + }); + + it('smtpIdentityFromConfig maps a stored config (fromName + email → from)', async () => { + const { smtpIdentityFromConfig } = await import('./smtp.provider'); + expect(smtpIdentityFromConfig({ host: 'h', port: 465, secure: true, user: 'u', pass: 'p', fromEmail: 'a@b.com', fromName: 'Acme' })) + .toEqual({ host: 'h', port: 465, secure: true, user: 'u', pass: 'p', from: 'Acme ' }); + expect(smtpIdentityFromConfig({ host: 'h', user: 'u', pass: 'p' })).toBeNull(); + }); +}); diff --git a/packages/iios-service/src/capability/smtp.provider.ts b/packages/iios-service/src/capability/smtp.provider.ts index 2485b3a..cebd86c 100644 --- a/packages/iios-service/src/capability/smtp.provider.ts +++ b/packages/iios-service/src/capability/smtp.provider.ts @@ -84,6 +84,29 @@ function identityFrom(env: NodeJS.ProcessEnv, prefix: string): SmtpIdentity | nu }; } +/** Map a tenant's stored SMTP config (BYO) to a sending identity, or null if incomplete. */ +export function smtpIdentityFromConfig(c: Record | null | undefined): SmtpIdentity | null { + if (!c) return null; + const s = (v: unknown): string => (typeof v === 'string' ? v.trim() : ''); + const host = s(c.host); + const user = s(c.user); + const pass = s(c.pass); + const fromEmail = s(c.fromEmail); + if (!host || !user || !pass || !fromEmail) return null; + const fromName = s(c.fromName); + return { + host, + port: Number(c.port) || 587, + secure: c.secure === true || c.secure === 'true', + user, + pass, + from: fromName ? `${fromName} <${fromEmail}>` : fromEmail, + }; +} + +/** Resolve a scope's own SMTP identity (BYO), decrypting the stored credential. Null when unset. */ +export type SmtpCredResolver = (scopeId: string) => Promise; + interface EmailPayload { subject?: string; text?: string; @@ -107,10 +130,12 @@ export class SmtpProvider implements CapabilityProvider { private readonly makeTransport: (id: SmtpIdentity) => MailTransport; constructor( - private readonly primary: SmtpIdentity, + private readonly primary: SmtpIdentity | undefined, private readonly fallback?: SmtpIdentity, makeTransport?: (id: SmtpIdentity) => MailTransport, private readonly resolveAttachment?: AttachmentResolver, + /** BYO: resolve the tenant's own SMTP identity per scope; used before the env identity. */ + private readonly credResolver?: SmtpCredResolver, ) { this.makeTransport = makeTransport ?? defaultTransport; } @@ -119,6 +144,13 @@ export class SmtpProvider implements CapabilityProvider { const started = Date.now(); const p = (req.payload ?? {}) as EmailPayload; + // BYO SMTP: a tenant's own identity wins over the platform env identity. The env fallback + // (accounts@ → ceo@) applies ONLY to the platform identity, never to a tenant's own server. + const tenant = req.scopeId && this.credResolver ? await this.credResolver(req.scopeId).catch(() => null) : null; + const identity = tenant ?? this.primary; + if (!identity) return this.failed('NOT_CONFIGURED', started); + const fallback = tenant ? undefined : this.fallback; + // Resolve attachment bytes up front. Fail CLOSED — never send an invoice/receipt email missing // its file; a FAILED command retries instead. Resolved once so a fallback retry doesn't re-fetch. let attachments: MailAttachment[] | undefined; @@ -145,13 +177,13 @@ export class SmtpProvider implements CapabilityProvider { }); try { - const info = await attempt(this.primary); + const info = await attempt(identity); return { providerRef: info.messageId, outcome: 'SENT', latencyMs: Date.now() - started }; } catch (err) { // Retry via the fallback identity ONLY if the primary never got the message accepted. - if (this.fallback && isPreAcceptanceFailure(err)) { + if (fallback && isPreAcceptanceFailure(err)) { try { - const info = await attempt(this.fallback); + const info = await attempt(fallback); return { providerRef: `fallback:${info.messageId}`, outcome: 'SENT', latencyMs: Date.now() - started }; } catch (err2) { return { providerRef: `smtp-error-${randomUUID().slice(0, 8)}`, outcome: 'FAILED', errorCode: codeOf(err2), latencyMs: Date.now() - started }; diff --git a/packages/iios-service/src/search/meili.adapter.ts b/packages/iios-service/src/search/meili.adapter.ts new file mode 100644 index 0000000..7974484 --- /dev/null +++ b/packages/iios-service/src/search/meili.adapter.ts @@ -0,0 +1,101 @@ +import { Logger } from '@nestjs/common'; +import { MeiliSearch } from 'meilisearch'; +import type { MessageSearchPort, SearchDoc, SearchHit } from './message-search.port'; + +const INDEX = 'iios_messages'; + +/** A no-op search port — bound when Meilisearch isn't configured, so search degrades to empty. */ +export const noopMessageSearch: MessageSearchPort = { + ready: () => false, + index: async () => undefined, + remove: async () => undefined, + search: async () => [], +}; + +/** + * Meilisearch-backed message search. Configured via MEILI_URL (+ optional MEILI_KEY). The index is + * created + configured on first use (searchable text/subject; filterable scopeId/threadId/source; + * sortable at). Filtering by scopeId AND threadId is the permission fence — the caller only ever + * passes thread ids they belong to. + */ +export class MeiliMessageSearch implements MessageSearchPort { + private readonly client: MeiliSearch; + private readonly logger = new Logger(MeiliMessageSearch.name); + private settingsReady?: Promise; + + constructor(host: string, apiKey?: string) { + this.client = new MeiliSearch({ host, ...(apiKey ? { apiKey } : {}) }); + } + + ready(): boolean { + return true; + } + + /** Idempotently ensure the index + its attribute settings exist (runs once, memoised). */ + private ensure(): Promise { + if (!this.settingsReady) { + this.settingsReady = (async () => { + await this.client.createIndex(INDEX, { primaryKey: 'id' }).catch(() => undefined); + await this.client.index(INDEX).updateSettings({ + searchableAttributes: ['text', 'subject'], + filterableAttributes: ['scopeId', 'threadId', 'source'], + sortableAttributes: ['at'], + }); + })().catch((err) => { + this.settingsReady = undefined; // let a later call retry + throw err; + }); + } + return this.settingsReady; + } + + async index(docs: SearchDoc[]): Promise { + if (docs.length === 0) return; + await this.ensure(); + await this.client.index(INDEX).addDocuments(docs, { primaryKey: 'id' }); + } + + async remove(ids: string[]): Promise { + if (ids.length === 0) return; + await this.client.index(INDEX).deleteDocuments(ids); + } + + async search(scopeId: string, threadIds: string[], query: string, limit: number): Promise { + if (threadIds.length === 0 || !query.trim()) return []; + await this.ensure(); + const quoted = threadIds.map((t) => JSON.stringify(t)).join(', '); + const res = await this.client.index(INDEX).search(query, { + filter: `scopeId = ${JSON.stringify(scopeId)} AND threadId IN [${quoted}]`, + limit, + sort: ['at:desc'], + attributesToHighlight: ['text'], + highlightPreTag: '', + highlightPostTag: '', + attributesToCrop: ['text'], + cropLength: 30, + }); + return res.hits.map((h) => { + const doc = h as unknown as SearchDoc & { _formatted?: { text?: string } }; + return { + id: doc.id, + threadId: doc.threadId, + text: doc.text, + subject: doc.subject ?? null, + source: doc.source ?? null, + at: doc.at, + snippet: doc._formatted?.text ?? doc.text, + }; + }); + } +} + +/** Build the search port from env: MEILI_URL set → Meilisearch; else the no-op. */ +export function messageSearchFromEnv(env: NodeJS.ProcessEnv = process.env): MessageSearchPort { + const host = env.MEILI_URL?.trim(); + if (!host) { + new Logger('MessageSearch').log('MEILI_URL unset — message search disabled (no-op)'); + return noopMessageSearch; + } + new Logger('MessageSearch').log(`using Meilisearch @ ${host}`); + return new MeiliMessageSearch(host, env.MEILI_KEY?.trim() || undefined); +} diff --git a/packages/iios-service/src/search/message-search.port.ts b/packages/iios-service/src/search/message-search.port.ts new file mode 100644 index 0000000..6d5991f --- /dev/null +++ b/packages/iios-service/src/search/message-search.port.ts @@ -0,0 +1,39 @@ +/** One indexed message document. `at` is epoch-ms so Meilisearch can sort recency-first. */ +export interface SearchDoc { + id: string; // interactionId + scopeId: string; + threadId: string; + text: string; + subject: string | null; + source: string | null; // opaque app source tag (e.g. crm-messenger / crm-mail) — drives the surface + actorId: string | null; + at: number; +} + +/** A search hit with a highlighted snippet. */ +export interface SearchHit { + id: string; + threadId: string; + text: string; + subject: string | null; + source: string | null; + at: number; + /** The match with `…` around the query terms (from the engine's highlighter). */ + snippet: string; +} + +/** + * The message-search seam. IIOS owns egress-style search: a message index + a permission-scoped + * query. The concrete engine (Meilisearch) sits behind this; an unconfigured deployment binds a + * no-op so search degrades to empty rather than erroring. + */ +export interface MessageSearchPort { + /** True only when a real engine is configured (else index/search are inert). */ + ready(): boolean; + index(docs: SearchDoc[]): Promise; + remove(ids: string[]): Promise; + /** Search WITHIN the given scope and thread set only (the permission fence — enforced here). */ + search(scopeId: string, threadIds: string[], query: string, limit: number): Promise; +} + +export const MESSAGE_SEARCH_PORT = Symbol('MESSAGE_SEARCH_PORT'); diff --git a/packages/iios-service/src/search/search.controller.ts b/packages/iios-service/src/search/search.controller.ts new file mode 100644 index 0000000..6e81e17 --- /dev/null +++ b/packages/iios-service/src/search/search.controller.ts @@ -0,0 +1,36 @@ +import { BadRequestException, Body, Controller, Headers, Post } from '@nestjs/common'; +import { SessionVerifier } from '../platform/session.verifier'; +import { ActorResolver, type MessagePrincipal } from '../identity/actor.resolver'; +import { SearchService } from './search.service'; + +interface SearchDto { query?: string; limit?: number } + +/** Message search (session-auth). Results are permission-scoped inside the service to the caller's + * own scope + threads. Reindex backfills the caller's scope (idempotent). */ +@Controller('v1/search') +export class SearchController { + constructor( + private readonly search: SearchService, + private readonly session: SessionVerifier, + private readonly actors: ActorResolver, + ) {} + + @Post() + async run(@Body() body: SearchDto, @Headers('authorization') authorization?: string) { + const principal = this.principal(authorization); + return this.search.search(principal, { query: body.query ?? '', ...(body.limit != null ? { limit: body.limit } : {}) }); + } + + @Post('reindex') + async reindex(@Headers('authorization') authorization?: string) { + const principal = this.principal(authorization); + const scope = await this.actors.findScope(principal); + return { indexed: await this.search.reindexAll(scope?.id) }; + } + + private principal(authorization?: string): MessagePrincipal { + const token = (authorization ?? '').replace(/^Bearer\s+/i, ''); + if (!token) throw new BadRequestException('Authorization bearer token is required'); + return this.session.verify(token); + } +} diff --git a/packages/iios-service/src/search/search.module.ts b/packages/iios-service/src/search/search.module.ts new file mode 100644 index 0000000..1312e42 --- /dev/null +++ b/packages/iios-service/src/search/search.module.ts @@ -0,0 +1,24 @@ +import { Module } from '@nestjs/common'; +import { OutboxModule } from '../outbox/outbox.module'; +import { SearchService } from './search.service'; +import { SearchProjector } from './search.projector'; +import { SearchController } from './search.controller'; +import { MESSAGE_SEARCH_PORT } from './message-search.port'; +import { messageSearchFromEnv } from './meili.adapter'; + +/** + * Message search (Meilisearch). The engine binds from MEILI_URL; unset → a no-op port so search + * returns empty rather than erroring. The projector indexes on `message.sent`; the service enforces + * the permission fence (scope + caller's threads). SessionVerifier + ActorResolver are global. + */ +@Module({ + imports: [OutboxModule], + controllers: [SearchController], + providers: [ + SearchService, + SearchProjector, + { provide: MESSAGE_SEARCH_PORT, useFactory: () => messageSearchFromEnv() }, + ], + exports: [SearchService], +}) +export class SearchModule {} diff --git a/packages/iios-service/src/search/search.projector.ts b/packages/iios-service/src/search/search.projector.ts new file mode 100644 index 0000000..2577099 --- /dev/null +++ b/packages/iios-service/src/search/search.projector.ts @@ -0,0 +1,35 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { CloudEvent, IIOS_EVENTS } from '@insignia/iios-contracts'; +import { OutboxBus } from '../outbox/outbox.bus'; +import { DlqService } from '../outbox/dlq.service'; +import { ProjectionCursorService } from '../projection/projection-cursor.service'; +import { SearchService } from './search.service'; + +/** + * Indexes messages into the search engine as they are sent. Reacts to the `message.sent` event on + * the outbox bus (same pattern as the inbox projector). Indexing is idempotent (upsert by id), so + * no per-event claim is needed — a redelivered event just re-indexes the same doc. + */ +@Injectable() +export class SearchProjector implements OnModuleInit { + private readonly consumer = 'search-projector'; + private readonly logger = new Logger(SearchProjector.name); + + constructor( + private readonly search: SearchService, + private readonly bus: OutboxBus, + private readonly dlq: DlqService, + private readonly cursor: ProjectionCursorService, + ) {} + + onModuleInit(): void { + this.dlq.registerHandler(this.consumer, (e) => this.onMessageSent(e)); + this.bus.on(IIOS_EVENTS.messageSent, (p) => void this.onMessageSent(p as CloudEvent).catch((err) => this.dlq.onConsumerFailure(this.consumer, p as CloudEvent, err))); + } + + async onMessageSent(event: CloudEvent): Promise { + const data = event.data as { interactionId?: string }; + if (data.interactionId) await this.search.indexInteraction(data.interactionId); + await this.cursor.advance(this.consumer, event); + } +} diff --git a/packages/iios-service/src/search/search.service.spec.ts b/packages/iios-service/src/search/search.service.spec.ts new file mode 100644 index 0000000..f6f7089 --- /dev/null +++ b/packages/iios-service/src/search/search.service.spec.ts @@ -0,0 +1,57 @@ +import { describe, it, expect, vi } from 'vitest'; +import { SearchService } from './search.service'; +import type { MessageSearchPort, SearchHit } from './message-search.port'; +import type { ActorResolver, MessagePrincipal } from '../identity/actor.resolver'; +import type { PrismaService } from '../prisma/prisma.service'; + +const principal: MessagePrincipal = { userId: 'u1', orgId: 'org', appId: 'app' }; + +function make(opts: { ready?: boolean; threadIds?: string[]; hits?: SearchHit[]; scope?: { id: string } | null } = {}) { + const engine: MessageSearchPort = { + ready: () => opts.ready ?? true, + index: vi.fn(async () => undefined), + remove: vi.fn(async () => undefined), + search: vi.fn(async () => opts.hits ?? []), + }; + const actors = { + findScope: vi.fn(async () => (opts.scope === undefined ? { id: 'scope_1' } : opts.scope)), + resolveActor: vi.fn(async () => ({ id: 'actor_1' })), + } as unknown as ActorResolver; + const prisma = { + iiosThreadParticipant: { findMany: vi.fn(async () => (opts.threadIds ?? ['t1', 't2']).map((threadId) => ({ threadId }))) }, + } as unknown as PrismaService; + return { svc: new SearchService(engine, prisma, actors), engine }; +} + +describe('SearchService (permission fence)', () => { + it('searches only within the caller scope + their thread ids', async () => { + const { svc, engine } = make({ threadIds: ['t1', 't2', 't3'], hits: [{ id: 'i1', threadId: 't2', text: 'hello world', subject: 'General', source: 'crm-messenger', at: 1, snippet: 'hello' }] }); + const res = await svc.search(principal, { query: 'hello' }); + expect(res[0]!.id).toBe('i1'); + expect(engine.search).toHaveBeenCalledWith('scope_1', ['t1', 't2', 't3'], 'hello', 20); + }); + + it('returns empty (no engine hit) for a blank query', async () => { + const { svc, engine } = make(); + expect(await svc.search(principal, { query: ' ' })).toEqual([]); + expect(engine.search).not.toHaveBeenCalled(); + }); + + it('returns empty when the caller belongs to no threads', async () => { + const { svc, engine } = make({ threadIds: [] }); + expect(await svc.search(principal, { query: 'hello' })).toEqual([]); + expect(engine.search).not.toHaveBeenCalled(); + }); + + it('returns empty when search is not configured (no-op engine)', async () => { + const { svc, engine } = make({ ready: false }); + expect(await svc.search(principal, { query: 'hello' })).toEqual([]); + expect(engine.search).not.toHaveBeenCalled(); + }); + + it('caps the limit at 50', async () => { + const { svc, engine } = make(); + await svc.search(principal, { query: 'x', limit: 999 }); + expect((engine.search as ReturnType).mock.calls[0]![3]).toBe(50); + }); +}); diff --git a/packages/iios-service/src/search/search.service.ts b/packages/iios-service/src/search/search.service.ts new file mode 100644 index 0000000..ce06aa1 --- /dev/null +++ b/packages/iios-service/src/search/search.service.ts @@ -0,0 +1,83 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { PrismaService } from '../prisma/prisma.service'; +import { ActorResolver, type MessagePrincipal } from '../identity/actor.resolver'; +import { MESSAGE_SEARCH_PORT, type MessageSearchPort, type SearchDoc, type SearchHit } from './message-search.port'; + +/** + * Message search. The permission fence lives HERE: a query only ever runs against the caller's own + * scope AND the set of threads the caller currently belongs to (resolved live from participant + * rows, so membership changes are reflected immediately). The engine never sees a thread the caller + * isn't in. Text is drawn from the message's TEXT part; every conversation kind (chat, mail, sms…) + * is a message with parts, so all are searchable through one index. + */ +@Injectable() +export class SearchService { + constructor( + @Inject(MESSAGE_SEARCH_PORT) private readonly engine: MessageSearchPort, + private readonly prisma: PrismaService, + private readonly actors: ActorResolver, + ) {} + + async search(principal: MessagePrincipal, opts: { query: string; limit?: number }): Promise { + const q = (opts.query ?? '').trim(); + if (!q || !this.engine.ready()) return []; + const scope = await this.actors.findScope(principal); + if (!scope) return []; + const actor = await this.actors.resolveActor(scope.id, principal); + const memberships = await this.prisma.iiosThreadParticipant.findMany({ where: { actorId: actor.id }, select: { threadId: true } }); + const threadIds = memberships.map((m) => m.threadId); + if (threadIds.length === 0) return []; + return this.engine.search(scope.id, threadIds, q, Math.min(opts.limit ?? 20, 50)); + } + + /** Build + index the search doc for one interaction (called by the projector on message.sent). */ + async indexInteraction(interactionId: string): Promise { + if (!this.engine.ready()) return; + const doc = await this.buildDoc(interactionId); + if (doc) await this.engine.index([doc]); + } + + /** Remove an interaction from the index (retention / redaction hook). */ + async removeInteraction(interactionId: string): Promise { + if (this.engine.ready()) await this.engine.remove([interactionId]); + } + + /** Backfill: (re)index every text message, optionally for one scope. Returns the count indexed. */ + async reindexAll(scopeId?: string, batch = 500): Promise { + if (!this.engine.ready()) return 0; + const interactions = await this.prisma.iiosInteraction.findMany({ + where: { ...(scopeId ? { scopeId } : {}), threadId: { not: null }, parts: { some: { kind: 'TEXT' } } }, + select: { id: true }, + }); + let indexed = 0; + for (let i = 0; i < interactions.length; i += batch) { + const docs = (await Promise.all(interactions.slice(i, i + batch).map((x) => this.buildDoc(x.id)))).filter((d): d is SearchDoc => d !== null); + if (docs.length > 0) { + await this.engine.index(docs); + indexed += docs.length; + } + } + return indexed; + } + + private async buildDoc(interactionId: string): Promise { + const i = await this.prisma.iiosInteraction.findUnique({ + where: { id: interactionId }, + include: { parts: { where: { kind: 'TEXT' }, take: 1 }, thread: true }, + }); + if (!i || !i.threadId || !i.thread) return null; + const text = i.parts[0]?.bodyText?.trim(); + if (!text) return null; // nothing searchable + const meta = (i.thread.metadata as { source?: string } | null) ?? {}; + return { + id: i.id, + scopeId: i.scopeId, + threadId: i.threadId, + text, + subject: i.thread.subject ?? null, + source: meta.source ?? null, + actorId: i.actorId ?? null, + at: i.occurredAt.getTime(), + }; + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4aac1a1..7f9f3b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -415,6 +415,9 @@ importers: jwks-rsa: specifier: ^4.1.0 version: 4.1.0 + meilisearch: + specifier: ^0.45.0 + version: 0.45.0 nodemailer: specifier: ^9.0.3 version: 9.0.3 @@ -2715,6 +2718,9 @@ packages: resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} engines: {node: '>= 0.8'} + meilisearch@0.45.0: + resolution: {integrity: sha512-+zCzEqE+CumY4icB0Vox180adZqaNtnr60hJWGiEdmol5eWmksfY8rYsTcz87styXC2ZOg+2yF56gdH6oyIBTA==} + memfs@3.5.3: resolution: {integrity: sha512-UERzLsxzllchadvbPs5aolHh65ISpKpM+ccLbOJ8/vvpBKmAWf+la7dXFy7Mr0ySHbdHrFv5kGFCUHHe6GFEmw==} engines: {node: '>= 4.0.0'} @@ -5902,6 +5908,8 @@ snapshots: media-typer@1.1.0: {} + meilisearch@0.45.0: {} + memfs@3.5.3: dependencies: fs-monkey: 1.1.0