feat: add worker queue fabric — classify, dnc, p1, review, digest, thread-resolve, outbox, AI modules

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-17 15:40:28 +05:30
parent 6aec093516
commit 47f345c4bf
21 changed files with 1097 additions and 0 deletions
@@ -0,0 +1,100 @@
+import { Client } from 'pg';
+import { Queue } from 'bullmq';
+import { ClassifyJobData, ThreadResolveJobData } from '@tower/types';
+import { createLogger } from '@tower/logger';
+
+const logger = createLogger('outbox-listener');
+const MAX_ATTEMPTS = 5;
+
+async function processEvent(
+  event: { id: string; type: string; payload: unknown },
+  classifyQueue: Queue<ClassifyJobData>,
+  threadResolveQueue: Queue<ThreadResolveJobData>,
+  prisma: any,
+): Promise<void> {
+  if (event.type === 'classify') {
+    await classifyQueue.add('classify', event.payload as ClassifyJobData, {
+      attempts: 3,
+      backoff: { type: 'exponential', delay: 1000 },
+    });
+  } else if (event.type === 'thread_resolve') {
+    await threadResolveQueue.add('thread-resolve', event.payload as ThreadResolveJobData, {
+      attempts: 2,
+    });
+  } else {
+    logger.warn({ type: event.type, eventId: event.id }, 'Unknown outbox event type — skipping');
+  }
+  await prisma.outboxEvent.delete({ where: { id: event.id } });
+}
+
+// One-time sweep at startup: drain events that arrived while the worker was offline
+async function drainPending(
+  prisma: any,
+  classifyQueue: Queue<ClassifyJobData>,
+  threadResolveQueue: Queue<ThreadResolveJobData>,
+): Promise<void> {
+  const events = await prisma.outboxEvent.findMany({
+    where: { attempts: { lt: MAX_ATTEMPTS } },
+    orderBy: { createdAt: 'asc' },
+    take: 500,
+  });
+
+  if (events.length > 0) {
+    logger.info({ count: events.length }, 'Draining pending outbox events from downtime');
+  }
+
+  for (const event of events) {
+    try {
+      await processEvent(event, classifyQueue, threadResolveQueue, prisma);
+      logger.info({ eventId: event.id, type: event.type }, 'Startup drain: processed');
+    } catch (err) {
+      await prisma.outboxEvent.update({
+        where: { id: event.id },
+        data: { attempts: { increment: 1 }, lastError: String(err) },
+      });
+      logger.warn({ err, eventId: event.id }, 'Startup drain: event failed');
+    }
+  }
+}
+
+export async function startOutboxListener(
+  databaseUrl: string,
+  prisma: any,
+  classifyQueue: Queue<ClassifyJobData>,
+  threadResolveQueue: Queue<ThreadResolveJobData>,
+): Promise<Client> {
+  await drainPending(prisma, classifyQueue, threadResolveQueue);
+
+  // Dedicated pg connection for LISTEN — Prisma does not support LISTEN/NOTIFY
+  const pgClient = new Client({ connectionString: databaseUrl });
+  await pgClient.connect();
+  await pgClient.query('LISTEN outbox_event');
+
+  pgClient.on('notification', (msg) => {
+    const eventId = msg.payload;
+    if (!eventId) return;
+
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+    prisma.outboxEvent
+      .findUnique({ where: { id: eventId } })
+      .then(async (event: { id: string; type: string; payload: unknown; attempts: number } | null) => {
+        if (!event || event.attempts >= MAX_ATTEMPTS) return;
+        try {
+          await processEvent(event, classifyQueue, threadResolveQueue, prisma);
+          logger.info({ eventId, type: event.type }, 'Outbox event processed');
+        } catch (err) {
+          await prisma.outboxEvent.update({
+            where: { id: event.id },
+            data: { attempts: { increment: 1 }, lastError: String(err) },
+          });
+          logger.warn({ err, eventId }, 'Outbox event failed — incremented attempts');
+        }
+      })
+      .catch((err: unknown) => logger.error({ err, eventId }, 'Outbox notification handler error'));
+  });
+
+  pgClient.on('error', (err) => logger.error({ err }, 'Outbox pg client error'));
+
+  logger.info('Outbox listener ready');
+  return pgClient;
+}