feat: add spam detection lane (tower.classify.spam) with content-aware filtering
- Add SpamJobData + sync missing job types (DncJobData, ClassifyJobData, P1JobData, etc.) from dist back to source in @tower/types - spam-detector.ts: emoji-only, short-greeting, link-only detection + duplicate-hash (1h window) - spam.queue.ts / spam.processor.ts: BullMQ tower.classify.spam lane - classify.processor.ts: spam check (Step 0) before rule engine, accepts spamQueue param - main.ts: wire spamQueue + spamWorker; spam pre-filter before ingestQueue.add - dnc.processor.ts: labelled reason logging Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,8 @@ import { createForwardQueue } from './queues/forward.queue';
|
||||
import { createForwardWorker } from './queues/forward.processor';
|
||||
import { createIndexQueue } from './queues/index.queue';
|
||||
import { createIndexWorker } from './queues/index.processor';
|
||||
import { createSpamQueue } from './queues/spam.queue';
|
||||
import { createSpamWorker } from './queues/spam.processor';
|
||||
import { WhatsAppSessionPool } from './whatsapp/session-pool';
|
||||
import { matchContentRules, TenantRuleRow } from './whatsapp/match-rules';
|
||||
import { syncGroups } from './whatsapp/group-sync';
|
||||
@@ -15,6 +17,7 @@ import { handleReaction } from './core/approval';
|
||||
import { approveMessage } from './core/approve-message';
|
||||
import { startOtpSenderLoop } from './whatsapp/otp-sender';
|
||||
import { handleCommand } from './whatsapp/command-handler';
|
||||
import { detectSpam, checkDuplicateHash } from './spam/spam-detector';
|
||||
|
||||
const logger = createLogger('tower-worker');
|
||||
|
||||
@@ -30,11 +33,13 @@ async function bootstrap() {
|
||||
const ingestQueue = createIngestQueue(env.REDIS_URL);
|
||||
const forwardQueue = createForwardQueue(env.REDIS_URL);
|
||||
const indexQueue = createIndexQueue(env.REDIS_URL);
|
||||
const spamQueue = createSpamQueue(env.REDIS_URL);
|
||||
const pool = new WhatsAppSessionPool();
|
||||
|
||||
const ingestWorker = createIngestWorker(env.REDIS_URL, prisma, pool, forwardQueue, indexQueue);
|
||||
const forwardWorker = createForwardWorker(env.REDIS_URL, pool);
|
||||
const indexWorker = createIndexWorker(env.REDIS_URL, meiliClient);
|
||||
const spamWorker = createSpamWorker(env.REDIS_URL, prisma);
|
||||
|
||||
ingestWorker.on('completed', (job) => logger.info({ jobId: job.id }, 'Ingest job completed'));
|
||||
ingestWorker.on('failed', (job, err) => logger.error({ jobId: job?.id, err }, 'Ingest job failed'));
|
||||
@@ -42,6 +47,7 @@ async function bootstrap() {
|
||||
forwardWorker.on('failed', (job, err) => logger.error({ jobId: job?.id, err }, 'Forward job failed'));
|
||||
indexWorker.on('completed', (job) => logger.info({ jobId: job.id }, 'Index job completed'));
|
||||
indexWorker.on('failed', (job, err) => logger.error({ jobId: job?.id, err }, 'Index job failed'));
|
||||
spamWorker.on('failed', (job, err) => logger.warn({ jobId: job?.id, err }, 'Spam job failed'));
|
||||
|
||||
const groupMaps = new Map<string, Map<string, string>>();
|
||||
|
||||
@@ -94,6 +100,25 @@ async function bootstrap() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Spam pre-filter — runs before rule engine; spam messages are never stored
|
||||
const spamResult = detectSpam(msg.content);
|
||||
const isDupe = spamResult.isSpam
|
||||
? false
|
||||
: await checkDuplicateHash(spamResult.contentHash, group.tenantId, sourceGroupId, 60 * 60 * 1000, prisma);
|
||||
if (spamResult.isSpam || isDupe) {
|
||||
const reason = isDupe ? 'duplicate_hash' : (spamResult.reason ?? 'emoji_only');
|
||||
logger.debug({ senderJid: msg.senderJid, reason }, 'Spam filtered before ingest');
|
||||
await spamQueue.add('spam', {
|
||||
tenantId: group.tenantId,
|
||||
messageId: 'pre-ingest',
|
||||
sourceGroupId,
|
||||
senderJid: msg.senderJid,
|
||||
reason,
|
||||
contentHash: spamResult.contentHash,
|
||||
}, { attempts: 1 }).catch(() => void 0);
|
||||
return;
|
||||
}
|
||||
|
||||
// Load active rules for this tenant
|
||||
const ruleRows: TenantRuleRow[] = await prisma.tenantRule.findMany({
|
||||
where: { tenantId: group.tenantId, isActive: true },
|
||||
@@ -279,9 +304,11 @@ async function bootstrap() {
|
||||
await ingestWorker.close();
|
||||
await forwardWorker.close();
|
||||
await indexWorker.close();
|
||||
await spamWorker.close();
|
||||
await ingestQueue.close();
|
||||
await forwardQueue.close();
|
||||
await indexQueue.close();
|
||||
await spamQueue.close();
|
||||
await prisma.$disconnect();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user