feat: add spam detection lane (tower.classify.spam) with content-aware filtering
- Add SpamJobData + sync missing job types (DncJobData, ClassifyJobData, P1JobData, etc.) from dist back to source in @tower/types - spam-detector.ts: emoji-only, short-greeting, link-only detection + duplicate-hash (1h window) - spam.queue.ts / spam.processor.ts: BullMQ tower.classify.spam lane - classify.processor.ts: spam check (Step 0) before rule engine, accepts spamQueue param - main.ts: wire spamQueue + spamWorker; spam pre-filter before ingestQueue.add - dnc.processor.ts: labelled reason logging Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
export type SpamReason = 'emoji_only' | 'short_greeting' | 'link_only' | 'forward_flood' | 'duplicate_hash';
|
||||
|
||||
export interface SpamResult {
|
||||
isSpam: boolean;
|
||||
reason?: SpamReason;
|
||||
contentHash: string;
|
||||
}
|
||||
|
||||
const GREETING_PHRASES = [
|
||||
'good morning', 'good evening', 'good night', 'good afternoon',
|
||||
'gm', 'gn', 'subah ki shayari', 'शुभ प्रभात', 'शुभ रात्रि',
|
||||
'happy', 'jai hind', 'jai shree ram', 'jai mata di',
|
||||
'forwarded as received', 'please share', 'must share',
|
||||
];
|
||||
|
||||
const EMOJI_RE = /^[\p{Emoji}\p{Emoji_Presentation}\s]+$/u;
|
||||
const URL_RE = /https?:\/\/\S+/gi;
|
||||
const TOKEN_MIN = 4;
|
||||
|
||||
export function hashContent(content: string): string {
|
||||
return createHash('sha256').update(content.trim().toLowerCase()).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
export function detectSpam(content: string): SpamResult {
|
||||
const trimmed = content.trim();
|
||||
const contentHash = hashContent(trimmed);
|
||||
|
||||
if (EMOJI_RE.test(trimmed)) {
|
||||
return { isSpam: true, reason: 'emoji_only', contentHash };
|
||||
}
|
||||
|
||||
const lower = trimmed.toLowerCase();
|
||||
const withoutUrls = lower.replace(URL_RE, '').trim();
|
||||
const tokens = withoutUrls.split(/\s+/).filter(Boolean);
|
||||
|
||||
if (tokens.length < TOKEN_MIN) {
|
||||
if (GREETING_PHRASES.some((phrase) => lower.includes(phrase))) {
|
||||
return { isSpam: true, reason: 'short_greeting', contentHash };
|
||||
}
|
||||
}
|
||||
|
||||
const urls = trimmed.match(URL_RE) ?? [];
|
||||
if (urls.length > 0 && tokens.length === 0) {
|
||||
return { isSpam: true, reason: 'link_only', contentHash };
|
||||
}
|
||||
|
||||
return { isSpam: false, contentHash };
|
||||
}
|
||||
|
||||
export async function checkDuplicateHash(
|
||||
contentHash: string,
|
||||
tenantId: string,
|
||||
sourceGroupId: string,
|
||||
windowMs: number,
|
||||
prisma: any,
|
||||
): Promise<boolean> {
|
||||
const since = new Date(Date.now() - windowMs);
|
||||
const count = await prisma.message.count({
|
||||
where: {
|
||||
tenantId,
|
||||
sourceGroupId,
|
||||
content: { startsWith: '' },
|
||||
createdAt: { gte: since },
|
||||
},
|
||||
});
|
||||
if (count === 0) return false;
|
||||
|
||||
const recent = await prisma.message.findMany({
|
||||
where: { tenantId, sourceGroupId, createdAt: { gte: since } },
|
||||
select: { content: true },
|
||||
take: 100,
|
||||
});
|
||||
|
||||
return recent.some((m: { content: string }) => hashContent(m.content) === contentHash);
|
||||
}
|
||||
Reference in New Issue
Block a user