From 2ce3395006ac893dbfaad79726f5c1e7cf8eaa5f Mon Sep 17 00:00:00 2001 From: Satyam Rastogi Date: Sat, 30 May 2026 20:33:01 +0530 Subject: [PATCH] feat(ai-chat): use openai/gpt-oss-120b and token-budget the role context Switch the assistant model from qwen/qwen3-32b to openai/gpt-oss-120b. Cap every per-role context list, hard-clamp the assembled system prompt to ~4000 tokens at a line boundary, and bound re-sent chat history to 12 messages so the model context window can't be exceeded regardless of mock-data volume. --- README.md | 2 +- src/components/Chatbot.jsx | 70 ++++++++++++++++++++++++++++++-------- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 3a66e14..1733f09 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ - **Maps**: react-leaflet - **Charts**: Recharts - **Icons**: lucide-react -- **AI**: Groq SDK (qwen-32b) with dynamic role-aware system prompts +- **AI**: Groq SDK (openai/gpt-oss-120b) with dynamic role-aware system prompts (token-budgeted) - **Notifications**: sonner (toasts) ### Setup diff --git a/src/components/Chatbot.jsx b/src/components/Chatbot.jsx index 654a5fb..b416105 100644 --- a/src/components/Chatbot.jsx +++ b/src/components/Chatbot.jsx @@ -14,6 +14,40 @@ const groq = new Groq({ dangerouslyAllowBrowser: true // Allowed for client-side demo }); +// Model used for the assistant. gpt-oss-120b is a non-reasoning instruct model +// (no blocks) with a 131K context window — clean streaming for the demo. +const AI_MODEL = 'openai/gpt-oss-120b'; + +// --- CONTEXT BUDGETING ------------------------------------------------------- +// Keep the role context comprehensive but bounded: cap every list so no single +// section can balloon, then hard-clamp the whole system prompt to a token budget. +// This keeps requests fast for the live demo and makes a context-window overflow +// structurally impossible regardless of how much mock data grows. +const SYSTEM_CONTEXT_TOKEN_BUDGET = 4000; // ~16K chars of summarized role data +const HISTORY_TURN_LIMIT = 12; // most-recent messages re-sent each turn + +// Rough token estimate (~4 chars/token for English) — good enough for budgeting. +const estimateTokens = (str = '') => Math.ceil(str.length / 4); + +// Map the first `max` items; if more exist, append a count line so the model +// knows the list was truncated — and no section ever grows unbounded. +const capList = (items = [], max, fmt) => { + const shown = items.slice(0, max).map(fmt); + if (items.length > max) shown.push(` …and ${items.length - max} more`); + return shown.join('\n'); +}; + +// Hard safety net: clamp the assembled system prompt to the token budget at a +// line boundary so the context window can never be exceeded. +const clampToTokenBudget = (text = '', tokenBudget = SYSTEM_CONTEXT_TOKEN_BUDGET) => { + if (estimateTokens(text) <= tokenBudget) return text; + const maxChars = tokenBudget * 4; + const cut = text.slice(0, maxChars); + const lastNl = cut.lastIndexOf('\n'); + const trimmed = lastNl > 0 ? cut.slice(0, lastNl) : cut; + return `${trimmed}\n\n[context truncated to stay within the model limit]`; +}; + // --- CONTEXT GENERATOR (Deep, role-aware with cross-role visibility) --- const generateRoleContext = (user, store) => { @@ -79,11 +113,11 @@ Contact: 866-259-6533`; const totalStaff = personnel.length; // Project details - const projDetails = myProjects.map(p => { + const projDetails = capList(myProjects, 10, p => { const variance = p.budget > 0 ? Math.round(((p.spent - p.budget) / p.budget) * 100) : 0; const nextMs = (p.milestones || []).find(m => m.status !== 'completed'); return ` - ${p.address} (${p.projectType}): ${p.status.toUpperCase()}, ${p.completionPercentage || 0}% done, Budget $${p.budget.toLocaleString()}, Spent $${p.spent.toLocaleString()} (${variance > 0 ? '+' : ''}${variance}%), Next: ${nextMs ? `${nextMs.name} by ${nextMs.dueDate}` : 'N/A'}`; - }).join('\n'); + }); // Upcoming milestones across all owner projects const upcomingMs = myProjects.flatMap(p => (p.milestones || []).filter(m => { @@ -196,7 +230,7 @@ PROJECTS OVERVIEW (Cross-Role: Admin sees ALL): VENDOR COMPLIANCE: - Total Vendors: ${vendors.length} | Non-Compliant: ${nonCompliant.length} -${nonCompliant.map(v => ` - ${v.vendorName}: Status=${v.status}`).join('\n') || ' All compliant.'} +${capList(nonCompliant, 8, v => ` - ${v.vendorName}: Status=${v.status}`) || ' All compliant.'} STAFF: ${activeStaff.length} active / ${personnel.length} total @@ -224,8 +258,11 @@ INSTRUCTIONS: const goldenLeads = properties.filter(p => p.propertyData?.yearBuilt < 2000 && p.propertyData?.currentEstimatedMarketValue > 400000).slice(0, 3) .map(p => ` - ${p.propertyData.propertyAddress} (Built: ${p.propertyData.yearBuilt})`).join('\n'); - const pendingSigs = properties.filter(p => p.pendingSignature || p.propertyData?.pendingSignature) - .map(p => ` - ${p.propertyData?.propertyAddress || 'N/A'} (Owner: ${p.ownerData?.fullName || 'N/A'})`).join('\n'); + const pendingSigs = capList( + properties.filter(p => p.pendingSignature || p.propertyData?.pendingSignature), + 8, + p => ` - ${p.propertyData?.propertyAddress || 'N/A'} (Owner: ${p.ownerData?.fullName || 'N/A'})` + ); const myWins = meetings.filter(m => m.agentId === user.id && m.status === 'Converted'); const myCompleted = meetings.filter(m => m.agentId === user.id && (m.status === 'Converted' || m.status === 'Completed')); @@ -267,10 +304,10 @@ INSTRUCTIONS: const totalSpent = myProjects.reduce((s, p) => s + (p.spent || 0), 0); const pendingInvoices = myProjects.reduce((s, p) => s + (p.invoices || []).filter(i => i.status === 'pending').reduce((is, i) => is + i.amount, 0), 0); - const projDetails = myProjects.map(p => { + const projDetails = capList(myProjects, 10, p => { const nextMs = (p.milestones || []).find(m => m.status !== 'completed'); return ` - ${p.address} (${p.projectType}): ${p.status.toUpperCase()}, ${p.completionPercentage || 0}% done, Budget $${p.budget.toLocaleString()}, Spent $${p.spent.toLocaleString()}, Next: ${nextMs ? `${nextMs.name} by ${nextMs.dueDate}` : 'All complete'}`; - }).join('\n'); + }); // Upcoming milestones const upcomingMs = myProjects.flatMap(p => (p.milestones || []).filter(m => { @@ -278,7 +315,7 @@ INSTRUCTIONS: return diff >= -3 && diff <= 14 && m.status !== 'completed'; }).map(m => ({ ...m, address: p.address }))).sort((a, b) => new Date(a.dueDate) - new Date(b.dueDate)); - const msText = upcomingMs.map(m => ` - ${m.name} @ ${m.address} — due ${m.dueDate} (${m.status})`).join('\n'); + const msText = capList(upcomingMs, 10, m => ` - ${m.name} @ ${m.address} — due ${m.dueDate} (${m.status})`); // My documents / compliance const myDocs = documents.filter(d => d.entityId === user.id || d.entityId === 'con_001'); @@ -330,7 +367,7 @@ INSTRUCTIONS: const pendingTasks = myTasks.filter(t => t.status === 'pending'); const completedTasks = myTasks.filter(t => t.status === 'completed'); - const taskDetails = myTasks.filter(t => t.status !== 'completed').map(t => ` - ${t.name} @ ${t.address} — ${t.status} (due ${t.dueDate})`).join('\n'); + const taskDetails = capList(myTasks.filter(t => t.status !== 'completed'), 10, t => ` - ${t.name} @ ${t.address} — ${t.status} (due ${t.dueDate})`); // Pending payouts (from invoices on projects I'm on) const pendingPay = myProjects.reduce((s, p) => s + (p.invoices || []).filter(i => i.status === 'pending' && (i.payeeTo === myId || i.payeeTo === 'sub_001')).reduce((is, i) => is + i.amount, 0), 0); @@ -562,16 +599,21 @@ const Chatbot = (props) => { // --- Generate deep, role-aware context --- - // 1. Build Dynamic Context - let systemContext = generateRoleContext(user, storeData); + // 1. Build Dynamic Context, then hard-clamp so the context window can + // never be exceeded regardless of how much mock data exists. + let systemContext = clampToTokenBudget(generateRoleContext(user, storeData)); - // 2. Call Groq + // 2. Only re-send the most recent turns so multi-turn chats stay bounded. + const recentMessages = newMessages.slice(-HISTORY_TURN_LIMIT); + logger.info?.(`[AI] system≈${estimateTokens(systemContext)} tok · history=${recentMessages.length} msgs · model=${AI_MODEL}`); + + // 3. Call Groq const chatCompletion = await groq.chat.completions.create({ messages: [ { role: 'system', content: systemContext }, - ...newMessages + ...recentMessages ], - model: 'qwen/qwen3-32b', + model: AI_MODEL, temperature: 0.7, max_tokens: 1024, });