neogenia/neo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP - using deep agent to create complex workflow

Browse Source
This commit is contained in:
Francisco Gaona
2026-01-18 04:45:15 +01:00
parent 20fc90a3fb
commit 8b192ba7f5
4 changed files with 546 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
backend/package-lock.json generated
View File

@@ -11,7 +11,7 @@
"dependencies": {
"@casl/ability": "^6.7.5",
"@fastify/websocket": "^10.0.1",
"@langchain/core": "^1.1.12",
"@langchain/core": "^1.1.15",
"@langchain/langgraph": "^1.0.15",
"@langchain/openai": "^1.2.1",
"@nestjs/bullmq": "^10.1.0",
@@ -29,9 +29,10 @@
"bullmq": "^5.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.1",
"deepagents": "^1.5.0",
"ioredis": "^5.3.2",
"knex": "^3.1.0",
"langchain": "^1.2.7",
"langchain": "^1.2.10",
"mysql2": "^3.15.3",
"objection": "^3.1.5",
"openai": "^6.15.0",
@@ -228,6 +229,26 @@
"tslib": "^2.1.0"
}
},
"node_modules/@anthropic-ai/sdk": {
"version": "0.71.2",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.71.2.tgz",
"integrity": "sha512-TGNDEUuEstk/DKu0/TflXAEt+p+p/WhTlFzEnoosvbaDU2LTjm42igSdlL0VijrKpWejtOKxX0b8A7uc+XiSAQ==",
"license": "MIT",
"dependencies": {
"json-schema-to-ts": "^3.1.1"
},
"bin": {
"anthropic-ai-sdk": "bin/cli"
},
"peerDependencies": {
"zod": "^3.25.0 || ^4.0.0"
},
"peerDependenciesMeta": {
"zod": {
"optional": true
}
}
},
"node_modules/@babel/code-frame": {
"version": "7.27.1",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz",
@@ -689,6 +710,15 @@
"@babel/core": "^7.0.0-0"
}
},
"node_modules/@babel/runtime": {
"version": "7.28.6",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz",
"integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==",
"license": "MIT",
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/template": {
"version": "7.27.2",
"resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz",
@@ -1689,10 +1719,26 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@langchain/anthropic": {
"version": "1.3.10",
"resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.10.tgz",
"integrity": "sha512-VXq5fsEJ4FB5XGrnoG+bfm0I7OlmYLI4jZ6cX9RasyqhGo9wcDyKw1+uEQ1H7Og7jWrTa1bfXCun76wttewJnw==",
"license": "MIT",
"dependencies": {
"@anthropic-ai/sdk": "^0.71.0",
"zod": "^3.25.76 || ^4"
},
"engines": {
"node": ">=20"
},
"peerDependencies": {
"@langchain/core": "1.1.15"
}
},
"node_modules/@langchain/core": {
"version": "1.1.12",
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.12.tgz",
"integrity": "sha512-sHWLvhyLi3fntlg3MEPB89kCjxEX7/+imlIYJcp6uFGCAZfGxVWklqp22HwjT1szorUBYrkO8u0YA554ReKxGQ==",
"version": "1.1.15",
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.15.tgz",
"integrity": "sha512-b8RN5DkWAmDAlMu/UpTZEluYwCLpm63PPWniRKlE8ie3KkkE7IuMQ38pf4kV1iaiI+d99BEQa2vafQHfCujsRA==",
"license": "MIT",
"dependencies": {
"@cfworker/json-schema": "^4.0.2",
@@ -2487,7 +2533,6 @@
"version": "2.1.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
"integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
"dev": true,
"license": "MIT",
"dependencies": {
"@nodelib/fs.stat": "2.0.5",
@@ -2501,7 +2546,6 @@
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
"integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 8"
@@ -2511,7 +2555,6 @@
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
"integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@nodelib/fs.scandir": "2.1.5",
@@ -4029,7 +4072,6 @@
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
"integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
"dev": true,
"license": "MIT",
"dependencies": {
"fill-range": "^7.1.1"
@@ -4754,6 +4796,22 @@
"dev": true,
"license": "MIT"
},
"node_modules/deepagents": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/deepagents/-/deepagents-1.5.0.tgz",
"integrity": "sha512-tjZLOISPMpqfk+k/iE1uIZavXW9j4NrhopUmH5ARqzmk95EEtGDyN++tgnY+tdVOOZTjE2LHjOVV7or58dtx8A==",
"license": "MIT",
"dependencies": {
"@langchain/anthropic": "^1.3.7",
"@langchain/core": "^1.1.12",
"@langchain/langgraph": "^1.0.14",
"fast-glob": "^3.3.3",
"langchain": "^1.2.7",
"micromatch": "^4.0.8",
"yaml": "^2.8.2",
"zod": "^4.3.5"
}
},
"node_modules/deepmerge": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
@@ -5514,7 +5572,6 @@
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
"integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@nodelib/fs.stat": "^2.0.2",
@@ -5740,7 +5797,6 @@
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
"integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
"dev": true,
"license": "MIT",
"dependencies": {
"to-regex-range": "^5.0.1"
@@ -6145,7 +6201,6 @@
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
"integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
"dev": true,
"license": "ISC",
"dependencies": {
"is-glob": "^4.0.1"
@@ -6594,7 +6649,6 @@
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
"integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.10.0"
@@ -6623,7 +6677,6 @@
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
"integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
"dev": true,
"license": "MIT",
"dependencies": {
"is-extglob": "^2.1.1"
@@ -6658,7 +6711,6 @@
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
"integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=0.12.0"
@@ -7609,6 +7661,19 @@
"fast-deep-equal": "^3.1.3"
}
},
"node_modules/json-schema-to-ts": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz",
"integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.18.3",
"ts-algebra": "^2.0.0"
},
"engines": {
"node": ">=16"
}
},
"node_modules/json-schema-traverse": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
@@ -7811,9 +7876,9 @@
}
},
"node_modules/langchain": {
"version": "1.2.7",
"resolved": "https://registry.npmjs.org/langchain/-/langchain-1.2.7.tgz",
"integrity": "sha512-G+3Ftz/08CurJaE7LukQGBf3mCSz7XM8LZeAaFPg391Ru4lT8eLYfG6Fv4ZI0u6EBsPVcOQfaS9ig8nCRmJeqA==",
"version": "1.2.10",
"resolved": "https://registry.npmjs.org/langchain/-/langchain-1.2.10.tgz",
"integrity": "sha512-9uVxOJE/RTECvNutQfOLwH7f6R9mcq0G/IMHwA2eptDA86R/Yz2zWMz4vARVFPxPrdSJ9nJFDPAqRQlRFwdHBw==",
"license": "MIT",
"dependencies": {
"@langchain/langgraph": "^1.0.0",
@@ -7826,7 +7891,7 @@
"node": ">=20"
},
"peerDependencies": {
"@langchain/core": "1.1.12"
"@langchain/core": "1.1.15"
}
},
"node_modules/langchain/node_modules/uuid": {
@@ -8201,7 +8266,6 @@
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">= 8"
@@ -8211,7 +8275,6 @@
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
"integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
"dev": true,
"license": "MIT",
"dependencies": {
"braces": "^3.0.3",
@@ -8225,7 +8288,6 @@
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
"integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8.6"
@@ -9388,7 +9450,6 @@
"version": "1.2.3",
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
"integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
"dev": true,
"funding": [
{
"type": "github",
@@ -9727,7 +9788,6 @@
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
"integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
"dev": true,
"funding": [
{
"type": "github",
@@ -10657,7 +10717,6 @@
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
"integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"is-number": "^7.0.0"
@@ -10709,6 +10768,12 @@
"tree-kill": "cli.js"
}
},
"node_modules/ts-algebra": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz",
"integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==",
"license": "MIT"
},
"node_modules/ts-api-utils": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.4.3.tgz",
@@ -11455,6 +11520,21 @@
"dev": true,
"license": "ISC"
},
"node_modules/yaml": {
"version": "2.8.2",
"resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz",
"integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==",
"license": "ISC",
"bin": {
"yaml": "bin.mjs"
},
"engines": {
"node": ">= 14.6"
},
"funding": {
"url": "https://github.com/sponsors/eemeli"
}
},
"node_modules/yargs": {
"version": "17.7.2",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",
@@ -11508,9 +11588,9 @@
}
},
"node_modules/zod": {
"version": "3.25.76",
"resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz",
"integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==",
"version": "4.3.5",
"resolved": "https://registry.npmjs.org/zod/-/zod-4.3.5.tgz",
"integrity": "sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/colinhacks"

5
backend/package.json
View File

@@ -28,7 +28,7 @@
"dependencies": {
"@casl/ability": "^6.7.5",
"@fastify/websocket": "^10.0.1",
"@langchain/core": "^1.1.12",
"@langchain/core": "^1.1.15",
"@langchain/langgraph": "^1.0.15",
"@langchain/openai": "^1.2.1",
"@nestjs/bullmq": "^10.1.0",
@@ -46,9 +46,10 @@
"bullmq": "^5.1.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.1",
"deepagents": "^1.5.0",
"ioredis": "^5.3.2",
"knex": "^3.1.0",
"langchain": "^1.2.7",
"langchain": "^1.2.10",
"mysql2": "^3.15.3",
"objection": "^3.1.5",
"openai": "^6.15.0",

459
backend/src/ai-assistant/ai-assistant.service.ts
View File

@@ -1,8 +1,9 @@
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import { JsonOutputParser } from '@langchain/core/output_parsers';
import { HumanMessage, SystemMessage } from '@langchain/core/messages';
import { AIMessage, BaseMessage, HumanMessage, SystemMessage } from '@langchain/core/messages';
import { ChatOpenAI } from '@langchain/openai';
import { Annotation, END, START, StateGraph } from '@langchain/langgraph';
import { createDeepAgent } from 'deepagents';
import { ObjectService } from '../object/object.service';
import { PageLayoutService } from '../page-layout/page-layout.service';
import { TenantDatabaseService } from '../tenant/tenant-database.service';
@@ -68,32 +69,275 @@ export class AiAssistantService {
const prior = this.conversationState.get(conversationKey);
const trimmedHistory = Array.isArray(history) ? history.slice(-6) : [];
const initialState: AiAssistantState = {
message: this.combineHistory(trimmedHistory, message),
history: trimmedHistory,
context: context || {},
extractedFields: prior?.fields,
};
// Use Deep Agent as the main coordinator
const result = await this.runDeepAgent(tenantId, userId, message, trimmedHistory, context, prior);
const finalState = await this.runAssistantGraph(tenantId, userId, initialState);
if (finalState.record) {
// Update conversation state based on result
if (result.record) {
this.conversationState.delete(conversationKey);
} else if (finalState.extractedFields && Object.keys(finalState.extractedFields).length > 0) {
} else if ('extractedFields' in result && result.extractedFields && Object.keys(result.extractedFields).length > 0) {
this.conversationState.set(conversationKey, {
fields: finalState.extractedFields,
fields: result.extractedFields,
updatedAt: Date.now(),
});
}
return {
reply: finalState.reply || 'How can I help?',
action: finalState.action,
missingFields: finalState.missingFields,
record: finalState.record,
reply: result.reply || 'How can I help?',
action: result.action,
missingFields: result.missingFields,
record: result.record,
};
}
private async runDeepAgent(
tenantId: string,
userId: string,
message: string,
history: AiAssistantState['history'],
context: AiAssistantState['context'],
prior?: { fields: Record<string, any>; updatedAt: number },
): Promise<AiAssistantReply & { extractedFields?: Record<string, any> }> {
const openAiConfig = await this.getOpenAiConfig(tenantId);
if (!openAiConfig) {
this.logger.warn('No OpenAI config found; using fallback graph execution.');
// Fallback to direct graph execution if no OpenAI config
const initialState: AiAssistantState = {
message: this.combineHistory(history, message),
history: history,
context: context || {},
extractedFields: prior?.fields,
};
const graph = this.buildResolveOrCreateRecordGraph(tenantId, userId);
const result = await graph.invoke(initialState);
return {
reply: result.reply || 'How can I help?',
action: result.action,
missingFields: result.missingFields,
record: result.record,
};
}
// Build the compiled subagent
const compiledSubagent = this.buildResolveOrCreateRecordGraph(tenantId, userId);
// Create Deep Agent with the subagent
const mainModel = new ChatOpenAI({
apiKey: openAiConfig.apiKey,
model: this.normalizeChatModel(openAiConfig.model),
temperature: 0.3,
});
const systemPrompt = this.buildDeepAgentSystemPrompt(context);
const agent = createDeepAgent({
model: mainModel,
systemPrompt,
tools: [],
subagents: [
{
name: 'resolve-or-create-record',
description: [
'ALWAYS use this subagent for ANY operation involving CRM records (create, find, or lookup).',
'',
'This subagent handles:',
'- Looking up existing records by name or other fields',
'- Creating new records with the provided field values',
'- Resolving related records (e.g., finding an Account to link a Contact to)',
'- Validating required fields before creating records',
'',
'IMPORTANT: When invoking this subagent, include in your message:',
'- The exact user request',
'- The object type (Account, Contact, etc.) if known from context',
'- Any previously collected information',
'',
'Example invocations:',
'- "Create Account named Acme Corp" (objectApiName: Account)',
'- "Create Contact John Doe" (objectApiName: Contact)',
'- "Add phone number 555-1234 for Contact John" (objectApiName: ContactDetail)',
].join('\n'),
runnable: compiledSubagent,
},
],
});
// Convert history to messages format
const messages: BaseMessage[] = [];
if (history && history.length > 0) {
for (const entry of history) {
if (entry.role === 'user') {
messages.push(new HumanMessage(entry.text));
} else if (entry.role === 'assistant') {
messages.push(new AIMessage(entry.text));
}
}
}
messages.push(new HumanMessage(message));
// Include context information in the first message if available
let contextInfo = '';
if (context?.objectApiName) {
contextInfo += `\n[System Context: User is working with ${context.objectApiName} object`;
if (context.recordId) {
contextInfo += `, record ID: ${context.recordId}`;
}
contextInfo += ']';
}
if (prior?.fields && Object.keys(prior.fields).length > 0) {
contextInfo += `\n[Previously collected field values: ${JSON.stringify(prior.fields)}]`;
}
if (contextInfo && messages.length > 0) {
const lastMessage = messages[messages.length - 1];
if (lastMessage instanceof HumanMessage) {
messages[messages.length - 1] = new HumanMessage(
lastMessage.content + contextInfo,
);
}
}
try {
console.log('=== DEEP AGENT: Starting invocation ===');
console.log('Messages:', messages.map(m => ({ role: m._getType(), content: m.content })));
const result = await agent.invoke({ messages });
console.log('=== DEEP AGENT: Result received ===');
console.log('Result messages count:', result.messages.length);
// Look for subagent results in the messages
let subagentResult: any = null;
for (let i = result.messages.length - 1; i >= 0; i--) {
const msg = result.messages[i];
console.log(`Message ${i}:`, {
type: msg._getType(),
content: typeof msg.content === 'string' ? msg.content.substring(0, 200) : msg.content,
additional_kwargs: msg.additional_kwargs,
});
// Check if this message has subagent output data
if (msg.additional_kwargs?.action || msg.additional_kwargs?.record) {
subagentResult = msg.additional_kwargs;
console.log('Found subagent result in message additional_kwargs:', subagentResult);
break;
}
}
const lastMsg = result.messages[result.messages.length - 1];
const replyText = typeof lastMsg.content === 'string'
? lastMsg.content
: 'How can I help?';
console.log('Final reply text:', replyText);
// If we found subagent results, use them; otherwise use defaults
if (subagentResult) {
console.log('=== DEEP AGENT: Using subagent result ===');
return {
reply: replyText,
action: subagentResult.action || 'clarify',
missingFields: subagentResult.missingFields || [],
record: subagentResult.record,
extractedFields: subagentResult.extractedFields,
};
}
console.log('=== DEEP AGENT: No subagent result found, using defaults ===');
return {
reply: replyText,
action: 'clarify',
missingFields: [],
record: undefined,
};
} catch (error) {
this.logger.error(`Deep Agent execution failed: ${error.message}`, error.stack);
// Fallback to direct graph execution
const initialState: AiAssistantState = {
message: this.combineHistory(history, message),
history: history,
context: context || {},
extractedFields: prior?.fields,
};
const graph = this.buildResolveOrCreateRecordGraph(tenantId, userId);
const result = await graph.invoke(initialState);
return {
reply: result.reply || 'How can I help?',
action: result.action,
missingFields: result.missingFields,
record: result.record,
};
}
}
private buildDeepAgentSystemPrompt(context?: AiAssistantState['context']): string {
const contextInfo = context?.objectApiName
? ` The user is currently working with the ${context.objectApiName} object.`
: '';
return [
'You are an AI assistant helping users interact with a CRM system through natural conversation.',
'Your role is to understand user requests and coordinate with specialized subagents to fulfill them.',
'',
'Core Responsibilities:',
'- Parse user requests to understand what records they want to create or find',
'- Identify the record type (Account, Contact, ContactDetail, etc.)',
'- Break down complex multi-step requests into manageable tasks',
'- ALWAYS delegate to the "resolve-or-create-record" subagent for ANY record operation',
'',
'Understanding Record Relationships:',
'- When user says "Create X under Y account":',
' 1. X is NOT an Account - it is a Contact or child record',
' 2. Y is the parent Account name',
' 3. You must: (a) First ensure Y account exists, (b) Then create X as a Contact under Y',
'- Accounts are top-level organizations/companies',
'- Contacts are people/entities that belong to Accounts',
'- ContactDetails are phone/email/address records for Contacts or Accounts',
'',
'Multi-Step Example:',
'- User: "Create Chipi under Jeannete Staley account"',
' Step 1: Invoke subagent to find/create Account "Jeannete Staley" (objectApiName: Account)',
' Step 2: Once account exists, invoke subagent to create Contact "Chipi" linked to that Account (objectApiName: Contact)',
'',
'When invoking the subagent:',
'- Include the record type in parentheses: "Find Account Jeannete Staley (objectApiName: Account)"',
'- For child records: "Create Contact Chipi for Account <id> (objectApiName: Contact)"',
'- Pass the user\'s full request with context',
'',
'DO NOT try to create records yourself - ALWAYS use the subagent for:',
' * Finding existing records',
' * Creating new records',
' * Resolving related records',
'',
'Important Patterns:',
'- When a user says "Create X under/for Y", this means:',
' 1. You need to first find or verify record Y exists',
' 2. Then create record X with a reference to Y',
' Example: "Create Max under John Doe Account" means find the Account named "John Doe",',
' then create a record named "Max" that references that Account.',
'',
'- For polymorphic relationships (records that can reference multiple types):',
' * ContactDetail records can reference either Account or Contact',
' * Infer the correct type from context clues in the user\'s message',
'',
'CRITICAL - Checking Results:',
'- After the subagent responds, CHECK if it actually completed the action',
'- Look for phrases like "I still need" or "missing fields" which indicate incomplete work',
'- If the subagent asks for more information, relay that to the user - DO NOT claim success',
'- Only report success if the subagent explicitly confirms record creation',
'- DO NOT fabricate success messages if the subagent indicates it needs more data',
'',
'Response Style:',
'- Be conversational and helpful',
'- Confirm what you\'re doing: "I\'ll create Max as a Contact under the John Doe Account"',
'- Ask for clarification when the request is ambiguous',
'- Report success clearly ONLY when confirmed: "Created Contact Max under John Doe Account"',
'- If the subagent needs more info, ask the user for that specific information',
'',
contextInfo,
].join('\n');
}
async searchRecords(
tenantId: string,
userId: string,
@@ -196,13 +440,13 @@ export class AiAssistantService {
};
}
private async runAssistantGraph(
private buildResolveOrCreateRecordGraph(
tenantId: string,
userId: string,
state: AiAssistantState,
): Promise<AiAssistantState> {
) {
const AssistantState = Annotation.Root({
message: Annotation<string>(),
messages: Annotation<BaseMessage[]>(),
history: Annotation<AiAssistantState['history']>(),
context: Annotation<AiAssistantState['context']>(),
objectDefinition: Annotation<any>(),
@@ -215,33 +459,174 @@ export class AiAssistantService {
reply: Annotation<string>(),
});
// Entry node to transform Deep Agent messages into our state format
const transformInput = async (state: any): Promise<AiAssistantState> => {
console.log('=== SUBAGENT: Transform Input ===');
console.log('Received state keys:', Object.keys(state));
console.log('Has messages:', state.messages ? state.messages.length : 'no');
// If invoked by Deep Agent, state will have messages array
if (state.messages && Array.isArray(state.messages)) {
const lastMessage = state.messages[state.messages.length - 1];
const messageText = typeof lastMessage.content === 'string'
? lastMessage.content
: '';
console.log('Extracted message from Deep Agent:', messageText);
// Try to extract context from message (Deep Agent should include it)
const contextMatch = messageText.match(/\[System Context: User is working with (\w+) object(?:, record ID: ([^\]]+))?\]/);
const priorFieldsMatch = messageText.match(/\[Previously collected field values: ({[^\]]+})\]/);
let extractedContext: AiAssistantState['context'] = {};
let extractedFields: Record<string, any> | undefined;
if (contextMatch) {
extractedContext.objectApiName = contextMatch[1];
if (contextMatch[2]) {
extractedContext.recordId = contextMatch[2];
}
console.log('Extracted context from annotations:', extractedContext);
} else {
// Fallback: Try to infer object type from the message itself
console.log('No context annotation found, attempting to infer from message...');
// Check for explicit objectApiName mentions in parentheses
const explicitMatch = messageText.match(/\(objectApiName:\s*(\w+)\)/);
if (explicitMatch) {
extractedContext.objectApiName = explicitMatch[1];
console.log('Found explicit objectApiName:', extractedContext.objectApiName);
} else {
// Try to infer from keywords and patterns
const lowerMsg = messageText.toLowerCase();
// Pattern: "Create X under/for Y account" - X is NOT an account, it's a child record
const underAccountMatch = messageText.match(/create\s+([\w\s]+?)\s+(?:under|for)\s+([\w\s]+?)\s+account/i);
if (underAccountMatch) {
// The thing being created is likely a Contact or ContactDetail
console.log('Detected "under account" pattern - inferring child record type');
// Check if it's a contact detail (phone/email)
if (lowerMsg.includes('phone') || lowerMsg.includes('email') || lowerMsg.includes('address')) {
extractedContext.objectApiName = 'ContactDetail';
} else {
// Default to Contact for things created under accounts
extractedContext.objectApiName = 'Contact';
}
console.log('Inferred child object type:', extractedContext.objectApiName);
} else if (lowerMsg.includes('account') && (lowerMsg.includes('create') || lowerMsg.includes('add'))) {
extractedContext.objectApiName = 'Account';
} else if (lowerMsg.includes('contact') && !lowerMsg.includes('contact detail')) {
extractedContext.objectApiName = 'Contact';
} else if (lowerMsg.includes('contact detail') || lowerMsg.includes('contactdetail')) {
extractedContext.objectApiName = 'ContactDetail';
} else if (lowerMsg.includes('phone') || lowerMsg.includes('email')) {
extractedContext.objectApiName = 'ContactDetail';
}
if (extractedContext.objectApiName) {
console.log('Inferred objectApiName from keywords:', extractedContext.objectApiName);
} else {
console.warn('Could not infer objectApiName from message!');
}
}
}
if (priorFieldsMatch) {
try {
extractedFields = JSON.parse(priorFieldsMatch[1]);
console.log('Extracted prior fields:', extractedFields);
} catch (e) {
console.warn('Failed to parse prior fields');
}
}
// Clean the message text from system context annotations
const cleanMessage = messageText
.replace(/\[System Context:[^\]]+\]/g, '')
.replace(/\[Previously collected field values:[^\]]+\]/g, '')
.replace(/\(objectApiName:\s*\w+\)/g, '')
.trim();
console.log('Final transformed state:', {
message: cleanMessage,
context: extractedContext,
hasExtractedFields: !!extractedFields,
});
return {
message: cleanMessage,
messages: state.messages,
history: [],
context: extractedContext,
extractedFields,
} as AiAssistantState;
}
// If invoked directly (fallback or testing), use the state as-is
console.log('Using direct state (not from Deep Agent)');
return state as AiAssistantState;
};
const workflow = new StateGraph(AssistantState)
.addNode('transformInput', transformInput)
.addNode('loadContext', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Load Context ===');
return this.loadContext(tenantId, current);
})
.addNode('extractFields', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Extract Fields ===');
return this.extractFields(tenantId, current);
})
.addNode('decideNext', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Decide Next ===');
return this.decideNextStep(current);
})
.addNode('createRecord', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Create Record ===');
return this.createRecord(tenantId, userId, current);
})
.addNode('respondMissing', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Respond Missing ===');
return this.respondWithMissingFields(current);
})
.addEdge(START, 'loadContext')
.addNode('formatOutput', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Format Output ===');
console.log('Final state before output:', {
action: current.action,
record: current.record,
reply: current.reply,
missingFields: current.missingFields,
});
// Format the output for Deep Agent to understand
const outputMessage = new AIMessage({
content: current.reply || 'Completed.',
additional_kwargs: {
action: current.action,
record: current.record,
missingFields: current.missingFields,
extractedFields: current.extractedFields,
},
});
return {
...current,
messages: [...(current.messages || []), outputMessage],
} as AiAssistantState;
})
.addEdge(START, 'transformInput')
.addEdge('transformInput', 'loadContext')
.addEdge('loadContext', 'extractFields')
.addEdge('extractFields', 'decideNext')
.addConditionalEdges('decideNext', (current: AiAssistantState) => {
return current.action === 'create_record' ? 'createRecord' : 'respondMissing';
})
.addEdge('createRecord', END)
.addEdge('respondMissing', END);
.addEdge('createRecord', 'formatOutput')
.addEdge('respondMissing', 'formatOutput')
.addEdge('formatOutput', END);
const graph = workflow.compile();
return graph.invoke(state);
return workflow.compile();
}
private async loadContext(
@@ -451,6 +836,8 @@ export class AiAssistantService {
userId,
);
console.log('record',record);
const nameValue = enrichedState.extractedFields.name || record?.name || record?.id;
const label = enrichedState.objectDefinition.label || enrichedState.objectDefinition.apiName;
@@ -619,6 +1006,26 @@ export class AiAssistantService {
const phoneField = fieldDefinitions.find((field) =>
field.apiName.toLowerCase().includes('phone') || field.label.toLowerCase().includes('phone'),
);
// Check for Account lookup field (for Contacts)
const accountField = fieldDefinitions.find((field) =>
field.apiName === 'accountId' || field.apiName.toLowerCase().includes('account'),
);
// Pattern: "Create X under/for Y account" - extract name and account reference
const underAccountMatch = message.match(/create\s+([^\s]+(?:\s+[^\s]+)?)\s+(?:under|for)\s+(.+?)\s+account/i);
if (underAccountMatch && nameField) {
const recordName = underAccountMatch[1].trim();
const accountName = underAccountMatch[2].trim();
extracted[nameField.apiName] = recordName;
if (accountField) {
// Store the account name for lookup
extracted[accountField.apiName] = accountName;
console.log('Extracted hierarchical pattern:', { name: recordName, account: accountName });
}
}
// Generic pattern matching for any field: "label: value" or "set label to value"
for (const field of fieldDefinitions) {
@@ -628,7 +1035,7 @@ export class AiAssistantService {
}
}
if (nameField) {
if (nameField && !extracted[nameField.apiName]) {
const nameMatch = message.match(/add\s+([^\n]+?)(?:\s+with\s+|$)/i);
if (nameMatch?.[1]) {
extracted[nameField.apiName] = nameMatch[1].trim();
@@ -646,6 +1053,8 @@ export class AiAssistantService {
extracted[nameField.apiName] = message.replace(/^add\s+/i, '').trim();
}
console.log('Heuristic extraction result:', extracted);
return extracted;
}

1
backend/src/ai-assistant/ai-assistant.types.ts
View File

@@ -19,6 +19,7 @@ export interface AiAssistantReply {
export interface AiAssistantState {
message: string;
messages?: any[]; // BaseMessage[] from langchain - used when invoked by Deep Agent
history?: AiChatMessage[];
context: AiChatContext;
objectDefinition?: any;