WIP - using deep agent to create complex workflow

This commit is contained in:
Francisco Gaona
2026-01-18 04:45:15 +01:00
parent 20fc90a3fb
commit 8b192ba7f5
4 changed files with 546 additions and 55 deletions

View File

@@ -1,8 +1,9 @@
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import { JsonOutputParser } from '@langchain/core/output_parsers';
import { HumanMessage, SystemMessage } from '@langchain/core/messages';
import { AIMessage, BaseMessage, HumanMessage, SystemMessage } from '@langchain/core/messages';
import { ChatOpenAI } from '@langchain/openai';
import { Annotation, END, START, StateGraph } from '@langchain/langgraph';
import { createDeepAgent } from 'deepagents';
import { ObjectService } from '../object/object.service';
import { PageLayoutService } from '../page-layout/page-layout.service';
import { TenantDatabaseService } from '../tenant/tenant-database.service';
@@ -68,32 +69,275 @@ export class AiAssistantService {
const prior = this.conversationState.get(conversationKey);
const trimmedHistory = Array.isArray(history) ? history.slice(-6) : [];
const initialState: AiAssistantState = {
message: this.combineHistory(trimmedHistory, message),
history: trimmedHistory,
context: context || {},
extractedFields: prior?.fields,
};
// Use Deep Agent as the main coordinator
const result = await this.runDeepAgent(tenantId, userId, message, trimmedHistory, context, prior);
const finalState = await this.runAssistantGraph(tenantId, userId, initialState);
if (finalState.record) {
// Update conversation state based on result
if (result.record) {
this.conversationState.delete(conversationKey);
} else if (finalState.extractedFields && Object.keys(finalState.extractedFields).length > 0) {
} else if ('extractedFields' in result && result.extractedFields && Object.keys(result.extractedFields).length > 0) {
this.conversationState.set(conversationKey, {
fields: finalState.extractedFields,
fields: result.extractedFields,
updatedAt: Date.now(),
});
}
return {
reply: finalState.reply || 'How can I help?',
action: finalState.action,
missingFields: finalState.missingFields,
record: finalState.record,
reply: result.reply || 'How can I help?',
action: result.action,
missingFields: result.missingFields,
record: result.record,
};
}
private async runDeepAgent(
tenantId: string,
userId: string,
message: string,
history: AiAssistantState['history'],
context: AiAssistantState['context'],
prior?: { fields: Record<string, any>; updatedAt: number },
): Promise<AiAssistantReply & { extractedFields?: Record<string, any> }> {
const openAiConfig = await this.getOpenAiConfig(tenantId);
if (!openAiConfig) {
this.logger.warn('No OpenAI config found; using fallback graph execution.');
// Fallback to direct graph execution if no OpenAI config
const initialState: AiAssistantState = {
message: this.combineHistory(history, message),
history: history,
context: context || {},
extractedFields: prior?.fields,
};
const graph = this.buildResolveOrCreateRecordGraph(tenantId, userId);
const result = await graph.invoke(initialState);
return {
reply: result.reply || 'How can I help?',
action: result.action,
missingFields: result.missingFields,
record: result.record,
};
}
// Build the compiled subagent
const compiledSubagent = this.buildResolveOrCreateRecordGraph(tenantId, userId);
// Create Deep Agent with the subagent
const mainModel = new ChatOpenAI({
apiKey: openAiConfig.apiKey,
model: this.normalizeChatModel(openAiConfig.model),
temperature: 0.3,
});
const systemPrompt = this.buildDeepAgentSystemPrompt(context);
const agent = createDeepAgent({
model: mainModel,
systemPrompt,
tools: [],
subagents: [
{
name: 'resolve-or-create-record',
description: [
'ALWAYS use this subagent for ANY operation involving CRM records (create, find, or lookup).',
'',
'This subagent handles:',
'- Looking up existing records by name or other fields',
'- Creating new records with the provided field values',
'- Resolving related records (e.g., finding an Account to link a Contact to)',
'- Validating required fields before creating records',
'',
'IMPORTANT: When invoking this subagent, include in your message:',
'- The exact user request',
'- The object type (Account, Contact, etc.) if known from context',
'- Any previously collected information',
'',
'Example invocations:',
'- "Create Account named Acme Corp" (objectApiName: Account)',
'- "Create Contact John Doe" (objectApiName: Contact)',
'- "Add phone number 555-1234 for Contact John" (objectApiName: ContactDetail)',
].join('\n'),
runnable: compiledSubagent,
},
],
});
// Convert history to messages format
const messages: BaseMessage[] = [];
if (history && history.length > 0) {
for (const entry of history) {
if (entry.role === 'user') {
messages.push(new HumanMessage(entry.text));
} else if (entry.role === 'assistant') {
messages.push(new AIMessage(entry.text));
}
}
}
messages.push(new HumanMessage(message));
// Include context information in the first message if available
let contextInfo = '';
if (context?.objectApiName) {
contextInfo += `\n[System Context: User is working with ${context.objectApiName} object`;
if (context.recordId) {
contextInfo += `, record ID: ${context.recordId}`;
}
contextInfo += ']';
}
if (prior?.fields && Object.keys(prior.fields).length > 0) {
contextInfo += `\n[Previously collected field values: ${JSON.stringify(prior.fields)}]`;
}
if (contextInfo && messages.length > 0) {
const lastMessage = messages[messages.length - 1];
if (lastMessage instanceof HumanMessage) {
messages[messages.length - 1] = new HumanMessage(
lastMessage.content + contextInfo,
);
}
}
try {
console.log('=== DEEP AGENT: Starting invocation ===');
console.log('Messages:', messages.map(m => ({ role: m._getType(), content: m.content })));
const result = await agent.invoke({ messages });
console.log('=== DEEP AGENT: Result received ===');
console.log('Result messages count:', result.messages.length);
// Look for subagent results in the messages
let subagentResult: any = null;
for (let i = result.messages.length - 1; i >= 0; i--) {
const msg = result.messages[i];
console.log(`Message ${i}:`, {
type: msg._getType(),
content: typeof msg.content === 'string' ? msg.content.substring(0, 200) : msg.content,
additional_kwargs: msg.additional_kwargs,
});
// Check if this message has subagent output data
if (msg.additional_kwargs?.action || msg.additional_kwargs?.record) {
subagentResult = msg.additional_kwargs;
console.log('Found subagent result in message additional_kwargs:', subagentResult);
break;
}
}
const lastMsg = result.messages[result.messages.length - 1];
const replyText = typeof lastMsg.content === 'string'
? lastMsg.content
: 'How can I help?';
console.log('Final reply text:', replyText);
// If we found subagent results, use them; otherwise use defaults
if (subagentResult) {
console.log('=== DEEP AGENT: Using subagent result ===');
return {
reply: replyText,
action: subagentResult.action || 'clarify',
missingFields: subagentResult.missingFields || [],
record: subagentResult.record,
extractedFields: subagentResult.extractedFields,
};
}
console.log('=== DEEP AGENT: No subagent result found, using defaults ===');
return {
reply: replyText,
action: 'clarify',
missingFields: [],
record: undefined,
};
} catch (error) {
this.logger.error(`Deep Agent execution failed: ${error.message}`, error.stack);
// Fallback to direct graph execution
const initialState: AiAssistantState = {
message: this.combineHistory(history, message),
history: history,
context: context || {},
extractedFields: prior?.fields,
};
const graph = this.buildResolveOrCreateRecordGraph(tenantId, userId);
const result = await graph.invoke(initialState);
return {
reply: result.reply || 'How can I help?',
action: result.action,
missingFields: result.missingFields,
record: result.record,
};
}
}
private buildDeepAgentSystemPrompt(context?: AiAssistantState['context']): string {
const contextInfo = context?.objectApiName
? ` The user is currently working with the ${context.objectApiName} object.`
: '';
return [
'You are an AI assistant helping users interact with a CRM system through natural conversation.',
'Your role is to understand user requests and coordinate with specialized subagents to fulfill them.',
'',
'Core Responsibilities:',
'- Parse user requests to understand what records they want to create or find',
'- Identify the record type (Account, Contact, ContactDetail, etc.)',
'- Break down complex multi-step requests into manageable tasks',
'- ALWAYS delegate to the "resolve-or-create-record" subagent for ANY record operation',
'',
'Understanding Record Relationships:',
'- When user says "Create X under Y account":',
' 1. X is NOT an Account - it is a Contact or child record',
' 2. Y is the parent Account name',
' 3. You must: (a) First ensure Y account exists, (b) Then create X as a Contact under Y',
'- Accounts are top-level organizations/companies',
'- Contacts are people/entities that belong to Accounts',
'- ContactDetails are phone/email/address records for Contacts or Accounts',
'',
'Multi-Step Example:',
'- User: "Create Chipi under Jeannete Staley account"',
' Step 1: Invoke subagent to find/create Account "Jeannete Staley" (objectApiName: Account)',
' Step 2: Once account exists, invoke subagent to create Contact "Chipi" linked to that Account (objectApiName: Contact)',
'',
'When invoking the subagent:',
'- Include the record type in parentheses: "Find Account Jeannete Staley (objectApiName: Account)"',
'- For child records: "Create Contact Chipi for Account <id> (objectApiName: Contact)"',
'- Pass the user\'s full request with context',
'',
'DO NOT try to create records yourself - ALWAYS use the subagent for:',
' * Finding existing records',
' * Creating new records',
' * Resolving related records',
'',
'Important Patterns:',
'- When a user says "Create X under/for Y", this means:',
' 1. You need to first find or verify record Y exists',
' 2. Then create record X with a reference to Y',
' Example: "Create Max under John Doe Account" means find the Account named "John Doe",',
' then create a record named "Max" that references that Account.',
'',
'- For polymorphic relationships (records that can reference multiple types):',
' * ContactDetail records can reference either Account or Contact',
' * Infer the correct type from context clues in the user\'s message',
'',
'CRITICAL - Checking Results:',
'- After the subagent responds, CHECK if it actually completed the action',
'- Look for phrases like "I still need" or "missing fields" which indicate incomplete work',
'- If the subagent asks for more information, relay that to the user - DO NOT claim success',
'- Only report success if the subagent explicitly confirms record creation',
'- DO NOT fabricate success messages if the subagent indicates it needs more data',
'',
'Response Style:',
'- Be conversational and helpful',
'- Confirm what you\'re doing: "I\'ll create Max as a Contact under the John Doe Account"',
'- Ask for clarification when the request is ambiguous',
'- Report success clearly ONLY when confirmed: "Created Contact Max under John Doe Account"',
'- If the subagent needs more info, ask the user for that specific information',
'',
contextInfo,
].join('\n');
}
async searchRecords(
tenantId: string,
userId: string,
@@ -196,13 +440,13 @@ export class AiAssistantService {
};
}
private async runAssistantGraph(
private buildResolveOrCreateRecordGraph(
tenantId: string,
userId: string,
state: AiAssistantState,
): Promise<AiAssistantState> {
) {
const AssistantState = Annotation.Root({
message: Annotation<string>(),
messages: Annotation<BaseMessage[]>(),
history: Annotation<AiAssistantState['history']>(),
context: Annotation<AiAssistantState['context']>(),
objectDefinition: Annotation<any>(),
@@ -215,33 +459,174 @@ export class AiAssistantService {
reply: Annotation<string>(),
});
// Entry node to transform Deep Agent messages into our state format
const transformInput = async (state: any): Promise<AiAssistantState> => {
console.log('=== SUBAGENT: Transform Input ===');
console.log('Received state keys:', Object.keys(state));
console.log('Has messages:', state.messages ? state.messages.length : 'no');
// If invoked by Deep Agent, state will have messages array
if (state.messages && Array.isArray(state.messages)) {
const lastMessage = state.messages[state.messages.length - 1];
const messageText = typeof lastMessage.content === 'string'
? lastMessage.content
: '';
console.log('Extracted message from Deep Agent:', messageText);
// Try to extract context from message (Deep Agent should include it)
const contextMatch = messageText.match(/\[System Context: User is working with (\w+) object(?:, record ID: ([^\]]+))?\]/);
const priorFieldsMatch = messageText.match(/\[Previously collected field values: ({[^\]]+})\]/);
let extractedContext: AiAssistantState['context'] = {};
let extractedFields: Record<string, any> | undefined;
if (contextMatch) {
extractedContext.objectApiName = contextMatch[1];
if (contextMatch[2]) {
extractedContext.recordId = contextMatch[2];
}
console.log('Extracted context from annotations:', extractedContext);
} else {
// Fallback: Try to infer object type from the message itself
console.log('No context annotation found, attempting to infer from message...');
// Check for explicit objectApiName mentions in parentheses
const explicitMatch = messageText.match(/\(objectApiName:\s*(\w+)\)/);
if (explicitMatch) {
extractedContext.objectApiName = explicitMatch[1];
console.log('Found explicit objectApiName:', extractedContext.objectApiName);
} else {
// Try to infer from keywords and patterns
const lowerMsg = messageText.toLowerCase();
// Pattern: "Create X under/for Y account" - X is NOT an account, it's a child record
const underAccountMatch = messageText.match(/create\s+([\w\s]+?)\s+(?:under|for)\s+([\w\s]+?)\s+account/i);
if (underAccountMatch) {
// The thing being created is likely a Contact or ContactDetail
console.log('Detected "under account" pattern - inferring child record type');
// Check if it's a contact detail (phone/email)
if (lowerMsg.includes('phone') || lowerMsg.includes('email') || lowerMsg.includes('address')) {
extractedContext.objectApiName = 'ContactDetail';
} else {
// Default to Contact for things created under accounts
extractedContext.objectApiName = 'Contact';
}
console.log('Inferred child object type:', extractedContext.objectApiName);
} else if (lowerMsg.includes('account') && (lowerMsg.includes('create') || lowerMsg.includes('add'))) {
extractedContext.objectApiName = 'Account';
} else if (lowerMsg.includes('contact') && !lowerMsg.includes('contact detail')) {
extractedContext.objectApiName = 'Contact';
} else if (lowerMsg.includes('contact detail') || lowerMsg.includes('contactdetail')) {
extractedContext.objectApiName = 'ContactDetail';
} else if (lowerMsg.includes('phone') || lowerMsg.includes('email')) {
extractedContext.objectApiName = 'ContactDetail';
}
if (extractedContext.objectApiName) {
console.log('Inferred objectApiName from keywords:', extractedContext.objectApiName);
} else {
console.warn('Could not infer objectApiName from message!');
}
}
}
if (priorFieldsMatch) {
try {
extractedFields = JSON.parse(priorFieldsMatch[1]);
console.log('Extracted prior fields:', extractedFields);
} catch (e) {
console.warn('Failed to parse prior fields');
}
}
// Clean the message text from system context annotations
const cleanMessage = messageText
.replace(/\[System Context:[^\]]+\]/g, '')
.replace(/\[Previously collected field values:[^\]]+\]/g, '')
.replace(/\(objectApiName:\s*\w+\)/g, '')
.trim();
console.log('Final transformed state:', {
message: cleanMessage,
context: extractedContext,
hasExtractedFields: !!extractedFields,
});
return {
message: cleanMessage,
messages: state.messages,
history: [],
context: extractedContext,
extractedFields,
} as AiAssistantState;
}
// If invoked directly (fallback or testing), use the state as-is
console.log('Using direct state (not from Deep Agent)');
return state as AiAssistantState;
};
const workflow = new StateGraph(AssistantState)
.addNode('transformInput', transformInput)
.addNode('loadContext', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Load Context ===');
return this.loadContext(tenantId, current);
})
.addNode('extractFields', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Extract Fields ===');
return this.extractFields(tenantId, current);
})
.addNode('decideNext', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Decide Next ===');
return this.decideNextStep(current);
})
.addNode('createRecord', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Create Record ===');
return this.createRecord(tenantId, userId, current);
})
.addNode('respondMissing', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Respond Missing ===');
return this.respondWithMissingFields(current);
})
.addEdge(START, 'loadContext')
.addNode('formatOutput', async (current: AiAssistantState) => {
console.log('=== SUBAGENT: Format Output ===');
console.log('Final state before output:', {
action: current.action,
record: current.record,
reply: current.reply,
missingFields: current.missingFields,
});
// Format the output for Deep Agent to understand
const outputMessage = new AIMessage({
content: current.reply || 'Completed.',
additional_kwargs: {
action: current.action,
record: current.record,
missingFields: current.missingFields,
extractedFields: current.extractedFields,
},
});
return {
...current,
messages: [...(current.messages || []), outputMessage],
} as AiAssistantState;
})
.addEdge(START, 'transformInput')
.addEdge('transformInput', 'loadContext')
.addEdge('loadContext', 'extractFields')
.addEdge('extractFields', 'decideNext')
.addConditionalEdges('decideNext', (current: AiAssistantState) => {
return current.action === 'create_record' ? 'createRecord' : 'respondMissing';
})
.addEdge('createRecord', END)
.addEdge('respondMissing', END);
.addEdge('createRecord', 'formatOutput')
.addEdge('respondMissing', 'formatOutput')
.addEdge('formatOutput', END);
const graph = workflow.compile();
return graph.invoke(state);
return workflow.compile();
}
private async loadContext(
@@ -451,6 +836,8 @@ export class AiAssistantService {
userId,
);
console.log('record',record);
const nameValue = enrichedState.extractedFields.name || record?.name || record?.id;
const label = enrichedState.objectDefinition.label || enrichedState.objectDefinition.apiName;
@@ -619,6 +1006,26 @@ export class AiAssistantService {
const phoneField = fieldDefinitions.find((field) =>
field.apiName.toLowerCase().includes('phone') || field.label.toLowerCase().includes('phone'),
);
// Check for Account lookup field (for Contacts)
const accountField = fieldDefinitions.find((field) =>
field.apiName === 'accountId' || field.apiName.toLowerCase().includes('account'),
);
// Pattern: "Create X under/for Y account" - extract name and account reference
const underAccountMatch = message.match(/create\s+([^\s]+(?:\s+[^\s]+)?)\s+(?:under|for)\s+(.+?)\s+account/i);
if (underAccountMatch && nameField) {
const recordName = underAccountMatch[1].trim();
const accountName = underAccountMatch[2].trim();
extracted[nameField.apiName] = recordName;
if (accountField) {
// Store the account name for lookup
extracted[accountField.apiName] = accountName;
console.log('Extracted hierarchical pattern:', { name: recordName, account: accountName });
}
}
// Generic pattern matching for any field: "label: value" or "set label to value"
for (const field of fieldDefinitions) {
@@ -628,7 +1035,7 @@ export class AiAssistantService {
}
}
if (nameField) {
if (nameField && !extracted[nameField.apiName]) {
const nameMatch = message.match(/add\s+([^\n]+?)(?:\s+with\s+|$)/i);
if (nameMatch?.[1]) {
extracted[nameField.apiName] = nameMatch[1].trim();
@@ -646,6 +1053,8 @@ export class AiAssistantService {
extracted[nameField.apiName] = message.replace(/^add\s+/i, '').trim();
}
console.log('Heuristic extraction result:', extracted);
return extracted;
}