feat: Improve captain conversation handling (#12599)

Co-authored-by: Pranav <pranavrajs@gmail.com>

app/javascript/dashboard/featureFlags.js

@@ -49,6 +49,5 @@ export const PREMIUM_FEATURES = [
   FEATURE_FLAGS.CUSTOM_ROLES,
   FEATURE_FLAGS.AUDIT_LOGS,
   FEATURE_FLAGS.HELP_CENTER,
-  FEATURE_FLAGS.CAPTAIN_V2,
   FEATURE_FLAGS.SAML,
 ];

config/initializers/ai_agents.rb

@@ -15,6 +15,7 @@ Rails.application.config.after_initialize do
         config.openai_api_base = api_base
       end
       config.default_model = model
+      config.max_turns = 30
       config.debug = false
     end
   end

enterprise/app/jobs/captain/conversation/response_builder_job.rb

@@ -49,10 +49,15 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
       .where(message_type: [:incoming, :outgoing])
       .where(private: false)
       .map do |message|
-      {
+      message_hash = {
         content: prepare_multimodal_message_content(message),
         role: determine_role(message)
       }
+
+      # Include agent_name if present in additional_attributes
+      message_hash[:agent_name] = message.additional_attributes['agent_name'] if message.additional_attributes&.dig('agent_name').present?
+
+      message_hash
     end
   end
 
@@ -79,25 +84,31 @@ class Captain::Conversation::ResponseBuilderJob < ApplicationJob
   end
 
   def create_handoff_message
-    create_outgoing_message(@assistant.config['handoff_message'].presence || I18n.t('conversations.captain.handoff'))
+    create_outgoing_message(
+      @assistant.config['handoff_message'].presence || I18n.t('conversations.captain.handoff')
+    )
   end
 
   def create_messages
     validate_message_content!(@response['response'])
-    create_outgoing_message(@response['response'])
+    create_outgoing_message(@response['response'], agent_name: @response['agent_name'])
   end
 
   def validate_message_content!(content)
     raise ArgumentError, 'Message content cannot be blank' if content.blank?
   end
 
-  def create_outgoing_message(message_content)
+  def create_outgoing_message(message_content, agent_name: nil)
+    additional_attrs = {}
+    additional_attrs[:agent_name] = agent_name if agent_name.present?
+
     @conversation.messages.create!(
       message_type: :outgoing,
       account_id: account.id,
       inbox_id: inbox.id,
       sender: @assistant,
-      content: message_content
+      content: message_content,
+      additional_attributes: additional_attrs
     )
   end
 

enterprise/app/models/captain/assistant.rb

@@ -105,6 +105,7 @@ class Captain::Assistant < ApplicationRecord
       product_name: config['product_name'] || 'this product',
       scenarios: scenarios.enabled.map do |scenario|
         {
+          title: scenario.title,
           key: scenario.title.parameterize.underscore,
           description: scenario.description
         }

enterprise/app/models/captain/scenario.rb

@@ -38,7 +38,7 @@ class Captain::Scenario < ApplicationRecord
 
   scope :enabled, -> { where(enabled: true) }
 
-  delegate :temperature, :feature_faq, :feature_memory, :product_name, to: :assistant
+  delegate :temperature, :feature_faq, :feature_memory, :product_name, :response_guidelines, :guardrails, to: :assistant
 
   before_save :resolve_tool_references
 
@@ -46,7 +46,10 @@ class Captain::Scenario < ApplicationRecord
     {
       title: title,
       instructions: resolved_instructions,
-      tools: resolved_tools
+      tools: resolved_tools,
+      assistant_name: assistant.name.downcase.gsub(/\s+/, '_'),
+      response_guidelines: response_guidelines || [],
+      guardrails: guardrails || []
     }
   end
 
@@ -61,9 +64,7 @@ class Captain::Scenario < ApplicationRecord
   end
 
   def resolved_instructions
-    instruction.gsub(TOOL_REFERENCE_REGEX) do |match|
+    instruction.gsub(TOOL_REFERENCE_REGEX, '`\1` tool')
-      "#{match} tool "
-    end
   end
 
   def resolved_tools

enterprise/app/models/concerns/toolable.rb

@@ -70,7 +70,7 @@ module Concerns::Toolable
     return raw_response_body if response_template.blank?
 
     response_data = parse_response_body(raw_response_body)
-    render_template(response_template, { 'response' => response_data })
+    render_template(response_template, { 'response' => response_data, 'r' => response_data })
   end
 
   private

enterprise/app/services/captain/assistant/agent_runner_service.rb

@@ -74,7 +74,12 @@ class Captain::Assistant::AgentRunnerService
   # Response formatting methods
   def process_agent_result(result)
     Rails.logger.info "[Captain V2] Agent result: #{result.inspect}"
-    format_response(result.output)
+    response = format_response(result.output)
+
+    # Extract agent name from context
+    response['agent_name'] = result.context&.dig(:current_agent)
+
+    response
   end
 
   def format_response(output)

enterprise/lib/captain/prompts/assistant.liquid

@@ -2,12 +2,13 @@
 You are part of Captain, a multi-agent AI system designed for seamless agent coordination and task execution. You can transfer conversations to specialized agents using handoff functions (e.g., `handoff_to_[agent_name]`). These transfers happen in the background - never mention or draw attention to them in your responses.
 
 # Your Identity
-You are {{name}}, a helpful and knowledgeable assistant. Your role is to provide accurate information, assist with tasks, and ensure users get the help they need.
+You are {{name}}, a helpful and knowledgeable assistant. Your role is to primarily act as a orchestrator handling multiple scenarios by using handoff tools. Your job also involves providing accurate information, assisting with tasks, and ensuring the customer get the help they need.
 
 {{ description }}
 
-Don't digress away from your instructions, and use all the available tools at your disposal for solving customer issues. If you are to state something factual about {{product_name}} ensure you source that information from the FAQs only. Use the faq_lookup tool for this.
+Don't digress away from your instructions, and use all the available tools at your disposal for solving customer issues. If you are to state something factual about {{product_name}} ensure you source that information from the FAQs only. Use the `captain--tools--faq_lookup` tool for this.
 
+{% if conversation || contact -%}
 # Current Context
 
 Here's the metadata we have about the current conversation and the contact associated with it:
@@ -19,12 +20,16 @@ Here's the metadata we have about the current conversation and the contact assoc
 {% if contact -%}
 {% render 'contact' %}
 {% endif -%}
+{% endif -%}
 
 {% if response_guidelines.size > 0 -%}
 # Response Guidelines
 Your responses should follow these guidelines:
 {% for guideline in response_guidelines -%}
 - {{ guideline }}
+- Be conversational but professional
+- Provide actionable information
+- Include relevant details from tool responses
 {% endfor %}
 {% endif -%}
 
@@ -45,30 +50,26 @@ First, understand what the user is asking:
 - **Complexity**: Can you handle it or does it need specialized expertise?
 
 ## 2. Check for Specialized Scenarios First
-Before using any tools, check if the request matches any of these scenarios. If unclear, ask clarifying questions to determine if a scenario applies:
+
+Before using any tools, check if the request matches any of these scenarios. If it seems like a particular scenario matches, use the specific handoff tool to transfer the conversation to the specific agent. The following are the scenario agents that are available to you.
 
 {% for scenario in scenarios -%}
-### handoff_to_{{ scenario.key }}
+- {{ scenario.title }}: {{ scenario.description }}, use the `handoff_to_{{ scenario.key }}` tool to transfer the conversation to the {{ scenario.title }} agent.
-{{ scenario.description }}
+{% endfor %}
-{% endfor -%}
+If unclear, ask clarifying questions to determine if a scenario applies:
 
 ## 3. Handle the Request
-If no specialized scenario clearly matches, handle it yourself:
+If no specialized scenario clearly matches, handle it yourself in the following way
 
 ### For Questions and Information Requests
-1. **First, check existing knowledge**: Use `faq_lookup` tool to search for relevant information
+1. **First, check existing knowledge**: Use `captain--tools--faq_lookup` tool to search for relevant information
-2. **If not found in FAQs**: Provide your best answer based on available context
+2. **If not found in FAQs**: Try to ask clarifying questions to gather more information
-3. **If unable to answer**: Use `handoff` tool to transfer to a human expert
+3. **If unable to answer**: Use `captain--tools--handoff` tool to transfer to a human expert
 
 ### For Complex or Unclear Requests
 1. **Ask clarifying questions**: Gather more information if needed
 2. **Break down complex tasks**: Handle step by step or hand off if too complex
-3. **Escalate when necessary**: Use `handoff` tool for issues beyond your capabilities
+3. **Escalate when necessary**: Use `captain--tools--handoff` tool for issues beyond your capabilities
-
-## Response Best Practices
-- Be conversational but professional
-- Provide actionable information
-- Include relevant details from tool responses
 
 # Human Handoff Protocol
 Transfer to a human agent when:
@@ -77,4 +78,4 @@ Transfer to a human agent when:
 - The issue requires specialized knowledge or permissions you don't have
 - Multiple attempts to help have been unsuccessful
 
-When using the `handoff` tool, provide a clear reason that helps the human agent understand the context.
+When using the `captain--tools--handoff` tool, provide a clear reason that helps the human agent understand the context.

enterprise/lib/captain/prompts/scenario.liquid

@@ -1,20 +1,44 @@
 # System context
-You are part of a multi-agent system where you've been handed off a conversation to handle a specific task.
+You are part of a multi-agent system where you've been handed off a conversation to handle a specific task. The handoff was seamless - the user is not aware of any transfer. Continue the conversation naturally.
-The handoff was seamless - the user is not aware of any transfer. Continue the conversation naturally.
 
 # Your Role
-You are a specialized agent called {{ title }}, your task is to handle the following scenario:
+You are a specialized agent called "{{ title }}", your task is to handle the following scenario:
 
 {{ instructions }}
 
+If you believe the user's request is not within the scope of your role, you can assign this conversation back to the orchestrator agent using the `handoff_to_{{ assistant_name }}` tool
+
+{% if conversation || contact %}
+# Current Context
+
+Here's the metadata we have about the current conversation and the contact associated with it:
+
 {% if conversation -%}
 {% render 'conversation' %}
+{% endif -%}
 
 {% if contact -%}
 {% render 'contact' %}
 {% endif -%}
 {% endif -%}
 
+
+{% if response_guidelines.size > 0 -%}
+# Response Guidelines
+Your responses should follow these guidelines:
+{% for guideline in response_guidelines -%}
+- {{ guideline }}
+{% endfor %}
+{% endif -%}
+
+{% if guardrails.size > 0 -%}
+# Guardrails
+Always respect these boundaries:
+{% for guardrail in guardrails -%}
+- {{ guardrail }}
+{% endfor %}
+{% endif -%}
+
 {% if tools.size > 0 -%}
 # Available Tools
 You have access to these tools:

spec/enterprise/services/captain/assistant/agent_runner_service_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
   let(:mock_runner) { instance_double(Agents::Runner) }
   let(:mock_agent) { instance_double(Agents::Agent) }
   let(:mock_scenario_agent) { instance_double(Agents::Agent) }
-  let(:mock_result) { instance_double(Agents::RunResult, output: { 'response' => 'Test response' }) }
+  let(:mock_result) { instance_double(Agents::RunResult, output: { 'response' => 'Test response' }, context: nil) }
 
   let(:message_history) do
     [
@@ -99,7 +99,7 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
     it 'processes and formats agent result' do
       result = service.generate_response(message_history: message_history)
 
-      expect(result).to eq({ 'response' => 'Test response' })
+      expect(result).to eq({ 'response' => 'Test response', 'agent_name' => nil })
     end
 
     context 'when no scenarios are enabled' do
@@ -118,14 +118,15 @@ RSpec.describe Captain::Assistant::AgentRunnerService do
     end
 
     context 'when agent result is a string' do
-      let(:mock_result) { instance_double(Agents::RunResult, output: 'Simple string response') }
+      let(:mock_result) { instance_double(Agents::RunResult, output: 'Simple string response', context: nil) }
 
       it 'formats string response correctly' do
         result = service.generate_response(message_history: message_history)
 
         expect(result).to eq({
                                'response' => 'Simple string response',
-                               'reasoning' => 'Processed by agent'
+                               'reasoning' => 'Processed by agent',
+                               'agent_name' => nil
                              })
       end
     end