feat: Response Bot using GPT and Webpage Sources (#7518)

This commit introduces the ability to associate response sources to an inbox, allowing external webpages to be parsed by Chatwoot. The parsed data is converted into embeddings for use with GPT models when managing customer queries.

The implementation relies on the `pgvector` extension for PostgreSQL. Database migrations related to this feature are handled separately by `Features::ResponseBotService`. A future update will integrate these migrations into the default rails migrations, once compatibility with Postgres extensions across all self-hosted installation options is confirmed.

Additionally, a new GitHub action has been added to the CI pipeline to ensure the execution of specs related to this feature.

.circleci/config.yml

@@ -12,7 +12,7 @@ defaults: &defaults
     # Specify service dependencies here if necessary
     # CircleCI maintains a library of pre-built images
     # documented at https://circleci.com/docs/2.0/circleci-images/
-    - image: cimg/postgres:14.1
+    - image: cimg/postgres:15.3
     - image: cimg/redis:6.2.6
   environment:
       - RAILS_LOG_TO_STDOUT: false

.env.example

@@ -231,5 +231,10 @@ AZURE_APP_SECRET=
 # control the concurrency setting of sidekiq
 # SIDEKIQ_CONCURRENCY=10
 
+
+# AI powered features
+## OpenAI key
+# OPENAI_API_KEY=
+
 # Sentiment analysis model file path
 SENTIMENT_FILE_PATH=

.github/workflows/run_foss_spec.yml

@@ -18,11 +18,12 @@ jobs:
     runs-on: ubuntu-20.04
     services:
       postgres:
-        image: postgres:10.8
+        image: postgres:15.3
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: ""
           POSTGRES_DB: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
         ports:
           - 5432:5432
         # needed because the postgres container does not provide a healthcheck

.github/workflows/run_response_bot_spec.yml

@@ -0,0 +1,78 @@
+# #
+# # This workflow will run specs related to response bot
+# # This can only be activated in installations Where vector extension is available.
+# #
+
+name: Run Response Bot spec
+on:
+  push:
+    branches:
+      - develop
+      - master
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+    services:
+      postgres:
+        image: ankane/pgvector
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: ""
+          POSTGRES_DB: postgres
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a healthcheck
+        # tmpfs makes DB faster by using RAM
+        options: >-
+          --mount type=tmpfs,destination=/var/lib/postgresql/data
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: --entrypoint redis-server
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.ref }}
+        repository: ${{ github.event.pull_request.head.repo.full_name }}
+
+    - uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+
+    - name: yarn
+      run: yarn install
+
+    - name: Create database
+      run: bundle exec rake db:create
+
+    - name: Seed database
+      run: bundle exec rake db:schema:load
+
+    - name: Enable ResponseBotService in installation
+      run: RAILS_ENV=test bundle exec rails runner "Features::ResponseBotService.new.enable_in_installation"
+
+    # Run  Response Bot specs
+    - name: Run backend tests
+      run: |
+        bundle exec rspec spec/enterprise/controllers/api/v1/accounts/response_sources_controller_spec.rb spec/enterprise/controllers/enterprise/api/v1/accounts/inboxes_controller_spec.rb:47 --profile=10 --format documentation
+
+    - name: Upload rails log folder
+      uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: rails-log-folder
+        path: log

.rubocop.yml

@@ -87,6 +87,7 @@ Style/ClassAndModuleChildren:
   EnforcedStyle: compact
   Exclude:
     - 'config/application.rb'
+    - 'config/initializers/monkey_patches/*'
 Style/MapToHash:
   Enabled: false
 Style/HashSyntax:

Gemfile

@@ -165,6 +165,13 @@ gem 'omniauth'
 gem 'omniauth-google-oauth2'
 gem 'omniauth-rails_csrf_protection', '~> 1.0'
 
+## Gems for reponse bot
+# adds cosine similarity to postgres using vector extension
+gem 'neighbor'
+gem 'pgvector'
+# Convert Website HTML to Markdown
+gem 'reverse_markdown'
+
 # Sentiment analysis
 gem 'informers'
 

Gemfile.lock

@@ -459,6 +459,8 @@ GEM
     multi_json (1.15.0)
     multi_xml (0.6.0)
     multipart-post (2.3.0)
+    neighbor (0.2.3)
+      activerecord (>= 5.2)
     net-http-persistent (4.0.2)
       connection_pool (~> 2.2)
     net-imap (0.3.6)
@@ -532,6 +534,7 @@ GEM
     pg_search (2.3.6)
       activerecord (>= 5.2)
       activesupport (>= 5.2)
+    pgvector (0.1.1)
     procore-sift (1.0.0)
       activerecord (>= 6.1)
     pry (0.14.2)
@@ -617,6 +620,8 @@ GEM
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     retriable (3.1.2)
+    reverse_markdown (2.1.1)
+      nokogiri
     rexml (3.2.5)
     rspec-core (3.12.2)
       rspec-support (~> 3.12.0)
@@ -884,6 +889,7 @@ DEPENDENCIES
   lograge (~> 0.12.0)
   maxminddb
   mock_redis
+  neighbor
   newrelic-sidekiq-metrics
   newrelic_rpm
   omniauth
@@ -892,6 +898,7 @@ DEPENDENCIES
   omniauth-rails_csrf_protection (~> 1.0)
   pg
   pg_search
+  pgvector
   procore-sift
   pry-rails
   puma
@@ -905,6 +912,7 @@ DEPENDENCIES
   redis-namespace
   responders
   rest-client
+  reverse_markdown
   rspec-rails
   rspec_junit_formatter
   rubocop

app/models/account.rb

@@ -151,5 +151,5 @@ class Account < ApplicationRecord
 end
 
 Account.prepend_mod_with('Account')
-Account.include_mod_with('EnterpriseAccountConcern')
+Account.include_mod_with('Concerns::Account')
 Account.include_mod_with('Audit::Account')

app/models/inbox.rb

@@ -172,3 +172,4 @@ end
 
 Inbox.prepend_mod_with('Inbox')
 Inbox.include_mod_with('Audit::Inbox')
+Inbox.include_mod_with('Concerns::Inbox')

app/policies/inbox_policy.rb

@@ -38,6 +38,10 @@ class InboxPolicy < ApplicationPolicy
     @account_user.administrator?
   end
 
+  def response_sources?
+    @account_user.administrator?
+  end
+
   def create?
     @account_user.administrator?
   end

app/policies/response_source_policy.rb

@@ -0,0 +1,17 @@
+class ResponseSourcePolicy < ApplicationPolicy
+  def parse?
+    @account_user.administrator?
+  end
+
+  def create?
+    @account_user.administrator?
+  end
+
+  def add_document?
+    @account_user.administrator?
+  end
+
+  def remove_document?
+    @account_user.administrator?
+  end
+end

app/services/message_templates/hook_execution_service.rb

@@ -70,3 +70,4 @@ class MessageTemplates::HookExecutionService
     true
   end
 end
+MessageTemplates::HookExecutionService.prepend_mod_with('MessageTemplates::HookExecutionService')

app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder

@@ -0,0 +1,3 @@
+json.array! @response_sources do |response_source|
+  json.partial! 'api/v1/models/response_source', formats: [:json], resource: response_source
+end

app/views/api/v1/accounts/response_sources/add_document.json.jbuilder

@@ -0,0 +1 @@
+json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source

app/views/api/v1/accounts/response_sources/create.json.jbuilder

@@ -0,0 +1 @@
+json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source

app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder

@@ -0,0 +1 @@
+json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source

app/views/api/v1/models/_response_source.json.jbuilder

@@ -0,0 +1,16 @@
+json.id resource.id
+json.name resource.name
+json.source_link resource.source_link
+json.source_type resource.source_type
+json.inbox_id resource.inbox_id
+json.account_id resource.account_id
+json.created_at resource.created_at.to_i
+json.updated_at resource.updated_at.to_i
+json.response_documents do
+  json.array! resource.response_documents do |response_document|
+    json.id response_document.id
+    json.document_link response_document.document_link
+    json.created_at response_document.created_at.to_i
+    json.updated_at response_document.updated_at.to_i
+  end
+end

config/features.yml

@@ -55,3 +55,5 @@
   enabled: false
 - name: audit_logs
   enabled: false
+- name: response_bot
+  enabled: false

config/initializers/monkey_patches/schema_dumper.rb

@@ -0,0 +1,35 @@
+# When working with experimental extensions, which doesn't have support on all providers
+# This monkey patch will help us to ignore the extensions when dumping the schema
+# Additionally we will also ignore the tables associated with those features and exentions
+
+# Once the feature stabilizes, we can remove the tables/extension from the ignore list
+# Ensure you write appropriate migrations when you do that.
+
+module ActiveRecord
+  module ConnectionAdapters
+    module PostgreSQL
+      class SchemaDumper < ConnectionAdapters::SchemaDumper
+        cattr_accessor :ignore_extentions, default: []
+
+        private
+
+        def extensions(stream)
+          extensions = @connection.extensions
+          return unless extensions.any?
+
+          stream.puts '  # These are extensions that must be enabled in order to support this database'
+          extensions.sort.each do |extension|
+            stream.puts "  enable_extension #{extension.inspect}" unless ignore_extentions.include?(extension)
+          end
+          stream.puts
+        end
+      end
+    end
+  end
+end
+
+## Extentions / Tables to be ignored
+ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaDumper.ignore_extentions << 'vector'
+ActiveRecord::SchemaDumper.ignore_tables << 'responses'
+ActiveRecord::SchemaDumper.ignore_tables << 'response_sources'
+ActiveRecord::SchemaDumper.ignore_tables << 'response_documents'

config/routes.rb

@@ -140,6 +140,7 @@ Rails.application.routes.draw do
           resources :inboxes, only: [:index, :show, :create, :update, :destroy] do
             get :assignable_agents, on: :member
             get :campaigns, on: :member
+            get :response_sources, on: :member
             get :agent_bot, on: :member
             post :set_agent_bot, on: :member
             delete :avatar, on: :member
@@ -151,6 +152,15 @@ Rails.application.routes.draw do
             end
           end
           resources :labels, only: [:index, :show, :create, :update, :destroy]
+          resources :response_sources, only: [:create] do
+            collection do
+              post :parse
+            end
+            member do
+              post :add_document
+              post :remove_document
+            end
+          end
 
           resources :notifications, only: [:index, :update] do
             collection do

enterprise/app/controllers/api/v1/accounts/response_sources_controller.rb

@@ -0,0 +1,34 @@
+class Api::V1::Accounts::ResponseSourcesController < Api::V1::Accounts::BaseController
+  before_action :current_account
+  before_action :check_authorization
+  before_action :find_response_source, only: [:add_document, :remove_document]
+
+  def parse
+    links = PageCrawlerService.new(params[:link]).page_links
+    render json: { links: links }
+  end
+
+  def create
+    @response_source = Current.account.response_sources.new(response_source_params)
+    @response_source.save!
+  end
+
+  def add_document
+    @response_source.response_documents.create!(document_link: params[:document_link])
+  end
+
+  def remove_document
+    @response_source.response_documents.find(params[:document_id]).destroy!
+  end
+
+  private
+
+  def find_response_source
+    @response_source = Current.account.response_sources.find(params[:id])
+  end
+
+  def response_source_params
+    params.require(:response_source).permit(:name, :source_link, :inbox_id,
+                                            response_documents_attributes: [:document_link])
+  end
+end

enterprise/app/controllers/enterprise/api/v1/accounts/inboxes_controller.rb

@@ -1,4 +1,8 @@
 module Enterprise::Api::V1::Accounts::InboxesController
+  def response_sources
+    @response_sources = @inbox.response_sources
+  end
+
   def inbox_attributes
     super + ee_inbox_attributes
   end

enterprise/app/jobs/response_bot_job.rb

@@ -0,0 +1,7 @@
+class ResponseBotJob < ApplicationJob
+  queue_as :medium
+
+  def perform(conversation)
+    ::Enterprise::MessageTemplates::ResponseBotService.new(conversation: conversation).perform
+  end
+end

enterprise/app/jobs/response_builder_job.rb

@@ -0,0 +1,76 @@
+class ResponseBuilderJob < ApplicationJob
+  queue_as :default
+
+  def perform(response_document)
+    reset_previous_responses(response_document)
+    data = prepare_data(response_document)
+    response = post_request(data)
+    create_responses(response, response_document)
+  end
+
+  private
+
+  def reset_previous_responses(response_document)
+    response_document.responses.destroy_all
+  end
+
+  def prepare_data(response_document)
+    {
+      model: 'gpt-3.5-turbo',
+      messages: [
+        {
+          role: 'system',
+          content: system_message_content
+        },
+        {
+          role: 'user',
+          content: response_document.content
+        }
+      ]
+    }
+  end
+
+  def system_message_content
+    <<~SYSTEM_MESSAGE_CONTENT
+       You are a content writer looking to convert user content into short FAQs which can be added to your website's helper centre.
+       Format the webpage content provided in the message to FAQ format like the following example.#{' '}
+       Ensure that you only generate faqs from the information provider in the message.#{' '}
+       Ensure that output is always valid json.#{'  '}
+       If no match is available, return an empty JSON.
+       ```
+      [ { "question": "What is the pricing?",
+         "answer" : " There are different pricing tiers available."
+       }]
+       ```
+    SYSTEM_MESSAGE_CONTENT
+  end
+
+  def post_request(data)
+    headers = prepare_headers
+    HTTParty.post(
+      'https://api.openai.com/v1/chat/completions',
+      headers: headers,
+      body: data.to_json
+    )
+  end
+
+  def prepare_headers
+    {
+      'Content-Type' => 'application/json',
+      'Authorization' => "Bearer #{ENV.fetch('OPENAI_API_KEY')}"
+    }
+  end
+
+  def create_responses(response, response_document)
+    response_body = JSON.parse(response.body)
+    faqs = JSON.parse(response_body['choices'][0]['message']['content'].strip)
+
+    faqs.each do |faq|
+      response_document.responses.create!(
+        question: faq['question'],
+        answer: faq['answer'],
+        account_id: response_document.account_id
+      )
+    end
+  end
+end

enterprise/app/jobs/response_document_content_job.rb

@@ -0,0 +1,10 @@
+# app/jobs/response_document_content_job.rb
+class ResponseDocumentContentJob < ApplicationJob
+  queue_as :default
+
+  def perform(response_document)
+    # Replace the selector with the actual one you need.
+    content = PageCrawlerService.new(response_document.document_link).body_text_content
+    response_document.update!(content: content[0..15_000])
+  end
+end

enterprise/app/models/enterprise/concerns/account.rb

@@ -0,0 +1,15 @@
+module Enterprise::Concerns::Account
+  extend ActiveSupport::Concern
+
+  included do
+    has_many :sla_policies, dependent: :destroy_async
+
+    def self.add_response_related_associations
+      has_many :response_sources, dependent: :destroy_async
+      has_many :response_documents, dependent: :destroy_async
+      has_many :responses, dependent: :destroy_async
+    end
+
+    add_response_related_associations if Features::ResponseBotService.new.vector_extension_enabled?
+  end
+end

enterprise/app/models/enterprise/concerns/inbox.rb

@@ -0,0 +1,13 @@
+module Enterprise::Concerns::Inbox
+  extend ActiveSupport::Concern
+
+  included do
+    def self.add_response_related_associations
+      has_many :response_sources, dependent: :destroy_async
+      has_many :response_documents, dependent: :destroy_async
+      has_many :responses, dependent: :destroy_async
+    end
+
+    add_response_related_associations if Features::ResponseBotService.new.vector_extension_enabled?
+  end
+end

enterprise/app/models/enterprise/enterprise_account_concern.rb

@@ -1,7 +0,0 @@
-module Enterprise::EnterpriseAccountConcern
-  extend ActiveSupport::Concern
-
-  included do
-    has_many :sla_policies, dependent: :destroy_async
-  end
-end

enterprise/app/models/enterprise/inbox.rb

@@ -5,6 +5,19 @@ module Enterprise::Inbox
     super - overloaded_agent_ids
   end
 
+  def get_responses(query)
+    embedding = Openai::EmbeddingsService.new.get_embedding(query)
+    responses.nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5)
+  end
+
+  def active_bot?
+    super || response_bot_enabled?
+  end
+
+  def response_bot_enabled?
+    account.feature_enabled?('response_bot') && response_sources.any?
+  end
+
   private
 
   def get_agent_ids_over_assignment_limit(limit)

enterprise/app/models/response.rb

@@ -0,0 +1,36 @@
+# == Schema Information
+#
+# Table name: responses
+#
+#  id                   :bigint           not null, primary key
+#  answer               :text             not null
+#  embedding            :vector(1536)
+#  question             :string           not null
+#  created_at           :datetime         not null
+#  updated_at           :datetime         not null
+#  account_id           :bigint           not null
+#  response_document_id :bigint
+#
+# Indexes
+#
+#  index_responses_on_embedding             (embedding) USING ivfflat
+#  index_responses_on_response_document_id  (response_document_id)
+#
+class Response < ApplicationRecord
+  belongs_to :response_document
+  belongs_to :account
+  has_neighbors :embedding, normalize: true
+
+  before_save :update_response_embedding
+
+  def self.search(query)
+    embedding = Openai::EmbeddingsService.new.get_embedding(query)
+    nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5)
+  end
+
+  private
+
+  def update_response_embedding
+    self.embedding = Openai::EmbeddingsService.new.get_embedding("#{question}: #{answer}")
+  end
+end

enterprise/app/models/response_document.rb

@@ -0,0 +1,46 @@
+# == Schema Information
+#
+# Table name: response_documents
+#
+#  id                 :bigint           not null, primary key
+#  content            :text
+#  document_link      :string
+#  document_type      :string
+#  created_at         :datetime         not null
+#  updated_at         :datetime         not null
+#  account_id         :bigint           not null
+#  document_id        :bigint
+#  response_source_id :bigint           not null
+#
+# Indexes
+#
+#  index_response_documents_on_document            (document_type,document_id)
+#  index_response_documents_on_response_source_id  (response_source_id)
+#
+class ResponseDocument < ApplicationRecord
+  has_many :responses, dependent: :destroy
+  belongs_to :account
+  belongs_to :response_source
+
+  before_validation :set_account
+  after_create :ensure_content
+  after_update :handle_content_change
+
+  private
+
+  def set_account
+    self.account = response_source.account
+  end
+
+  def ensure_content
+    return unless content.nil?
+
+    ResponseDocumentContentJob.perform_later(self)
+  end
+
+  def handle_content_change
+    return unless saved_change_to_content? && content.present?
+
+    ResponseBuilderJob.perform_later(self)
+  end
+end

enterprise/app/models/response_source.rb

@@ -0,0 +1,28 @@
+# == Schema Information
+#
+# Table name: response_sources
+#
+#  id                :bigint           not null, primary key
+#  name              :string           not null
+#  source_link       :string
+#  source_model_type :string
+#  source_type       :integer          default("external"), not null
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#  account_id        :bigint           not null
+#  inbox_id          :bigint           not null
+#  source_model_id   :bigint
+#
+# Indexes
+#
+#  index_response_sources_on_source_model  (source_model_type,source_model_id)
+#
+class ResponseSource < ApplicationRecord
+  enum source_type: { external: 0, kbase: 1, inbox: 2 }
+  belongs_to :account
+  belongs_to :inbox
+  has_many :response_documents, dependent: :destroy
+  has_many :responses, through: :response_documents
+
+  accepts_nested_attributes_for :response_documents
+end

enterprise/app/services/enterprise/message_templates/hook_execution_service.rb

@@ -0,0 +1,10 @@
+module Enterprise::MessageTemplates::HookExecutionService
+  def trigger_templates
+    super
+    ResponseBotJob.perform_later(conversation) if should_process_response_bot?
+  end
+
+  def should_process_response_bot?
+    conversation.pending? && message.incoming? && inbox.response_bot_enabled?
+  end
+end

enterprise/app/services/enterprise/message_templates/response_bot_service.rb

@@ -0,0 +1,121 @@
+class Enterprise::MessageTemplates::ResponseBotService
+  pattr_initialize [:conversation!]
+
+  def perform
+    ActiveRecord::Base.transaction do
+      response = get_response(conversation.messages.last.content)
+      process_response(conversation.messages.last, response)
+    end
+  rescue StandardError => e
+    ChatwootExceptionTracker.new(e, account: conversation.account).capture_exception
+    true
+  end
+
+  private
+
+  delegate :contact, :account, :inbox, to: :conversation
+
+  def get_response(content)
+    previous_messages = []
+    get_previous_messages(previous_messages)
+    ChatGpt.new(response_sections(content)).generate_response('', previous_messages)
+  end
+
+  def get_previous_messages(previous_messages)
+    conversation.messages.where(message_type: [:outgoing, :incoming]).where(private: false).find_each do |message|
+      next if message.content_type != 'text'
+
+      role = determine_role(message)
+      previous_messages << { content: message.content, role: role }
+    end
+  end
+
+  def determine_role(message)
+    message.message_type == 'incoming' ? 'user' : 'system'
+  end
+
+  def response_sections(content)
+    sections = ''
+
+    inbox.get_responses(content).each do |response|
+      sections += "{context_id: #{response.id}, context: #{response.question} ? #{response.answer}}"
+    end
+    sections
+  end
+
+  def process_response(message, response)
+    if response == 'conversation_handoff'
+      process_action(message, 'handoff')
+    else
+      create_messages(response, conversation)
+    end
+  end
+
+  def process_action(_message, action)
+    case action
+    when 'handoff'
+      conversation.messages.create!('message_type': :outgoing, 'account_id': conversation.account_id, 'inbox_id': conversation.inbox_id,
+                                    'content': 'passing to an agent')
+      conversation.update(status: :open)
+    end
+  end
+
+  def create_messages(response, conversation)
+    response, article_ids = process_response_content(response)
+    create_outgoing_message(response, conversation)
+    create_outgoing_message_with_cards(article_ids, conversation) if article_ids.present?
+  end
+
+  def process_response_content(response)
+    # Regular expression to match '{context_ids: [ids]}'
+    regex = /{context_ids: \[(\d+(?:, *\d+)*)\]}/
+
+    # Extract ids from string
+    id_string = response[regex, 1] # This will give you '42, 43'
+    article_ids = id_string.split(',').map(&:to_i) if id_string # This will give you [42, 43]
+
+    # Remove '{context_ids: [ids]}' from string
+    response = response.sub(regex, '')
+
+    [response, article_ids]
+  end
+
+  def create_outgoing_message(response, conversation)
+    conversation.messages.create!(
+      {
+        message_type: :outgoing,
+        account_id: conversation.account_id,
+        inbox_id: conversation.inbox_id,
+        content: response
+      }
+    )
+  end
+
+  def create_outgoing_message_with_cards(article_ids, conversation)
+    content_attributes = get_article_hash(article_ids.uniq)
+    return if content_attributes.blank?
+
+    conversation.messages.create!(
+      {
+        message_type: :outgoing,
+        account_id: conversation.account_id,
+        inbox_id: conversation.inbox_id,
+        content: 'suggested articles',
+        content_type: 'article',
+        content_attributes: content_attributes
+      }
+    )
+  end
+
+  def get_article_hash(article_ids)
+    items = []
+    article_ids.each do |article_id|
+      response = Response.find(article_id)
+      next if response.nil?
+
+      items << { title: response.question, description: response.answer[0, 120], link: response.response_document.document_link }
+    end
+
+    items.present? ? { items: items } : {}
+  end
+end

enterprise/app/services/features/response_bot_service.rb

@@ -0,0 +1,83 @@
+class Features::ResponseBotService
+  MIGRATION_VERSION = ActiveRecord::Migration[7.0]
+
+  def enable_in_installation
+    enable_vector_extension
+    create_tables
+  end
+
+  def enable_vector_extension
+    MIGRATION_VERSION.enable_extension 'vector'
+  rescue ActiveRecord::StatementInvalid
+    print 'Vector extension not available'
+  end
+
+  def disable_vector_extension
+    MIGRATION_VERSION.disable_extension 'vector'
+  end
+
+  def vector_extension_enabled?
+    ActiveRecord::Base.connection.extension_enabled?('vector')
+  end
+
+  def create_tables
+    return unless vector_extension_enabled?
+
+    %i[response_sources response_documents responses].each do |table|
+      send("create_#{table}_table")
+    end
+  end
+
+  def drop_tables
+    %i[responses response_documents response_sources].each do |table|
+      MIGRATION_VERSION.drop_table table if MIGRATION_VERSION.table_exists?(table)
+    end
+  end
+
+  private
+
+  def create_response_sources_table
+    return if MIGRATION_VERSION.table_exists?(:response_sources)
+
+    MIGRATION_VERSION.create_table :response_sources do |t|
+      t.integer :source_type, null: false, default: 0
+      t.string :name, null: false
+      t.string :source_link
+      t.references :source_model, polymorphic: true
+      t.bigint :account_id, null: false
+      t.bigint :inbox_id, null: false
+      t.timestamps
+    end
+  end
+
+  def create_response_documents_table
+    return if MIGRATION_VERSION.table_exists?(:response_documents)
+
+    MIGRATION_VERSION.create_table :response_documents do |t|
+      t.bigint :response_source_id, null: false
+      t.string :document_link
+      t.references :document, polymorphic: true
+      t.text :content
+      t.bigint :account_id, null: false
+      t.timestamps
+    end
+
+    MIGRATION_VERSION.add_index :response_documents, :response_source_id
+  end
+
+  def create_responses_table
+    return if MIGRATION_VERSION.table_exists?(:responses)
+
+    MIGRATION_VERSION.create_table :responses do |t|
+      t.bigint :response_document_id
+      t.string :question, null: false
+      t.text :answer, null: false
+      t.bigint :account_id, null: false
+      t.vector :embedding, limit: 1536
+      t.timestamps
+    end
+
+    MIGRATION_VERSION.add_index :responses, :response_document_id
+    MIGRATION_VERSION.add_index :responses, :embedding, using: :ivfflat, opclass: :vector_l2_ops
+  end
+end

enterprise/app/services/openai/embeddings_service.rb

@@ -0,0 +1,22 @@
+class Openai::EmbeddingsService
+  def get_embedding(content)
+    fetch_embeddings(content)
+  end
+
+  private
+
+  def fetch_embeddings(input)
+    url = 'https://api.openai.com/v1/embeddings'
+    headers = {
+      'Authorization' => "Bearer #{ENV.fetch('OPENAI_API_KEY')}",
+      'Content-Type' => 'application/json'
+    }
+    data = {
+      input: input,
+      model: 'text-embedding-ada-002'
+    }
+
+    response = Net::HTTP.post(URI(url), data.to_json, headers)
+    JSON.parse(response.body)['data'].pick('embedding')
+  end
+end

enterprise/app/services/page_crawler_service.rb

@@ -0,0 +1,38 @@
+class PageCrawlerService
+  attr_reader :external_link
+
+  def initialize(external_link)
+    @external_link = external_link
+    @doc = Nokogiri::HTML(HTTParty.get(external_link).body)
+  end
+
+  def page_links
+    sitemap? ? extract_links_from_sitemap : extract_links_from_html
+  end
+
+  def page_title
+    title_element = @doc.at_xpath('//title')
+    title_element&.text&.strip
+  end
+
+  def body_text_content
+    ReverseMarkdown.convert @doc.at_xpath('//body'), unknown_tags: :bypass, github_flavored: true
+  end
+
+  private
+
+  def sitemap?
+    @external_link.end_with?('.xml')
+  end
+
+  def extract_links_from_sitemap
+    @doc.xpath('//loc').to_set(&:text)
+  end
+
+  def extract_links_from_html
+    @doc.xpath('//a/@href').to_set do |link|
+      absolute_url = URI.join(@external_link, link.value).to_s
+      absolute_url
+    end
+  end
+end

enterprise/lib/chat_gpt.rb

@@ -0,0 +1,41 @@
+class ChatGpt
+  def self.base_uri
+    'https://api.openai.com'
+  end
+
+  def initialize(context_sections = '')
+    @model = 'gpt-4'
+    system_message = { 'role': 'system',
+                       'content': 'You are a very enthusiastic customer support representative who loves ' \
+                                  'to help people! Given the following Context sections from the ' \
+                                  'documentation, continue the conversation with only that information, ' \
+                                  "outputed in markdown format along with context_ids in format 'response \n {context_ids: [values] }'  " \
+                                  "\n If you are unsure and the answer is not explicitly written in the documentation,  " \
+                                  "say 'Sorry, I don't know how to help with that. Do you want to chat with a human agent?' " \
+                                  "If they ask to Chat with human agent return text 'conversation_handoff'." \
+                                  "Context sections: \n" \
+                                  "\n\n #{context_sections}}" }
+
+    @messages = [
+      system_message
+    ]
+  end
+
+  def generate_response(input, previous_messages = [])
+    previous_messages.each do |message|
+      @messages << message
+    end
+
+    @messages << { 'role': 'user', 'content': input } if input.present?
+    headers = { 'Content-Type' => 'application/json',
+                'Authorization' => "Bearer #{ENV.fetch('OPENAI_API_KEY')}" }
+    body = {
+      model: @model,
+      messages: @messages
+    }.to_json
+
+    response = HTTParty.post("#{self.class.base_uri}/v1/chat/completions", headers: headers, body: body)
+    response_body = JSON.parse(response.body)
+    response_body['choices'][0]['message']['content'].strip
+  end
+end

spec/enterprise/controllers/api/v1/accounts/response_sources_controller_spec.rb

@@ -0,0 +1,135 @@
+require 'rails_helper'
+
+RSpec.describe 'Response Sources API', type: :request do
+  let!(:account) { create(:account) }
+  let!(:admin) { create(:user, account: account, role: :administrator) }
+  let!(:inbox) { create(:inbox, account: account) }
+
+  before do
+    skip('Skipping since vector is not enabled in this environment') unless Features::ResponseBotService.new.vector_extension_enabled?
+  end
+
+  describe 'POST /api/v1/accounts/{account.id}/response_sources/parse' do
+    let(:valid_params) do
+      {
+        link: 'http://test.test'
+      }
+    end
+
+    context 'when it is an unauthenticated user' do
+      it 'returns unauthorized' do
+        post "/api/v1/accounts/#{account.id}/response_sources/parse", params: valid_params
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'when it is an authenticated user' do
+      it 'returns links in the webpage' do
+        crawler = double
+        allow(PageCrawlerService).to receive(:new).and_return(crawler)
+        allow(crawler).to receive(:page_links).and_return(['http://test.test'])
+
+        post "/api/v1/accounts/#{account.id}/response_sources/parse", headers: admin.create_new_auth_token,
+                                                                      params: valid_params
+        expect(response).to have_http_status(:success)
+        expect(response.parsed_body['links']).to eq(['http://test.test'])
+      end
+    end
+  end
+
+  describe 'POST /api/v1/accounts/{account.id}/response_sources' do
+    let(:valid_params) do
+      {
+        response_source: {
+          name: 'Test',
+          source_link: 'http://test.test',
+          inbox_id: inbox.id,
+          response_documents_attributes: [
+            { document_link: 'http://test1.test' },
+            { document_link: 'http://test2.test' }
+          ]
+        }
+      }
+    end
+
+    context 'when it is an unauthenticated user' do
+      it 'returns unauthorized' do
+        expect { post "/api/v1/accounts/#{account.id}/response_sources", params: valid_params }.not_to change(ResponseSource, :count)
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'when it is an authenticated user' do
+      it 'creates the response sources and documents' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources", headers: admin.create_new_auth_token,
+                                                                  params: valid_params
+        end.to change(ResponseSource, :count).by(1)
+
+        expect(ResponseDocument.count).to eq(2)
+        expect(response).to have_http_status(:success)
+      end
+    end
+  end
+
+  describe 'POST /api/v1/accounts/{account.id}/response_sources/{response_source.id}/add_document' do
+    let!(:response_source) { create(:response_source, account: account, inbox: inbox) }
+    let(:valid_params) do
+      { document_link: 'http://test.test' }
+    end
+
+    context 'when it is an unauthenticated user' do
+      it 'returns unauthorized' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/add_document",
+               params: valid_params
+        end.not_to change(ResponseDocument, :count)
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'when it is an authenticated user' do
+      it 'creates the response sources and documents' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/add_document", headers: admin.create_new_auth_token,
+                                                                                                     params: valid_params
+        end.to change(ResponseDocument, :count).by(1)
+        expect(response).to have_http_status(:success)
+      end
+    end
+  end
+
+  describe 'POST /api/v1/accounts/{account.id}/response_sources/{response_source.id}/remove_document' do
+    let!(:response_source) { create(:response_source, account: account, inbox: inbox) }
+    let!(:response_document) { response_source.response_documents.create!(document_link: 'http://test.test') }
+    let(:valid_params) do
+      { document_id: response_document.id }
+    end
+
+    context 'when it is an unauthenticated user' do
+      it 'returns unauthorized' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/remove_document",
+               params: valid_params
+        end.not_to change(ResponseDocument, :count)
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'when it is an authenticated user' do
+      it 'creates the response sources and documents' do
+        expect do
+          post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/remove_document", headers: admin.create_new_auth_token,
+                                                                                                        params: valid_params
+        end.to change(ResponseDocument, :count).by(-1)
+        expect(response).to have_http_status(:success)
+
+        expect { response_document.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+  end
+end

spec/enterprise/controllers/enterprise/api/v1/accounts/inboxes_controller_spec.rb

@@ -43,4 +43,44 @@ RSpec.describe 'Enterprise Inboxes API', type: :request do
       end
     end
   end
+
+  describe 'GET /api/v1/accounts/{account.id}/inboxes/{inbox.id}/response_sources' do
+    let(:inbox) { create(:inbox, account: account) }
+    let(:agent) { create(:user, account: account, role: :agent) }
+    let(:administrator) { create(:user, account: account, role: :administrator) }
+
+    before do
+      skip('Skipping since vector is not enabled in this environment') unless Features::ResponseBotService.new.vector_extension_enabled?
+    end
+
+    context 'when it is an unauthenticated user' do
+      it 'returns unauthorized' do
+        get "/api/v1/accounts/#{account.id}/inboxes/#{inbox.id}/response_sources"
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+    end
+
+    context 'when it is an authenticated user' do
+      it 'returns unauthorized for agents' do
+        get "/api/v1/accounts/#{account.id}/inboxes/#{inbox.id}/response_sources",
+            headers: agent.create_new_auth_token,
+            as: :json
+
+        expect(response).to have_http_status(:unauthorized)
+      end
+
+      it 'returns all response_sources belonging to the inbox to administrators' do
+        response_source = create(:response_source, account: account, inbox: inbox)
+        get "/api/v1/accounts/#{account.id}/inboxes/#{inbox.id}/response_sources",
+            headers: administrator.create_new_auth_token,
+            as: :json
+
+        expect(response).to have_http_status(:success)
+        body = JSON.parse(response.body, symbolize_names: true)
+        expect(body.first[:id]).to eq(response_source.id)
+        expect(body.length).to eq(1)
+      end
+    end
+  end
 end

spec/factories/response_source.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :response_source do
+    name { Faker::Name.name }
+    source_link { Faker::Internet.url }
+    account
+  end
+end