diff --git a/.circleci/config.yml b/.circleci/config.yml index 607f33872..e2c6dc583 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ defaults: &defaults # Specify service dependencies here if necessary # CircleCI maintains a library of pre-built images # documented at https://circleci.com/docs/2.0/circleci-images/ - - image: cimg/postgres:14.1 + - image: cimg/postgres:15.3 - image: cimg/redis:6.2.6 environment: - RAILS_LOG_TO_STDOUT: false diff --git a/.env.example b/.env.example index acda72798..a3a5bf7cc 100644 --- a/.env.example +++ b/.env.example @@ -231,5 +231,10 @@ AZURE_APP_SECRET= # control the concurrency setting of sidekiq # SIDEKIQ_CONCURRENCY=10 + +# AI powered features +## OpenAI key +# OPENAI_API_KEY= + # Sentiment analysis model file path SENTIMENT_FILE_PATH= diff --git a/.github/workflows/run_foss_spec.yml b/.github/workflows/run_foss_spec.yml index 51688944c..71009f5a0 100644 --- a/.github/workflows/run_foss_spec.yml +++ b/.github/workflows/run_foss_spec.yml @@ -18,11 +18,12 @@ jobs: runs-on: ubuntu-20.04 services: postgres: - image: postgres:10.8 + image: postgres:15.3 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: "" POSTGRES_DB: postgres + POSTGRES_HOST_AUTH_METHOD: trust ports: - 5432:5432 # needed because the postgres container does not provide a healthcheck diff --git a/.github/workflows/run_response_bot_spec.yml b/.github/workflows/run_response_bot_spec.yml new file mode 100644 index 000000000..daa89494a --- /dev/null +++ b/.github/workflows/run_response_bot_spec.yml @@ -0,0 +1,78 @@ +# # +# # This workflow will run specs related to response bot +# # This can only be activated in installations Where vector extension is available. +# # + +name: Run Response Bot spec +on: + push: + branches: + - develop + - master + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-20.04 + services: + postgres: + image: ankane/pgvector + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: "" + POSTGRES_DB: postgres + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + # needed because the postgres container does not provide a healthcheck + # tmpfs makes DB faster by using RAM + options: >- + --mount type=tmpfs,destination=/var/lib/postgresql/data + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis + ports: + - 6379:6379 + options: --entrypoint redis-server + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + + - uses: actions/setup-node@v3 + with: + node-version: 16 + + - name: yarn + run: yarn install + + - name: Create database + run: bundle exec rake db:create + + - name: Seed database + run: bundle exec rake db:schema:load + + - name: Enable ResponseBotService in installation + run: RAILS_ENV=test bundle exec rails runner "Features::ResponseBotService.new.enable_in_installation" + + # Run Response Bot specs + - name: Run backend tests + run: | + bundle exec rspec spec/enterprise/controllers/api/v1/accounts/response_sources_controller_spec.rb spec/enterprise/controllers/enterprise/api/v1/accounts/inboxes_controller_spec.rb:47 --profile=10 --format documentation + + - name: Upload rails log folder + uses: actions/upload-artifact@v3 + if: always() + with: + name: rails-log-folder + path: log diff --git a/.rubocop.yml b/.rubocop.yml index 914308551..2befbac4b 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -87,6 +87,7 @@ Style/ClassAndModuleChildren: EnforcedStyle: compact Exclude: - 'config/application.rb' + - 'config/initializers/monkey_patches/*' Style/MapToHash: Enabled: false Style/HashSyntax: diff --git a/Gemfile b/Gemfile index 743818b98..bc92fae31 100644 --- a/Gemfile +++ b/Gemfile @@ -165,6 +165,13 @@ gem 'omniauth' gem 'omniauth-google-oauth2' gem 'omniauth-rails_csrf_protection', '~> 1.0' +## Gems for reponse bot +# adds cosine similarity to postgres using vector extension +gem 'neighbor' +gem 'pgvector' +# Convert Website HTML to Markdown +gem 'reverse_markdown' + # Sentiment analysis gem 'informers' diff --git a/Gemfile.lock b/Gemfile.lock index 1051e724e..462eff774 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -459,6 +459,8 @@ GEM multi_json (1.15.0) multi_xml (0.6.0) multipart-post (2.3.0) + neighbor (0.2.3) + activerecord (>= 5.2) net-http-persistent (4.0.2) connection_pool (~> 2.2) net-imap (0.3.6) @@ -532,6 +534,7 @@ GEM pg_search (2.3.6) activerecord (>= 5.2) activesupport (>= 5.2) + pgvector (0.1.1) procore-sift (1.0.0) activerecord (>= 6.1) pry (0.14.2) @@ -617,6 +620,8 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) retriable (3.1.2) + reverse_markdown (2.1.1) + nokogiri rexml (3.2.5) rspec-core (3.12.2) rspec-support (~> 3.12.0) @@ -884,6 +889,7 @@ DEPENDENCIES lograge (~> 0.12.0) maxminddb mock_redis + neighbor newrelic-sidekiq-metrics newrelic_rpm omniauth @@ -892,6 +898,7 @@ DEPENDENCIES omniauth-rails_csrf_protection (~> 1.0) pg pg_search + pgvector procore-sift pry-rails puma @@ -905,6 +912,7 @@ DEPENDENCIES redis-namespace responders rest-client + reverse_markdown rspec-rails rspec_junit_formatter rubocop diff --git a/app/models/account.rb b/app/models/account.rb index 564d2f7e2..f837776eb 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -151,5 +151,5 @@ class Account < ApplicationRecord end Account.prepend_mod_with('Account') -Account.include_mod_with('EnterpriseAccountConcern') +Account.include_mod_with('Concerns::Account') Account.include_mod_with('Audit::Account') diff --git a/app/models/inbox.rb b/app/models/inbox.rb index fcafc2ff0..07ca46cbf 100644 --- a/app/models/inbox.rb +++ b/app/models/inbox.rb @@ -172,3 +172,4 @@ end Inbox.prepend_mod_with('Inbox') Inbox.include_mod_with('Audit::Inbox') +Inbox.include_mod_with('Concerns::Inbox') diff --git a/app/policies/inbox_policy.rb b/app/policies/inbox_policy.rb index 851cac7d0..891b3414a 100644 --- a/app/policies/inbox_policy.rb +++ b/app/policies/inbox_policy.rb @@ -38,6 +38,10 @@ class InboxPolicy < ApplicationPolicy @account_user.administrator? end + def response_sources? + @account_user.administrator? + end + def create? @account_user.administrator? end diff --git a/app/policies/response_source_policy.rb b/app/policies/response_source_policy.rb new file mode 100644 index 000000000..e6083ba22 --- /dev/null +++ b/app/policies/response_source_policy.rb @@ -0,0 +1,17 @@ +class ResponseSourcePolicy < ApplicationPolicy + def parse? + @account_user.administrator? + end + + def create? + @account_user.administrator? + end + + def add_document? + @account_user.administrator? + end + + def remove_document? + @account_user.administrator? + end +end diff --git a/app/services/message_templates/hook_execution_service.rb b/app/services/message_templates/hook_execution_service.rb index 9315a889e..d7d90f198 100644 --- a/app/services/message_templates/hook_execution_service.rb +++ b/app/services/message_templates/hook_execution_service.rb @@ -70,3 +70,4 @@ class MessageTemplates::HookExecutionService true end end +MessageTemplates::HookExecutionService.prepend_mod_with('MessageTemplates::HookExecutionService') diff --git a/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder b/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder new file mode 100644 index 000000000..aff6e551c --- /dev/null +++ b/app/views/api/v1/accounts/inboxes/response_sources.json.jbuilder @@ -0,0 +1,3 @@ +json.array! @response_sources do |response_source| + json.partial! 'api/v1/models/response_source', formats: [:json], resource: response_source +end diff --git a/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder b/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder new file mode 100644 index 000000000..ab0af26d3 --- /dev/null +++ b/app/views/api/v1/accounts/response_sources/add_document.json.jbuilder @@ -0,0 +1 @@ +json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source diff --git a/app/views/api/v1/accounts/response_sources/create.json.jbuilder b/app/views/api/v1/accounts/response_sources/create.json.jbuilder new file mode 100644 index 000000000..ab0af26d3 --- /dev/null +++ b/app/views/api/v1/accounts/response_sources/create.json.jbuilder @@ -0,0 +1 @@ +json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source diff --git a/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder b/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder new file mode 100644 index 000000000..ab0af26d3 --- /dev/null +++ b/app/views/api/v1/accounts/response_sources/remove_document.json.jbuilder @@ -0,0 +1 @@ +json.partial! 'api/v1/models/response_source', formats: [:json], resource: @response_source diff --git a/app/views/api/v1/models/_response_source.json.jbuilder b/app/views/api/v1/models/_response_source.json.jbuilder new file mode 100644 index 000000000..570f3687e --- /dev/null +++ b/app/views/api/v1/models/_response_source.json.jbuilder @@ -0,0 +1,16 @@ +json.id resource.id +json.name resource.name +json.source_link resource.source_link +json.source_type resource.source_type +json.inbox_id resource.inbox_id +json.account_id resource.account_id +json.created_at resource.created_at.to_i +json.updated_at resource.updated_at.to_i +json.response_documents do + json.array! resource.response_documents do |response_document| + json.id response_document.id + json.document_link response_document.document_link + json.created_at response_document.created_at.to_i + json.updated_at response_document.updated_at.to_i + end +end diff --git a/config/features.yml b/config/features.yml index 6132bda68..a9e0697ff 100644 --- a/config/features.yml +++ b/config/features.yml @@ -55,3 +55,5 @@ enabled: false - name: audit_logs enabled: false +- name: response_bot + enabled: false diff --git a/config/initializers/monkey_patches/schema_dumper.rb b/config/initializers/monkey_patches/schema_dumper.rb new file mode 100644 index 000000000..fd1865d0e --- /dev/null +++ b/config/initializers/monkey_patches/schema_dumper.rb @@ -0,0 +1,35 @@ +# When working with experimental extensions, which doesn't have support on all providers +# This monkey patch will help us to ignore the extensions when dumping the schema +# Additionally we will also ignore the tables associated with those features and exentions + +# Once the feature stabilizes, we can remove the tables/extension from the ignore list +# Ensure you write appropriate migrations when you do that. + +module ActiveRecord + module ConnectionAdapters + module PostgreSQL + class SchemaDumper < ConnectionAdapters::SchemaDumper + cattr_accessor :ignore_extentions, default: [] + + private + + def extensions(stream) + extensions = @connection.extensions + return unless extensions.any? + + stream.puts ' # These are extensions that must be enabled in order to support this database' + extensions.sort.each do |extension| + stream.puts " enable_extension #{extension.inspect}" unless ignore_extentions.include?(extension) + end + stream.puts + end + end + end + end +end + +## Extentions / Tables to be ignored +ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaDumper.ignore_extentions << 'vector' +ActiveRecord::SchemaDumper.ignore_tables << 'responses' +ActiveRecord::SchemaDumper.ignore_tables << 'response_sources' +ActiveRecord::SchemaDumper.ignore_tables << 'response_documents' diff --git a/config/routes.rb b/config/routes.rb index ab1fc4115..cb589b562 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -140,6 +140,7 @@ Rails.application.routes.draw do resources :inboxes, only: [:index, :show, :create, :update, :destroy] do get :assignable_agents, on: :member get :campaigns, on: :member + get :response_sources, on: :member get :agent_bot, on: :member post :set_agent_bot, on: :member delete :avatar, on: :member @@ -151,6 +152,15 @@ Rails.application.routes.draw do end end resources :labels, only: [:index, :show, :create, :update, :destroy] + resources :response_sources, only: [:create] do + collection do + post :parse + end + member do + post :add_document + post :remove_document + end + end resources :notifications, only: [:index, :update] do collection do diff --git a/enterprise/app/controllers/api/v1/accounts/response_sources_controller.rb b/enterprise/app/controllers/api/v1/accounts/response_sources_controller.rb new file mode 100644 index 000000000..5c84933cf --- /dev/null +++ b/enterprise/app/controllers/api/v1/accounts/response_sources_controller.rb @@ -0,0 +1,34 @@ +class Api::V1::Accounts::ResponseSourcesController < Api::V1::Accounts::BaseController + before_action :current_account + before_action :check_authorization + before_action :find_response_source, only: [:add_document, :remove_document] + + def parse + links = PageCrawlerService.new(params[:link]).page_links + render json: { links: links } + end + + def create + @response_source = Current.account.response_sources.new(response_source_params) + @response_source.save! + end + + def add_document + @response_source.response_documents.create!(document_link: params[:document_link]) + end + + def remove_document + @response_source.response_documents.find(params[:document_id]).destroy! + end + + private + + def find_response_source + @response_source = Current.account.response_sources.find(params[:id]) + end + + def response_source_params + params.require(:response_source).permit(:name, :source_link, :inbox_id, + response_documents_attributes: [:document_link]) + end +end diff --git a/enterprise/app/controllers/enterprise/api/v1/accounts/inboxes_controller.rb b/enterprise/app/controllers/enterprise/api/v1/accounts/inboxes_controller.rb index b39db609d..81435390f 100644 --- a/enterprise/app/controllers/enterprise/api/v1/accounts/inboxes_controller.rb +++ b/enterprise/app/controllers/enterprise/api/v1/accounts/inboxes_controller.rb @@ -1,4 +1,8 @@ module Enterprise::Api::V1::Accounts::InboxesController + def response_sources + @response_sources = @inbox.response_sources + end + def inbox_attributes super + ee_inbox_attributes end diff --git a/enterprise/app/jobs/response_bot_job.rb b/enterprise/app/jobs/response_bot_job.rb new file mode 100644 index 000000000..5ad703bab --- /dev/null +++ b/enterprise/app/jobs/response_bot_job.rb @@ -0,0 +1,7 @@ +class ResponseBotJob < ApplicationJob + queue_as :medium + + def perform(conversation) + ::Enterprise::MessageTemplates::ResponseBotService.new(conversation: conversation).perform + end +end diff --git a/enterprise/app/jobs/response_builder_job.rb b/enterprise/app/jobs/response_builder_job.rb new file mode 100644 index 000000000..a459b887d --- /dev/null +++ b/enterprise/app/jobs/response_builder_job.rb @@ -0,0 +1,76 @@ +class ResponseBuilderJob < ApplicationJob + queue_as :default + + def perform(response_document) + reset_previous_responses(response_document) + data = prepare_data(response_document) + response = post_request(data) + create_responses(response, response_document) + end + + private + + def reset_previous_responses(response_document) + response_document.responses.destroy_all + end + + def prepare_data(response_document) + { + model: 'gpt-3.5-turbo', + messages: [ + { + role: 'system', + content: system_message_content + }, + { + role: 'user', + content: response_document.content + } + ] + } + end + + def system_message_content + <<~SYSTEM_MESSAGE_CONTENT + You are a content writer looking to convert user content into short FAQs which can be added to your website's helper centre. + Format the webpage content provided in the message to FAQ format like the following example.#{' '} + Ensure that you only generate faqs from the information provider in the message.#{' '} + Ensure that output is always valid json.#{' '} + If no match is available, return an empty JSON. + ``` + [ { "question": "What is the pricing?", + "answer" : " There are different pricing tiers available." + }] + ``` + SYSTEM_MESSAGE_CONTENT + end + + def post_request(data) + headers = prepare_headers + HTTParty.post( + 'https://api.openai.com/v1/chat/completions', + headers: headers, + body: data.to_json + ) + end + + def prepare_headers + { + 'Content-Type' => 'application/json', + 'Authorization' => "Bearer #{ENV.fetch('OPENAI_API_KEY')}" + } + end + + def create_responses(response, response_document) + response_body = JSON.parse(response.body) + faqs = JSON.parse(response_body['choices'][0]['message']['content'].strip) + + faqs.each do |faq| + response_document.responses.create!( + question: faq['question'], + answer: faq['answer'], + account_id: response_document.account_id + ) + end + end +end diff --git a/enterprise/app/jobs/response_document_content_job.rb b/enterprise/app/jobs/response_document_content_job.rb new file mode 100644 index 000000000..bced3a12f --- /dev/null +++ b/enterprise/app/jobs/response_document_content_job.rb @@ -0,0 +1,10 @@ +# app/jobs/response_document_content_job.rb +class ResponseDocumentContentJob < ApplicationJob + queue_as :default + + def perform(response_document) + # Replace the selector with the actual one you need. + content = PageCrawlerService.new(response_document.document_link).body_text_content + response_document.update!(content: content[0..15_000]) + end +end diff --git a/enterprise/app/models/enterprise/concerns/account.rb b/enterprise/app/models/enterprise/concerns/account.rb new file mode 100644 index 000000000..699e5b813 --- /dev/null +++ b/enterprise/app/models/enterprise/concerns/account.rb @@ -0,0 +1,15 @@ +module Enterprise::Concerns::Account + extend ActiveSupport::Concern + + included do + has_many :sla_policies, dependent: :destroy_async + + def self.add_response_related_associations + has_many :response_sources, dependent: :destroy_async + has_many :response_documents, dependent: :destroy_async + has_many :responses, dependent: :destroy_async + end + + add_response_related_associations if Features::ResponseBotService.new.vector_extension_enabled? + end +end diff --git a/enterprise/app/models/enterprise/concerns/inbox.rb b/enterprise/app/models/enterprise/concerns/inbox.rb new file mode 100644 index 000000000..f7c15b150 --- /dev/null +++ b/enterprise/app/models/enterprise/concerns/inbox.rb @@ -0,0 +1,13 @@ +module Enterprise::Concerns::Inbox + extend ActiveSupport::Concern + + included do + def self.add_response_related_associations + has_many :response_sources, dependent: :destroy_async + has_many :response_documents, dependent: :destroy_async + has_many :responses, dependent: :destroy_async + end + + add_response_related_associations if Features::ResponseBotService.new.vector_extension_enabled? + end +end diff --git a/enterprise/app/models/enterprise/enterprise_account_concern.rb b/enterprise/app/models/enterprise/enterprise_account_concern.rb deleted file mode 100644 index 8e430dbe7..000000000 --- a/enterprise/app/models/enterprise/enterprise_account_concern.rb +++ /dev/null @@ -1,7 +0,0 @@ -module Enterprise::EnterpriseAccountConcern - extend ActiveSupport::Concern - - included do - has_many :sla_policies, dependent: :destroy_async - end -end diff --git a/enterprise/app/models/enterprise/inbox.rb b/enterprise/app/models/enterprise/inbox.rb index 6fe3ff2f2..70c37456b 100644 --- a/enterprise/app/models/enterprise/inbox.rb +++ b/enterprise/app/models/enterprise/inbox.rb @@ -5,6 +5,19 @@ module Enterprise::Inbox super - overloaded_agent_ids end + def get_responses(query) + embedding = Openai::EmbeddingsService.new.get_embedding(query) + responses.nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5) + end + + def active_bot? + super || response_bot_enabled? + end + + def response_bot_enabled? + account.feature_enabled?('response_bot') && response_sources.any? + end + private def get_agent_ids_over_assignment_limit(limit) diff --git a/enterprise/app/models/response.rb b/enterprise/app/models/response.rb new file mode 100644 index 000000000..1fc82e94c --- /dev/null +++ b/enterprise/app/models/response.rb @@ -0,0 +1,36 @@ +# == Schema Information +# +# Table name: responses +# +# id :bigint not null, primary key +# answer :text not null +# embedding :vector(1536) +# question :string not null +# created_at :datetime not null +# updated_at :datetime not null +# account_id :bigint not null +# response_document_id :bigint +# +# Indexes +# +# index_responses_on_embedding (embedding) USING ivfflat +# index_responses_on_response_document_id (response_document_id) +# +class Response < ApplicationRecord + belongs_to :response_document + belongs_to :account + has_neighbors :embedding, normalize: true + + before_save :update_response_embedding + + def self.search(query) + embedding = Openai::EmbeddingsService.new.get_embedding(query) + nearest_neighbors(:embedding, embedding, distance: 'cosine').first(5) + end + + private + + def update_response_embedding + self.embedding = Openai::EmbeddingsService.new.get_embedding("#{question}: #{answer}") + end +end diff --git a/enterprise/app/models/response_document.rb b/enterprise/app/models/response_document.rb new file mode 100644 index 000000000..6e547a64c --- /dev/null +++ b/enterprise/app/models/response_document.rb @@ -0,0 +1,46 @@ +# == Schema Information +# +# Table name: response_documents +# +# id :bigint not null, primary key +# content :text +# document_link :string +# document_type :string +# created_at :datetime not null +# updated_at :datetime not null +# account_id :bigint not null +# document_id :bigint +# response_source_id :bigint not null +# +# Indexes +# +# index_response_documents_on_document (document_type,document_id) +# index_response_documents_on_response_source_id (response_source_id) +# +class ResponseDocument < ApplicationRecord + has_many :responses, dependent: :destroy + belongs_to :account + belongs_to :response_source + + before_validation :set_account + after_create :ensure_content + after_update :handle_content_change + + private + + def set_account + self.account = response_source.account + end + + def ensure_content + return unless content.nil? + + ResponseDocumentContentJob.perform_later(self) + end + + def handle_content_change + return unless saved_change_to_content? && content.present? + + ResponseBuilderJob.perform_later(self) + end +end diff --git a/enterprise/app/models/response_source.rb b/enterprise/app/models/response_source.rb new file mode 100644 index 000000000..99db2ac3d --- /dev/null +++ b/enterprise/app/models/response_source.rb @@ -0,0 +1,28 @@ +# == Schema Information +# +# Table name: response_sources +# +# id :bigint not null, primary key +# name :string not null +# source_link :string +# source_model_type :string +# source_type :integer default("external"), not null +# created_at :datetime not null +# updated_at :datetime not null +# account_id :bigint not null +# inbox_id :bigint not null +# source_model_id :bigint +# +# Indexes +# +# index_response_sources_on_source_model (source_model_type,source_model_id) +# +class ResponseSource < ApplicationRecord + enum source_type: { external: 0, kbase: 1, inbox: 2 } + belongs_to :account + belongs_to :inbox + has_many :response_documents, dependent: :destroy + has_many :responses, through: :response_documents + + accepts_nested_attributes_for :response_documents +end diff --git a/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb b/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb new file mode 100644 index 000000000..5eadd2416 --- /dev/null +++ b/enterprise/app/services/enterprise/message_templates/hook_execution_service.rb @@ -0,0 +1,10 @@ +module Enterprise::MessageTemplates::HookExecutionService + def trigger_templates + super + ResponseBotJob.perform_later(conversation) if should_process_response_bot? + end + + def should_process_response_bot? + conversation.pending? && message.incoming? && inbox.response_bot_enabled? + end +end diff --git a/enterprise/app/services/enterprise/message_templates/response_bot_service.rb b/enterprise/app/services/enterprise/message_templates/response_bot_service.rb new file mode 100644 index 000000000..4082c3cb2 --- /dev/null +++ b/enterprise/app/services/enterprise/message_templates/response_bot_service.rb @@ -0,0 +1,121 @@ +class Enterprise::MessageTemplates::ResponseBotService + pattr_initialize [:conversation!] + + def perform + ActiveRecord::Base.transaction do + response = get_response(conversation.messages.last.content) + process_response(conversation.messages.last, response) + end + rescue StandardError => e + ChatwootExceptionTracker.new(e, account: conversation.account).capture_exception + true + end + + private + + delegate :contact, :account, :inbox, to: :conversation + + def get_response(content) + previous_messages = [] + get_previous_messages(previous_messages) + ChatGpt.new(response_sections(content)).generate_response('', previous_messages) + end + + def get_previous_messages(previous_messages) + conversation.messages.where(message_type: [:outgoing, :incoming]).where(private: false).find_each do |message| + next if message.content_type != 'text' + + role = determine_role(message) + previous_messages << { content: message.content, role: role } + end + end + + def determine_role(message) + message.message_type == 'incoming' ? 'user' : 'system' + end + + def response_sections(content) + sections = '' + + inbox.get_responses(content).each do |response| + sections += "{context_id: #{response.id}, context: #{response.question} ? #{response.answer}}" + end + sections + end + + def process_response(message, response) + if response == 'conversation_handoff' + process_action(message, 'handoff') + else + create_messages(response, conversation) + end + end + + def process_action(_message, action) + case action + when 'handoff' + conversation.messages.create!('message_type': :outgoing, 'account_id': conversation.account_id, 'inbox_id': conversation.inbox_id, + 'content': 'passing to an agent') + conversation.update(status: :open) + end + end + + def create_messages(response, conversation) + response, article_ids = process_response_content(response) + create_outgoing_message(response, conversation) + create_outgoing_message_with_cards(article_ids, conversation) if article_ids.present? + end + + def process_response_content(response) + # Regular expression to match '{context_ids: [ids]}' + regex = /{context_ids: \[(\d+(?:, *\d+)*)\]}/ + + # Extract ids from string + id_string = response[regex, 1] # This will give you '42, 43' + article_ids = id_string.split(',').map(&:to_i) if id_string # This will give you [42, 43] + + # Remove '{context_ids: [ids]}' from string + response = response.sub(regex, '') + + [response, article_ids] + end + + def create_outgoing_message(response, conversation) + conversation.messages.create!( + { + message_type: :outgoing, + account_id: conversation.account_id, + inbox_id: conversation.inbox_id, + content: response + } + ) + end + + def create_outgoing_message_with_cards(article_ids, conversation) + content_attributes = get_article_hash(article_ids.uniq) + return if content_attributes.blank? + + conversation.messages.create!( + { + message_type: :outgoing, + account_id: conversation.account_id, + inbox_id: conversation.inbox_id, + content: 'suggested articles', + content_type: 'article', + content_attributes: content_attributes + } + ) + end + + def get_article_hash(article_ids) + items = [] + article_ids.each do |article_id| + response = Response.find(article_id) + next if response.nil? + + items << { title: response.question, description: response.answer[0, 120], link: response.response_document.document_link } + end + + items.present? ? { items: items } : {} + end +end diff --git a/enterprise/app/services/features/response_bot_service.rb b/enterprise/app/services/features/response_bot_service.rb new file mode 100644 index 000000000..857ddfedd --- /dev/null +++ b/enterprise/app/services/features/response_bot_service.rb @@ -0,0 +1,83 @@ +class Features::ResponseBotService + MIGRATION_VERSION = ActiveRecord::Migration[7.0] + + def enable_in_installation + enable_vector_extension + create_tables + end + + def enable_vector_extension + MIGRATION_VERSION.enable_extension 'vector' + rescue ActiveRecord::StatementInvalid + print 'Vector extension not available' + end + + def disable_vector_extension + MIGRATION_VERSION.disable_extension 'vector' + end + + def vector_extension_enabled? + ActiveRecord::Base.connection.extension_enabled?('vector') + end + + def create_tables + return unless vector_extension_enabled? + + %i[response_sources response_documents responses].each do |table| + send("create_#{table}_table") + end + end + + def drop_tables + %i[responses response_documents response_sources].each do |table| + MIGRATION_VERSION.drop_table table if MIGRATION_VERSION.table_exists?(table) + end + end + + private + + def create_response_sources_table + return if MIGRATION_VERSION.table_exists?(:response_sources) + + MIGRATION_VERSION.create_table :response_sources do |t| + t.integer :source_type, null: false, default: 0 + t.string :name, null: false + t.string :source_link + t.references :source_model, polymorphic: true + t.bigint :account_id, null: false + t.bigint :inbox_id, null: false + t.timestamps + end + end + + def create_response_documents_table + return if MIGRATION_VERSION.table_exists?(:response_documents) + + MIGRATION_VERSION.create_table :response_documents do |t| + t.bigint :response_source_id, null: false + t.string :document_link + t.references :document, polymorphic: true + t.text :content + t.bigint :account_id, null: false + t.timestamps + end + + MIGRATION_VERSION.add_index :response_documents, :response_source_id + end + + def create_responses_table + return if MIGRATION_VERSION.table_exists?(:responses) + + MIGRATION_VERSION.create_table :responses do |t| + t.bigint :response_document_id + t.string :question, null: false + t.text :answer, null: false + t.bigint :account_id, null: false + t.vector :embedding, limit: 1536 + t.timestamps + end + + MIGRATION_VERSION.add_index :responses, :response_document_id + MIGRATION_VERSION.add_index :responses, :embedding, using: :ivfflat, opclass: :vector_l2_ops + end +end diff --git a/enterprise/app/services/openai/embeddings_service.rb b/enterprise/app/services/openai/embeddings_service.rb new file mode 100644 index 000000000..2ace41f1c --- /dev/null +++ b/enterprise/app/services/openai/embeddings_service.rb @@ -0,0 +1,22 @@ +class Openai::EmbeddingsService + def get_embedding(content) + fetch_embeddings(content) + end + + private + + def fetch_embeddings(input) + url = 'https://api.openai.com/v1/embeddings' + headers = { + 'Authorization' => "Bearer #{ENV.fetch('OPENAI_API_KEY')}", + 'Content-Type' => 'application/json' + } + data = { + input: input, + model: 'text-embedding-ada-002' + } + + response = Net::HTTP.post(URI(url), data.to_json, headers) + JSON.parse(response.body)['data'].pick('embedding') + end +end diff --git a/enterprise/app/services/page_crawler_service.rb b/enterprise/app/services/page_crawler_service.rb new file mode 100644 index 000000000..84b9f46b4 --- /dev/null +++ b/enterprise/app/services/page_crawler_service.rb @@ -0,0 +1,38 @@ +class PageCrawlerService + attr_reader :external_link + + def initialize(external_link) + @external_link = external_link + @doc = Nokogiri::HTML(HTTParty.get(external_link).body) + end + + def page_links + sitemap? ? extract_links_from_sitemap : extract_links_from_html + end + + def page_title + title_element = @doc.at_xpath('//title') + title_element&.text&.strip + end + + def body_text_content + ReverseMarkdown.convert @doc.at_xpath('//body'), unknown_tags: :bypass, github_flavored: true + end + + private + + def sitemap? + @external_link.end_with?('.xml') + end + + def extract_links_from_sitemap + @doc.xpath('//loc').to_set(&:text) + end + + def extract_links_from_html + @doc.xpath('//a/@href').to_set do |link| + absolute_url = URI.join(@external_link, link.value).to_s + absolute_url + end + end +end diff --git a/enterprise/lib/chat_gpt.rb b/enterprise/lib/chat_gpt.rb new file mode 100644 index 000000000..86c1b33bd --- /dev/null +++ b/enterprise/lib/chat_gpt.rb @@ -0,0 +1,41 @@ +class ChatGpt + def self.base_uri + 'https://api.openai.com' + end + + def initialize(context_sections = '') + @model = 'gpt-4' + system_message = { 'role': 'system', + 'content': 'You are a very enthusiastic customer support representative who loves ' \ + 'to help people! Given the following Context sections from the ' \ + 'documentation, continue the conversation with only that information, ' \ + "outputed in markdown format along with context_ids in format 'response \n {context_ids: [values] }' " \ + "\n If you are unsure and the answer is not explicitly written in the documentation, " \ + "say 'Sorry, I don't know how to help with that. Do you want to chat with a human agent?' " \ + "If they ask to Chat with human agent return text 'conversation_handoff'." \ + "Context sections: \n" \ + "\n\n #{context_sections}}" } + + @messages = [ + system_message + ] + end + + def generate_response(input, previous_messages = []) + previous_messages.each do |message| + @messages << message + end + + @messages << { 'role': 'user', 'content': input } if input.present? + headers = { 'Content-Type' => 'application/json', + 'Authorization' => "Bearer #{ENV.fetch('OPENAI_API_KEY')}" } + body = { + model: @model, + messages: @messages + }.to_json + + response = HTTParty.post("#{self.class.base_uri}/v1/chat/completions", headers: headers, body: body) + response_body = JSON.parse(response.body) + response_body['choices'][0]['message']['content'].strip + end +end diff --git a/spec/enterprise/controllers/api/v1/accounts/response_sources_controller_spec.rb b/spec/enterprise/controllers/api/v1/accounts/response_sources_controller_spec.rb new file mode 100644 index 000000000..74767da56 --- /dev/null +++ b/spec/enterprise/controllers/api/v1/accounts/response_sources_controller_spec.rb @@ -0,0 +1,135 @@ +require 'rails_helper' + +RSpec.describe 'Response Sources API', type: :request do + let!(:account) { create(:account) } + let!(:admin) { create(:user, account: account, role: :administrator) } + let!(:inbox) { create(:inbox, account: account) } + + before do + skip('Skipping since vector is not enabled in this environment') unless Features::ResponseBotService.new.vector_extension_enabled? + end + + describe 'POST /api/v1/accounts/{account.id}/response_sources/parse' do + let(:valid_params) do + { + link: 'http://test.test' + } + end + + context 'when it is an unauthenticated user' do + it 'returns unauthorized' do + post "/api/v1/accounts/#{account.id}/response_sources/parse", params: valid_params + + expect(response).to have_http_status(:unauthorized) + end + end + + context 'when it is an authenticated user' do + it 'returns links in the webpage' do + crawler = double + allow(PageCrawlerService).to receive(:new).and_return(crawler) + allow(crawler).to receive(:page_links).and_return(['http://test.test']) + + post "/api/v1/accounts/#{account.id}/response_sources/parse", headers: admin.create_new_auth_token, + params: valid_params + expect(response).to have_http_status(:success) + expect(response.parsed_body['links']).to eq(['http://test.test']) + end + end + end + + describe 'POST /api/v1/accounts/{account.id}/response_sources' do + let(:valid_params) do + { + response_source: { + name: 'Test', + source_link: 'http://test.test', + inbox_id: inbox.id, + response_documents_attributes: [ + { document_link: 'http://test1.test' }, + { document_link: 'http://test2.test' } + ] + } + } + end + + context 'when it is an unauthenticated user' do + it 'returns unauthorized' do + expect { post "/api/v1/accounts/#{account.id}/response_sources", params: valid_params }.not_to change(ResponseSource, :count) + + expect(response).to have_http_status(:unauthorized) + end + end + + context 'when it is an authenticated user' do + it 'creates the response sources and documents' do + expect do + post "/api/v1/accounts/#{account.id}/response_sources", headers: admin.create_new_auth_token, + params: valid_params + end.to change(ResponseSource, :count).by(1) + + expect(ResponseDocument.count).to eq(2) + expect(response).to have_http_status(:success) + end + end + end + + describe 'POST /api/v1/accounts/{account.id}/response_sources/{response_source.id}/add_document' do + let!(:response_source) { create(:response_source, account: account, inbox: inbox) } + let(:valid_params) do + { document_link: 'http://test.test' } + end + + context 'when it is an unauthenticated user' do + it 'returns unauthorized' do + expect do + post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/add_document", + params: valid_params + end.not_to change(ResponseDocument, :count) + + expect(response).to have_http_status(:unauthorized) + end + end + + context 'when it is an authenticated user' do + it 'creates the response sources and documents' do + expect do + post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/add_document", headers: admin.create_new_auth_token, + params: valid_params + end.to change(ResponseDocument, :count).by(1) + expect(response).to have_http_status(:success) + end + end + end + + describe 'POST /api/v1/accounts/{account.id}/response_sources/{response_source.id}/remove_document' do + let!(:response_source) { create(:response_source, account: account, inbox: inbox) } + let!(:response_document) { response_source.response_documents.create!(document_link: 'http://test.test') } + let(:valid_params) do + { document_id: response_document.id } + end + + context 'when it is an unauthenticated user' do + it 'returns unauthorized' do + expect do + post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/remove_document", + params: valid_params + end.not_to change(ResponseDocument, :count) + + expect(response).to have_http_status(:unauthorized) + end + end + + context 'when it is an authenticated user' do + it 'creates the response sources and documents' do + expect do + post "/api/v1/accounts/#{account.id}/response_sources/#{response_source.id}/remove_document", headers: admin.create_new_auth_token, + params: valid_params + end.to change(ResponseDocument, :count).by(-1) + expect(response).to have_http_status(:success) + + expect { response_document.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end +end diff --git a/spec/enterprise/controllers/enterprise/api/v1/accounts/inboxes_controller_spec.rb b/spec/enterprise/controllers/enterprise/api/v1/accounts/inboxes_controller_spec.rb index 724a7b0cb..fbde6917f 100644 --- a/spec/enterprise/controllers/enterprise/api/v1/accounts/inboxes_controller_spec.rb +++ b/spec/enterprise/controllers/enterprise/api/v1/accounts/inboxes_controller_spec.rb @@ -43,4 +43,44 @@ RSpec.describe 'Enterprise Inboxes API', type: :request do end end end + + describe 'GET /api/v1/accounts/{account.id}/inboxes/{inbox.id}/response_sources' do + let(:inbox) { create(:inbox, account: account) } + let(:agent) { create(:user, account: account, role: :agent) } + let(:administrator) { create(:user, account: account, role: :administrator) } + + before do + skip('Skipping since vector is not enabled in this environment') unless Features::ResponseBotService.new.vector_extension_enabled? + end + + context 'when it is an unauthenticated user' do + it 'returns unauthorized' do + get "/api/v1/accounts/#{account.id}/inboxes/#{inbox.id}/response_sources" + + expect(response).to have_http_status(:unauthorized) + end + end + + context 'when it is an authenticated user' do + it 'returns unauthorized for agents' do + get "/api/v1/accounts/#{account.id}/inboxes/#{inbox.id}/response_sources", + headers: agent.create_new_auth_token, + as: :json + + expect(response).to have_http_status(:unauthorized) + end + + it 'returns all response_sources belonging to the inbox to administrators' do + response_source = create(:response_source, account: account, inbox: inbox) + get "/api/v1/accounts/#{account.id}/inboxes/#{inbox.id}/response_sources", + headers: administrator.create_new_auth_token, + as: :json + + expect(response).to have_http_status(:success) + body = JSON.parse(response.body, symbolize_names: true) + expect(body.first[:id]).to eq(response_source.id) + expect(body.length).to eq(1) + end + end + end end diff --git a/spec/factories/response_source.rb b/spec/factories/response_source.rb new file mode 100644 index 000000000..4e1882e39 --- /dev/null +++ b/spec/factories/response_source.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :response_source do + name { Faker::Name.name } + source_link { Faker::Internet.url } + account + end +end