diff --git a/.env.example b/.env.example index f405ea03d..acda72798 100644 --- a/.env.example +++ b/.env.example @@ -230,3 +230,6 @@ AZURE_APP_SECRET= ## Change these values to fine tune performance # control the concurrency setting of sidekiq # SIDEKIQ_CONCURRENCY=10 + +# Sentiment analysis model file path +SENTIMENT_FILE_PATH= diff --git a/Gemfile b/Gemfile index b340d9be0..ffde3d3e3 100644 --- a/Gemfile +++ b/Gemfile @@ -165,6 +165,9 @@ gem 'omniauth' gem 'omniauth-google-oauth2' gem 'omniauth-rails_csrf_protection', '~> 1.0' +# Sentiment analysis +gem 'informers' + ### Gems required only in specific deployment environments ### ############################################################## diff --git a/Gemfile.lock b/Gemfile.lock index ccd3c39cd..6b52c1e67 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -145,6 +145,7 @@ GEM statsd-ruby (~> 1.1) bcrypt (3.1.18) bindex (0.8.1) + blingfire (0.1.8) bootsnap (1.16.0) msgpack (~> 1.2) brakeman (5.4.1) @@ -361,6 +362,10 @@ GEM image_processing (1.12.2) mini_magick (>= 4.9.5, < 5) ruby-vips (>= 2.0.17, < 3) + informers (0.2.0) + blingfire (>= 0.1.7) + numo-narray + onnxruntime (>= 0.5.1) jbuilder (2.11.5) actionview (>= 5.0.0) activesupport (>= 5.0.0) @@ -474,6 +479,7 @@ GEM racc (~> 1.4) nokogiri (1.15.2-x86_64-linux) racc (~> 1.4) + numo-narray (0.9.2.1) oauth (1.1.0) oauth-tty (~> 1.0, >= 1.0.1) snaky_hash (~> 2.0) @@ -502,6 +508,14 @@ GEM omniauth-rails_csrf_protection (1.0.1) actionpack (>= 4.2) omniauth (~> 2.0) + onnxruntime (0.7.6) + ffi + onnxruntime (0.7.6-arm64-darwin) + ffi + onnxruntime (0.7.6-x86_64-darwin) + ffi + onnxruntime (0.7.6-x86_64-linux) + ffi openssl (3.1.0) orm_adapter (0.5.0) os (1.1.4) @@ -846,6 +860,7 @@ DEPENDENCIES hashie html2text! image_processing + informers jbuilder json_refs json_schemer diff --git a/app/models/conversation.rb b/app/models/conversation.rb index efcca1c9b..25fa48c0d 100644 --- a/app/models/conversation.rb +++ b/app/models/conversation.rb @@ -304,3 +304,4 @@ class Conversation < ApplicationRecord end Conversation.include_mod_with('EnterpriseConversationConcern') +Conversation.include_mod_with('SentimentAnalysisHelper') diff --git a/app/models/message.rb b/app/models/message.rb index c025746c7..75117a2cd 100644 --- a/app/models/message.rb +++ b/app/models/message.rb @@ -12,6 +12,7 @@ # private :boolean default(FALSE) # processed_message_content :text # sender_type :string +# sentiment :jsonb # status :integer default("sent") # created_at :datetime not null # updated_at :datetime not null @@ -246,6 +247,7 @@ class Message < ApplicationRecord reopen_conversation notify_via_mail set_conversation_activity + update_message_sentiments dispatch_create_events send_reply execute_message_template_hooks @@ -371,4 +373,10 @@ class Message < ApplicationRecord conversation.update_columns(last_activity_at: created_at) # rubocop:enable Rails/SkipsModelValidations end + + def update_message_sentiments + # override in the enterprise ::Enterprise::SentimentAnalysisJob.perform_later(self) + end end + +Message.prepend_mod_with('Message') diff --git a/db/migrate/20230706090122_sentiment_column_to_messages.rb b/db/migrate/20230706090122_sentiment_column_to_messages.rb new file mode 100644 index 000000000..2d933e6d2 --- /dev/null +++ b/db/migrate/20230706090122_sentiment_column_to_messages.rb @@ -0,0 +1,5 @@ +class SentimentColumnToMessages < ActiveRecord::Migration[7.0] + def change + add_column :messages, :sentiment, :jsonb, default: {} + end +end diff --git a/db/schema.rb b/db/schema.rb index b8c552ab4..baa433702 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 2023_06_20_212340) do +ActiveRecord::Schema[7.0].define(version: 2023_07_06_090122) do # These are extensions that must be enabled in order to support this database enable_extension "pg_stat_statements" enable_extension "pg_trgm" @@ -669,6 +669,7 @@ ActiveRecord::Schema[7.0].define(version: 2023_06_20_212340) do t.jsonb "external_source_ids", default: {} t.jsonb "additional_attributes", default: {} t.text "processed_message_content" + t.jsonb "sentiment", default: {} t.index "((additional_attributes -> 'campaign_id'::text))", name: "index_messages_on_additional_attributes_campaign_id", using: :gin t.index ["account_id", "inbox_id"], name: "index_messages_on_account_id_and_inbox_id" t.index ["account_id"], name: "index_messages_on_account_id" diff --git a/enterprise/app/jobs/enterprise/sentiment_analysis_job.rb b/enterprise/app/jobs/enterprise/sentiment_analysis_job.rb new file mode 100644 index 000000000..856bdef9a --- /dev/null +++ b/enterprise/app/jobs/enterprise/sentiment_analysis_job.rb @@ -0,0 +1,39 @@ +class Enterprise::SentimentAnalysisJob < ApplicationJob + queue_as :default + + def perform(message) + return if message.account.locale != 'en' + return if valid_incoming_message?(message) + + save_message_sentiment(message) + rescue StandardError => e + Rails.logger.error("Sentiment Analysis Error for message #{message.id}: #{e}") + ChatwootExceptionTracker.new(e, account: message.account).capture_exception + end + + def save_message_sentiment(message) + # We are truncating the data here to avoind the OnnxRuntime::Error + # Indices element out of data bounds, idx=512 must be within the inclusive range [-512,511] + # While gathering the maningfull node the Array/tensor index is going out of bound + + text = message.content&.truncate(2900) + sentiment = model.predict(text) + message.sentiment = sentiment.merge(value: label_val(sentiment)) + + message.save! + end + + # Model initializes OnnxRuntime::Model, with given file for inference session and to create the tensor + def model + model_path = ENV.fetch('SENTIMENT_FILE_PATH', nil) + Informers::SentimentAnalysis.new(model_path) if model_path.present? + end + + def label_val(sentiment) + sentiment[:label] == 'positive' ? 1 : -1 + end + + def valid_incoming_message?(message) + !message.incoming? || message.private? + end +end diff --git a/enterprise/app/models/enterprise/message.rb b/enterprise/app/models/enterprise/message.rb new file mode 100644 index 000000000..30c49da3d --- /dev/null +++ b/enterprise/app/models/enterprise/message.rb @@ -0,0 +1,5 @@ +module Enterprise::Message + def update_message_sentiments + ::Enterprise::SentimentAnalysisJob.perform_later(self) + end +end diff --git a/enterprise/app/models/enterprise/sentiment_analysis_helper.rb b/enterprise/app/models/enterprise/sentiment_analysis_helper.rb new file mode 100644 index 000000000..6c26d5c07 --- /dev/null +++ b/enterprise/app/models/enterprise/sentiment_analysis_helper.rb @@ -0,0 +1,45 @@ +module Enterprise::SentimentAnalysisHelper + extend ActiveSupport::Concern + + included do + def opening_sentiments + records = incoming_messages.first(average_message_count) + average_sentiment(records) + end + + def closing_sentiments + return unless resolved? + + records = incoming_messages.last(average_message_count) + average_sentiment(records) + end + + def average_sentiment(records) + { + label: average_sentiment_label(records), + score: average_sentiment_score(records) + } + end + + private + + def average_sentiment_label(records) + value = records.pluck(:sentiment).sum { |a| a['value'].to_i } + value.negative? ? 'negative' : 'positive' + end + + def average_sentiment_score(records) + total = records.pluck(:sentiment).sum { |a| a['score'].to_f } + total / average_message_count + end + + def average_message_count + # incoming_messages.count >= 10 ? 5 : ((incoming_messages.count / 2) - 1) + 5 + end + + def incoming_messages + messages.incoming.where(private: false) + end + end +end diff --git a/spec/enterprise/jobs/enterprise/sentiment_analysis_job_spec.rb b/spec/enterprise/jobs/enterprise/sentiment_analysis_job_spec.rb new file mode 100644 index 000000000..d56f1bef5 --- /dev/null +++ b/spec/enterprise/jobs/enterprise/sentiment_analysis_job_spec.rb @@ -0,0 +1,82 @@ +require 'rails_helper' + +RSpec.describe Enterprise::SentimentAnalysisJob do + context 'when account locale set to english language' do + let(:account) { create(:account, locale: 'en') } + let(:message) { build(:message, content_type: nil, account: account) } + + context 'when update the message sentiments' do + let(:model_path) { 'sentiment-analysis.onnx' } + let(:model) { double } + + before do + allow(Informers::SentimentAnalysis).to receive(:new).with(model_path).and_return(model) + allow(model).to receive(:predict).and_return({ label: 'positive', score: '0.6' }) + end + + it 'with incoming message' do + with_modified_env SENTIMENT_FILE_PATH: 'sentiment-analysis.onnx' do + message.update(message_type: :incoming) + + described_class.perform_now(message) + + expect(message.sentiment).not_to be_empty + end + end + + it 'update sentiment label for positive message' do + with_modified_env SENTIMENT_FILE_PATH: 'sentiment-analysis.onnx' do + message.update(message_type: :incoming, content: 'I like your product') + + described_class.perform_now(message) + + expect(message.sentiment).not_to be_empty + expect(message.sentiment['label']).to eq('positive') + expect(message.sentiment['value']).to eq(1) + end + end + + it 'update sentiment label for negative message' do + with_modified_env SENTIMENT_FILE_PATH: 'sentiment-analysis.onnx' do + message.update(message_type: :incoming, content: 'I did not like your product') + allow(model).to receive(:predict).and_return({ label: 'negative', score: '0.6' }) + + described_class.perform_now(message) + + expect(message.sentiment).not_to be_empty + expect(message.sentiment['label']).to eq('negative') + expect(message.sentiment['value']).to eq(-1) + end + end + end + + context 'when does not update the message sentiments' do + it 'with outgoing message' do + message.update(message_type: :outgoing) + + described_class.perform_now(message) + + expect(message.sentiment).to be_empty + end + + it 'with private message' do + message.update(private: true) + + described_class.perform_now(message) + + expect(message.sentiment).to be_empty + end + end + end + + context 'when account locale is not set to english language' do + let(:account) { create(:account, locale: 'es') } + let(:message) { build(:message, content_type: nil, account: account) } + + it 'does not update the message sentiments' do + described_class.perform_now(message) + + expect(message.sentiment).to be_empty + end + end +end diff --git a/spec/enterprise/models/conversation_spec.rb b/spec/enterprise/models/conversation_spec.rb index 14f9e060a..e91f3f885 100644 --- a/spec/enterprise/models/conversation_spec.rb +++ b/spec/enterprise/models/conversation_spec.rb @@ -4,4 +4,34 @@ RSpec.describe Conversation, type: :model do describe 'associations' do it { is_expected.to belong_to(:sla_policy).optional } end + + describe 'conversation sentiments' do + include ActiveJob::TestHelper + + let(:conversation) { create(:conversation, additional_attributes: { referer: 'https://www.chatwoot.com/' }) } + + before do + 10.times do + message = create(:message, conversation_id: conversation.id, account_id: conversation.account_id, message_type: 'incoming') + message.update(sentiment: { 'label': 'positive', score: '0.4' }) + end + end + + it 'returns opening sentiments' do + sentiments = conversation.opening_sentiments + expect(sentiments[:label]).to eq('positive') + end + + it 'returns closing sentiments if conversation is not resolved' do + sentiments = conversation.closing_sentiments + expect(sentiments).to be_nil + end + + it 'returns closing sentiments if it is resolved' do + conversation.resolved! + + sentiments = conversation.closing_sentiments + expect(sentiments[:label]).to eq('positive') + end + end end diff --git a/spec/enterprise/models/message_spec.rb b/spec/enterprise/models/message_spec.rb new file mode 100644 index 000000000..4da7e6947 --- /dev/null +++ b/spec/enterprise/models/message_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'rails_helper' +require Rails.root.join 'spec/models/concerns/liquidable_shared.rb' + +RSpec.describe Message do + context 'with sentiment analysis' do + let(:message) { build(:message, message_type: :incoming, content_type: nil, account: create(:account)) } + + it 'calls SentimentAnalysisJob' do + allow(Enterprise::SentimentAnalysisJob).to receive(:perform_later).and_return(:perform_later).with(message) + + message.save! + + expect(Enterprise::SentimentAnalysisJob).to have_received(:perform_later) + end + end +end