diff --git a/.github/workflows/run_mfa_spec.yml b/.github/workflows/run_mfa_spec.yml index 61b406f8a..69d019cc9 100644 --- a/.github/workflows/run_mfa_spec.yml +++ b/.github/workflows/run_mfa_spec.yml @@ -70,6 +70,7 @@ jobs: spec/services/mfa/authentication_service_spec.rb \ spec/requests/api/v1/profile/mfa_controller_spec.rb \ spec/controllers/devise_overrides/sessions_controller_spec.rb \ + spec/models/application_record_external_credentials_encryption_spec.rb \ --profile=10 \ --format documentation env: diff --git a/app/javascript/dashboard/i18n/locale/en/report.json b/app/javascript/dashboard/i18n/locale/en/report.json index 7c42fdfba..c622170b0 100644 --- a/app/javascript/dashboard/i18n/locale/en/report.json +++ b/app/javascript/dashboard/i18n/locale/en/report.json @@ -51,6 +51,7 @@ }, "DATE_RANGE_OPTIONS": { "LAST_7_DAYS": "Last 7 days", + "LAST_14_DAYS": "Last 14 days", "LAST_30_DAYS": "Last 30 days", "LAST_3_MONTHS": "Last 3 months", "LAST_6_MONTHS": "Last 6 months", @@ -266,6 +267,8 @@ "NO_ENOUGH_DATA": "We've not received enough data points to generate report, Please try again later.", "DOWNLOAD_INBOX_REPORTS": "Download inbox reports", "FILTER_DROPDOWN_LABEL": "Select Inbox", + "ALL_INBOXES": "All Inboxes", + "SEARCH_INBOX": "Search Inbox", "METRICS": { "CONVERSATIONS": { "NAME": "Conversations", @@ -467,6 +470,13 @@ "CONVERSATIONS": "{count} conversations", "DOWNLOAD_REPORT": "Download report" }, + "RESOLUTION_HEATMAP": { + "HEADER": "Resolutions", + "NO_CONVERSATIONS": "No conversations", + "CONVERSATION": "{count} conversation", + "CONVERSATIONS": "{count} conversations", + "DOWNLOAD_REPORT": "Download report" + }, "AGENT_CONVERSATIONS": { "HEADER": "Conversations by agents", "LOADING_MESSAGE": "Loading agent metrics...", diff --git a/app/javascript/dashboard/routes/dashboard/settings/reports/LiveReports.vue b/app/javascript/dashboard/routes/dashboard/settings/reports/LiveReports.vue index a0eeb3ab9..1eb9640ac 100644 --- a/app/javascript/dashboard/routes/dashboard/settings/reports/LiveReports.vue +++ b/app/javascript/dashboard/routes/dashboard/settings/reports/LiveReports.vue @@ -1,6 +1,7 @@ - - diff --git a/app/javascript/dashboard/routes/dashboard/settings/reports/components/HeatmapContainer.vue b/app/javascript/dashboard/routes/dashboard/settings/reports/components/HeatmapContainer.vue deleted file mode 100644 index 280e1d6ee..000000000 --- a/app/javascript/dashboard/routes/dashboard/settings/reports/components/HeatmapContainer.vue +++ /dev/null @@ -1,119 +0,0 @@ - - - diff --git a/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/BaseHeatmap.vue b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/BaseHeatmap.vue new file mode 100644 index 000000000..3f9dd9db4 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/BaseHeatmap.vue @@ -0,0 +1,214 @@ + + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/BaseHeatmapContainer.vue b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/BaseHeatmapContainer.vue new file mode 100644 index 000000000..2a692b12a --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/BaseHeatmapContainer.vue @@ -0,0 +1,265 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/ConversationHeatmapContainer.vue b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/ConversationHeatmapContainer.vue new file mode 100644 index 000000000..394b7cdc2 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/ConversationHeatmapContainer.vue @@ -0,0 +1,18 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/HeatmapTooltip.vue b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/HeatmapTooltip.vue new file mode 100644 index 000000000..79377a6a3 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/HeatmapTooltip.vue @@ -0,0 +1,57 @@ + + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/ResolutionHeatmapContainer.vue b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/ResolutionHeatmapContainer.vue new file mode 100644 index 000000000..24530d0c7 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/ResolutionHeatmapContainer.vue @@ -0,0 +1,18 @@ + + + diff --git a/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/composables/useHeatmapTooltip.js b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/composables/useHeatmapTooltip.js new file mode 100644 index 000000000..28b050542 --- /dev/null +++ b/app/javascript/dashboard/routes/dashboard/settings/reports/components/heatmaps/composables/useHeatmapTooltip.js @@ -0,0 +1,34 @@ +import { ref } from 'vue'; + +export function useHeatmapTooltip() { + const visible = ref(false); + const x = ref(0); + const y = ref(0); + const value = ref(null); + + let timeoutId = null; + + const show = (event, cellValue) => { + clearTimeout(timeoutId); + + // Update position immediately for smooth movement + const rect = event.target.getBoundingClientRect(); + x.value = rect.left + rect.width / 2; + y.value = rect.top; + + // Only delay content update and visibility + timeoutId = setTimeout(() => { + value.value = cellValue; + visible.value = true; + }, 100); + }; + + const hide = () => { + clearTimeout(timeoutId); + timeoutId = setTimeout(() => { + visible.value = false; + }, 50); + }; + + return { visible, x, y, value, show, hide }; +} diff --git a/app/javascript/dashboard/store/modules/conversationStats.js b/app/javascript/dashboard/store/modules/conversationStats.js index 353c1e59a..ba3e5c455 100644 --- a/app/javascript/dashboard/store/modules/conversationStats.js +++ b/app/javascript/dashboard/store/modules/conversationStats.js @@ -26,12 +26,12 @@ const fetchMetaData = async (commit, params) => { }; const debouncedFetchMetaData = debounce(fetchMetaData, 500, false, 1500); -const longDebouncedFetchMetaData = debounce(fetchMetaData, 1000, false, 8000); +const longDebouncedFetchMetaData = debounce(fetchMetaData, 5000, false, 10000); const superLongDebouncedFetchMetaData = debounce( fetchMetaData, - 1500, + 10000, false, - 10000 + 20000 ); export const actions = { diff --git a/app/javascript/dashboard/store/modules/reports.js b/app/javascript/dashboard/store/modules/reports.js index bb5364bb5..99b2acf18 100644 --- a/app/javascript/dashboard/store/modules/reports.js +++ b/app/javascript/dashboard/store/modules/reports.js @@ -57,11 +57,13 @@ const state = { uiFlags: { isFetchingAccountConversationMetric: false, isFetchingAccountConversationsHeatmap: false, + isFetchingAccountResolutionsHeatmap: false, isFetchingAgentConversationMetric: false, isFetchingTeamConversationMetric: false, }, accountConversationMetric: {}, accountConversationHeatmap: [], + accountResolutionHeatmap: [], agentConversationMetric: [], teamConversationMetric: [], }, @@ -89,6 +91,9 @@ const getters = { getAccountConversationHeatmapData(_state) { return _state.overview.accountConversationHeatmap; }, + getAccountResolutionHeatmapData(_state) { + return _state.overview.accountResolutionHeatmap; + }, getAgentConversationMetric(_state) { return _state.overview.agentConversationMetric; }, @@ -130,6 +135,16 @@ export const actions = { commit(types.default.TOGGLE_HEATMAP_LOADING, false); }); }, + fetchAccountResolutionHeatmap({ commit }, reportObj) { + commit(types.default.TOGGLE_RESOLUTION_HEATMAP_LOADING, true); + Report.getReports({ ...reportObj, groupBy: 'hour' }).then(heatmapData => { + let { data } = heatmapData; + data = clampDataBetweenTimeline(data, reportObj.from, reportObj.to); + + commit(types.default.SET_RESOLUTION_HEATMAP_DATA, data); + commit(types.default.TOGGLE_RESOLUTION_HEATMAP_LOADING, false); + }); + }, fetchAccountSummary({ commit }, reportObj) { commit(types.default.SET_ACCOUNT_SUMMARY_STATUS, STATUS.FETCHING); Report.getSummary( @@ -287,6 +302,9 @@ const mutations = { [types.default.SET_HEATMAP_DATA](_state, heatmapData) { _state.overview.accountConversationHeatmap = heatmapData; }, + [types.default.SET_RESOLUTION_HEATMAP_DATA](_state, heatmapData) { + _state.overview.accountResolutionHeatmap = heatmapData; + }, [types.default.TOGGLE_ACCOUNT_REPORT_LOADING](_state, { metric, value }) { _state.accountReport.isFetching[metric] = value; }, @@ -299,6 +317,9 @@ const mutations = { [types.default.TOGGLE_HEATMAP_LOADING](_state, flag) { _state.overview.uiFlags.isFetchingAccountConversationsHeatmap = flag; }, + [types.default.TOGGLE_RESOLUTION_HEATMAP_LOADING](_state, flag) { + _state.overview.uiFlags.isFetchingAccountResolutionsHeatmap = flag; + }, [types.default.SET_ACCOUNT_SUMMARY](_state, summaryData) { _state.accountSummary = summaryData; }, diff --git a/app/javascript/dashboard/store/mutation-types.js b/app/javascript/dashboard/store/mutation-types.js index 4f361e140..68ff79e66 100644 --- a/app/javascript/dashboard/store/mutation-types.js +++ b/app/javascript/dashboard/store/mutation-types.js @@ -187,6 +187,8 @@ export default { SET_ACCOUNT_REPORTS: 'SET_ACCOUNT_REPORTS', SET_HEATMAP_DATA: 'SET_HEATMAP_DATA', TOGGLE_HEATMAP_LOADING: 'TOGGLE_HEATMAP_LOADING', + SET_RESOLUTION_HEATMAP_DATA: 'SET_RESOLUTION_HEATMAP_DATA', + TOGGLE_RESOLUTION_HEATMAP_LOADING: 'TOGGLE_RESOLUTION_HEATMAP_LOADING', SET_ACCOUNT_SUMMARY: 'SET_ACCOUNT_SUMMARY', SET_BOT_SUMMARY: 'SET_BOT_SUMMARY', TOGGLE_ACCOUNT_REPORT_LOADING: 'TOGGLE_ACCOUNT_REPORT_LOADING', diff --git a/app/jobs/agent_bots/webhook_job.rb b/app/jobs/agent_bots/webhook_job.rb index d0c4e7959..b3a3d6cc1 100644 --- a/app/jobs/agent_bots/webhook_job.rb +++ b/app/jobs/agent_bots/webhook_job.rb @@ -1,3 +1,7 @@ class AgentBots::WebhookJob < WebhookJob queue_as :high + + def perform(url, payload, webhook_type = :agent_bot_webhook) + super(url, payload, webhook_type) + end end diff --git a/app/models/channel/email.rb b/app/models/channel/email.rb index a8fadb61e..b1124dd75 100644 --- a/app/models/channel/email.rb +++ b/app/models/channel/email.rb @@ -40,6 +40,12 @@ class Channel::Email < ApplicationRecord AUTHORIZATION_ERROR_THRESHOLD = 10 + # TODO: Remove guard once encryption keys become mandatory (target 3-4 releases out). + if Chatwoot.encryption_configured? + encrypts :imap_password + encrypts :smtp_password + end + self.table_name = 'channel_email' EDITABLE_ATTRS = [:email, :imap_enabled, :imap_login, :imap_password, :imap_address, :imap_port, :imap_enable_ssl, :smtp_enabled, :smtp_login, :smtp_password, :smtp_address, :smtp_port, :smtp_domain, :smtp_enable_starttls_auto, diff --git a/app/models/channel/facebook_page.rb b/app/models/channel/facebook_page.rb index 1b6e151cb..1866d245b 100644 --- a/app/models/channel/facebook_page.rb +++ b/app/models/channel/facebook_page.rb @@ -21,6 +21,12 @@ class Channel::FacebookPage < ApplicationRecord include Channelable include Reauthorizable + # TODO: Remove guard once encryption keys become mandatory (target 3-4 releases out). + if Chatwoot.encryption_configured? + encrypts :page_access_token + encrypts :user_access_token + end + self.table_name = 'channel_facebook_pages' validates :page_id, uniqueness: { scope: :account_id } diff --git a/app/models/channel/instagram.rb b/app/models/channel/instagram.rb index 964a4c1a2..7e6444b30 100644 --- a/app/models/channel/instagram.rb +++ b/app/models/channel/instagram.rb @@ -19,6 +19,9 @@ class Channel::Instagram < ApplicationRecord include Reauthorizable self.table_name = 'channel_instagram' + # TODO: Remove guard once encryption keys become mandatory (target 3-4 releases out). + encrypts :access_token if Chatwoot.encryption_configured? + AUTHORIZATION_ERROR_THRESHOLD = 1 validates :access_token, presence: true diff --git a/app/models/channel/line.rb b/app/models/channel/line.rb index a417dbf64..63b0924da 100644 --- a/app/models/channel/line.rb +++ b/app/models/channel/line.rb @@ -18,6 +18,12 @@ class Channel::Line < ApplicationRecord include Channelable + # TODO: Remove guard once encryption keys become mandatory (target 3-4 releases out). + if Chatwoot.encryption_configured? + encrypts :line_channel_secret + encrypts :line_channel_token + end + self.table_name = 'channel_line' EDITABLE_ATTRS = [:line_channel_id, :line_channel_secret, :line_channel_token].freeze diff --git a/app/models/channel/telegram.rb b/app/models/channel/telegram.rb index b00897614..b18c6dbc9 100644 --- a/app/models/channel/telegram.rb +++ b/app/models/channel/telegram.rb @@ -17,6 +17,9 @@ class Channel::Telegram < ApplicationRecord include Channelable + # TODO: Remove guard once encryption keys become mandatory (target 3-4 releases out). + encrypts :bot_token, deterministic: true if Chatwoot.encryption_configured? + self.table_name = 'channel_telegram' EDITABLE_ATTRS = [:bot_token].freeze diff --git a/app/models/channel/twilio_sms.rb b/app/models/channel/twilio_sms.rb index 73e5c873e..2f9130cbb 100644 --- a/app/models/channel/twilio_sms.rb +++ b/app/models/channel/twilio_sms.rb @@ -28,6 +28,9 @@ class Channel::TwilioSms < ApplicationRecord self.table_name = 'channel_twilio_sms' + # TODO: Remove guard once encryption keys become mandatory (target 3-4 releases out). + encrypts :auth_token if Chatwoot.encryption_configured? + validates :account_sid, presence: true # The same parameter is used to store api_key_secret if api_key authentication is opted validates :auth_token, presence: true diff --git a/app/models/channel/twitter_profile.rb b/app/models/channel/twitter_profile.rb index d0f765e9f..4ec167ce5 100644 --- a/app/models/channel/twitter_profile.rb +++ b/app/models/channel/twitter_profile.rb @@ -19,6 +19,12 @@ class Channel::TwitterProfile < ApplicationRecord include Channelable + # TODO: Remove guard once encryption keys become mandatory (target 3-4 releases out). + if Chatwoot.encryption_configured? + encrypts :twitter_access_token + encrypts :twitter_access_token_secret + end + self.table_name = 'channel_twitter_profiles' validates :profile_id, uniqueness: { scope: :account_id } diff --git a/app/models/integrations/hook.rb b/app/models/integrations/hook.rb index ca77fa13d..97d3f91ae 100644 --- a/app/models/integrations/hook.rb +++ b/app/models/integrations/hook.rb @@ -21,6 +21,9 @@ class Integrations::Hook < ApplicationRecord before_validation :ensure_hook_type after_create :trigger_setup_if_crm + # TODO: Remove guard once encryption keys become mandatory (target 3-4 releases out). + encrypts :access_token, deterministic: true if Chatwoot.encryption_configured? + validates :account_id, presence: true validates :app_id, presence: true validates :inbox_id, presence: true, if: -> { hook_type == 'inbox' } diff --git a/app/services/whatsapp/incoming_message_service_helpers.rb b/app/services/whatsapp/incoming_message_service_helpers.rb index e40dc408f..705babbba 100644 --- a/app/services/whatsapp/incoming_message_service_helpers.rb +++ b/app/services/whatsapp/incoming_message_service_helpers.rb @@ -47,6 +47,15 @@ module Whatsapp::IncomingMessageServiceHelpers %w[reaction ephemeral unsupported request_welcome].include?(message_type) end + def argentina_phone_number?(phone_number) + phone_number.match(/^54/) + end + + def normalised_argentina_mobil_number(phone_number) + # Remove 9 before country code + phone_number.sub(/^549/, '54') + end + def processed_waid(waid) Whatsapp::PhoneNumberNormalizationService.new(inbox).normalize_and_find_contact(waid) end diff --git a/app/services/whatsapp/phone_normalizers/argentina_phone_normalizer.rb b/app/services/whatsapp/phone_normalizers/argentina_phone_normalizer.rb new file mode 100644 index 000000000..109a0683f --- /dev/null +++ b/app/services/whatsapp/phone_normalizers/argentina_phone_normalizer.rb @@ -0,0 +1,18 @@ +# Handles Argentina phone number normalization +# +# Argentina phone numbers can appear with or without "9" after country code +# This normalizer removes the "9" when present to create consistent format: 54 + area + number +class Whatsapp::PhoneNormalizers::ArgentinaPhoneNormalizer < Whatsapp::PhoneNormalizers::BasePhoneNormalizer + def normalize(waid) + return waid unless handles_country?(waid) + + # Remove "9" after country code if present (549 → 54) + waid.sub(/^549/, '54') + end + + private + + def country_code_pattern + /^54/ + end +end diff --git a/app/services/whatsapp/phone_number_normalization_service.rb b/app/services/whatsapp/phone_number_normalization_service.rb index b8e416794..cd10db0d0 100644 --- a/app/services/whatsapp/phone_number_normalization_service.rb +++ b/app/services/whatsapp/phone_number_normalization_service.rb @@ -1,5 +1,5 @@ # Service to handle phone number normalization for WhatsApp messages -# Currently supports Brazil phone number format variations +# Currently supports Brazil and Argentina phone number format variations # Designed to be extensible for additional countries in future PRs # # Usage: Whatsapp::PhoneNumberNormalizationService.new(inbox).normalize_and_find_contact(waid) @@ -34,6 +34,7 @@ class Whatsapp::PhoneNumberNormalizationService end NORMALIZERS = [ - Whatsapp::PhoneNormalizers::BrazilPhoneNormalizer + Whatsapp::PhoneNormalizers::BrazilPhoneNormalizer, + Whatsapp::PhoneNormalizers::ArgentinaPhoneNormalizer ].freeze end diff --git a/config/application.rb b/config/application.rb index d644dd28f..aa150794a 100644 --- a/config/application.rb +++ b/config/application.rb @@ -75,7 +75,11 @@ module Chatwoot config.active_record.encryption.primary_key = ENV['ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY'] config.active_record.encryption.deterministic_key = ENV.fetch('ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY', nil) config.active_record.encryption.key_derivation_salt = ENV.fetch('ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT', nil) + # TODO: Remove once encryption is mandatory and legacy plaintext is migrated. config.active_record.encryption.support_unencrypted_data = true + # Extend deterministic queries so they match both encrypted and plaintext rows + config.active_record.encryption.extend_queries = true + # Store a per-row key reference to support future key rotation config.active_record.encryption.store_key_references = true end end @@ -94,6 +98,8 @@ module Chatwoot end def self.encryption_configured? + # TODO: Once Active Record encryption keys are mandatory (target 3-4 releases out), + # remove this guard and assume encryption is always enabled. # Check if proper encryption keys are configured # MFA/2FA features should only be enabled when proper keys are set ENV['ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY'].present? && diff --git a/config/locales/en.yml b/config/locales/en.yml index 6afab9253..ad54b8dfa 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -202,6 +202,8 @@ en: captain: resolved: 'Conversation was marked resolved by %{user_name} due to inactivity' open: 'Conversation was marked open by %{user_name}' + agent_bot: + error_moved_to_open: 'Conversation was marked open by system due to an error with the agent bot.' status: resolved: 'Conversation was marked resolved by %{user_name}' contact_resolved: 'Conversation was resolved by %{contact_name}' diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 50a47a20b..138cf78b3 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -27,6 +27,7 @@ - purgable - housekeeping - async_database_migration + - bulk_reindex_low - active_storage_analysis - active_storage_purge - action_mailbox_incineration diff --git a/lib/seeders/reports/conversation_creator.rb b/lib/seeders/reports/conversation_creator.rb index b6259de7d..1cd11ef33 100644 --- a/lib/seeders/reports/conversation_creator.rb +++ b/lib/seeders/reports/conversation_creator.rb @@ -16,8 +16,11 @@ class Seeders::Reports::ConversationCreator @priorities = [nil, 'urgent', 'high', 'medium', 'low'] end + # rubocop:disable Metrics/MethodLength def create_conversation(created_at:) conversation = nil + should_resolve = false + resolution_time = nil ActiveRecord::Base.transaction do travel_to(created_at) do @@ -26,14 +29,35 @@ class Seeders::Reports::ConversationCreator add_labels_to_conversation(conversation) create_messages_for_conversation(conversation) - resolve_conversation_if_needed(conversation) + + # Determine if should resolve but don't update yet + should_resolve = rand > 0.3 + if should_resolve + resolution_delay = rand((30.minutes)..(24.hours)) + resolution_time = created_at + resolution_delay + end end travel_back end + # Now resolve outside of time travel if needed + if should_resolve && resolution_time + # rubocop:disable Rails/SkipsModelValidations + conversation.update_column(:status, :resolved) + conversation.update_column(:updated_at, resolution_time) + # rubocop:enable Rails/SkipsModelValidations + + # Trigger the event with proper timestamp + travel_to(resolution_time) do + trigger_conversation_resolved_event(conversation) + end + travel_back + end + conversation end + # rubocop:enable Metrics/MethodLength private @@ -85,16 +109,6 @@ class Seeders::Reports::ConversationCreator message_creator.create_messages end - def resolve_conversation_if_needed(conversation) - return unless rand < 0.7 - - resolution_delay = rand((30.minutes)..(24.hours)) - travel(resolution_delay) - conversation.update!(status: :resolved) - - trigger_conversation_resolved_event(conversation) - end - def trigger_conversation_resolved_event(conversation) event_data = { conversation: conversation } diff --git a/lib/webhooks/trigger.rb b/lib/webhooks/trigger.rb index 41b3a415d..95c399d54 100644 --- a/lib/webhooks/trigger.rb +++ b/lib/webhooks/trigger.rb @@ -31,14 +31,33 @@ class Webhooks::Trigger end def handle_error(error) - return unless should_handle_error? + return unless SUPPORTED_ERROR_HANDLE_EVENTS.include?(@payload[:event]) return unless message - update_message_status(error) + case @webhook_type + when :agent_bot_webhook + conversation = message.conversation + return unless conversation&.pending? + + conversation.open! + create_agent_bot_error_activity(conversation) + when :api_inbox_webhook + update_message_status(error) + end end - def should_handle_error? - @webhook_type == :api_inbox_webhook && SUPPORTED_ERROR_HANDLE_EVENTS.include?(@payload[:event]) + def create_agent_bot_error_activity(conversation) + content = I18n.t('conversations.activity.agent_bot.error_moved_to_open') + Conversations::ActivityMessageJob.perform_later(conversation, activity_message_params(conversation, content)) + end + + def activity_message_params(conversation, content) + { + account_id: conversation.account_id, + inbox_id: conversation.inbox_id, + message_type: :activity, + content: content + } end def update_message_status(error) diff --git a/script/bulk_reindex_messages.rb b/script/bulk_reindex_messages.rb new file mode 100644 index 000000000..1e19f70a7 --- /dev/null +++ b/script/bulk_reindex_messages.rb @@ -0,0 +1,58 @@ +# Bulk reindex all messages with throttling to prevent DB overload +# This creates jobs slowly to avoid overwhelming the database connection pool +# Usage: RAILS_ENV=production POSTGRES_STATEMENT_TIMEOUT=6000s bundle exec rails runner script/bulk_reindex_messages.rb + +JOBS_PER_MINUTE = 50 # Adjust based on your DB capacity +BATCH_SIZE = 1000 # Messages per job + +batch_count = 0 +total_batches = (Message.count / BATCH_SIZE.to_f).ceil +start_time = Time.zone.now + +index_name = Message.searchkick_index.name + +puts '=' * 80 +puts "Bulk Reindex Started at #{start_time}" +puts '=' * 80 +puts "Total messages: #{Message.count}" +puts "Batch size: #{BATCH_SIZE}" +puts "Total batches: #{total_batches}" +puts "Index name: #{index_name}" +puts "Rate: #{JOBS_PER_MINUTE} jobs/minute (#{JOBS_PER_MINUTE * BATCH_SIZE} messages/minute)" +puts "Estimated time: #{(total_batches / JOBS_PER_MINUTE.to_f / 60).round(2)} hours" +puts '=' * 80 +puts '' + +sleep(15) + +Message.find_in_batches(batch_size: BATCH_SIZE).with_index do |batch, index| + batch_count += 1 + + # Enqueue to low priority queue with proper format + Searchkick::BulkReindexJob.set(queue: :bulk_reindex_low).perform_later( + class_name: 'Message', + index_name: index_name, + batch_id: index, + record_ids: batch.map(&:id) # Keep as integers like Message.reindex does + ) + + # Throttle: wait after every N jobs + if (batch_count % JOBS_PER_MINUTE).zero? + elapsed = Time.zone.now - start_time + progress = (batch_count.to_f / total_batches * 100).round(2) + queue_size = Sidekiq::Queue.new('bulk_reindex_low').size + + puts "[#{Time.zone.now.strftime('%Y-%m-%d %H:%M:%S')}] Progress: #{batch_count}/#{total_batches} (#{progress}%)" + puts " Queue size: #{queue_size}" + puts " Elapsed: #{(elapsed / 3600).round(2)} hours" + puts " ETA: #{((elapsed / batch_count * (total_batches - batch_count)) / 3600).round(2)} hours remaining" + puts '' + + sleep(60) + end +end + +puts '=' * 80 +puts "Done! Created #{batch_count} jobs" +puts "Total time: #{((Time.zone.now - start_time) / 3600).round(2)} hours" +puts '=' * 80 diff --git a/script/monitor_reindex.rb b/script/monitor_reindex.rb new file mode 100644 index 000000000..6a2c1ee6c --- /dev/null +++ b/script/monitor_reindex.rb @@ -0,0 +1,19 @@ +# Monitor bulk reindex progress +# RAILS_ENV=production bundle exec rails runner script/monitor_reindex.rb + +puts 'Monitoring bulk reindex progress (Ctrl+C to stop)...' +puts '' + +loop do + bulk_queue = Sidekiq::Queue.new('bulk_reindex_low') + prod_queue = Sidekiq::Queue.new('async_database_migration') + retry_set = Sidekiq::RetrySet.new + + puts "[#{Time.zone.now.strftime('%Y-%m-%d %H:%M:%S')}]" + puts " Bulk Reindex Queue: #{bulk_queue.size} jobs" + puts " Production Queue: #{prod_queue.size} jobs" + puts " Retry Queue: #{retry_set.size} jobs" + puts " #{('-' * 60)}" + + sleep(30) +end diff --git a/script/reindex_single_account.rb b/script/reindex_single_account.rb new file mode 100644 index 000000000..cb7dd8c87 --- /dev/null +++ b/script/reindex_single_account.rb @@ -0,0 +1,58 @@ +# Reindex messages for a single account +# Usage: bundle exec rails runner script/reindex_single_account.rb ACCOUNT_ID [DAYS_BACK] + +#account_id = ARGV[0]&.to_i +days_back = (ARGV[1] || 30).to_i + +# if account_id.nil? || account_id.zero? +# puts "Usage: bundle exec rails runner script/reindex_single_account.rb ACCOUNT_ID [DAYS_BACK]" +# puts "Example: bundle exec rails runner script/reindex_single_account.rb 93293 30" +# exit 1 +# end + +# account = Account.find(account_id) +# puts "=" * 80 +# puts "Reindexing messages for: #{account.name} (ID: #{account.id})" +# puts "=" * 80 + +# Enable feature if not already enabled +# unless account.feature_enabled?('advanced_search_indexing') +# puts "Enabling advanced_search_indexing feature..." +# account.enable_features(:advanced_search_indexing) +# account.save! +# end + +# Get messages to index +# messages = Message.where(account_id: account.id) +# .where(message_type: [0, 1]) # incoming/outgoing only +# .where('created_at >= ?', days_back.days.ago) + +messages = Message.where('created_at >= ?', days_back.days.ago) + +puts "Found #{messages.count} messages to index (last #{days_back} days)" +puts '' + +sleep(15) + +# Create bulk reindex jobs +index_name = Message.searchkick_index.name +batch_count = 0 + +messages.find_in_batches(batch_size: 1000).with_index do |batch, index| + Searchkick::BulkReindexJob.set(queue: :bulk_reindex_low).perform_later( + class_name: 'Message', + index_name: index_name, + batch_id: index, + record_ids: batch.map(&:id) + ) + + batch_count += 1 + print '.' + sleep(0.5) # Small delay +end + +puts '' +puts '=' * 80 +puts "Done! Created #{batch_count} bulk reindex jobs" +puts 'Messages will be indexed shortly via the bulk_reindex_low queue' +puts '=' * 80 diff --git a/spec/lib/webhooks/trigger_spec.rb b/spec/lib/webhooks/trigger_spec.rb index 8ff2a21a5..224a35e07 100644 --- a/spec/lib/webhooks/trigger_spec.rb +++ b/spec/lib/webhooks/trigger_spec.rb @@ -1,6 +1,8 @@ require 'rails_helper' describe Webhooks::Trigger do + include ActiveJob::TestHelper + subject(:trigger) { described_class } let!(:account) { create(:account) } @@ -8,8 +10,18 @@ describe Webhooks::Trigger do let!(:conversation) { create(:conversation, inbox: inbox) } let!(:message) { create(:message, account: account, inbox: inbox, conversation: conversation) } - let!(:webhook_type) { :api_inbox_webhook } + let(:webhook_type) { :api_inbox_webhook } let!(:url) { 'https://test.com' } + let(:agent_bot_error_content) { I18n.t('conversations.activity.agent_bot.error_moved_to_open') } + + before do + ActiveJob::Base.queue_adapter = :test + end + + after do + clear_enqueued_jobs + clear_performed_jobs + end describe '#execute' do it 'triggers webhook' do @@ -54,6 +66,57 @@ describe Webhooks::Trigger do ).and_raise(RestClient::ExceptionWithResponse.new('error', 500)).once expect { trigger.execute(url, payload, webhook_type) }.to change { message.reload.status }.from('sent').to('failed') end + + context 'when webhook type is agent bot' do + let(:webhook_type) { :agent_bot_webhook } + + it 'reopens conversation and enqueues activity message if pending' do + conversation.update(status: :pending) + payload = { event: 'message_created', conversation: { id: conversation.id }, id: message.id } + + expect(RestClient::Request).to receive(:execute) + .with( + method: :post, + url: url, + payload: payload.to_json, + headers: { content_type: :json, accept: :json }, + timeout: 5 + ).and_raise(RestClient::ExceptionWithResponse.new('error', 500)).once + + expect do + perform_enqueued_jobs do + trigger.execute(url, payload, webhook_type) + end + end.not_to(change { message.reload.status }) + + expect(conversation.reload.status).to eq('open') + + activity_message = conversation.reload.messages.order(:created_at).last + expect(activity_message.message_type).to eq('activity') + expect(activity_message.content).to eq(agent_bot_error_content) + end + + it 'does not change message status or enqueue activity when conversation is not pending' do + payload = { event: 'message_created', conversation: { id: conversation.id }, id: message.id } + + expect(RestClient::Request).to receive(:execute) + .with( + method: :post, + url: url, + payload: payload.to_json, + headers: { content_type: :json, accept: :json }, + timeout: 5 + ).and_raise(RestClient::ExceptionWithResponse.new('error', 500)).once + + expect do + trigger.execute(url, payload, webhook_type) + end.not_to(change { message.reload.status }) + + expect(Conversations::ActivityMessageJob).not_to have_been_enqueued + + expect(conversation.reload.status).to eq('open') + end + end end it 'does not update message status if webhook fails for other events' do diff --git a/spec/models/application_record_external_credentials_encryption_spec.rb b/spec/models/application_record_external_credentials_encryption_spec.rb new file mode 100644 index 000000000..65c347434 --- /dev/null +++ b/spec/models/application_record_external_credentials_encryption_spec.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe ApplicationRecord do + it_behaves_like 'encrypted external credential', + factory: :channel_email, + attribute: :smtp_password, + value: 'smtp-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_email, + attribute: :imap_password, + value: 'imap-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_twilio_sms, + attribute: :auth_token, + value: 'twilio-secret' + + it_behaves_like 'encrypted external credential', + factory: :integrations_hook, + attribute: :access_token, + value: 'hook-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_facebook_page, + attribute: :page_access_token, + value: 'fb-page-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_facebook_page, + attribute: :user_access_token, + value: 'fb-user-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_instagram, + attribute: :access_token, + value: 'ig-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_line, + attribute: :line_channel_secret, + value: 'line-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_line, + attribute: :line_channel_token, + value: 'line-token-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_telegram, + attribute: :bot_token, + value: 'telegram-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_twitter_profile, + attribute: :twitter_access_token, + value: 'twitter-access-secret' + + it_behaves_like 'encrypted external credential', + factory: :channel_twitter_profile, + attribute: :twitter_access_token_secret, + value: 'twitter-secret-secret' + + context 'when backfilling legacy plaintext' do + before do + skip('encryption keys missing; see run_mfa_spec workflow') unless Chatwoot.encryption_configured? + end + + it 'reads existing plaintext and encrypts on update' do + account = create(:account) + channel = create(:channel_email, account: account, smtp_password: nil) + + # Simulate legacy plaintext by updating the DB directly + sql = ActiveRecord::Base.send( + :sanitize_sql_array, + ['UPDATE channel_email SET smtp_password = ? WHERE id = ?', 'legacy-plain', channel.id] + ) + ActiveRecord::Base.connection.execute(sql) + + legacy_record = Channel::Email.find(channel.id) + expect(legacy_record.smtp_password).to eq('legacy-plain') + + legacy_record.update!(smtp_password: 'encrypted-now') + + stored_value = legacy_record.reload.read_attribute_before_type_cast(:smtp_password) + expect(stored_value).to be_present + expect(stored_value).not_to include('encrypted-now') + expect(legacy_record.smtp_password).to eq('encrypted-now') + end + end + + context 'when looking up telegram legacy records' do + before do + skip('encryption keys missing; see run_mfa_spec workflow') unless Chatwoot.encryption_configured? + end + + it 'finds plaintext records via fallback lookup' do + channel = create(:channel_telegram, bot_token: 'legacy-token') + + # Simulate legacy plaintext by updating the DB directly + sql = ActiveRecord::Base.send( + :sanitize_sql_array, + ['UPDATE channel_telegram SET bot_token = ? WHERE id = ?', 'legacy-token', channel.id] + ) + ActiveRecord::Base.connection.execute(sql) + + found = Channel::Telegram.find_by(bot_token: 'legacy-token') + expect(found).to eq(channel) + end + end +end diff --git a/spec/services/whatsapp/incoming_message_service_spec.rb b/spec/services/whatsapp/incoming_message_service_spec.rb index ede1ba824..6c23e9b71 100644 --- a/spec/services/whatsapp/incoming_message_service_spec.rb +++ b/spec/services/whatsapp/incoming_message_service_spec.rb @@ -341,6 +341,58 @@ describe Whatsapp::IncomingMessageService do end end + describe 'When the incoming waid is an Argentine number with 9 after country code' do + let(:wa_id) { '5491123456789' } + + it 'creates appropriate conversations, message and contacts if contact does not exist' do + described_class.new(inbox: whatsapp_channel.inbox, params: params).perform + expect(whatsapp_channel.inbox.conversations.count).not_to eq(0) + expect(Contact.all.first.name).to eq('Sojan Jose') + expect(whatsapp_channel.inbox.messages.first.content).to eq('Test') + expect(whatsapp_channel.inbox.contact_inboxes.first.source_id).to eq(wa_id) + end + + it 'appends to existing contact if contact inbox exists with normalized format' do + # Normalized format removes the 9 after country code + normalized_wa_id = '541123456789' + contact_inbox = create(:contact_inbox, inbox: whatsapp_channel.inbox, source_id: normalized_wa_id) + last_conversation = create(:conversation, inbox: whatsapp_channel.inbox, contact_inbox: contact_inbox) + described_class.new(inbox: whatsapp_channel.inbox, params: params).perform + # no new conversation should be created + expect(whatsapp_channel.inbox.conversations.count).to eq(1) + # message appended to the last conversation + expect(last_conversation.messages.last.content).to eq(params[:messages].first[:text][:body]) + # should use the normalized wa_id from existing contact + expect(whatsapp_channel.inbox.contact_inboxes.first.source_id).to eq(normalized_wa_id) + end + end + + describe 'When incoming waid is an Argentine number without 9 after country code' do + let(:wa_id) { '541123456789' } + + context 'when a contact inbox exists with the same format' do + it 'appends to existing contact' do + contact_inbox = create(:contact_inbox, inbox: whatsapp_channel.inbox, source_id: wa_id) + last_conversation = create(:conversation, inbox: whatsapp_channel.inbox, contact_inbox: contact_inbox) + described_class.new(inbox: whatsapp_channel.inbox, params: params).perform + # no new conversation should be created + expect(whatsapp_channel.inbox.conversations.count).to eq(1) + # message appended to the last conversation + expect(last_conversation.messages.last.content).to eq(params[:messages].first[:text][:body]) + end + end + + context 'when a contact inbox does not exist' do + it 'creates contact inbox with the incoming waid' do + described_class.new(inbox: whatsapp_channel.inbox, params: params).perform + expect(whatsapp_channel.inbox.conversations.count).not_to eq(0) + expect(Contact.all.first.name).to eq('Sojan Jose') + expect(whatsapp_channel.inbox.messages.first.content).to eq('Test') + expect(whatsapp_channel.inbox.contact_inboxes.first.source_id).to eq(wa_id) + end + end + end + describe 'when message processing is in progress' do it 'ignores the current message creation request' do params = { 'contacts' => [{ 'profile' => { 'name' => 'Kedar' }, 'wa_id' => '919746334593' }], diff --git a/spec/support/examples/encrypted_external_credential_examples.rb b/spec/support/examples/encrypted_external_credential_examples.rb new file mode 100644 index 000000000..c67d814a9 --- /dev/null +++ b/spec/support/examples/encrypted_external_credential_examples.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'encrypted external credential' do |factory:, attribute:, value: 'secret-token'| + before do + skip('encryption keys missing; see run_mfa_spec workflow') unless Chatwoot.encryption_configured? + if defined?(Facebook::Messenger::Subscriptions) + allow(Facebook::Messenger::Subscriptions).to receive(:subscribe).and_return(true) + allow(Facebook::Messenger::Subscriptions).to receive(:unsubscribe).and_return(true) + end + end + + it "encrypts #{attribute} at rest" do + record = create(factory, attribute => value) + + raw_stored_value = record.reload.read_attribute_before_type_cast(attribute).to_s + expect(raw_stored_value).to be_present + expect(raw_stored_value).not_to include(value) + expect(record.public_send(attribute)).to eq(value) + expect(record.encrypted_attribute?(attribute)).to be(true) + end +end