diff --git a/.env.example b/.env.example index 642a81fa9..f405ea03d 100644 --- a/.env.example +++ b/.env.example @@ -222,20 +222,10 @@ STRIPE_WEBHOOK_SECRET= # Make sure to follow https://edgeguides.rubyonrails.org/active_storage_overview.html#cross-origin-resource-sharing-cors-configuration on the cloud storage after setting this to true. DIRECT_UPLOADS_ENABLED= -# MS OAUTH creds +#MS OAUTH creds AZURE_APP_ID= AZURE_APP_SECRET= -## MS Azure Tenant ID -# Set the following id to the id of your Azure 'tenant'. -# This will enable single tenant applications to work. -# If the following id is set, Chatwoot will use the Microsoft Graph API -# to send and receive emails, as that seems to be required for single -# tenant applications. -# -# https://learn.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant -AZURE_TENANT_ID= - ## Advanced configurations ## Change these values to fine tune performance # control the concurrency setting of sidekiq diff --git a/app/controllers/api/v1/accounts/microsoft/authorizations_controller.rb b/app/controllers/api/v1/accounts/microsoft/authorizations_controller.rb index 7bdd88aa2..bee47b213 100644 --- a/app/controllers/api/v1/accounts/microsoft/authorizations_controller.rb +++ b/app/controllers/api/v1/accounts/microsoft/authorizations_controller.rb @@ -4,7 +4,13 @@ class Api::V1::Accounts::Microsoft::AuthorizationsController < Api::V1::Accounts def create email = params[:authorization][:email] - redirect_url = microsoft_client.auth_code.authorize_url(auth_params) + redirect_url = microsoft_client.auth_code.authorize_url( + { + redirect_uri: "#{base_url}/microsoft/callback", + scope: 'offline_access https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/SMTP.Send openid profile', + prompt: 'consent' + } + ) if redirect_url email = email.downcase ::Redis::Alfred.setex(email, Current.account.id, 5.minutes) @@ -19,31 +25,4 @@ class Api::V1::Accounts::Microsoft::AuthorizationsController < Api::V1::Accounts def check_authorization raise Pundit::NotAuthorizedError unless Current.account_user.administrator? end - - # SMTP, Pop and IMAP are being deprecated by Outlook. - # https://learn.microsoft.com/en-us/exchange/clients-and-mobile-in-exchange-online/deprecation-of-basic-authentication-exchange-online - # - # As such, Microsoft has made it a real pain to use them. - # If AZURE_TENANT_ID is set, we will use the MS Graph API instead. - def auth_params - return graph_auth_params if ENV.fetch('AZURE_TENANT_ID', false) - - standard_auth_params - end - - def standard_auth_params - { - redirect_uri: "#{base_url}/microsoft/callback", - scope: 'offline_access https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/SMTP.Send openid profile', - prompt: 'consent' - } - end - - def graph_auth_params - { - redirect_uri: "#{base_url}/microsoft/callback", - scope: 'offline_access https://graph.microsoft.com/Mail.Read https://graph.microsoft.com/Mail.Send openid profile', - prompt: 'consent' - } - end end diff --git a/app/controllers/concerns/microsoft_concern.rb b/app/controllers/concerns/microsoft_concern.rb index d1bcb49a6..3aa3e4e81 100644 --- a/app/controllers/concerns/microsoft_concern.rb +++ b/app/controllers/concerns/microsoft_concern.rb @@ -5,8 +5,8 @@ module MicrosoftConcern ::OAuth2::Client.new(ENV.fetch('AZURE_APP_ID', nil), ENV.fetch('AZURE_APP_SECRET', nil), { site: 'https://login.microsoftonline.com', - authorize_url: "https://login.microsoftonline.com/#{azure_tenant_id}/oauth2/v2.0/authorize", - token_url: "https://login.microsoftonline.com/#{azure_tenant_id}/oauth2/v2.0/token" + authorize_url: 'https://login.microsoftonline.com/common/oauth2/v2.0/authorize', + token_url: 'https://login.microsoftonline.com/common/oauth2/v2.0/token' }) end @@ -19,8 +19,4 @@ module MicrosoftConcern def base_url ENV.fetch('FRONTEND_URL', 'http://localhost:3000') end - - def azure_tenant_id - MicrosoftGraphAuth.azure_tenant_id - end end diff --git a/app/jobs/inboxes/fetch_imap_email_inboxes_job.rb b/app/jobs/inboxes/fetch_imap_email_inboxes_job.rb index c40abeabe..b360940e3 100644 --- a/app/jobs/inboxes/fetch_imap_email_inboxes_job.rb +++ b/app/jobs/inboxes/fetch_imap_email_inboxes_job.rb @@ -2,19 +2,8 @@ class Inboxes::FetchImapEmailInboxesJob < ApplicationJob queue_as :scheduled_jobs def perform - # check imap_enabled for channel Inbox.where(channel_type: 'Channel::Email').all.find_each(batch_size: 100) do |inbox| - next unless inbox.channel.imap_enabled? - - fetch_emails(inbox) - end - end - - def fetch_emails(inbox) - if inbox.channel.microsoft? && ENV.fetch('AZURE_TENANT_ID', false) - ::Inboxes::FetchMsGraphEmailForTenantJob.perform_later(inbox.channel) - else - ::Inboxes::FetchImapEmailsJob.perform_later(inbox.channel) + ::Inboxes::FetchImapEmailsJob.perform_later(inbox.channel) if inbox.channel.imap_enabled end end end diff --git a/app/jobs/inboxes/fetch_ms_graph_email_for_tenant_job.rb b/app/jobs/inboxes/fetch_ms_graph_email_for_tenant_job.rb deleted file mode 100644 index 7efd1784e..000000000 --- a/app/jobs/inboxes/fetch_ms_graph_email_for_tenant_job.rb +++ /dev/null @@ -1,101 +0,0 @@ -require 'net/http' - -class Inboxes::FetchMsGraphEmailForTenantJob < ApplicationJob - queue_as :scheduled_jobs - - def perform(channel) - process_email_for_channel(channel) - rescue EOFError => e - Rails.logger.error e - rescue StandardError => e - ChatwootExceptionTracker.new(e, account: channel.account).capture_exception - end - - private - - def should_fetch_email?(channel) - channel.imap_enabled? && channel.microsoft? && !channel.reauthorization_required? - end - - def process_email_for_channel(channel) - # fetching email for microsoft provider - fetch_mail_for_channel(channel) - - # clearing old failures like timeouts since the mail is now successfully processed - channel.reauthorized! - end - - def fetch_mail_for_channel(channel) - return if channel.provider_config['access_token'].blank? - - access_token = valid_access_token channel - - return unless access_token - - graph = graph_authenticate(access_token) - - process_mails(graph, channel) - end - - def process_mails(graph, channel) - response = graph.get_from_api('me/messages', {}, graph_query) - - unless response.is_a?(Net::HTTPSuccess) - channel.authorization_error! - return false - end - - json_response = JSON.parse(response.body) - json_response['value'].each do |message| - inbound_mail = Mail.read_from_string retrieve_mail_mime(graph, message['id']) - - next if channel.inbox.messages.find_by(source_id: inbound_mail.message_id).present? - - process_mail(inbound_mail, channel) - end - end - - def retrieve_mail_mime(graph, id) - response = graph.get_from_api("me/messages/#{id}/$value") - return unless response.is_a?(Net::HTTPSuccess) - - response.body - end - - def graph_authenticate(access_token) - MicrosoftGraphApi.new(access_token) - end - - def yesterday - (Time.zone.today - 1).strftime('%FT%TZ') - end - - def tomorrow - (Time.zone.today + 1).strftime('%FT%TZ') - end - - # Query to replicate the IMAP search used in Inboxes::FetchImapEmailsJob - # Selects the top 1000 records within the given filter, as that is the maximum - # page size for the API. Might need to look into paginating the requests later, - # for inboxes that receive more than 1000 emails a day? - # - # 1. https://learn.microsoft.com/en-us/graph/api/user-list-messages - # 2. https://learn.microsoft.com/en-us/graph/query-parameters - def graph_query - { - '$filter': "receivedDateTime ge #{yesterday} and receivedDateTime le #{tomorrow}", - '$top': '1000', '$select': 'id' - } - end - - def process_mail(inbound_mail, channel) - Imap::ImapMailbox.new.process(inbound_mail, channel) - rescue StandardError => e - ChatwootExceptionTracker.new(e, account: channel.account).capture_exception - end - - # Making sure the access token is valid for microsoft provider - def valid_access_token(channel) - Microsoft::RefreshOauthTokenService.new(channel: channel).access_token - end -end diff --git a/app/mailers/conversation_reply_mailer_helper.rb b/app/mailers/conversation_reply_mailer_helper.rb index ad45b64d6..346f4c9ca 100644 --- a/app/mailers/conversation_reply_mailer_helper.rb +++ b/app/mailers/conversation_reply_mailer_helper.rb @@ -23,7 +23,6 @@ module ConversationReplyMailerHelper def ms_smtp_settings return unless @inbox.email? && @channel.imap_enabled && @inbox.channel.provider == 'microsoft' - return ms_graph_settings if ENV.fetch('AZURE_TENANT_ID', false) smtp_settings = { address: 'smtp.office365.com', @@ -41,15 +40,6 @@ module ConversationReplyMailerHelper @options[:delivery_method_options] = smtp_settings end - def ms_graph_settings - graph_settings = { - token: @channel.provider_config['access_token'] - } - - @options[:delivery_method] = :microsoft_graph - @options[:delivery_method_options] = graph_settings - end - def set_delivery_method return unless @inbox.inbox_type == 'Email' && @channel.smtp_enabled diff --git a/config/initializers/delivery_methods.rb b/config/initializers/delivery_methods.rb deleted file mode 100644 index 2c5bcd777..000000000 --- a/config/initializers/delivery_methods.rb +++ /dev/null @@ -1,3 +0,0 @@ -require 'microsoft_graph_delivery_method' - -ActionMailer::Base.add_delivery_method :microsoft_graph, MicrosoftGraphDeliveryMethod diff --git a/lib/microsoft_graph_api.rb b/lib/microsoft_graph_api.rb deleted file mode 100644 index 47d05e964..000000000 --- a/lib/microsoft_graph_api.rb +++ /dev/null @@ -1,62 +0,0 @@ -# Simple HTTPS API helper class for interacting with MS Graph. -# Uses the standard ruby HTTP library for interacting with the API. - -require 'uri' -require 'net/http' - -class MicrosoftGraphApi - API_VERSION = 'v1.0'.freeze - API_PORT = 443 - API_URL = "https://graph.microsoft.com/#{API_VERSION}".freeze - - def initialize(token) - @token = token - end - - # Simple get request to the endpoint - # - # 'queries' are the get variables after the main url - # eg. foo/bar?query=myquery - def get_from_api(endpoint, headers = {}, query = {}) - uri = endpoint_to_uri(endpoint, query) - https = setup_https(uri.host) - request = Net::HTTP::Get.new(uri.request_uri) - - # Assign each header to the request - headers.each { |key, value| request[key.to_s] = value.to_s } - request['Authorization'] = "Bearer #{@token}" - - https.request(request) - end - - # Simple post request to the endpoint - def post_to_api(endpoint, headers = {}, body = '') - uri = endpoint_to_uri(endpoint) - https = setup_https(uri.host) - request = Net::HTTP::Post.new(uri.path) - - # Assign each header to the request - headers.each { |key, value| request[key.to_s] = value.to_s } - request['Authorization'] = "Bearer #{@token}" - - request.body = body - https.request(request) - end - - private - - def setup_https(host) - https = Net::HTTP.new(host, API_PORT) - https.use_ssl = true - https - end - - def endpoint_to_uri(endpoint, query = {}) - endpoint.delete_prefix('/') - uri = URI("#{API_URL}/#{endpoint}") - return uri if query.empty? - - uri.query = URI.encode_www_form(query) - uri - end -end diff --git a/lib/microsoft_graph_auth.rb b/lib/microsoft_graph_auth.rb index 8c6016aeb..a1c0bbea0 100644 --- a/lib/microsoft_graph_auth.rb +++ b/lib/microsoft_graph_auth.rb @@ -9,18 +9,6 @@ require 'omniauth-oauth2' # Implements an OmniAuth strategy to get a Microsoft Graph # compatible token from Azure AD class MicrosoftGraphAuth < OmniAuth::Strategies::OAuth2 - # Microsoft Azure Tenant - # For single tenant applications, meant to be used by - # organisations for their own apps, the 'common' endpoint is not allowed. - # If the environment variable 'AZURE_TENANT_ID' is set, - # this will return it's value, otherwise, it will default to 'common'. - # - # The tenant id for your Azure organization can be obtained by - # by accessing 'Tenant properties' from the Azure portal. - def self.azure_tenant_id - ENV.fetch('AZURE_TENANT_ID', 'common') - end - option :name, :microsoft_graph_auth DEFAULT_SCOPE = 'offline_access https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/SMTP.Send' @@ -28,8 +16,8 @@ class MicrosoftGraphAuth < OmniAuth::Strategies::OAuth2 # Configure the Microsoft identity platform endpoints option :client_options, site: 'https://login.microsoftonline.com', - authorize_url: "/#{azure_tenant_id}/oauth2/v2.0/authorize", - token_url: "/#{azure_tenant_id}/oauth2/v2.0/token" + authorize_url: '/common/oauth2/v2.0/authorize', + token_url: '/common/oauth2/v2.0/token' option :pcke, true # Send the scope parameter during authorize diff --git a/lib/microsoft_graph_delivery_method.rb b/lib/microsoft_graph_delivery_method.rb deleted file mode 100644 index 7d03495a3..000000000 --- a/lib/microsoft_graph_delivery_method.rb +++ /dev/null @@ -1,26 +0,0 @@ -# Recently (around Feb/Mar 2023), Microsoft has made sending -# email through SMTP with Outlook near impossible, at least -# for single tenant applications. -# -# As such, adding a delivery method to use the Microsoft Graph -# API allows for emails to be sent again. -require 'base64' - -class MicrosoftGraphDeliveryMethod - def initialize(config) - @config = config - end - - def deliver!(mail) - # Create a new API connection, and post the mail to the `me/sendMail` endpoint. - # https://learn.microsoft.com/en-us/graph/api/user-sendmail#example-4-send-a-new-message-using-mime-format - - headers = { - 'Content-Type' => 'text/plain' - } - body = Base64.encode64(mail.to_s) - - graph = MicrosoftGraphApi.new(@config[:token]) - graph.post_to_api('me/sendMail', headers, body) - end -end diff --git a/spec/controllers/api/v1/accounts/microsoft/authorization_controller_spec.rb b/spec/controllers/api/v1/accounts/microsoft/authorization_controller_spec.rb index 91fb060e0..853cf2850 100644 --- a/spec/controllers/api/v1/accounts/microsoft/authorization_controller_spec.rb +++ b/spec/controllers/api/v1/accounts/microsoft/authorization_controller_spec.rb @@ -43,29 +43,6 @@ RSpec.describe 'Microsoft Authorization API', type: :request do expect(response.parsed_body['url']).to eq response_url expect(Redis::Alfred.get(administrator.email)).to eq(account.id.to_s) end - - it 'creates a new authorization and returns the redirect url for single tenant' do - with_modified_env AZURE_TENANT_ID: 'azure_tenant_id' do - post "/api/v1/accounts/#{account.id}/microsoft/authorization", - headers: administrator.create_new_auth_token, - params: { email: administrator.email }, - as: :json - - microsoft_service = Class.new { extend MicrosoftConcern } - - response_url = microsoft_service.microsoft_client.auth_code.authorize_url( - { - redirect_uri: "#{ENV.fetch('FRONTEND_URL', 'http://localhost:3000')}/microsoft/callback", - scope: 'offline_access https://graph.microsoft.com/Mail.Read https://graph.microsoft.com/Mail.Send openid profile', - prompt: 'consent' - } - ) - expect(response.parsed_body['url']).to eq response_url - end - - expect(response).to have_http_status(:success) - expect(Redis::Alfred.get(administrator.email)).to eq(account.id.to_s) - end end end end diff --git a/spec/jobs/inboxes/fetch_imap_email_inboxes_job_spec.rb b/spec/jobs/inboxes/fetch_imap_email_inboxes_job_spec.rb index 57ee00bef..ae4f540d4 100644 --- a/spec/jobs/inboxes/fetch_imap_email_inboxes_job_spec.rb +++ b/spec/jobs/inboxes/fetch_imap_email_inboxes_job_spec.rb @@ -7,12 +7,6 @@ RSpec.describe Inboxes::FetchImapEmailInboxesJob do imap_password: 'password', account: account) end let(:email_inbox) { create(:inbox, channel: imap_email_channel, account: account) } - let(:microsoft_imap_email_channel) do - create(:channel_email, provider: 'microsoft', imap_enabled: true, imap_address: 'outlook.office365.com', - imap_port: 993, imap_login: 'imap@outlook.com', imap_password: 'password', account: account, - provider_config: { access_token: 'access_token' }) - end - let(:ms_email_inbox) { create(:inbox, channel: microsoft_imap_email_channel, account: account) } it 'enqueues the job' do expect { described_class.perform_later }.to have_enqueued_job(described_class) @@ -26,24 +20,4 @@ RSpec.describe Inboxes::FetchImapEmailInboxesJob do described_class.perform_now end end - - context 'when microsoft inbox' do - it 'calls fetch ms graph email job for single tenant app' do - stub_request(:get, 'https://graph.microsoft.com/v1.0/me/messages?$filter=receivedDateTime%20ge%202023-05-23T00:00:00Z%20and%20receivedDateTime%20le%202023-05-25T00:00:00Z&$select=id&$top=1000') - - with_modified_env AZURE_TENANT_ID: 'azure_tenant_id' do - expect(Inboxes::FetchMsGraphEmailForTenantJob).to receive(:perform_later).with(microsoft_imap_email_channel).once - - described_class.perform_now - end - end - - it 'calls fetch imap email job for multi tenant app' do - with_modified_env AZURE_TENANT_ID: nil do - expect(Inboxes::FetchImapEmailsJob).to receive(:perform_later).with(microsoft_imap_email_channel).once - - described_class.perform_now - end - end - end end diff --git a/spec/jobs/inboxes/fetch_ms_graph_email_for_tenant_job_spec.rb b/spec/jobs/inboxes/fetch_ms_graph_email_for_tenant_job_spec.rb deleted file mode 100644 index 79160a800..000000000 --- a/spec/jobs/inboxes/fetch_ms_graph_email_for_tenant_job_spec.rb +++ /dev/null @@ -1,68 +0,0 @@ -require 'rails_helper' - -RSpec.describe Inboxes::FetchMsGraphEmailForTenantJob do - include ActionMailbox::TestHelper - - let(:account) { create(:account) } - let(:microsoft_imap_email_channel) do - create(:channel_email, provider: 'microsoft', imap_enabled: true, imap_address: 'outlook.office365.com', - imap_port: 993, imap_login: 'imap@outlook.com', imap_password: 'password', account: account, - provider_config: { access_token: 'access_token' }) - end - let(:ms_email_inbox) { create(:inbox, channel: microsoft_imap_email_channel, account: account) } - let(:inbound_mail) { create_inbound_email_from_mail(from: 'testemail@gmail.com', to: 'imap@outlook.com', subject: 'Hello!') } - let(:yesterday) { (Time.zone.today - 1).strftime('%FT%TZ') } - let(:tomorrow) { (Time.zone.today + 1).strftime('%FT%TZ') } - - it 'enqueues the job' do - expect { described_class.perform_later }.to have_enqueued_job(described_class) - .on_queue('scheduled_jobs') - end - - context 'when imap fetch new emails for microsoft mailer' do - before do - stub_request(:get, "https://graph.microsoft.com/v1.0/me/messages?$filter=receivedDateTime%20ge%20#{yesterday}%20and%20receivedDateTime%20le%20#{tomorrow}&$select=id&$top=1000") - .with( - headers: { - 'Accept' => '*/*', - 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Authorization' => 'Bearer access_token', - 'User-Agent' => 'Ruby' - } - ) - .to_return(status: 200, body: '{"value":[{"id":"1"}]}', headers: {}) - - stub_request(:get, 'https://graph.microsoft.com/v1.0/me/messages/1/$value') - .with( - headers: { - 'Accept' => '*/*', - 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Authorization' => 'Bearer access_token', - 'User-Agent' => 'Ruby' - } - ) - .to_return(status: 200, body: '', headers: {}) - end - - it 'fetch and process all emails' do - ms_imap_email_inbox = double - - with_modified_env AZURE_TENANT_ID: 'azure_tenant_id' do - email = Mail.new do - to 'test@outlook.com' - from 'test@gmail.com' - subject :test.to_s - body 'hello' - end - imap_fetch_mail = Net::IMAP::FetchData.new - imap_fetch_mail.attr = { RFC822: email }.with_indifferent_access - - allow(Mail).to receive(:read_from_string).and_return(inbound_mail) - allow(Imap::ImapMailbox).to receive(:new).and_return(ms_imap_email_inbox) - expect(ms_imap_email_inbox).to receive(:process).with(inbound_mail, microsoft_imap_email_channel).once - - described_class.perform_now(microsoft_imap_email_channel) - end - end - end -end diff --git a/spec/lib/microsoft_graph_api_spec.rb b/spec/lib/microsoft_graph_api_spec.rb deleted file mode 100644 index f16291c5e..000000000 --- a/spec/lib/microsoft_graph_api_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -require 'rails_helper' -# explicitly requiring since we are loading apms conditionally in application.rb -require 'sentry-ruby' - -describe MicrosoftGraphApi do - let(:yesterday) { (Time.zone.today - 1).strftime('%FT%TZ') } - let(:tomorrow) { (Time.zone.today + 1).strftime('%FT%TZ') } - - describe '#get_from_api' do - before do - stub_request(:get, "https://graph.microsoft.com/v1.0/me/messages?$filter=receivedDateTime%20ge%20#{yesterday}%20and%20receivedDateTime%20le%20#{tomorrow}&$select=id&$top=1000") - .with( - headers: { - 'Accept' => '*/*', - 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Authorization' => 'Bearer access_token', - 'User-Agent' => 'Ruby' - } - ) - .to_return(status: 200, body: '{"value":[{"id":"1"}]}', headers: {}) - - stub_request(:get, 'https://graph.microsoft.com/v1.0/me/messages/1/$value') - .with( - headers: { - 'Accept' => '*/*', - 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Authorization' => 'Bearer access_token', - 'User-Agent' => 'Ruby' - } - ) - .to_return(status: 200, body: '', headers: {}) - - stub_request(:post, 'https://graph.microsoft.com/v1.0/me/sendMail') - .with( - body: 'email body', - headers: { - 'Accept' => '*/*', - 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', - 'Authorization' => 'Bearer access_token', - 'User-Agent' => 'Ruby' - } - ) - .to_return(status: 200, body: 'email body', headers: { 'Content-Type' => 'text/plain' }) - end - - it 'fetch emails' do - graph_query = { :$filter => "receivedDateTime ge #{yesterday} and receivedDateTime le #{tomorrow}", :$top => '1000', :$select => 'id' } - response = described_class.new('access_token').get_from_api('me/messages', {}, graph_query) - - json_response = JSON.parse(response.body) - expect(json_response['value'][0]['id']).to eq '1' - end - - it 'post emails' do - response = described_class.new('access_token').post_to_api('me/sendMail', {}, 'email body') - - expect(response.is_a?(Net::HTTPSuccess)).to be true - expect(response.body).to eq('email body') - end - end -end