chore: fetch mails with multiple attachments (#7030)

app/jobs/inboxes/fetch_imap_emails_job.rb

@@ -33,23 +33,30 @@ class Inboxes::FetchImapEmailsJob < ApplicationJob
   end
 
   def fetch_mail_for_channel(channel)
-    # TODO: rather than setting this as default method for all mail objects, lets if can do new mail object
+    imap_inbox = authenticated_imap_inbox(channel, channel.imap_password, 'PLAIN')
-    # using Mail.retriever_method.new(params)
+    last_email_time = DateTime.parse(Net::IMAP.format_datetime(last_email_time(channel)))
-    Mail.defaults do
-      retriever_method :imap, address: channel.imap_address,
-                              port: channel.imap_port,
-                              user_name: channel.imap_login,
-                              password: channel.imap_password,
-                              enable_ssl: channel.imap_enable_ssl
-    end
 
-    Mail.find(what: :last, count: 10, order: :asc).each do |inbound_mail|
+    received_mails(imap_inbox).each do |message_id|
-      next if channel.inbox.messages.find_by(source_id: inbound_mail.message_id).present?
+      inbound_mail = Mail.read_from_string imap_inbox.fetch(message_id, 'RFC822')[0].attr['RFC822']
+
+      next if email_already_present?(channel, inbound_mail, last_email_time)
 
       process_mail(inbound_mail, channel)
     end
   end
 
+  def email_already_present?(channel, inbound_mail, last_email_time)
+    processed_email?(inbound_mail, last_email_time) || channel.inbox.messages.find_by(source_id: inbound_mail.message_id).present?
+  end
+
+  def received_mails(imap_inbox)
+    imap_inbox.search(['BEFORE', tomorrow, 'SINCE', yesterday])
+  end
+
+  def processed_email?(current_email, last_email_time)
+    current_email.date < last_email_time
+  end
+
   def fetch_mail_for_ms_provider(channel)
     return if channel.provider_config['access_token'].blank?
 
@@ -57,14 +64,14 @@ class Inboxes::FetchImapEmailsJob < ApplicationJob
 
     return unless access_token
 
-    imap = imap_authenticate(channel, access_token)
+    imap_inbox = authenticated_imap_inbox(channel, access_token, 'XOAUTH2')
 
-    process_mails(imap, channel)
+    process_mails(imap_inbox, channel)
   end
 
-  def process_mails(imap, channel)
+  def process_mails(imap_inbox, channel)
-    imap.search(['BEFORE', tomorrow, 'SINCE', yesterday]).each do |message_id|
+    received_mails(imap_inbox).each do |message_id|
-      inbound_mail = Mail.read_from_string imap.fetch(message_id, 'RFC822')[0].attr['RFC822']
+      inbound_mail = Mail.read_from_string imap_inbox.fetch(message_id, 'RFC822')[0].attr['RFC822']
 
       next if channel.inbox.messages.find_by(source_id: inbound_mail.message_id).present?
 
@@ -72,13 +79,19 @@ class Inboxes::FetchImapEmailsJob < ApplicationJob
     end
   end
 
-  def imap_authenticate(channel, access_token)
+  def authenticated_imap_inbox(channel, access_token, auth_method)
     imap = Net::IMAP.new(channel.imap_address, channel.imap_port, true)
-    imap.authenticate('XOAUTH2', channel.imap_login, access_token)
+    imap.authenticate(auth_method, channel.imap_login, access_token)
     imap.select('INBOX')
     imap
   end
 
+  def last_email_time(channel)
+    time = 1.hour.ago.to_s
+    time = channel.inbox.messages.incoming.last.content_attributes['email']['date'] if channel.inbox.messages.any?
+    DateTime.parse(time)
+  end
+
   def yesterday
     (Time.zone.today - 1).strftime('%d-%b-%Y')
   end

app/mailboxes/mailbox_helper.rb

@@ -23,7 +23,7 @@ module MailboxHelper
   def add_attachments_to_message
     return if @message.blank?
 
-    processed_mail.attachments.each do |mail_attachment|
+    processed_mail.attachments.last(Message::NUMBER_OF_PERMITTED_ATTACHMENTS).each do |mail_attachment|
       attachment = @message.attachments.new(
         account_id: @conversation.account_id,
         file_type: 'file'

spec/fixtures/files/multiple_attachments.eml

@@ -0,0 +1,270 @@
+From: test@gmail.com
+Date: Thu, 4 May 2023 10:35:52 +0530
+Message-ID: <6215d536e0484_10bc6191402183@tejaswinis-MacBook-Pro.local.mail>
+Subject: multiple attachments
+To: test@outlook.com
+Content-Type: multipart/mixed; boundary="0000000000002488f405fad721cc"
+
+--0000000000002488f405fad721cc
+Content-Type: multipart/alternative; boundary="0000000000002488f205fad721ca"
+
+--0000000000002488f205fad721ca
+Content-Type: text/plain; charset="UTF-8"
+Content-Transfer-Encoding: quoted-printable
+
+Attaching 30 files:
+
+Hi people!
+
+We are excited to inform you that we have recently released some new
+features and several updates to our platform. These features and updates
+are designed to enhance your experience and make your trading journey
+seamless and efficient.
+
+
+
+> Okay noted
+
+
+--0000000000002488f205fad721ca--
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 3.32.04 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 3.32.04 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8l3>
+X-Attachment-Id: f_lh8nwk8l3
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 2.52.11 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 2.52.11 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8l2>
+X-Attachment-Id: f_lh8nwk8l2
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 3.32.12 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 3.32.12 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8l4>
+X-Attachment-Id: f_lh8nwk8l4
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 2.46.52 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 2.46.52 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8h0>
+X-Attachment-Id: f_lh8nwk8h0
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 2.49.26 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 2.49.26 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8k1>
+X-Attachment-Id: f_lh8nwk8k1
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 7.58.33 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 7.58.33 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8n7>
+X-Attachment-Id: f_lh8nwk8n7
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 6.32.57 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 6.32.57 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8m5>
+X-Attachment-Id: f_lh8nwk8m5
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 7.58.04 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 7.58.04 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8m6>
+X-Attachment-Id: f_lh8nwk8m6
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-04-26 at 1.37.06 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-04-26 at 1.37.06 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8n8>
+X-Attachment-Id: f_lh8nwk8n8
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-04-26 at 1.50.55 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-04-26 at 1.50.55 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8n9>
+X-Attachment-Id: f_lh8nwk8n9
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-04-29 at 10.17.21 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-04-29 at 10.17.21 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8o11>
+X-Attachment-Id: f_lh8nwk8o11
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-02 at 11.44.34 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-02 at 11.44.34 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8o12>
+X-Attachment-Id: f_lh8nwk8o12
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-02 at 11.44.50 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-02 at 11.44.50 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8o13>
+X-Attachment-Id: f_lh8nwk8o13
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 2.46.52 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 2.46.52 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsii15>
+X-Attachment-Id: f_lh8nwsii15
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 2.49.26 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 2.49.26 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsij16>
+X-Attachment-Id: f_lh8nwsij16
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 2.52.11 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 2.52.11 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsik17>
+X-Attachment-Id: f_lh8nwsik17
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-02 at 11.49.19 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-02 at 11.49.19 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8o14>
+X-Attachment-Id: f_lh8nwk8o14
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 3.32.04 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 3.32.04 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsil18>
+X-Attachment-Id: f_lh8nwsil18
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 3.32.12 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 3.32.12 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsil19>
+X-Attachment-Id: f_lh8nwsil19
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-04-26 at 9.43.29 AM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-04-26 at 9.43.29 AM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwk8n10>
+X-Attachment-Id: f_lh8nwk8n10
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 6.32.57 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 6.32.57 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsim20>
+X-Attachment-Id: f_lh8nwsim20
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 7.58.04 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 7.58.04 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsim21>
+X-Attachment-Id: f_lh8nwsim21
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-03 at 7.58.33 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-03 at 7.58.33 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsim22>
+X-Attachment-Id: f_lh8nwsim22
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-04-26 at 1.37.06 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-04-26 at 1.37.06 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsim23>
+X-Attachment-Id: f_lh8nwsim23
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-04-29 at 10.17.21 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-04-29 at 10.17.21 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsin26>
+X-Attachment-Id: f_lh8nwsin26
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-02 at 11.44.34 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-02 at 11.44.34 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsio27>
+X-Attachment-Id: f_lh8nwsio27
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-04-26 at 1.50.55 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-04-26 at 1.50.55 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsin24>
+X-Attachment-Id: f_lh8nwsin24
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-02 at 11.44.50 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-02 at 11.44.50 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsio28>
+X-Attachment-Id: f_lh8nwsio28
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-05-02 at 11.49.19 PM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-05-02 at 11.49.19 PM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsio29>
+X-Attachment-Id: f_lh8nwsio29
+
+
+--0000000000002488f405fad721cc
+Content-Type: image/png; name="Screenshot 2023-04-26 at 9.43.29 AM.png"
+Content-Disposition: attachment; filename="Screenshot 2023-04-26 at 9.43.29 AM.png"
+Content-Transfer-Encoding: base64
+Content-ID: <f_lh8nwsin25>
+X-Attachment-Id: f_lh8nwsin25
+
+
+--0000000000002488f405fad721cc--

spec/jobs/inboxes/fetch_imap_emails_job_spec.rb

@@ -16,6 +16,7 @@ RSpec.describe Inboxes::FetchImapEmailsJob, type: :job do
   let(:ms_email_inbox) { create(:inbox, channel: microsoft_imap_email_channel, account: account) }
   let!(:conversation) { create(:conversation, inbox: imap_email_channel.inbox, account: account) }
   let(:inbound_mail) { create_inbound_email_from_mail(from: 'testemail@gmail.com', to: 'imap@outlook.com', subject: 'Hello!') }
+  let(:inbound_mail_with_attachments) { create_inbound_email_from_fixture('multiple_attachments.eml') }
 
   it 'enqueues the job' do
     expect { described_class.perform_later }.to have_enqueued_job(described_class)
@@ -31,15 +32,62 @@ RSpec.describe Inboxes::FetchImapEmailsJob, type: :job do
         body 'hello'
       end
 
-      allow(Mail).to receive(:find).and_return([email])
+      imap_fetch_mail = Net::IMAP::FetchData.new
+      imap_fetch_mail.attr = { seqno: 1, RFC822: email }.with_indifferent_access
+
+      imap = double
+
+      allow(Net::IMAP).to receive(:new).and_return(imap)
+      allow(imap).to receive(:authenticate)
+      allow(imap).to receive(:select)
+      allow(imap).to receive(:search).and_return([1])
+      allow(imap).to receive(:fetch).and_return([imap_fetch_mail])
+
+      read_mail = Mail::Message.new(date: DateTime.now, from: 'testemail@gmail.com', to: 'imap@outlook.com', subject: 'Hello!')
+      allow(Mail).to receive(:read_from_string).and_return(inbound_mail.mail)
+
       imap_mailbox = double
+
       allow(Imap::ImapMailbox).to receive(:new).and_return(imap_mailbox)
-      expect(imap_mailbox).to receive(:process).with(email, imap_email_channel).once
+      expect(imap_mailbox).to receive(:process).with(read_mail, imap_email_channel).once
 
       described_class.perform_now(imap_email_channel)
     end
   end
 
+  context 'when imap fetch new emails with more than 15 attachments' do
+    it 'process the email' do
+      email = Mail.new do
+        to 'test@outlook.com'
+        from 'test@gmail.com'
+        subject :test.to_s
+        body 'hello'
+      end
+
+      imap_fetch_mail = Net::IMAP::FetchData.new
+      imap_fetch_mail.attr = { seqno: 1, RFC822: email }.with_indifferent_access
+
+      imap = double
+
+      allow(Net::IMAP).to receive(:new).and_return(imap)
+      allow(imap).to receive(:authenticate)
+      allow(imap).to receive(:select)
+      allow(imap).to receive(:search).and_return([1])
+      allow(imap).to receive(:fetch).and_return([imap_fetch_mail])
+
+      inbound_mail_with_attachments.mail.date = DateTime.now
+
+      allow(Mail).to receive(:read_from_string).and_return(inbound_mail_with_attachments.mail)
+
+      imap_mailbox = Imap::ImapMailbox.new
+
+      allow(Imap::ImapMailbox).to receive(:new).and_return(imap_mailbox)
+
+      described_class.perform_now(imap_email_channel)
+      expect(Message.last.attachments.count).to eq(15)
+    end
+  end
+
   context 'when imap fetch new emails for microsoft mailer' do
     it 'fetch and process all emails' do
       email = Mail.new do