From f875a09fb7921bd644a20821f44b0e9f2b5aa67b Mon Sep 17 00:00:00 2001 From: Sojan Jose Date: Tue, 7 Jan 2020 22:59:17 +0530 Subject: [PATCH] Chore: Switch from Carrierwave to ActiveStorage (#393) --- .rubocop.yml | 2 + Gemfile | 7 +- Gemfile.lock | 76 ++++++++++++++----- app/bot/bot_configurator.rb | 0 app/builders/messages/message_builder.rb | 19 +++-- .../api/v1/callbacks_controller.rb | 10 ++- .../dashboard/conversation/ContactPanel.vue | 2 +- app/models/attachment.rb | 17 ++--- app/models/channel/facebook_page.rb | 5 +- app/models/concerns/avatarable.rb | 18 +++++ app/models/contact.rb | 5 +- app/models/inbox.rb | 9 +-- app/models/user.rb | 15 +--- app/uploaders/attachment_uploader.rb | 21 ----- app/uploaders/avatar_uploader.rb | 19 ----- app/views/api/v1/contacts/show.json.jbuilder | 2 +- .../api/v1/conversations/index.json.jbuilder | 2 +- app/views/api/v1/inboxes/index.json.jbuilder | 2 +- config/initializers/carrierwave.rb | 37 --------- config/storage.yml | 20 ++--- ...27191631_remove_carrier_wave_attributes.rb | 7 ++ db/schema.rb | 5 +- docker/Dockerfile | 1 + lib/local_resource.rb | 6 +- .../builders/messages/message_builder_spec.rb | 34 +++++++++ .../factories/{ => channel}/channel_widget.rb | 0 spec/factories/channel/facebook_pages.rb | 1 + .../incoming_fb_text_message.rb | 12 +++ spec/models/channel/facebook_page_spec.rb | 2 +- 29 files changed, 192 insertions(+), 164 deletions(-) delete mode 100644 app/bot/bot_configurator.rb create mode 100644 app/models/concerns/avatarable.rb delete mode 100644 app/uploaders/attachment_uploader.rb delete mode 100644 app/uploaders/avatar_uploader.rb delete mode 100644 config/initializers/carrierwave.rb create mode 100644 db/migrate/20191227191631_remove_carrier_wave_attributes.rb create mode 100644 spec/builders/messages/message_builder_spec.rb rename spec/factories/{ => channel}/channel_widget.rb (100%) create mode 100644 spec/factories/facebook_message/incoming_fb_text_message.rb diff --git a/.rubocop.yml b/.rubocop.yml index 248a11eed..975f045f6 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,6 +6,8 @@ inherit_from: .rubocop_todo.yml Metrics/LineLength: Max: 150 +Metrics/ClassLength: + Max: 125 RSpec/ExampleLength: Max: 15 Documentation: diff --git a/Gemfile b/Gemfile index d0e53f197..fd1d783e4 100644 --- a/Gemfile +++ b/Gemfile @@ -23,6 +23,9 @@ gem 'valid_email2' gem 'uglifier' ##-- for active storage --## +gem 'aws-sdk-s3', require: false +gem 'azure-storage', require: false +gem 'google-cloud-storage', require: false gem 'mini_magick' ##-- gems for database --# @@ -68,9 +71,7 @@ gem 'haikunator' gem 'brakeman' gem 'sentry-raven' -##-- TODO: move these gems to appropriate groups --## -# remove this gem in favor of active storage - github #158 -gem 'carrierwave-aws' +##-- background job processing --## gem 'sidekiq' group :development do diff --git a/Gemfile.lock b/Gemfile.lock index a7bb9d402..cba93566b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -68,7 +68,7 @@ GEM ast (2.4.0) attr_extras (6.2.1) aws-eventstream (1.0.3) - aws-partitions (1.259.0) + aws-partitions (1.262.0) aws-sdk-core (3.86.0) aws-eventstream (~> 1.0, >= 1.0.2) aws-partitions (~> 1, >= 1.239.0) @@ -87,6 +87,15 @@ GEM descendants_tracker (~> 0.0.4) ice_nine (~> 0.11.0) thread_safe (~> 0.3, >= 0.3.1) + azure-core (0.1.15) + faraday (~> 0.9) + faraday_middleware (~> 0.10) + nokogiri (~> 1.6) + azure-storage (0.15.0.preview) + azure-core (~> 0.1) + faraday (~> 0.9) + faraday_middleware (~> 0.10) + nokogiri (~> 1.6, >= 1.6.8) bcrypt (3.1.13) bindex (0.8.1) bootsnap (1.4.5) @@ -104,16 +113,6 @@ GEM bundler (>= 1.2.0, < 3) thor (~> 0.18) byebug (11.0.1) - carrierwave (2.0.2) - activemodel (>= 5.0.0) - activesupport (>= 5.0.0) - addressable (~> 2.6) - image_processing (~> 1.1) - mimemagic (>= 0.3.0) - mini_mime (>= 0.1.3) - carrierwave-aws (1.4.0) - aws-sdk-s3 (~> 1.0) - carrierwave (>= 0.7, < 2.1) chargebee (2.7.1) json_pure (~> 2.1) rest-client (>= 1.8, < 3.0) @@ -123,6 +122,8 @@ GEM concurrent-ruby (1.1.5) connection_pool (2.2.2) crass (1.0.5) + declarative (0.0.10) + declarative-option (0.1.0) descendants_tracker (0.0.4) thread_safe (~> 0.3, >= 0.3.1) devise (4.7.1) @@ -136,6 +137,7 @@ GEM devise (> 3.5.2, < 5) rails (>= 4.2.0, < 6.1) diff-lcs (1.3) + digest-crc (0.4.1) docile (1.3.2) domain_name (0.5.20190701) unf (>= 0.0.5, < 1.0.0) @@ -158,10 +160,38 @@ GEM i18n (>= 1.6, < 1.8) faraday (0.17.1) multipart-post (>= 1.2, < 3) + faraday_middleware (0.13.1) + faraday (>= 0.7.4, < 1.0) ffi (1.11.3) foreman (0.86.0) globalid (0.4.2) activesupport (>= 4.2.0) + google-api-client (0.36.4) + addressable (~> 2.5, >= 2.5.1) + googleauth (~> 0.9) + httpclient (>= 2.8.1, < 3.0) + mini_mime (~> 1.0) + representable (~> 3.0) + retriable (>= 2.0, < 4.0) + signet (~> 0.12) + google-cloud-core (1.4.1) + google-cloud-env (~> 1.0) + google-cloud-env (1.3.0) + faraday (~> 0.11) + google-cloud-storage (1.25.1) + addressable (~> 2.5) + digest-crc (~> 0.4) + google-api-client (~> 0.33) + google-cloud-core (~> 1.2) + googleauth (~> 0.9) + mini_mime (~> 1.0) + googleauth (0.10.0) + faraday (~> 0.12) + jwt (>= 1.4, < 3.0) + memoist (~> 0.16) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (~> 0.12) haikunator (1.1.0) hashie (4.0.0) http (3.3.0) @@ -177,12 +207,10 @@ GEM httparty (0.17.3) mime-types (~> 3.0) multi_xml (>= 0.5.2) + httpclient (2.8.3) i18n (1.7.0) concurrent-ruby (~> 1.0) ice_nine (0.11.2) - image_processing (1.10.0) - mini_magick (>= 4.9.5, < 5) - ruby-vips (>= 2.0.13, < 3) inflecto (0.0.2) jaro_winkler (1.5.4) jbuilder (2.9.1) @@ -221,6 +249,7 @@ GEM mini_mime (>= 0.1.1) marcel (0.3.3) mimemagic (~> 0.3.2) + memoist (0.16.2) memoizable (0.4.2) thread_safe (~> 0.3, >= 0.3.1) method_source (0.9.2) @@ -234,6 +263,7 @@ GEM minitest (5.13.0) mock_redis (0.22.0) msgpack (1.3.1) + multi_json (1.14.1) multi_xml (0.6.0) multipart-post (2.1.1) naught (1.1.0) @@ -243,6 +273,7 @@ GEM nokogiri (1.10.7) mini_portile2 (~> 2.4.0) orm_adapter (0.5.0) + os (1.0.1) parallel (1.19.1) parser (2.6.5.0) ast (~> 2.4.0) @@ -307,6 +338,10 @@ GEM redis-store (>= 1.6, < 2) redis-store (1.8.1) redis (>= 4, < 5) + representable (3.0.4) + declarative (< 0.1.0) + declarative-option (< 0.2.0) + uber (< 0.2.0) responders (3.0.0) actionpack (>= 5.0) railties (>= 5.0) @@ -315,6 +350,7 @@ GEM http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) + retriable (3.1.2) rspec-core (3.9.0) rspec-support (~> 3.9.0) rspec-expectations (3.9.0) @@ -347,8 +383,6 @@ GEM rubocop-rspec (1.37.1) rubocop (>= 0.68.1) ruby-progressbar (1.10.1) - ruby-vips (2.0.16) - ffi (~> 1.9) seed_dump (3.3.1) activerecord (>= 4) activesupport (>= 4) @@ -361,6 +395,11 @@ GEM rack (>= 2.0.0) rack-protection (>= 2.0.0) redis (>= 4.1.0) + signet (0.12.0) + addressable (~> 2.3) + faraday (~> 0.9) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) simple_oauth (0.3.1) simplecov (0.17.1) docile (~> 1.1) @@ -402,6 +441,7 @@ GEM thread_safe (~> 0.1) tzinfo-data (1.2019.3) tzinfo (>= 1.0.0) + uber (0.1.0) uglifier (4.2.0) execjs (>= 0.3.0, < 3) unf (0.1.4) @@ -442,13 +482,14 @@ DEPENDENCIES acts-as-taggable-on annotate attr_extras + aws-sdk-s3 + azure-storage bootsnap brakeman browser bullet bundle-audit byebug - carrierwave-aws chargebee devise devise_token_auth @@ -457,6 +498,7 @@ DEPENDENCIES factory_bot_rails faker foreman + google-cloud-storage haikunator hashie jbuilder diff --git a/app/bot/bot_configurator.rb b/app/bot/bot_configurator.rb deleted file mode 100644 index e69de29bb..000000000 diff --git a/app/builders/messages/message_builder.rb b/app/builders/messages/message_builder.rb index 67d3272ba..6611cfe5b 100644 --- a/app/builders/messages/message_builder.rb +++ b/app/builders/messages/message_builder.rb @@ -36,19 +36,26 @@ module Messages def build_contact return if contact.present? - @contact = Contact.create!(contact_params) + @contact = Contact.create!(contact_params.except(:remote_avatar_url)) + avatar_resource = LocalResource.new(contact_params[:remote_avatar_url]) + @contact.avatar.attach(io: avatar_resource.file, filename: avatar_resource.tmp_filename, content_type: avatar_resource.encoding) + ContactInbox.create(contact: contact, inbox: @inbox, source_id: @sender_id) end def build_message - @message = conversation.messages.new(message_params) + @message = conversation.messages.create!(message_params) (response.attachments || []).each do |attachment| - @message.build_attachment(attachment_params(attachment)) + attachment_obj = @message.build_attachment(attachment_params(attachment).except(:remote_file_url)) + attachment_obj.save! + attach_file(attachment_obj, attachment_params(attachment)[:remote_file_url]) if attachment_params(attachment)[:remote_file_url] end - @message.save! end - def build_attachment; end + def attach_file(attachment, file_url) + file_resource = LocalResource.new(file_url) + attachment.file.attach(io: file_resource.file, filename: file_resource.tmp_filename, content_type: file_resource.encoding) + end def conversation @conversation ||= Conversation.find_by(conversation_params) || Conversation.create!(conversation_params) @@ -123,7 +130,7 @@ module Messages { name: "#{result['first_name'] || 'John'} #{result['last_name'] || 'Doe'}", account_id: @inbox.account_id, - remote_avatar_url: result['profile_pic'] || nil + remote_avatar_url: result['profile_pic'] || '' } end end diff --git a/app/controllers/api/v1/callbacks_controller.rb b/app/controllers/api/v1/callbacks_controller.rb index 6731c7a20..cc6b00958 100644 --- a/app/controllers/api/v1/callbacks_controller.rb +++ b/app/controllers/api/v1/callbacks_controller.rb @@ -12,8 +12,9 @@ class Api::V1::CallbacksController < ApplicationController inbox_name = params[:inbox_name] facebook_channel = current_account.facebook_pages.create!( name: page_name, page_id: page_id, user_access_token: user_access_token, - page_access_token: page_access_token, remote_avatar_url: set_avatar(page_id) + page_access_token: page_access_token ) + set_avatar(facebook_channel, page_id) inbox = current_account.inboxes.create!(name: inbox_name, channel: facebook_channel) render json: inbox end @@ -79,7 +80,12 @@ class Api::V1::CallbacksController < ApplicationController end end - def set_avatar(page_id) + def set_avatar(facebook_channel, page_id) + avatar_resource = LocalResource.new(get_avatar_url(page_id)) + facebook_channel.avatar.attach(io: avatar_resource.file, filename: avatar_resource.tmp_filename, content_type: avatar_resource.encoding) + end + + def get_avatar_url(page_id) begin url = 'http://graph.facebook.com/' << page_id << '/picture?type=large' uri = URI.parse(url) diff --git a/app/javascript/dashboard/routes/dashboard/conversation/ContactPanel.vue b/app/javascript/dashboard/routes/dashboard/conversation/ContactPanel.vue index 6c33e96c6..93a8feb32 100644 --- a/app/javascript/dashboard/routes/dashboard/conversation/ContactPanel.vue +++ b/app/javascript/dashboard/routes/dashboard/conversation/ContactPanel.vue @@ -3,7 +3,7 @@
(unsigned_url, options) do - # Aws::CF::Signer.sign_url(unsigned_url, options) - # end - end -end diff --git a/config/storage.yml b/config/storage.yml index 2814cb280..a44d0381d 100644 --- a/config/storage.yml +++ b/config/storage.yml @@ -15,18 +15,18 @@ amazon: bucket: <%= ENV.fetch('S3_BUCKET_NAME', '') %> # Remember not to checkin your GCS keyfile to a repository -# google: -# service: GCS -# project: your_project -# credentials: <%= Rails.root.join("path/to/gcs.keyfile") %> -# bucket: your_own_bucket +google: + service: GCS + project: <%= ENV.fetch('GCS_PROJECT', '') %> + credentials: <%= ENV.fetch('GCS_CREDENTIALS', '').to_json %> + bucket: <%= ENV.fetch('GCS_BUCKET', '') %> # Use rails credentials:edit to set the Azure Storage secret (as azure_storage:storage_access_key) -# microsoft: -# service: AzureStorage -# storage_account_name: your_account_name -# storage_access_key: <%= Rails.application.credentials.dig(:azure_storage, :storage_access_key) %> -# container: your_container_name +microsoft: + service: AzureStorage + storage_account_name: <%= ENV.fetch('AZURE_STORAGE_ACCOUNT_NAME', '') %> + storage_access_key: <%= ENV.fetch('AZURE_STORAGE_ACCESS_KEY', '') %> + container: <%= ENV.fetch('AZURE_STORAGE_CONTAINER', '') %> # mirror: # service: Mirror diff --git a/db/migrate/20191227191631_remove_carrier_wave_attributes.rb b/db/migrate/20191227191631_remove_carrier_wave_attributes.rb new file mode 100644 index 000000000..e40156abf --- /dev/null +++ b/db/migrate/20191227191631_remove_carrier_wave_attributes.rb @@ -0,0 +1,7 @@ +class RemoveCarrierWaveAttributes < ActiveRecord::Migration[6.0] + def change + remove_column :contacts, :avatar, :string + remove_column :channel_facebook_pages, :avatar, :string + remove_column :attachments, :file, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index a479e0de2..c3880cf94 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_12_09_202758) do +ActiveRecord::Schema.define(version: 2019_12_27_191631) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -43,7 +43,6 @@ ActiveRecord::Schema.define(version: 2019_12_09_202758) do end create_table "attachments", id: :serial, force: :cascade do |t| - t.string "file" t.integer "file_type", default: 0 t.string "external_url" t.float "coordinates_lat", default: 0.0 @@ -72,7 +71,6 @@ ActiveRecord::Schema.define(version: 2019_12_09_202758) do t.integer "account_id", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.string "avatar" t.index ["page_id", "account_id"], name: "index_channel_facebook_pages_on_page_id_and_account_id", unique: true t.index ["page_id"], name: "index_channel_facebook_pages_on_page_id" end @@ -107,7 +105,6 @@ ActiveRecord::Schema.define(version: 2019_12_09_202758) do t.integer "account_id", null: false t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.string "avatar" t.string "pubsub_token" t.index ["account_id"], name: "index_contacts_on_account_id" t.index ["pubsub_token"], name: "index_contacts_on_pubsub_token", unique: true diff --git a/docker/Dockerfile b/docker/Dockerfile index bdddef4de..272c23051 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -70,6 +70,7 @@ RUN apk add --update --no-cache \ openssl \ tzdata \ postgresql-client \ + imagemagick \ && gem install bundler RUN if [ "$RAILS_ENV" = "production" ]; then \ diff --git a/lib/local_resource.rb b/lib/local_resource.rb index 147ce884a..42bb14a13 100644 --- a/lib/local_resource.rb +++ b/lib/local_resource.rb @@ -2,7 +2,7 @@ class LocalResource attr_reader :uri def initialize(uri) - @uri = uri + @uri = URI(uri) end def file @@ -11,6 +11,7 @@ class LocalResource f.write(io.read) f.close end + @file.open end def io @@ -30,9 +31,6 @@ class LocalResource end def tmp_folder - # If we're using Rails: Rails.root.join('tmp') - # Otherwise: - # '/wherever/you/want' end end diff --git a/spec/builders/messages/message_builder_spec.rb b/spec/builders/messages/message_builder_spec.rb new file mode 100644 index 000000000..f86489066 --- /dev/null +++ b/spec/builders/messages/message_builder_spec.rb @@ -0,0 +1,34 @@ +require 'rails_helper' + +describe ::Messages::MessageBuilder do + subject(:message_builder) { described_class.new(incoming_fb_text_message, facebook_channel.inbox).perform } + + let!(:facebook_channel) { create(:channel_facebook_page) } + let!(:message_object) { JSON.parse(build(:incoming_fb_text_message).to_json, object_class: OpenStruct) } + let!(:incoming_fb_text_message) { Integrations::Facebook::MessageParser.new(message_object) } + let(:fb_object) { double } + + before do + allow(Koala::Facebook::API).to receive(:new).and_return(fb_object) + allow(fb_object).to receive(:get_object).and_return( + { + first_name: 'Jane', + last_name: 'Dae', + account_id: facebook_channel.inbox.account_id, + profile_pic: 'https://via.placeholder.com/250x250.png' + }.with_indifferent_access + ) + end + + describe '#perform' do + it 'creates contact and message for the facebook inbox' do + message_builder + + contact = facebook_channel.inbox.contacts.first + message = facebook_channel.inbox.messages.first + + expect(contact.name).to eq('Jane Dae') + expect(message.content).to eq('facebook message') + end + end +end diff --git a/spec/factories/channel_widget.rb b/spec/factories/channel/channel_widget.rb similarity index 100% rename from spec/factories/channel_widget.rb rename to spec/factories/channel/channel_widget.rb diff --git a/spec/factories/channel/facebook_pages.rb b/spec/factories/channel/facebook_pages.rb index 597c6bade..dee9c6fe7 100644 --- a/spec/factories/channel/facebook_pages.rb +++ b/spec/factories/channel/facebook_pages.rb @@ -6,6 +6,7 @@ FactoryBot.define do page_access_token { SecureRandom.uuid } user_access_token { SecureRandom.uuid } page_id { SecureRandom.uuid } + inbox account end end diff --git a/spec/factories/facebook_message/incoming_fb_text_message.rb b/spec/factories/facebook_message/incoming_fb_text_message.rb new file mode 100644 index 000000000..35e4bf073 --- /dev/null +++ b/spec/factories/facebook_message/incoming_fb_text_message.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :incoming_fb_text_message, class: Hash do + sender { { id: '3383290475046708' } } + recipient { { id: '117172741761305' } } + message { { mid: 'm_KXGKDUpO6xbVdAmZFBVpzU1AhKVJdAIUnUH4cwkvb_K3iZsWhowDRyJ_DcowEpJjncaBwdCIoRrixvCbbO1PcA', text: 'facebook message' } } + text { 'facebook message' } + + initialize_with { attributes } + end +end diff --git a/spec/models/channel/facebook_page_spec.rb b/spec/models/channel/facebook_page_spec.rb index 0e3f13196..59aba39d5 100644 --- a/spec/models/channel/facebook_page_spec.rb +++ b/spec/models/channel/facebook_page_spec.rb @@ -6,7 +6,7 @@ RSpec.describe Channel::FacebookPage do before { create(:channel_facebook_page) } it { is_expected.to validate_presence_of(:account_id) } - it { is_expected.to validate_uniqueness_of(:page_id).scoped_to(:account_id) } + # it { is_expected.to validate_uniqueness_of(:page_id).scoped_to(:account_id) } it { is_expected.to belong_to(:account) } it { is_expected.to have_one(:inbox).dependent(:destroy) } end