Sidekiq to solid queue migration

.env.example

@@ -27,13 +27,6 @@ GOOGLE_OAUTH_CLIENT_SECRET='google-oauth-client-secret'
 # Application Monitoring
 NEW_RELIC_LICENSE_KEY='replace with newrelic licence key'
 
-# Redis url
-REDIS_URL='redis://127.0.0.1:6379/12'
-
-# Sidekiq Configuration
-SIDEKIQ_USERNAME: <some_username>
-SIDEKIQ_PASSWORD: <some_password>
-
 # Stripe
 STRIPE_PUBLISHABLE_KEY="stripe_publishable_key"
 STRIPE_SECRET_KEY="stripe_secret_key"
@@ -61,4 +54,8 @@ SEED_DATA_FROM_CSV=true
 
 VIRTUAL_VERIFIED_ADMIN_EMAILS=[]
 
-CI=true
+CI=true
+
+# SolidQueue Configuration
+SOLID_QUEUE_USERNAME: <some_username>
+SOLID_QUEUE_PASSWORD: <some_password>

Gemfile

@@ -114,11 +114,9 @@ gem "active_interaction"
 # For stripe payments
 gem "stripe"
 
-# Background job processing adapter
-gem "sidekiq", "~> 7.2"
-
-#  job scheduler extension for Sidekiq
-gem "sidekiq-scheduler"
+# Background job processing adapter and dashboard
+gem "mission_control-jobs"
+gem "solid_queue", "~> 0.3"
 
 # searchkick for elasticsearch
 gem "elasticsearch", "< 7.14" # select one
@@ -209,7 +207,6 @@ group :test, :ci do
   # Strategies for cleaning databases in Ruby.
   gem "database_cleaner", "~> 2.0"
   gem "hash_dot"
-  gem "rspec-sidekiq", git: "https://github.com/wspurgin/rspec-sidekiq", branch: "main"
 
   gem "rspec-buildkite"
   gem "rspec-retry"

Gemfile.lock

@@ -11,17 +11,6 @@ GIT
       rubocop
       smart_properties
 
-GIT
-  remote: https://github.com/wspurgin/rspec-sidekiq
-  revision: 0e710d7698c2fcea77245c7d458bb00b0fc50dae
-  branch: main
-  specs:
-    rspec-sidekiq (4.1.0)
-      rspec-core (~> 3.0)
-      rspec-expectations (~> 3.0)
-      rspec-mocks (~> 3.0)
-      sidekiq (>= 5, < 8)
-
 GEM
   remote: https://rubygems.org/
   specs:
@@ -222,7 +211,7 @@ GEM
       faraday (~> 1)
       multi_json
     erubi (1.12.0)
-    et-orbi (1.2.7)
+    et-orbi (1.2.11)
       tzinfo
     execjs (2.8.1)
     factory_bot (6.2.1)
@@ -257,8 +246,8 @@ GEM
     faraday-retry (1.0.3)
     ffi (1.15.5)
     foreman (0.87.2)
-    fugit (1.8.1)
-      et-orbi (~> 1, >= 1.2.7)
+    fugit (1.11.0)
+      et-orbi (~> 1, >= 1.2.11)
       raabro (~> 1.4)
     gems (1.2.0)
     globalid (1.1.0)
@@ -305,6 +294,10 @@ GEM
     image_processing (1.12.2)
       mini_magick (>= 4.9.5, < 5)
       ruby-vips (>= 2.0.17, < 3)
+    importmap-rails (2.0.1)
+      actionpack (>= 6.0.0)
+      activesupport (>= 6.0.0)
+      railties (>= 6.0.0)
     io-console (0.7.2)
     irb (1.12.0)
       rdoc
@@ -354,6 +347,11 @@ GEM
     mini_magick (4.12.0)
     mini_mime (1.1.5)
     minitest (5.23.1)
+    mission_control-jobs (0.2.1)
+      importmap-rails
+      rails (~> 7.1)
+      stimulus-rails
+      turbo-rails
     money (6.16.0)
       i18n (>= 0.6.4, <= 2)
     msgpack (1.6.0)
@@ -483,8 +481,6 @@ GEM
       railties (>= 3.2)
       tilt
     redis (4.8.0)
-    redis-client (0.22.1)
-      connection_pool
     regexp_parser (2.9.0)
     reline (0.5.3)
       io-console (~> 0.5)
@@ -552,8 +548,6 @@ GEM
     ruby_audit (2.2.0)
       bundler-audit (~> 0.9.0)
     rubyzip (2.3.2)
-    rufus-scheduler (3.8.2)
-      fugit (~> 1.1, >= 1.1.6)
     safely_block (0.4.0)
     sass-rails (6.0.0)
       sassc-rails (~> 2.1, >= 2.1.1)
@@ -589,15 +583,6 @@ GEM
       activesupport (>= 3)
     shoulda-matchers (5.3.0)
       activesupport (>= 5.2.0)
-    sidekiq (7.2.4)
-      concurrent-ruby (< 2)
-      connection_pool (>= 2.3.0)
-      rack (>= 2.2.4)
-      redis-client (>= 0.19.0)
-    sidekiq-scheduler (5.0.1)
-      rufus-scheduler (~> 3.2)
-      sidekiq (>= 4, < 8)
-      tilt (>= 1.4.0)
     signet (0.17.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
@@ -613,6 +598,12 @@ GEM
     snaky_hash (2.0.1)
       hashie
       version_gem (~> 1.1, >= 1.1.1)
+    solid_queue (0.3.3)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11.0)
+      railties (>= 7.1)
     spring (4.1.1)
     sprockets (4.2.1)
       concurrent-ruby (~> 1.0)
@@ -621,6 +612,8 @@ GEM
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
+    stimulus-rails (1.3.3)
+      railties (>= 6.0.0)
     stringio (3.1.0)
     stripe (8.2.0)
     strong_migrations (1.4.2)
@@ -630,6 +623,10 @@ GEM
     tilt (2.0.11)
     timeout (0.4.1)
     trailblazer-option (0.1.2)
+    turbo-rails (2.0.5)
+      actionpack (>= 6.0.0)
+      activejob (>= 6.0.0)
+      railties (>= 6.0.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     uber (0.1.0)
@@ -698,6 +695,7 @@ DEPENDENCIES
   jbuilder (~> 2.11)
   letter_opener
   letter_opener_web
+  mission_control-jobs
   money
   newrelic_rpm (~> 9.8.0)
   nokogiri (>= 1.16.2)
@@ -720,7 +718,6 @@ DEPENDENCIES
   rspec-buildkite
   rspec-rails (~> 6.1)
   rspec-retry
-  rspec-sidekiq!
   rubocop
   rubocop-performance
   rubocop-rails
@@ -735,9 +732,8 @@ DEPENDENCIES
   shakapacker (= 6.0.0)
   shoulda-callback-matchers (~> 1.1.1)
   shoulda-matchers (~> 5.1)
-  sidekiq (~> 7.2)
-  sidekiq-scheduler
   simplecov
+  solid_queue (~> 0.3)
   spring
   stripe
   strong_migrations

Procfile

@@ -1,2 +1,2 @@
 web: bundle exec puma -t 5:5 -p ${PORT:-3000} -e ${RACK_ENV:-production}
-worker: bundle exec sidekiq -e production -C config/sidekiq.yml
+worker: bundle exec rake solid_queue:start

Procfile.dev

@@ -1,3 +1,3 @@
 web: bin/rails s -b 0.0.0.0 -p 3000
 webpacker: bin/webpacker-dev-server
-sidekiq: bundle exec sidekiq -e development -C config/sidekiq.yml
+solidqueue: bundle exec rake solid_queue:start

app/controllers/mission_control_controller.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class MissionControlController < ApplicationController
+  before_action :authenticate!, if: :restricted_env?
+  skip_after_action :verify_authorized
+
+  private
+
+    def authenticate!
+      authenticate_or_request_with_http_basic do |username, password|
+        username == ENV.fetch("SOLID_QUEUE_USERNAME") && password == ENV.fetch("SOLID_QUEUE_PASSWORD")
+      end
+    end
+
+    def restricted_env?
+      Rails.env.production?
+    end
+end

config/application.rb

@@ -44,6 +44,7 @@ class Application < Rails::Application
     config.react.camelize_props = true
 
     # Use a real queuing backend for Active Job (and separate queues per environment).
-    config.active_job.queue_adapter = :sidekiq
+    config.active_job.queue_adapter = :solid_queue
+    config.mission_control.jobs.base_controller_class = "MissionControlController"
   end
 end

config/environments/production.rb

@@ -70,7 +70,7 @@
   # config.cache_store = :mem_cache_store
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
-  # config.active_job.queue_adapter     = :resque
+  config.active_job.queue_adapter = :solid_queue
   # config.active_job.queue_name_prefix = "miru_web_production"
 
   config.action_mailer.perform_caching = false
@@ -101,5 +101,4 @@
   host = ENV.fetch("APP_BASE_URL")
   config.action_mailer.default_url_options = { host: }
   config.action_mailer.asset_host = host
-  config.active_job.queue_adapter = :sidekiq
 end

config/routes.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
-require "sidekiq/web"
-
 class ActionDispatch::Routing::Mapper
   def draw(routes_name)
     instance_eval(File.read(Rails.root.join("config/routes/#{routes_name}.rb")))
   end
 end
 
 Rails.application.routes.draw do
+  mount MissionControl::Jobs::Engine, at: "/jobs"
+
   namespace :admin do
       resources :users
       resources :timesheet_entries
@@ -38,16 +38,6 @@ def draw(routes_name)
     mount LetterOpenerWeb::Engine, at: "/sent_emails"
   end
 
-  Sidekiq::Web.use Rack::Auth::Basic do |username, password|
-    ActiveSupport::SecurityUtils.secure_compare(
-      ::Digest::SHA256.hexdigest(username),
-      ::Digest::SHA256.hexdigest(ENV["SIDEKIQ_USERNAME"])) &
-      ActiveSupport::SecurityUtils.secure_compare(
-        ::Digest::SHA256.hexdigest(password),
-        ::Digest::SHA256.hexdigest(ENV["SIDEKIQ_PASSWORD"]))
-  end
-  mount Sidekiq::Web, at: "/sidekiq"
-
   draw(:internal_api)
   draw(:api)
 

config/solid_queue.yml

@@ -0,0 +1,25 @@
+default: &default
+  dispatchers:
+    - polling_interval: 1
+      batch_size: 500
+      recurring_tasks:
+        update_invoice_status:
+          class: UpdateInvoiceStatusToOverdueJob
+          schedule: '0 0 * * *'  # Runs every day at 12AM UTC
+        weekly_reminder:
+          class: WeeklyReminderToUserJob
+          schedule: '0 14 * * 1 Asia/Kolkata' # Runs every Monday at 14:00
+  workers:
+    - queues: "*"
+      threads: 5
+      processes: 1
+      polling_interval: 0.1
+
+development:
+ <<: *default
+
+test:
+ <<: *default
+
+production:
+ <<: *default