sidekiq to solid queue migration

.env.example

@@ -61,4 +61,8 @@ SEED_DATA_FROM_CSV=true
 
 VIRTUAL_VERIFIED_ADMIN_EMAILS=[]
 
-CI=true
+CI=true
+
+# SolidQueue Configuration
+SOLID_QUEUE_USERNAME: <some_username>
+SOLID_QUEUE_PASSWORD: <some_password>

Gemfile

@@ -114,6 +114,8 @@ gem "active_interaction"
 # For stripe payments
 gem "stripe"
 
+gem "mission_control-jobs"
+gem "solid_queue"
 # Background job processing adapter
 gem "sidekiq", "~> 7.2"
 

Gemfile.lock

@@ -305,6 +305,10 @@ GEM
     image_processing (1.12.2)
       mini_magick (>= 4.9.5, < 5)
       ruby-vips (>= 2.0.17, < 3)
+    importmap-rails (2.0.1)
+      actionpack (>= 6.0.0)
+      activesupport (>= 6.0.0)
+      railties (>= 6.0.0)
     io-console (0.7.2)
     irb (1.12.0)
       rdoc
@@ -354,6 +358,11 @@ GEM
     mini_magick (4.12.0)
     mini_mime (1.1.5)
     minitest (5.23.1)
+    mission_control-jobs (0.2.1)
+      importmap-rails
+      rails (~> 7.1)
+      stimulus-rails
+      turbo-rails
     money (6.16.0)
       i18n (>= 0.6.4, <= 2)
     msgpack (1.6.0)
@@ -613,6 +622,8 @@ GEM
     snaky_hash (2.0.1)
       hashie
       version_gem (~> 1.1, >= 1.1.1)
+    solid_queue (0.2.2)
+      rails (~> 7.1)
     spring (4.1.1)
     sprockets (4.2.1)
       concurrent-ruby (~> 1.0)
@@ -621,6 +632,8 @@ GEM
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
+    stimulus-rails (1.3.3)
+      railties (>= 6.0.0)
     stringio (3.1.0)
     stripe (8.2.0)
     strong_migrations (1.4.2)
@@ -630,6 +643,10 @@ GEM
     tilt (2.0.11)
     timeout (0.4.1)
     trailblazer-option (0.1.2)
+    turbo-rails (2.0.5)
+      actionpack (>= 6.0.0)
+      activejob (>= 6.0.0)
+      railties (>= 6.0.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     uber (0.1.0)
@@ -698,6 +715,7 @@ DEPENDENCIES
   jbuilder (~> 2.11)
   letter_opener
   letter_opener_web
+  mission_control-jobs
   money
   newrelic_rpm (~> 9.8.0)
   nokogiri (>= 1.16.2)
@@ -738,6 +756,7 @@ DEPENDENCIES
   sidekiq (~> 7.2)
   sidekiq-scheduler
   simplecov
+  solid_queue
   spring
   stripe
   strong_migrations

Procfile.dev

@@ -1,3 +1,5 @@
 web: bin/rails s -b 0.0.0.0 -p 3000
 webpacker: bin/webpacker-dev-server
 sidekiq: bundle exec sidekiq -e development -C config/sidekiq.yml
+solidqueueworker: bundle exec rake solid_queue:start
+

app/controllers/mission_control_controller.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class MissionControlController < ApplicationController
+  before_action :authenticate!, if: :restricted_env?
+  skip_after_action :verify_authorized
+
+  private
+
+    def authenticate!
+      authenticate_or_request_with_http_basic do |username, password|
+        username == ENV.fetch("SOLID_QUEUE_USERNAME") && password == ENV.fetch("SOLID_QUEUE_PASSWORD")
+      end
+    end
+
+    def restricted_env?
+      Rails.env.production?
+    end
+end

config/application.rb

@@ -44,6 +44,7 @@ class Application < Rails::Application
     config.react.camelize_props = true
 
     # Use a real queuing backend for Active Job (and separate queues per environment).
-    config.active_job.queue_adapter = :sidekiq
+    config.active_job.queue_adapter = :solid_queue
+    config.mission_control.jobs.base_controller_class = "MissionControlController"
   end
 end

config/environments/production.rb

@@ -70,7 +70,7 @@
   # config.cache_store = :mem_cache_store
 
   # Use a real queuing backend for Active Job (and separate queues per environment).
-  # config.active_job.queue_adapter     = :resque
+  config.active_job.queue_adapter = :solid_queue
   # config.active_job.queue_name_prefix = "miru_web_production"
 
   config.action_mailer.perform_caching = false
@@ -101,5 +101,5 @@
   host = ENV.fetch("APP_BASE_URL")
   config.action_mailer.default_url_options = { host: }
   config.action_mailer.asset_host = host
-  config.active_job.queue_adapter = :sidekiq
+  config.active_job.queue_adapter = :solid_queue
 end

config/routes.rb

@@ -9,6 +9,8 @@ def draw(routes_name)
 end
 
 Rails.application.routes.draw do
+  mount MissionControl::Jobs::Engine, at: "/jobs"
+
   namespace :admin do
       resources :users
       resources :timesheet_entries

config/solid_queue.yml

@@ -0,0 +1,28 @@
+default: &default
+  dispatchers:
+    - polling_interval: 1
+      batch_size: 500
+      recurring_tasks:
+        update_invoice_status:
+          class: UpdateInvoiceStatusToOverdueJob
+          schedule: every minute  # Runs every day at 12AM UTC
+        weekly_reminder:
+          class: WeeklyReminderToUserJob
+          schedule: every minute  # Runs every Monday at 14:00
+  workers:
+    - queues: "*"
+      threads: 5
+      processes: 1
+      polling_interval: 0.1
+  logger:
+    level: debug
+    formatter: detailed
+
+development:
+ <<: *default
+
+test:
+ <<: *default
+
+production:
+ <<: *default

db/migrate/20240701052603_create_solid_queue_tables.solid_queue.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+# This migration comes from solid_queue (originally 20231211200639)
+class CreateSolidQueueTables < ActiveRecord::Migration[7.0]
+  def change
+    create_table :solid_queue_jobs do |t|
+      t.string :queue_name, null: false
+      t.string :class_name, null: false, index: true
+      t.text :arguments
+      t.integer :priority, default: 0, null: false
+      t.string :active_job_id, index: true
+      t.datetime :scheduled_at
+      t.datetime :finished_at, index: true
+      t.string :concurrency_key
+
+      t.timestamps
+
+      t.index [ :queue_name, :finished_at ], name: "index_solid_queue_jobs_for_filtering"
+      t.index [ :scheduled_at, :finished_at ], name: "index_solid_queue_jobs_for_alerting"
+    end
+
+    create_table :solid_queue_scheduled_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.datetime :scheduled_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :scheduled_at, :priority, :job_id ], name: "index_solid_queue_dispatch_all"
+    end
+
+    create_table :solid_queue_ready_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :priority, :job_id ], name: "index_solid_queue_poll_all"
+      t.index [ :queue_name, :priority, :job_id ], name: "index_solid_queue_poll_by_queue"
+    end
+
+    create_table :solid_queue_claimed_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.bigint :process_id
+      t.datetime :created_at, null: false
+
+      t.index [ :process_id, :job_id ]
+    end
+
+    create_table :solid_queue_blocked_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.string :queue_name, null: false
+      t.integer :priority, default: 0, null: false
+      t.string :concurrency_key, null: false
+      t.datetime :expires_at, null: false
+
+      t.datetime :created_at, null: false
+
+      t.index [ :expires_at, :concurrency_key ], name: "index_solid_queue_blocked_executions_for_maintenance"
+    end
+
+    create_table :solid_queue_failed_executions do |t|
+      t.references :job, index: { unique: true }, null: false
+      t.text :error
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_pauses do |t|
+      t.string :queue_name, null: false, index: { unique: true }
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_processes do |t|
+      t.string :kind, null: false
+      t.datetime :last_heartbeat_at, null: false, index: true
+      t.bigint :supervisor_id, index: true
+
+      t.integer :pid, null: false
+      t.string :hostname
+      t.text :metadata
+
+      t.datetime :created_at, null: false
+    end
+
+    create_table :solid_queue_semaphores do |t|
+      t.string :key, null: false, index: { unique: true }
+      t.integer :value, default: 1, null: false
+      t.datetime :expires_at, null: false, index: true
+
+      t.timestamps
+
+      t.index [ :key, :value ], name: "index_solid_queue_semaphores_on_key_and_value"
+    end
+
+    add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade,
+      validate: false
+    add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade,
+      validate: false
+    add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade,
+      validate: false
+    add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade,
+      validate: false
+    add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade,
+      validate: false
+  end
+end

db/migrate/20240701052604_add_missing_index_to_blocked_executions.solid_queue.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# This migration comes from solid_queue (originally 20240110143450)
+class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1]
+  disable_ddl_transaction!
+
+  def change
+    add_index :solid_queue_blocked_executions, [ :concurrency_key, :priority, :job_id ],
+      name: "index_solid_queue_blocked_executions_for_release", algorithm: :concurrently
+  end
+end

db/migrate/20240701053139_validate_create_solid_queue_tables.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class ValidateCreateSolidQueueTables < ActiveRecord::Migration[7.0]
+  def change
+    validate_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs
+    validate_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs
+    validate_foreign_key :solid_queue_failed_executions, :solid_queue_jobs
+    validate_foreign_key :solid_queue_ready_executions, :solid_queue_jobs
+    validate_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs
+  end
+end