diff --git a/.env.example b/.env.example index 1c277032d0..1e634ea8f2 100644 --- a/.env.example +++ b/.env.example @@ -61,4 +61,8 @@ SEED_DATA_FROM_CSV=true VIRTUAL_VERIFIED_ADMIN_EMAILS=[] -CI=true \ No newline at end of file +CI=true + +# SolidQueue Configuration +SOLID_QUEUE_USERNAME: +SOLID_QUEUE_PASSWORD: \ No newline at end of file diff --git a/Gemfile b/Gemfile index d583ce6432..f4537755c1 100644 --- a/Gemfile +++ b/Gemfile @@ -114,6 +114,8 @@ gem "active_interaction" # For stripe payments gem "stripe" +gem "mission_control-jobs" +gem "solid_queue" # Background job processing adapter gem "sidekiq", "~> 7.2" diff --git a/Gemfile.lock b/Gemfile.lock index 3fec248c4b..9b35e8fdb9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -305,6 +305,10 @@ GEM image_processing (1.12.2) mini_magick (>= 4.9.5, < 5) ruby-vips (>= 2.0.17, < 3) + importmap-rails (2.0.1) + actionpack (>= 6.0.0) + activesupport (>= 6.0.0) + railties (>= 6.0.0) io-console (0.7.2) irb (1.12.0) rdoc @@ -354,6 +358,11 @@ GEM mini_magick (4.12.0) mini_mime (1.1.5) minitest (5.23.1) + mission_control-jobs (0.2.1) + importmap-rails + rails (~> 7.1) + stimulus-rails + turbo-rails money (6.16.0) i18n (>= 0.6.4, <= 2) msgpack (1.6.0) @@ -613,6 +622,8 @@ GEM snaky_hash (2.0.1) hashie version_gem (~> 1.1, >= 1.1.1) + solid_queue (0.2.2) + rails (~> 7.1) spring (4.1.1) sprockets (4.2.1) concurrent-ruby (~> 1.0) @@ -621,6 +632,8 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) + stimulus-rails (1.3.3) + railties (>= 6.0.0) stringio (3.1.0) stripe (8.2.0) strong_migrations (1.4.2) @@ -630,6 +643,10 @@ GEM tilt (2.0.11) timeout (0.4.1) trailblazer-option (0.1.2) + turbo-rails (2.0.5) + actionpack (>= 6.0.0) + activejob (>= 6.0.0) + railties (>= 6.0.0) tzinfo (2.0.6) concurrent-ruby (~> 1.0) uber (0.1.0) @@ -698,6 +715,7 @@ DEPENDENCIES jbuilder (~> 2.11) letter_opener letter_opener_web + mission_control-jobs money newrelic_rpm (~> 9.8.0) nokogiri (>= 1.16.2) @@ -738,6 +756,7 @@ DEPENDENCIES sidekiq (~> 7.2) sidekiq-scheduler simplecov + solid_queue spring stripe strong_migrations diff --git a/Procfile.dev b/Procfile.dev index ea1ced25b2..b0a11b4ab8 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,3 +1,5 @@ web: bin/rails s -b 0.0.0.0 -p 3000 webpacker: bin/webpacker-dev-server sidekiq: bundle exec sidekiq -e development -C config/sidekiq.yml +solidqueueworker: bundle exec rake solid_queue:start + diff --git a/app/controllers/mission_control_controller.rb b/app/controllers/mission_control_controller.rb new file mode 100644 index 0000000000..99465b3b85 --- /dev/null +++ b/app/controllers/mission_control_controller.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class MissionControlController < ApplicationController + before_action :authenticate!, if: :restricted_env? + skip_after_action :verify_authorized + + private + + def authenticate! + authenticate_or_request_with_http_basic do |username, password| + username == ENV.fetch("SOLID_QUEUE_USERNAME") && password == ENV.fetch("SOLID_QUEUE_PASSWORD") + end + end + + def restricted_env? + Rails.env.production? + end +end diff --git a/config/application.rb b/config/application.rb index 56a9aec35f..45d52a6abb 100644 --- a/config/application.rb +++ b/config/application.rb @@ -44,6 +44,7 @@ class Application < Rails::Application config.react.camelize_props = true # Use a real queuing backend for Active Job (and separate queues per environment). - config.active_job.queue_adapter = :sidekiq + config.active_job.queue_adapter = :solid_queue + config.mission_control.jobs.base_controller_class = "MissionControlController" end end diff --git a/config/environments/production.rb b/config/environments/production.rb index c2c0b2a67f..bab5207d25 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -70,7 +70,7 @@ # config.cache_store = :mem_cache_store # Use a real queuing backend for Active Job (and separate queues per environment). - # config.active_job.queue_adapter = :resque + config.active_job.queue_adapter = :solid_queue # config.active_job.queue_name_prefix = "miru_web_production" config.action_mailer.perform_caching = false @@ -101,5 +101,5 @@ host = ENV.fetch("APP_BASE_URL") config.action_mailer.default_url_options = { host: } config.action_mailer.asset_host = host - config.active_job.queue_adapter = :sidekiq + config.active_job.queue_adapter = :solid_queue end diff --git a/config/routes.rb b/config/routes.rb index a087ed0cc3..ccbdb952c3 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -9,6 +9,8 @@ def draw(routes_name) end Rails.application.routes.draw do + mount MissionControl::Jobs::Engine, at: "/jobs" + namespace :admin do resources :users resources :timesheet_entries diff --git a/config/solid_queue.yml b/config/solid_queue.yml new file mode 100644 index 0000000000..9b0a2cb37c --- /dev/null +++ b/config/solid_queue.yml @@ -0,0 +1,28 @@ +default: &default + dispatchers: + - polling_interval: 1 + batch_size: 500 + recurring_tasks: + update_invoice_status: + class: UpdateInvoiceStatusToOverdueJob + schedule: every minute # Runs every day at 12AM UTC + weekly_reminder: + class: WeeklyReminderToUserJob + schedule: every minute # Runs every Monday at 14:00 + workers: + - queues: "*" + threads: 5 + processes: 1 + polling_interval: 0.1 + logger: + level: debug + formatter: detailed + +development: + <<: *default + +test: + <<: *default + +production: + <<: *default diff --git a/db/migrate/20240701052603_create_solid_queue_tables.solid_queue.rb b/db/migrate/20240701052603_create_solid_queue_tables.solid_queue.rb new file mode 100644 index 0000000000..a4b60e6f97 --- /dev/null +++ b/db/migrate/20240701052603_create_solid_queue_tables.solid_queue.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +# This migration comes from solid_queue (originally 20231211200639) +class CreateSolidQueueTables < ActiveRecord::Migration[7.0] + def change + create_table :solid_queue_jobs do |t| + t.string :queue_name, null: false + t.string :class_name, null: false, index: true + t.text :arguments + t.integer :priority, default: 0, null: false + t.string :active_job_id, index: true + t.datetime :scheduled_at + t.datetime :finished_at, index: true + t.string :concurrency_key + + t.timestamps + + t.index [ :queue_name, :finished_at ], name: "index_solid_queue_jobs_for_filtering" + t.index [ :scheduled_at, :finished_at ], name: "index_solid_queue_jobs_for_alerting" + end + + create_table :solid_queue_scheduled_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.datetime :scheduled_at, null: false + + t.datetime :created_at, null: false + + t.index [ :scheduled_at, :priority, :job_id ], name: "index_solid_queue_dispatch_all" + end + + create_table :solid_queue_ready_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + + t.datetime :created_at, null: false + + t.index [ :priority, :job_id ], name: "index_solid_queue_poll_all" + t.index [ :queue_name, :priority, :job_id ], name: "index_solid_queue_poll_by_queue" + end + + create_table :solid_queue_claimed_executions do |t| + t.references :job, index: { unique: true }, null: false + t.bigint :process_id + t.datetime :created_at, null: false + + t.index [ :process_id, :job_id ] + end + + create_table :solid_queue_blocked_executions do |t| + t.references :job, index: { unique: true }, null: false + t.string :queue_name, null: false + t.integer :priority, default: 0, null: false + t.string :concurrency_key, null: false + t.datetime :expires_at, null: false + + t.datetime :created_at, null: false + + t.index [ :expires_at, :concurrency_key ], name: "index_solid_queue_blocked_executions_for_maintenance" + end + + create_table :solid_queue_failed_executions do |t| + t.references :job, index: { unique: true }, null: false + t.text :error + t.datetime :created_at, null: false + end + + create_table :solid_queue_pauses do |t| + t.string :queue_name, null: false, index: { unique: true } + t.datetime :created_at, null: false + end + + create_table :solid_queue_processes do |t| + t.string :kind, null: false + t.datetime :last_heartbeat_at, null: false, index: true + t.bigint :supervisor_id, index: true + + t.integer :pid, null: false + t.string :hostname + t.text :metadata + + t.datetime :created_at, null: false + end + + create_table :solid_queue_semaphores do |t| + t.string :key, null: false, index: { unique: true } + t.integer :value, default: 1, null: false + t.datetime :expires_at, null: false, index: true + + t.timestamps + + t.index [ :key, :value ], name: "index_solid_queue_semaphores_on_key_and_value" + end + + add_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade, + validate: false + add_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade, + validate: false + add_foreign_key :solid_queue_failed_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade, + validate: false + add_foreign_key :solid_queue_ready_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade, + validate: false + add_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs, column: :job_id, on_delete: :cascade, + validate: false + end +end diff --git a/db/migrate/20240701052604_add_missing_index_to_blocked_executions.solid_queue.rb b/db/migrate/20240701052604_add_missing_index_to_blocked_executions.solid_queue.rb new file mode 100644 index 0000000000..4fc4080c10 --- /dev/null +++ b/db/migrate/20240701052604_add_missing_index_to_blocked_executions.solid_queue.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# This migration comes from solid_queue (originally 20240110143450) +class AddMissingIndexToBlockedExecutions < ActiveRecord::Migration[7.1] + disable_ddl_transaction! + + def change + add_index :solid_queue_blocked_executions, [ :concurrency_key, :priority, :job_id ], + name: "index_solid_queue_blocked_executions_for_release", algorithm: :concurrently + end +end diff --git a/db/migrate/20240701053139_validate_create_solid_queue_tables.rb b/db/migrate/20240701053139_validate_create_solid_queue_tables.rb new file mode 100644 index 0000000000..048d1d649b --- /dev/null +++ b/db/migrate/20240701053139_validate_create_solid_queue_tables.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class ValidateCreateSolidQueueTables < ActiveRecord::Migration[7.0] + def change + validate_foreign_key :solid_queue_blocked_executions, :solid_queue_jobs + validate_foreign_key :solid_queue_claimed_executions, :solid_queue_jobs + validate_foreign_key :solid_queue_failed_executions, :solid_queue_jobs + validate_foreign_key :solid_queue_ready_executions, :solid_queue_jobs + validate_foreign_key :solid_queue_scheduled_executions, :solid_queue_jobs + end +end diff --git a/db/schema.rb b/db/schema.rb index 4e7bd6c553..5d30782689 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.1].define(version: 2024_05_16_054849) do +ActiveRecord::Schema[7.1].define(version: 2024_07_01_053139) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -459,6 +459,100 @@ t.datetime "updated_at", null: false end + create_table "solid_queue_blocked_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.string "concurrency_key", null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.index ["concurrency_key", "priority", "job_id"], name: "index_solid_queue_blocked_executions_for_release" + t.index ["expires_at", "concurrency_key"], name: "index_solid_queue_blocked_executions_for_maintenance" + t.index ["job_id"], name: "index_solid_queue_blocked_executions_on_job_id", unique: true + end + + create_table "solid_queue_claimed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.bigint "process_id" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_claimed_executions_on_job_id", unique: true + t.index ["process_id", "job_id"], name: "index_solid_queue_claimed_executions_on_process_id_and_job_id" + end + + create_table "solid_queue_failed_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.text "error" + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_failed_executions_on_job_id", unique: true + end + + create_table "solid_queue_jobs", force: :cascade do |t| + t.string "queue_name", null: false + t.string "class_name", null: false + t.text "arguments" + t.integer "priority", default: 0, null: false + t.string "active_job_id" + t.datetime "scheduled_at" + t.datetime "finished_at" + t.string "concurrency_key" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["active_job_id"], name: "index_solid_queue_jobs_on_active_job_id" + t.index ["class_name"], name: "index_solid_queue_jobs_on_class_name" + t.index ["finished_at"], name: "index_solid_queue_jobs_on_finished_at" + t.index ["queue_name", "finished_at"], name: "index_solid_queue_jobs_for_filtering" + t.index ["scheduled_at", "finished_at"], name: "index_solid_queue_jobs_for_alerting" + end + + create_table "solid_queue_pauses", force: :cascade do |t| + t.string "queue_name", null: false + t.datetime "created_at", null: false + t.index ["queue_name"], name: "index_solid_queue_pauses_on_queue_name", unique: true + end + + create_table "solid_queue_processes", force: :cascade do |t| + t.string "kind", null: false + t.datetime "last_heartbeat_at", null: false + t.bigint "supervisor_id" + t.integer "pid", null: false + t.string "hostname" + t.text "metadata" + t.datetime "created_at", null: false + t.index ["last_heartbeat_at"], name: "index_solid_queue_processes_on_last_heartbeat_at" + t.index ["supervisor_id"], name: "index_solid_queue_processes_on_supervisor_id" + end + + create_table "solid_queue_ready_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_ready_executions_on_job_id", unique: true + t.index ["priority", "job_id"], name: "index_solid_queue_poll_all" + t.index ["queue_name", "priority", "job_id"], name: "index_solid_queue_poll_by_queue" + end + + create_table "solid_queue_scheduled_executions", force: :cascade do |t| + t.bigint "job_id", null: false + t.string "queue_name", null: false + t.integer "priority", default: 0, null: false + t.datetime "scheduled_at", null: false + t.datetime "created_at", null: false + t.index ["job_id"], name: "index_solid_queue_scheduled_executions_on_job_id", unique: true + t.index ["scheduled_at", "priority", "job_id"], name: "index_solid_queue_dispatch_all" + end + + create_table "solid_queue_semaphores", force: :cascade do |t| + t.string "key", null: false + t.integer "value", default: 1, null: false + t.datetime "expires_at", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["expires_at"], name: "index_solid_queue_semaphores_on_expires_at" + t.index ["key", "value"], name: "index_solid_queue_semaphores_on_key_and_value" + t.index ["key"], name: "index_solid_queue_semaphores_on_key", unique: true + end + create_table "stripe_connected_accounts", force: :cascade do |t| t.string "account_id", null: false t.bigint "company_id", null: false @@ -605,6 +699,11 @@ add_foreign_key "project_members", "projects" add_foreign_key "project_members", "users" add_foreign_key "projects", "clients" + add_foreign_key "solid_queue_blocked_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_claimed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_failed_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_ready_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade + add_foreign_key "solid_queue_scheduled_executions", "solid_queue_jobs", column: "job_id", on_delete: :cascade add_foreign_key "stripe_connected_accounts", "companies" add_foreign_key "timeoff_entries", "leave_types" add_foreign_key "timeoff_entries", "users"