Run solid_queue with SQLite

.github/workflows/main.yml

@@ -9,7 +9,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby-version: [3.2.2]
-        database: [mysql]
+        database: [mysql, sqlite]
     services:
       mysql:
         image: mysql:8.0.31

test/dummy/config/database.yml

@@ -5,6 +5,22 @@
 #   gem "mysql2"
 #
 
+<% if ENV["TARGET_DB"] == "sqlite" %>
+default: &default
+  adapter: sqlite3
+  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 50 } %>
+  retries: 100
+
+development:
+  <<: *default
+  database: db/development.sqlite3
+
+test:
+  <<: *default
+  pool: 20
+  database: db/test.sqlite3
+
+<% else %>
 default: &default
   adapter: mysql2
   username: root
@@ -20,3 +36,5 @@ test:
   <<: *default
   pool: 20
   database: solid_queue_test
+<% end %>
+

test/dummy/config/initializers/sqlite3.rb

@@ -0,0 +1,30 @@
+module SqliteImmediateTransactions
+  def begin_db_transaction
+    log("begin immediate transaction", "TRANSACTION") do
+      with_raw_connection(allow_retry: true, materialize_transactions: false) do |conn|
+        conn.transaction(:immediate)
+      end
+    end
+  end
+end
+
+module SQLite3Configuration
+  private
+    def configure_connection
+      super
+
+      if @config[:retries]
+        retries = self.class.type_cast_config_to_integer(@config[:retries])
+        raw_connection.busy_handler do |count|
+          (count <= retries).tap { |result| sleep count * 0.001 if result }
+        end
+      end
+    end
+end
+
+ActiveSupport.on_load :active_record do
+  if defined?(ActiveRecord::ConnectionAdapters::SQLite3Adapter)
+    ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend SqliteImmediateTransactions
+    ActiveRecord::ConnectionAdapters::SQLite3Adapter.prepend SQLite3Configuration
+  end
+end

test/dummy/config/solid_queue.yml

@@ -5,7 +5,7 @@ default: &default
     default:
       pool_size: 5
   scheduler:
-    polling_interval: 300
+    polling_interval: 1
     batch_size: 500
 
 development:

test/integration/jobs_lifecycle_test.rb

@@ -44,7 +44,7 @@ class JobsLifecycleTest < ActiveSupport::TestCase
 
     travel_to 5.days.from_now
 
-    wait_for_jobs_to_finish_for(0.5.seconds)
+    wait_for_jobs_to_finish_for(5.seconds)
 
     assert_equal 2, JobBuffer.size
     assert_equal "I'm scheduled later", JobBuffer.last_value

test/integration/processes_lifecycle_test.rb

@@ -113,7 +113,7 @@ class ProcessLifecycleTest < ActiveSupport::TestCase
 
   test "term supervisor exceeding timeout while there are jobs in-flight" do
     no_pause = enqueue_store_result_job("no pause")
-    pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 0.1.second)
+    pause = enqueue_store_result_job("pause", pause: SolidQueue.shutdown_timeout + 1.second)
 
     signal_process(@pid, :TERM, wait: 0.1.second)
     wait_for_jobs_to_finish_for(SolidQueue.shutdown_timeout + 0.1.second)

test/test_helper.rb

@@ -15,6 +15,16 @@
   ActiveSupport::TestCase.fixtures :all
 end
 
+module BlockLogDeviceTimeoutExceptions
+  def write(...)
+    # Prevents Timeout exceptions from occurring during log writing, where they will be swallowed
+    # See https://bugs.ruby-lang.org/issues/9115
+    Thread.handle_interrupt(Timeout::Error => :never, Timeout::ExitException => :never) { super }
+  end
+end
+
+Logger::LogDevice.prepend(BlockLogDeviceTimeoutExceptions)
+
 class ActiveSupport::TestCase
   setup do
     SolidQueue.logger = ActiveSupport::Logger.new(nil)

test/unit/supervisor_test.rb

@@ -47,6 +47,7 @@ class SupervisorTest < ActiveSupport::TestCase
   end
 
   test "abort if there's already a pidfile for a supervisor" do
+    FileUtils.mkdir_p(File.dirname(@pidfile))
     File.write(@pidfile, ::Process.pid.to_s)
 
     pid = run_supervisor_as_fork(mode: :all)