Fix auth error in tests

sarc/testing/__init__.py

@@ -1,36 +1,45 @@
 import multiprocessing
 import os
 import subprocess
+import sys
 import time
 
 
-def popen_reader(state, function, args, shell=False):
+def popen_reader(state, function, args, env, shell=False):
     """Execute a command with the given formatter."""
     with subprocess.Popen(
         args,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         text=True,
+        env={
+            **os.environ,
+            **env
+        },
         shell=shell,
     ) as process:
         try:
             while process.poll() is None:
                 line = process.stdout.readline()
 
                 if len(line) > 0:
-                    function(line, state)
+                    function(line.strip(), state)
 
-            return process.poll()
+            return sys.exit(process.poll())
         except KeyboardInterrupt:
             print("Stopping due to user interrupt")
             process.kill()
 
-        return -1
+        return sys.exit(1)
 
 
 def is_ready(line, state):
-    state["ready"] = int("mongo_launch" in line)
-    state["last_line"] = line.strip()
+    # Warning: The script contains `set -v` which outputs the commands as they
+    # are executed (e.g. `function mongo_launch` or `mongo_launch &`). A more
+    # robust solution should probably be looked for rather than lokking into the
+    # stdout for the executed command
+    state["ready"] = int(state.get("ready", 0) or "mongo_launch" == line)
+    state["last_line"] = line
     state["error"] += int("MongoNetworkError" in line)
 
 
@@ -45,8 +54,8 @@ def __init__(
         admin_pass="admin_pass",
         write_name="write_name",
         write_pass="write_pass",
-        user_pass="user_pass",
         user_name="user_name",
+        user_pass="user_pass",
     ) -> None:
         self.env = {
             "MONGO_PORT": f"{port}",
@@ -73,26 +82,37 @@ def __init__(
         )
 
     def shutdown(self):
-        subprocess.call(["mongod", "--dbpath", self.path, "--shutdown"])
+        subprocess.call(["bash", "-c", f". {self.script} && mongo_stop"],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            env={
+                **os.environ,
+                **self.env,
+                **{"LAUNCH_MONGO": "0"}
+            }
+        )
 
     def __enter__(self):
         self.shutdown()
 
-        for k, v in self.env.items():
-            os.environ[k] = str(v)
-
         try:
             self.proc = multiprocessing.Process(
-                target=popen_reader, args=(self.state, is_ready, ["bash", self.script])
+                target=popen_reader, args=(self.state, is_ready, ["bash", self.script], self.env)
             )
             self.proc.start()
             while self.proc.is_alive() and self.state.get("ready", 0) != 1:
                 time.sleep(0.01)
 
                 if line := self.state.get("last_line"):
+                    # There is no real garanty that we will get last_line here,
+                    # it could already have been overriden by the process's call
+                    # to is_ready
                     print(line)
                     self.state["last_line"] = None
 
+            if self.proc.exitcode:
+                raise multiprocessing.ProcessError(f"Failed to start mongodb with env {self.env}")
+
         except:
             self.shutdown()
             raise

scripts/launch_mongod.sh

@@ -46,7 +46,7 @@ echo "====================================="
 ASCENDING=1
 DESCENDING=-1
 
-set -vm
+set -vm -o errexit
 
 
 if ! which mongosh >/dev/null 2>&1; then

tests/conftest.py

@@ -1,15 +1,16 @@
-import json
 import os
+import shutil
 import sys
 import tempfile
+import time
 import zoneinfo
 from pathlib import Path
-from unittest.mock import MagicMock, mock_open, patch
+from unittest.mock import MagicMock, mock_open
 
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import SimpleSpanProcessor
 from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
-from opentelemetry.trace import get_tracer_provider, set_tracer_provider
+from opentelemetry.trace import set_tracer_provider
 
 _tracer_provider = TracerProvider()
 _exporter = InMemorySpanExporter()
@@ -24,7 +25,6 @@
 from sarc.config import (
     ClusterConfig,
     Config,
-    MongoConfig,
     ScraperConfig,
     config,
     parse_config,
@@ -73,6 +73,17 @@ def disabled_cache():
         yield
 
 
+@pytest.fixture(scope="session", autouse=True)
+def clean_up_test_cache_before_run(standard_config_object, worker_id):
+    if worker_id in ("master", 0):
+        if standard_config_object.cache.exists():
+            shutil.rmtree(str(standard_config_object.cache))
+    else:
+        while standard_config_object.cache.exists():
+            time.sleep(1)
+    yield
+
+
 @pytest.fixture
 def tzlocal_is_mtl(monkeypatch):
     monkeypatch.setattr("sarc.config.TZLOCAL", zoneinfo.ZoneInfo("America/Montreal"))

tests/functional/jobs/test_func_sacct.py

@@ -281,6 +281,8 @@ def test_sacct_bin_and_accounts(test_config, remote):
 @patch("os.system")
 @pytest.mark.usefixtures("write_setup")
 def test_localhost(os_system, monkeypatch):
+    # This test requires write_setup.cache to be empty else it will never call
+    # mock_subprocess_run
     def mock_subprocess_run(*args, **kwargs):
         mock_subprocess_run.called += 1
         return subprocess.CompletedProcess(

tox.ini

@@ -14,8 +14,7 @@ commands_pre =
     podman run -dt --name testenv_mongo -p 27017:27017/tcp docker.io/library/mongo:latest
 commands = 
     poetry install --with dev
-    -poetry run coverage run --source sarc --parallel-mode -m pytest --doctest-modules --durations=50 --durations-min 1 -vv --timeout=20 -vvv tests/ {posargs}
-    poetry run coverage run --source sarc --parallel-mode -m pytest --doctest-modules --durations=50 --durations-min 1 -vv --timeout=20 -vvv --lf --last-failed-no-failures none --suppress-no-test-exit-code tests/ {posargs}
+    poetry run coverage run --source sarc --parallel-mode -m pytest --doctest-modules --durations=50 --durations-min 1 -vv --timeout=20 -vvv tests/ {posargs}
     poetry run coverage combine
     poetry run coverage report -m
 commands_post =