accept multiple directories for config definitions (#233)

Co-authored-by: adam.labrash <[email protected]>
Flared · Sep 21, 2022 · 43d5bb1 · 43d5bb1
1 parent f4638e8
commit 43d5bb1
Show file tree

Hide file tree

Showing 6 changed files with 17 additions and 13 deletions.
diff --git a/example/worker_manager b/example/worker_manager
@@ -4,7 +4,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT_DIR="$( cd ${DIR}/.. && pwd )"
 
 export SATURN_ENV="${SATURN_ENV:-development}"
-export SATURN_STATIC_DEFINITIONS_DIR="${SATURN_STATIC_DEFINITIONS_DIR:-${DIR}/definitions}"
+export SATURN_STATIC_DEFINITIONS_DIRS="${SATURN_STATIC_DEFINITIONS_DIRS:-${DIR}/definitions}"
 rm -f test.db
 ${ROOT_DIR}/bin/worker_manager
 
diff --git a/src/saturn_engine/config_definitions.py b/src/saturn_engine/config_definitions.py
@@ -21,7 +21,7 @@ class WorkerManagerConfig:
     flask_port: int
     database_url: str
     async_database_url: str
-    static_definitions_directory: str
+    static_definitions_directories: list[str]
     static_definitions_jobs_selector: t.Optional[str]
     work_items_per_worker: int
 

diff --git a/src/saturn_engine/default_config.py b/src/saturn_engine/default_config.py
@@ -42,9 +42,9 @@ class worker_manager(WorkerManagerConfig):
             .replace("sqlite:/", "sqlite+aiosqlite:/")
             .replace("postgresql:/", "postgresql+asyncpg:/")
         )
-        static_definitions_directory: str = os.environ.get(
-            "SATURN_STATIC_DEFINITIONS_DIR", "/opt/saturn/definitions"
-        )
+        static_definitions_directories: list[str] = os.environ.get(
+            "SATURN_STATIC_DEFINITIONS_DIRS", "/opt/saturn/definitions"
+        ).split(":")
         static_definitions_jobs_selector: t.Optional[str] = os.environ.get(
             "SATURN_STATIC_DEFINITIONS_JOBS_SELECTOR"
         )

diff --git a/src/saturn_engine/worker_manager/config/declarative.py b/src/saturn_engine/worker_manager/config/declarative.py
@@ -113,8 +113,12 @@ def load_definitions_from_str(definitions: str) -> StaticDefinitions:
     return compile_static_definitions(load_uncompiled_objects_from_str(definitions))
 
 
-def load_definitions_from_path(config_dir: str) -> StaticDefinitions:
-    return compile_static_definitions(load_uncompiled_objects_from_path(config_dir))
+def load_definitions_from_paths(config_dirs: list[str]) -> StaticDefinitions:
+    uncompiled_objects = []
+    for config_dir in config_dirs:
+        uncompiled_objects.extend(load_uncompiled_objects_from_path(config_dir))
+
+    return compile_static_definitions(uncompiled_objects)
 
 
 def filter_with_jobs_selector(

diff --git a/src/saturn_engine/worker_manager/context.py b/src/saturn_engine/worker_manager/context.py
@@ -2,7 +2,7 @@
 
 from saturn_engine.config import WorkerManagerConfig
 from saturn_engine.worker_manager.config.declarative import filter_with_jobs_selector
-from saturn_engine.worker_manager.config.declarative import load_definitions_from_path
+from saturn_engine.worker_manager.config.declarative import load_definitions_from_paths
 
 from .config.static_definitions import StaticDefinitions
 
@@ -25,8 +25,8 @@ def static_definitions(self) -> StaticDefinitions:
         - Jobs
         - JobDefinitions
         """
-        definitions = load_definitions_from_path(
-            self.config.static_definitions_directory
+        definitions = load_definitions_from_paths(
+            self.config.static_definitions_directories
         )
         if self.config.static_definitions_jobs_selector:
             definitions = filter_with_jobs_selector(

diff --git a/tests/worker_manager/config/test_declarative.py b/tests/worker_manager/config/test_declarative.py
@@ -7,18 +7,18 @@
 from saturn_engine.core.api import ResourceItem
 from saturn_engine.core.api import TopicItem
 from saturn_engine.worker_manager.config.declarative import filter_with_jobs_selector
-from saturn_engine.worker_manager.config.declarative import load_definitions_from_path
+from saturn_engine.worker_manager.config.declarative import load_definitions_from_paths
 from saturn_engine.worker_manager.config.declarative import load_definitions_from_str
 from saturn_engine.worker_manager.config.static_definitions import StaticDefinitions
 
 
-def test_load_definitions_from_path_simple() -> None:
+def test_load_definitions_from_paths_simple() -> None:
     test_dir = os.path.join(
         os.path.dirname(__file__),
         "testdata",
         "test_declarative_simple",
     )
-    static_definitions = load_definitions_from_path(test_dir)
+    static_definitions = load_definitions_from_paths([test_dir])
     assert (
         static_definitions.inventories["github-identifiers"].name
         == "github-identifiers"