merges run context and provider context, exposes init providers via r…

…un context

dlt/cli/deploy_command_helpers.py

@@ -16,7 +16,7 @@
 from dlt.common import git
 from dlt.common.configuration.exceptions import LookupTrace, ConfigFieldMissingException
 from dlt.common.configuration.providers import (
-    ConfigTomlProvider,
+    CONFIG_TOML,
     EnvironProvider,
     StringTomlProvider,
 )
@@ -71,7 +71,6 @@ def __init__(
         self.working_directory: str
         self.state: TPipelineState
 
-        self.config_prov = ConfigTomlProvider()
         self.env_prov = EnvironProvider()
         self.envs: List[LookupTrace] = []
         self.secret_envs: List[LookupTrace] = []
@@ -190,7 +189,7 @@ def _update_envs(self, trace: PipelineTrace) -> None:
                 # fmt.echo(f"{resolved_value.key}:{resolved_value.value}{type(resolved_value.value)} in {resolved_value.sections} is SECRET")
             else:
                 # move all config values that are not in config.toml into env
-                if resolved_value.provider_name != self.config_prov.name:
+                if resolved_value.provider_name != CONFIG_TOML:
                     self.envs.append(
                         LookupTrace(
                             self.env_prov.name,

dlt/cli/init_command.py

@@ -25,7 +25,7 @@
 from dlt.sources import SourceReference
 
 import dlt.reflection.names as n
-from dlt.reflection.script_inspector import inspect_pipeline_script
+from dlt.reflection.script_inspector import import_pipeline_script
 
 from dlt.cli import echo as fmt, pipeline_files as files_ops, source_detection
 from dlt.cli import utils
@@ -452,7 +452,7 @@ def init_command(
     )
 
     # inspect the script
-    inspect_pipeline_script(
+    import_pipeline_script(
         source_configuration.storage.storage_path,
         source_configuration.storage.to_relative_path(source_configuration.src_pipeline_script),
         ignore_missing_imports=True,

dlt/cli/telemetry_command.py

@@ -3,12 +3,12 @@
 
 from dlt.common.configuration.container import Container
 from dlt.common.configuration.providers.toml import ConfigTomlProvider
-from dlt.common.configuration.specs import RunConfiguration
+from dlt.common.configuration.specs import RuntimeConfiguration
 
 from dlt.cli import echo as fmt
 from dlt.cli.utils import get_telemetry_status
 from dlt.cli.config_toml_writer import WritableConfigValue, write_values
-from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext
+from dlt.common.configuration.specs import PluggableRunContext
 from dlt.common.runtime.anon_tracker import get_anonymous_id
 
 DLT_TELEMETRY_DOCS_URL = "https://dlthub.com/docs/reference/telemetry"
@@ -23,23 +23,24 @@ def telemetry_status_command() -> None:
 
 
 def change_telemetry_status_command(enabled: bool) -> None:
+    from dlt.common.runtime import run_context
+
     # value to write
     telemetry_value = [
-        WritableConfigValue("dlthub_telemetry", bool, enabled, (RunConfiguration.__section__,))
+        WritableConfigValue("dlthub_telemetry", bool, enabled, (RuntimeConfiguration.__section__,))
     ]
     # write local config
     # TODO: use designated (main) config provider (for non secret values) ie. taken from run context
-    config = ConfigTomlProvider(add_global_config=False)
+    run_ctx = run_context.current()
+    config = ConfigTomlProvider(run_ctx.settings_dir)
     if not config.is_empty:
         write_values(config._config_toml, telemetry_value, overwrite_existing=True)
         config.write_toml()
 
     # write global config
-    from dlt.common.runtime import run_context
-
-    global_path = run_context.current().global_dir
+    global_path = run_ctx.global_dir
     os.makedirs(global_path, exist_ok=True)
-    config = ConfigTomlProvider(settings_dir=global_path, add_global_config=False)
+    config = ConfigTomlProvider(settings_dir=global_path)
     write_values(config._config_toml, telemetry_value, overwrite_existing=True)
     config.write_toml()
 
@@ -48,5 +49,4 @@ def change_telemetry_status_command(enabled: bool) -> None:
     else:
         fmt.echo("Telemetry switched %s" % fmt.bold("OFF"))
     # reload config providers
-    if ConfigProvidersContext in Container():
-        del Container()[ConfigProvidersContext]
+    Container()[PluggableRunContext].reload_providers()

dlt/cli/utils.py

@@ -5,7 +5,7 @@
 from dlt.common.reflection.utils import set_ast_parents
 from dlt.common.typing import TFun
 from dlt.common.configuration import resolve_configuration
-from dlt.common.configuration.specs import RunConfiguration
+from dlt.common.configuration.specs import RuntimeConfiguration
 from dlt.common.runtime.telemetry import with_telemetry
 from dlt.common.runtime import run_context
 
@@ -60,7 +60,7 @@ def track_command(command: str, track_before: bool, *args: str) -> Callable[[TFu
 
 
 def get_telemetry_status() -> bool:
-    c = resolve_configuration(RunConfiguration())
+    c = resolve_configuration(RuntimeConfiguration())
     return c.dlthub_telemetry
 
 

dlt/common/configuration/accessors.py

@@ -6,7 +6,7 @@
 from dlt.common.configuration.providers.provider import ConfigProvider
 from dlt.common.configuration.specs import BaseConfiguration, is_base_configuration_inner_hint
 from dlt.common.configuration.utils import deserialize_value, log_traces, auto_cast
-from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext
+from dlt.common.configuration.specs import PluggableRunContext
 from dlt.common.typing import AnyType, ConfigValue, SecretValue, TSecretValue
 
 TConfigAny = TypeVar("TConfigAny", bound=Any)
@@ -54,7 +54,7 @@ def writable_provider(self) -> ConfigProvider:
         pass
 
     def _get_providers_from_context(self) -> Sequence[ConfigProvider]:
-        return Container()[ConfigProvidersContext].providers
+        return Container()[PluggableRunContext].providers.providers
 
     def _get_value(self, field: str, type_hint: Type[Any] = None) -> Tuple[Any, List[LookupTrace]]:
         # get default hint type, in case of dlt.secrets it it TSecretValue
@@ -85,7 +85,7 @@ def register_provider(provider: ConfigProvider) -> None:
         """Registers `provider` to participate in the configuration resolution. `provider`
         is added after all existing providers and will be used if all others do not resolve.
         """
-        Container()[ConfigProvidersContext].add_provider(provider)
+        Container()[PluggableRunContext].providers.add_provider(provider)
 
 
 class _ConfigAccessor(_Accessor):

dlt/common/configuration/providers/__init__.py

@@ -12,7 +12,6 @@
 )
 from .doc import CustomLoaderDocProvider
 from .vault import SECRETS_TOML_KEY
-from .google_secrets import GoogleSecretsProvider
 from .context import ContextProvider
 
 __all__ = [
@@ -26,7 +25,6 @@
     "SECRETS_TOML",
     "StringTomlProvider",
     "SECRETS_TOML_KEY",
-    "GoogleSecretsProvider",
     "ContextProvider",
     "CustomLoaderDocProvider",
 ]

dlt/common/configuration/providers/toml.py

@@ -1,7 +1,6 @@
 import os
 import tomlkit
 import tomlkit.items
-import functools
 from typing import Any, Optional
 
 from dlt.common.utils import update_dict_nested
@@ -45,12 +44,12 @@ def __init__(
         name: str,
         supports_secrets: bool,
         file_name: str,
-        settings_dir: str = None,
-        add_global_config: bool = False,
+        settings_dir: str,
+        global_dir: str = None,
     ) -> None:
         """Creates config provider from a `toml` file
 
-        The provider loads the `toml` file with specified name and from specified folder. If `add_global_config` flags is specified,
+        The provider loads the `toml` file with specified name and from specified folder. If `global_dir` is specified,
         it will additionally look for `file_name` in `dlt` global dir (home dir by default) and merge the content.
         The "settings" (`settings_dir`) values overwrite the "global" values.
 
@@ -61,19 +60,15 @@ def __init__(
             supports_secrets(bool): allows to store secret values in this provider
             file_name (str): The name of `toml` file to load
             settings_dir (str, optional): The location of `file_name`. If not specified, defaults to $cwd/.dlt
-            add_global_config (bool, optional): Looks for `file_name` in `dlt` home directory which in most cases is $HOME/.dlt
+            global_dir (bool, optional): Looks for `file_name` in global_dir (defaults to `dlt` home directory which in most cases is $HOME/.dlt)
 
         Raises:
             TomlProviderReadException: File could not be read, most probably `toml` parsing error
         """
-        from dlt.common.runtime import run_context
-
-        self._toml_path = os.path.join(
-            settings_dir or run_context.current().settings_dir, file_name
-        )
-        self._add_global_config = add_global_config
+        self._toml_path = os.path.join(settings_dir, file_name)
+        self._global_dir = os.path.join(global_dir, file_name) if global_dir else None
         self._config_toml = self._read_toml_files(
-            name, file_name, self._toml_path, add_global_config
+            name, file_name, self._toml_path, self._global_dir
         )
 
         super().__init__(
@@ -83,9 +78,7 @@ def __init__(
         )
 
     def write_toml(self) -> None:
-        assert (
-            not self._add_global_config
-        ), "Will not write configs when `add_global_config` flag was set"
+        assert not self._global_dir, "Will not write configs when `global_dir` was set"
         with open(self._toml_path, "w", encoding="utf-8") as f:
             tomlkit.dump(self._config_toml, f)
 
@@ -99,6 +92,10 @@ def set_value(self, key: str, value: Any, pipeline_name: Optional[str], *section
             value = value.unwrap()
         super().set_value(key, value, pipeline_name, *sections)
 
+    @property
+    def is_empty(self) -> bool:
+        return len(self._config_toml.body) == 0 and super().is_empty
+
     def set_fragment(
         self, key: Optional[str], value_or_fragment: str, pipeline_name: str, *sections: str
     ) -> None:
@@ -116,16 +113,12 @@ def to_toml(self) -> str:
 
     @staticmethod
     def _read_toml_files(
-        name: str, file_name: str, toml_path: str, add_global_config: bool
+        name: str, file_name: str, toml_path: str, global_path: str
     ) -> tomlkit.TOMLDocument:
         try:
             project_toml = SettingsTomlProvider._read_toml(toml_path)
-            if add_global_config:
-                from dlt.common.runtime import run_context
-
-                global_toml = SettingsTomlProvider._read_toml(
-                    os.path.join(run_context.current().global_dir, file_name)
-                )
+            if global_path:
+                global_toml = SettingsTomlProvider._read_toml(global_path)
                 project_toml = update_dict_nested(global_toml, project_toml)
             return project_toml
         except Exception as ex:
@@ -142,13 +135,13 @@ def _read_toml(toml_path: str) -> tomlkit.TOMLDocument:
 
 
 class ConfigTomlProvider(SettingsTomlProvider):
-    def __init__(self, settings_dir: str = None, add_global_config: bool = False) -> None:
+    def __init__(self, settings_dir: str, global_dir: str = None) -> None:
         super().__init__(
             CONFIG_TOML,
             False,
             CONFIG_TOML,
             settings_dir=settings_dir,
-            add_global_config=add_global_config,
+            global_dir=global_dir,
         )
 
     @property
@@ -157,13 +150,13 @@ def is_writable(self) -> bool:
 
 
 class SecretsTomlProvider(SettingsTomlProvider):
-    def __init__(self, settings_dir: str = None, add_global_config: bool = False) -> None:
+    def __init__(self, settings_dir: str, global_dir: str = None) -> None:
         super().__init__(
             SECRETS_TOML,
             True,
             SECRETS_TOML,
             settings_dir=settings_dir,
-            add_global_config=add_global_config,
+            global_dir=global_dir,
         )
 
     @property

dlt/common/configuration/resolve.py

@@ -26,7 +26,7 @@
 )
 from dlt.common.configuration.specs.config_section_context import ConfigSectionContext
 from dlt.common.configuration.specs.exceptions import NativeValueError
-from dlt.common.configuration.specs.config_providers_context import ConfigProvidersContext
+from dlt.common.configuration.specs.pluggable_run_context import PluggableRunContext
 from dlt.common.configuration.container import Container
 from dlt.common.configuration.utils import log_traces, deserialize_value
 from dlt.common.configuration.exceptions import (
@@ -417,7 +417,7 @@ def _resolve_single_value(
 
     container = Container()
     # get providers from container
-    providers_context = container[ConfigProvidersContext]
+    providers_context = container[PluggableRunContext].providers
     # we may be resolving context
     if is_context_inner_hint(inner_hint):
         # resolve context with context provider and do not look further

dlt/common/configuration/specs/__init__.py

@@ -1,4 +1,3 @@
-from .run_configuration import RunConfiguration
 from .base_configuration import (
     BaseConfiguration,
     CredentialsConfiguration,
@@ -36,16 +35,20 @@
     GcpServiceAccountCredentials as GcpClientCredentialsWithDefault,
 )
 
+from .pluggable_run_context import PluggableRunContext
+from .run_configuration import RuntimeConfiguration
+
 
 __all__ = [
-    "RunConfiguration",
+    "RuntimeConfiguration",
     "BaseConfiguration",
     "CredentialsConfiguration",
     "CredentialsWithDefault",
     "ContainerInjectableContext",
     "extract_inner_hint",
     "is_base_configuration_inner_hint",
     "configspec",
+    "PluggableRunContext",
     "ConfigSectionContext",
     "GcpServiceAccountCredentialsWithoutDefaults",
     "GcpServiceAccountCredentials",