Add tests to verify all settings via environment variables (#15657)

tests/test_settings.py

@@ -2,10 +2,12 @@
 import os
 import textwrap
 import warnings
+from datetime import timedelta
 from pathlib import Path
 
 import pydantic
 import pytest
+from pydantic_core import to_jsonable_python
 from sqlalchemy import make_url
 
 import prefect.context
@@ -42,13 +44,182 @@
     Profile,
     ProfilesCollection,
     Settings,
+    env_var_to_attr_name,
     get_current_settings,
     load_profile,
     load_profiles,
     save_profiles,
     temporary_settings,
 )
 
+SUPPORTED_SETTINGS = {
+    "PREFECT_API_BLOCKS_REGISTER_ON_START": {"test_value": True},
+    "PREFECT_API_DATABASE_CONNECTION_TIMEOUT": {"test_value": 10.0},
+    "PREFECT_API_DATABASE_CONNECTION_URL": {"test_value": "sqlite:///"},
+    "PREFECT_API_DATABASE_DRIVER": {"test_value": "sqlite+aiosqlite"},
+    "PREFECT_API_DATABASE_ECHO": {"test_value": True},
+    "PREFECT_API_DATABASE_HOST": {"test_value": "localhost"},
+    "PREFECT_API_DATABASE_MIGRATE_ON_START": {"test_value": True},
+    "PREFECT_API_DATABASE_NAME": {"test_value": "prefect"},
+    "PREFECT_API_DATABASE_PASSWORD": {"test_value": "password"},
+    "PREFECT_API_DATABASE_PORT": {"test_value": 5432},
+    "PREFECT_API_DATABASE_TIMEOUT": {"test_value": 10.0},
+    "PREFECT_API_DATABASE_USER": {"test_value": "user"},
+    "PREFECT_API_DEFAULT_LIMIT": {"test_value": 100},
+    "PREFECT_API_ENABLE_HTTP2": {"test_value": True},
+    "PREFECT_API_ENABLE_METRICS": {"test_value": True},
+    "PREFECT_API_EVENTS_RELATED_RESOURCE_CACHE_TTL": {
+        "test_value": timedelta(minutes=6)
+    },
+    "PREFECT_API_EVENTS_STREAM_OUT_ENABLED": {"test_value": True},
+    "PREFECT_API_KEY": {"test_value": "key"},
+    "PREFECT_API_LOG_RETRYABLE_ERRORS": {"test_value": True},
+    "PREFECT_API_MAX_FLOW_RUN_GRAPH_ARTIFACTS": {"test_value": 10},
+    "PREFECT_API_MAX_FLOW_RUN_GRAPH_NODES": {"test_value": 100},
+    "PREFECT_API_REQUEST_TIMEOUT": {"test_value": 10.0},
+    "PREFECT_API_SERVICES_CANCELLATION_CLEANUP_ENABLED": {"test_value": True},
+    "PREFECT_API_SERVICES_CANCELLATION_CLEANUP_LOOP_SECONDS": {"test_value": 10.0},
+    "PREFECT_API_SERVICES_EVENT_PERSISTER_BATCH_SIZE": {"test_value": 100},
+    "PREFECT_API_SERVICES_EVENT_PERSISTER_ENABLED": {"test_value": True},
+    "PREFECT_API_SERVICES_EVENT_PERSISTER_FLUSH_INTERVAL": {"test_value": 10.0},
+    "PREFECT_API_SERVICES_FLOW_RUN_NOTIFICATIONS_ENABLED": {"test_value": True},
+    "PREFECT_API_SERVICES_FOREMAN_DEPLOYMENT_LAST_POLLED_TIMEOUT_SECONDS": {
+        "test_value": 10,
+    },
+    "PREFECT_API_SERVICES_FOREMAN_ENABLED": {"test_value": True},
+    "PREFECT_API_SERVICES_FOREMAN_FALLBACK_HEARTBEAT_INTERVAL_SECONDS": {
+        "test_value": 10,
+    },
+    "PREFECT_API_SERVICES_FOREMAN_INACTIVITY_HEARTBEAT_MULTIPLE": {"test_value": 2},
+    "PREFECT_API_SERVICES_FOREMAN_LOOP_SECONDS": {"test_value": 10.0},
+    "PREFECT_API_SERVICES_FOREMAN_WORK_QUEUE_LAST_POLLED_TIMEOUT_SECONDS": {
+        "test_value": 10,
+    },
+    "PREFECT_API_SERVICES_LATE_RUNS_AFTER_SECONDS": {
+        "test_value": timedelta(seconds=20)
+    },
+    "PREFECT_API_SERVICES_LATE_RUNS_ENABLED": {"test_value": True},
+    "PREFECT_API_SERVICES_LATE_RUNS_LOOP_SECONDS": {"test_value": 10.0},
+    "PREFECT_API_SERVICES_PAUSE_EXPIRATIONS_ENABLED": {"test_value": True},
+    "PREFECT_API_SERVICES_PAUSE_EXPIRATIONS_LOOP_SECONDS": {"test_value": 10.0},
+    "PREFECT_API_SERVICES_SCHEDULER_DEPLOYMENT_BATCH_SIZE": {"test_value": 10},
+    "PREFECT_API_SERVICES_SCHEDULER_ENABLED": {"test_value": True},
+    "PREFECT_API_SERVICES_SCHEDULER_INSERT_BATCH_SIZE": {"test_value": 10},
+    "PREFECT_API_SERVICES_SCHEDULER_LOOP_SECONDS": {"test_value": 10.0},
+    "PREFECT_API_SERVICES_SCHEDULER_MAX_RUNS": {"test_value": 10},
+    "PREFECT_API_SERVICES_SCHEDULER_MAX_SCHEDULED_TIME": {
+        "test_value": timedelta(hours=10)
+    },
+    "PREFECT_API_SERVICES_SCHEDULER_MIN_RUNS": {"test_value": 10},
+    "PREFECT_API_SERVICES_SCHEDULER_MIN_SCHEDULED_TIME": {
+        "test_value": timedelta(minutes=10)
+    },
+    "PREFECT_API_SERVICES_TASK_RUN_RECORDER_ENABLED": {"test_value": True},
+    "PREFECT_API_SERVICES_TRIGGERS_ENABLED": {"test_value": True},
+    "PREFECT_API_SSL_CERT_FILE": {"test_value": "/path/to/cert"},
+    "PREFECT_API_TASK_CACHE_KEY_MAX_LENGTH": {"test_value": 10},
+    "PREFECT_API_TLS_INSECURE_SKIP_VERIFY": {"test_value": True},
+    "PREFECT_API_URL": {"test_value": "https://api.prefect.io"},
+    "PREFECT_ASYNC_FETCH_STATE_RESULT": {"test_value": True},
+    "PREFECT_CLIENT_CSRF_SUPPORT_ENABLED": {"test_value": True},
+    "PREFECT_CLIENT_ENABLE_METRICS": {"test_value": True},
+    "PREFECT_CLIENT_MAX_RETRIES": {"test_value": 3},
+    "PREFECT_CLIENT_METRICS_PORT": {"test_value": 9000},
+    "PREFECT_CLIENT_RETRY_EXTRA_CODES": {"test_value": "400"},
+    "PREFECT_CLIENT_RETRY_JITTER_FACTOR": {"test_value": 0.5},
+    "PREFECT_CLI_COLORS": {"test_value": True},
+    "PREFECT_CLI_PROMPT": {"test_value": True},
+    "PREFECT_CLI_WRAP_LINES": {"test_value": True},
+    "PREFECT_CLOUD_API_URL": {"test_value": "https://cloud.prefect.io"},
+    "PREFECT_CLOUD_UI_URL": {"test_value": "https://cloud.prefect.io"},
+    "PREFECT_DEBUG_MODE": {"test_value": True},
+    "PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE": {"test_value": "prefect"},
+    "PREFECT_DEFAULT_RESULT_STORAGE_BLOCK": {"test_value": "block"},
+    "PREFECT_DEFAULT_WORK_POOL_NAME": {"test_value": "default"},
+    "PREFECT_DEPLOYMENT_CONCURRENCY_SLOT_WAIT_SECONDS": {"test_value": 10.0},
+    "PREFECT_DEPLOYMENT_SCHEDULE_MAX_SCHEDULED_RUNS": {"test_value": 10},
+    "PREFECT_EVENTS_EXPIRED_BUCKET_BUFFER": {"test_value": timedelta(seconds=60)},
+    "PREFECT_EVENTS_MAXIMUM_LABELS_PER_RESOURCE": {"test_value": 10},
+    "PREFECT_EVENTS_MAXIMUM_RELATED_RESOURCES": {"test_value": 10},
+    "PREFECT_EVENTS_MAXIMUM_SIZE_BYTES": {"test_value": 10},
+    "PREFECT_EVENTS_MAXIMUM_WEBSOCKET_BACKFILL": {"test_value": timedelta(minutes=15)},
+    "PREFECT_EVENTS_PROACTIVE_GRANULARITY": {"test_value": timedelta(seconds=5)},
+    "PREFECT_EVENTS_RETENTION_PERIOD": {"test_value": timedelta(hours=7)},
+    "PREFECT_EVENTS_WEBSOCKET_BACKFILL_PAGE_SIZE": {"test_value": 10},
+    "PREFECT_EXPERIMENTAL_ENABLE_SCHEDULE_CONCURRENCY": {"test_value": True},
+    "PREFECT_EXPERIMENTAL_WARN": {"test_value": True},
+    "PREFECT_FLOW_DEFAULT_RETRIES": {"test_value": 10},
+    "PREFECT_FLOW_DEFAULT_RETRY_DELAY_SECONDS": {"test_value": 10},
+    "PREFECT_HOME": {"test_value": Path.home() / ".prefect" / "test"},
+    "PREFECT_LOCAL_STORAGE_PATH": {"test_value": Path("/path/to/storage")},
+    "PREFECT_LOGGING_COLORS": {"test_value": True},
+    "PREFECT_LOGGING_EXTRA_LOGGERS": {"test_value": "foo"},
+    "PREFECT_LOGGING_INTERNAL_LEVEL": {"test_value": "INFO"},
+    "PREFECT_LOGGING_LEVEL": {"test_value": "INFO"},
+    "PREFECT_LOGGING_LOG_PRINTS": {"test_value": True},
+    "PREFECT_LOGGING_MARKUP": {"test_value": True},
+    "PREFECT_LOGGING_SERVER_LEVEL": {"test_value": "INFO"},
+    "PREFECT_LOGGING_SETTINGS_PATH": {"test_value": Path("/path/to/settings.toml")},
+    "PREFECT_LOGGING_TO_API_BATCH_INTERVAL": {"test_value": 10.0},
+    "PREFECT_LOGGING_TO_API_BATCH_SIZE": {"test_value": 5_000_000},
+    "PREFECT_LOGGING_TO_API_ENABLED": {"test_value": True},
+    "PREFECT_LOGGING_TO_API_MAX_LOG_SIZE": {"test_value": 10},
+    "PREFECT_LOGGING_TO_API_WHEN_MISSING_FLOW": {"test_value": "ignore"},
+    "PREFECT_MEMOIZE_BLOCK_AUTO_REGISTRATION": {"test_value": True},
+    "PREFECT_MEMO_STORE_PATH": {"test_value": Path("/path/to/memo")},
+    "PREFECT_MESSAGING_BROKER": {"test_value": "broker"},
+    "PREFECT_MESSAGING_CACHE": {"test_value": "cache"},
+    "PREFECT_PROFILES_PATH": {"test_value": Path("/path/to/profiles.toml")},
+    "PREFECT_RESULTS_DEFAULT_SERIALIZER": {"test_value": "serializer"},
+    "PREFECT_RESULTS_PERSIST_BY_DEFAULT": {"test_value": True},
+    "PREFECT_RUNNER_POLL_FREQUENCY": {"test_value": 10},
+    "PREFECT_RUNNER_PROCESS_LIMIT": {"test_value": 10},
+    "PREFECT_RUNNER_SERVER_ENABLE": {"test_value": True},
+    "PREFECT_RUNNER_SERVER_HOST": {"test_value": "host"},
+    "PREFECT_RUNNER_SERVER_LOG_LEVEL": {"test_value": "INFO"},
+    "PREFECT_RUNNER_SERVER_MISSED_POLLS_TOLERANCE": {"test_value": 10},
+    "PREFECT_RUNNER_SERVER_PORT": {"test_value": 8080},
+    "PREFECT_SERVER_ALLOW_EPHEMERAL_MODE": {"test_value": True},
+    "PREFECT_SERVER_ANALYTICS_ENABLED": {"test_value": True},
+    "PREFECT_SERVER_API_HOST": {"test_value": "host"},
+    "PREFECT_SERVER_API_KEEPALIVE_TIMEOUT": {"test_value": 10},
+    "PREFECT_SERVER_API_PORT": {"test_value": 4200},
+    "PREFECT_SERVER_CORS_ALLOWED_HEADERS": {"test_value": "foo"},
+    "PREFECT_SERVER_CORS_ALLOWED_METHODS": {"test_value": "foo"},
+    "PREFECT_SERVER_CORS_ALLOWED_ORIGINS": {"test_value": "foo"},
+    "PREFECT_SERVER_CSRF_PROTECTION_ENABLED": {"test_value": True},
+    "PREFECT_SERVER_CSRF_TOKEN_EXPIRATION": {"test_value": timedelta(seconds=10)},
+    "PREFECT_SERVER_EPHEMERAL_STARTUP_TIMEOUT_SECONDS": {"test_value": 10},
+    "PREFECT_SILENCE_API_URL_MISCONFIGURATION": {"test_value": True},
+    "PREFECT_SQLALCHEMY_MAX_OVERFLOW": {"test_value": 10},
+    "PREFECT_SQLALCHEMY_POOL_SIZE": {"test_value": 10},
+    "PREFECT_TASKS_REFRESH_CACHE": {"test_value": True},
+    "PREFECT_TASK_DEFAULT_RETRIES": {"test_value": 10},
+    "PREFECT_TASK_DEFAULT_RETRY_DELAY_SECONDS": {"test_value": 10},
+    "PREFECT_TASK_RUN_TAG_CONCURRENCY_SLOT_WAIT_SECONDS": {"test_value": 10},
+    "PREFECT_TASK_SCHEDULING_DEFAULT_STORAGE_BLOCK": {"test_value": "block"},
+    "PREFECT_TASK_SCHEDULING_DELETE_FAILED_SUBMISSIONS": {"test_value": True},
+    "PREFECT_TASK_SCHEDULING_MAX_RETRY_QUEUE_SIZE": {"test_value": 10},
+    "PREFECT_TASK_SCHEDULING_MAX_SCHEDULED_QUEUE_SIZE": {"test_value": 10},
+    "PREFECT_TASK_SCHEDULING_PENDING_TASK_TIMEOUT": {
+        "test_value": timedelta(seconds=10)
+    },
+    "PREFECT_TEST_MODE": {"test_value": True},
+    "PREFECT_TEST_SETTING": {"test_value": "foo"},
+    "PREFECT_UI_API_URL": {"test_value": "https://api.prefect.io"},
+    "PREFECT_UI_ENABLED": {"test_value": True},
+    "PREFECT_UI_SERVE_BASE": {"test_value": "/base"},
+    "PREFECT_UI_STATIC_DIRECTORY": {"test_value": "/path/to/static"},
+    "PREFECT_UI_URL": {"test_value": "https://ui.prefect.io"},
+    "PREFECT_UNIT_TEST_LOOP_DEBUG": {"test_value": True},
+    "PREFECT_UNIT_TEST_MODE": {"test_value": True},
+    "PREFECT_WORKER_HEARTBEAT_SECONDS": {"test_value": 10.0},
+    "PREFECT_WORKER_PREFETCH_SECONDS": {"test_value": 10.0},
+    "PREFECT_WORKER_QUERY_SECONDS": {"test_value": 10.0},
+    "PREFECT_WORKER_WEBSERVER_HOST": {"test_value": "host"},
+    "PREFECT_WORKER_WEBSERVER_PORT": {"test_value": 8080},
+}
+
 
 class TestSettingClass:
     def test_setting_equality_with_value(self):
@@ -268,6 +439,11 @@ def test_loads_when_profile_path_is_not_a_toml_file(self, monkeypatch, tmp_path)
         with pytest.warns(UserWarning, match="Failed to load profiles from"):
             assert Settings().test_setting == "FOO"
 
+    def test_valid_setting_names_matches_supported_settings(self):
+        assert (
+            set(Settings().valid_setting_names()) == set(SUPPORTED_SETTINGS.keys())
+        ), "valid_setting_names output did not match supported settings. Please update SUPPORTED_SETTINGS if you are adding or removing a setting."
+
 
 class TestSettingAccess:
     def test_get_value_root_setting(self):
@@ -1133,3 +1309,55 @@ def test_equality(self):
                 Profile(name="bar", settings={}, source=Path("/new-path")),
             ]
         ), "Changed profile source should be inequal"
+
+
+class TestSettingValues:
+    @pytest.fixture(scope="function", params=list(SUPPORTED_SETTINGS.keys()))
+    def var_and_value(self, request):
+        var = request.param
+        return var, SUPPORTED_SETTINGS[var]["test_value"]
+
+    def test_set_via_env_var(self, var_and_value, monkeypatch):
+        for env_var in os.environ:
+            if env_var.startswith("PREFECT_"):
+                monkeypatch.delenv(env_var, raising=False)
+
+        var, value = var_and_value
+
+        if var == "PREFECT_TEST_SETTING":
+            monkeypatch.setenv("PREFECT_TEST_MODE", "True")
+
+        # mock set the env var
+        monkeypatch.setenv(var, str(value))
+
+        # create new root context to pick up the env var changes
+        warnings.filterwarnings("ignore", category=UserWarning)
+        with prefect.context.root_settings_context():
+            field_name = env_var_to_attr_name(var)
+            current_settings = get_current_settings()
+            # get value from settings object
+            settings_value = getattr(current_settings, field_name)
+
+            if isinstance(settings_value, pydantic.SecretStr):
+                settings_value = settings_value.get_secret_value()
+            if var == "PREFECT_CLIENT_RETRY_EXTRA_CODES":
+                assert settings_value == {int(value)}
+                assert getattr(prefect.settings, var).value() == {int(value)}
+                assert current_settings.to_environment_variables(exclude_unset=True)[
+                    var
+                ] == str([int(value)])
+
+            elif var == "PREFECT_LOGGING_EXTRA_LOGGERS":
+                assert settings_value == [value]
+                assert getattr(prefect.settings, var).value() == [value]
+                assert current_settings.to_environment_variables(exclude_unset=True)[
+                    var
+                ] == str([value])
+            else:
+                assert settings_value == value
+                # get value from legacy setting object
+                assert getattr(prefect.settings, var).value() == value
+                # ensure the value gets added to the environment variables
+                assert current_settings.to_environment_variables(exclude_unset=True)[
+                    var
+                ] == str(to_jsonable_python(value))