-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #10 from LamaAni/add_support_for_file_paths_and_se…
…cret_mask Add support for file paths and secret mask
- Loading branch information
Showing
17 changed files
with
167 additions
and
177 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| [flake8] | ||
| ignore = E203, W503 | ||
| max-line-length = 120 | ||
| exclude = | ||
| .* | ||
| build/* | ||
| dist/* | ||
| *.egg-info/* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,14 @@ | ||
| [[source]] | ||
| name = "pypi" | ||
| url = "https://pypi.org/simple" | ||
| verify_ssl = true | ||
| name = "pypi" | ||
|
|
||
| [packages] | ||
| zthreading=">=0.1.15" | ||
| zthreading = ">=0.1.15" | ||
|
|
||
| [dev-packages] | ||
| apache-airflow = ">=2.5.3" | ||
| pytest = "*" | ||
|
|
||
| [requires] | ||
| python_version = "3.8" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,74 +1,53 @@ | ||
| #!/usr/bin/env python | ||
| # -*- coding: utf-8 -*- | ||
| __title__ = "airflow_db_logger" | ||
| __author__ = "Zav Shotan" | ||
|
|
||
| import sys | ||
| import os | ||
| from copy import deepcopy | ||
| import airflow_db_logger.consts as consts | ||
|
|
||
|
|
||
| LOGGING_CONFIG = consts.DB_LOGGER_LOGGING_CONFIG | ||
| AIRFLOW_DEFAULT_LOGGING_CONFIG = consts.get_default_loggin_config() | ||
|
|
||
| from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG | ||
| from airflow_db_logger.config import ( # noqa | ||
| check_cli_for_init_db, # noqa: E402 | ||
| DB_LOGGER_TASK_FORMATTER, # noqa: E402 | ||
| DB_LOGGER_CONSOLE_FORMATTER, # noqa: E402 | ||
| DB_LOGGER_WRITE_DAG_PROCESSING_TO_DB, # noqa: E402 | ||
| DB_LOGGER_PROCESSER_LOG_LEVEL, | ||
| airflow_db_logger_log, | ||
| ) | ||
|
|
||
|
|
||
| def update_config_from_defaults(): | ||
| consts.IS_DB_LOGGER_LOADING_CONFIG | ||
|
|
||
| if consts.IS_DB_LOGGER_LOADING_CONFIG is True: | ||
| return | ||
|
|
||
| # Remove any other loads. | ||
| try: | ||
| consts.IS_DB_LOGGER_LOADING_CONFIG = True | ||
| def create_logging_config(): | ||
| config = deepcopy(DEFAULT_LOGGING_CONFIG) | ||
| processor_handler_config = { | ||
| "class": "airflow_db_logger.handlers.StreamHandler", | ||
| "formatter": DB_LOGGER_CONSOLE_FORMATTER, | ||
| "level": DB_LOGGER_PROCESSER_LOG_LEVEL, | ||
| } | ||
|
|
||
| if DB_LOGGER_WRITE_DAG_PROCESSING_TO_DB: | ||
| processor_handler_config = { | ||
| "class": "airflow_db_logger.handlers.StreamHandler", | ||
| "class": "airflow_db_logger.handlers.DBProcessLogHandler", | ||
| "formatter": DB_LOGGER_CONSOLE_FORMATTER, | ||
| "level": DB_LOGGER_PROCESSER_LOG_LEVEL, | ||
| } | ||
|
|
||
| if DB_LOGGER_WRITE_DAG_PROCESSING_TO_DB: | ||
| processor_handler_config = { | ||
| "class": "airflow_db_logger.handlers.DBProcessLogHandler", | ||
| "formatter": DB_LOGGER_CONSOLE_FORMATTER, | ||
| "level": DB_LOGGER_PROCESSER_LOG_LEVEL, | ||
| } | ||
|
|
||
| LOGGING_CONFIG.update(deepcopy(AIRFLOW_DEFAULT_LOGGING_CONFIG)) | ||
| LOGGING_CONFIG["handlers"] = { | ||
| "console": { | ||
| "class": "airflow_db_logger.handlers.StreamHandler", | ||
| "formatter": DB_LOGGER_CONSOLE_FORMATTER, | ||
| }, | ||
| "task": { | ||
| "class": "airflow_db_logger.handlers.DBTaskLogHandler", | ||
| "formatter": DB_LOGGER_TASK_FORMATTER, | ||
| }, | ||
| "processor": processor_handler_config, | ||
| } | ||
| config["handlers"] = { | ||
| "console": { | ||
| "class": "airflow_db_logger.handlers.StreamHandler", | ||
| "formatter": DB_LOGGER_CONSOLE_FORMATTER, | ||
| }, | ||
| "task": { | ||
| "class": "airflow_db_logger.handlers.DBTaskLogHandler", | ||
| "formatter": DB_LOGGER_TASK_FORMATTER, | ||
| }, | ||
| "processor": processor_handler_config, | ||
| } | ||
| return config | ||
|
|
||
| loggers = LOGGING_CONFIG.get("loggers", {}) | ||
|
|
||
| # for logger_name in loggers.keys(): | ||
| # loggers[logger_name]["level"] = LOG_LEVEL | ||
| LOGGING_CONFIG = create_logging_config() | ||
|
|
||
| # Checking for database initialization | ||
| check_cli_for_init_db() | ||
| # Checking for database initialization | ||
| check_cli_for_init_db() | ||
|
|
||
| finally: | ||
| consts.IS_DB_LOGGER_LOADING_CONFIG = False | ||
|
|
||
| if __name__ == "__main__": | ||
| import json | ||
|
|
||
| update_config_from_defaults() | ||
| # airflow_db_logger_log.info("airflow_db_logger initialized") | ||
| print(json.dumps(LOGGING_CONFIG, indent=2)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,38 +1 @@ | ||
| import os | ||
| from copy import deepcopy | ||
| from airflow_db_logger.shell_logging_config import create_shell_logging_config | ||
|
|
||
| global IS_DB_LOGGER_LOADING_CONFIG | ||
| IS_DB_LOGGER_LOADING_CONFIG = False | ||
|
|
||
| global DB_LOGGER_LOGGING_CONFIG | ||
| DB_LOGGER_LOGGING_CONFIG = create_shell_logging_config() | ||
|
|
||
|
|
||
| def get_default_loggin_config(): | ||
| from airflow.version import version as AIRFLOW_VERSION | ||
|
|
||
| AIRFLOW_VERSION_PARTS = AIRFLOW_VERSION.split(".") | ||
| AIRFLOW_VERSION_PARTS = [int(v) for v in AIRFLOW_VERSION_PARTS] | ||
|
|
||
| AIRFLOW_MAJOR_VERSION = AIRFLOW_VERSION_PARTS[0] | ||
|
|
||
| """Returns the airflow default logging config from the settings. | ||
| Start the ariflow system. settings.initialize should be called if the logging configuration is to be reset? | ||
| """ | ||
|
|
||
| config_env_name = ( | ||
| "AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS" if AIRFLOW_MAJOR_VERSION > 1 else "AIRFLOW__CORE__LOGGING_CONFIG_CLASS" | ||
| ) | ||
|
|
||
| action_logging_config_env = os.environ.get(config_env_name, None) | ||
|
|
||
| os.environ[config_env_name] = "airflow_db_logger.shell_logging_config.SIMPLE_LOGGING_CONFIG" | ||
|
|
||
| from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG # noqa | ||
|
|
||
| if action_logging_config_env is not None: | ||
| os.environ[config_env_name] = action_logging_config_env | ||
|
|
||
| return DEFAULT_LOGGING_CONFIG |
Oops, something went wrong.