diff --git a/ingestion/src/metadata/ingestion/connections/test_connections.py b/ingestion/src/metadata/ingestion/connections/test_connections.py index e98b67dbe32b..3b3f98756a69 100644 --- a/ingestion/src/metadata/ingestion/connections/test_connections.py +++ b/ingestion/src/metadata/ingestion/connections/test_connections.py @@ -40,6 +40,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.connections import kill_active_connections from metadata.profiler.orm.functions.conn_test import ConnTestFn +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import cli_logger from metadata.utils.timeout import timeout @@ -92,25 +93,24 @@ def _test_connection_steps( metadata: OpenMetadata, steps: List[TestConnectionStep], automation_workflow: Optional[AutomationWorkflow] = None, -) -> None: +) -> TestConnectionResult: """ Run all the function steps and raise any errors """ if automation_workflow: - _test_connection_steps_automation_workflow( + return _test_connection_steps_automation_workflow( metadata=metadata, steps=steps, automation_workflow=automation_workflow ) - else: - _test_connection_steps_during_ingestion(steps=steps) + return _test_connection_steps_and_raise(steps=steps) def _test_connection_steps_automation_workflow( metadata: OpenMetadata, steps: List[TestConnectionStep], automation_workflow: AutomationWorkflow, -) -> None: +) -> TestConnectionResult: """ Run the test connection as part of the automation workflow We need to update the automation workflow in each step @@ -187,33 +187,40 @@ def _test_connection_steps_automation_workflow( ) ) + return test_connection_result -def _test_connection_steps_during_ingestion(steps: List[TestConnectionStep]) -> None: - """ - Run the test connection as part of the ingestion workflow - Raise an exception if something fails - """ - test_connection_result = TestConnectionIngestionResult() + +def _test_connection_steps_during_ingestion( + steps: List[TestConnectionStep], +) -> TestConnectionResult: + """Run the test connection steps during ingestion""" + test_connection_result = TestConnectionResult( + status=StatusType.Running, + steps=[], + ) for step in steps: try: + logger.info(f"Running {step.name}...") step.function() - test_connection_result.success.append(f"'{step.name}': Pass") - - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.warning(f"{step.name}-{exc}") - if step.mandatory: - test_connection_result.failed.append( - f"'{step.name}': This is a mandatory step and we won't be able to extract" - f" necessary metadata. Failed due to: {exc}" + test_connection_result.steps.append( + TestConnectionStepResult( + name=step.name, + mandatory=step.mandatory, + passed=True, ) - - else: - test_connection_result.warning.append( - f"'{step.name}': This is a optional and the ingestion will continue to work as expected." - f"Failed due to: {exc}" + ) + except Exception as err: + logger.debug(traceback.format_exc()) + logger.error(f"{step.name}-{err}") + test_connection_result.steps.append( + TestConnectionStepResult( + name=step.name, + mandatory=step.mandatory, + passed=False, + message=step.error_message, + errorLog=str(err), ) - + ) if step.short_circuit: # break the workflow if the step is a short circuit step break @@ -221,10 +228,29 @@ def _test_connection_steps_during_ingestion(steps: List[TestConnectionStep]) -> logger.info("Test connection results:") logger.info(test_connection_result) - if test_connection_result.failed: - raise SourceConnectionException( - f"Some steps failed when testing the connection: [{test_connection_result}]" - ) + return test_connection_result + + +def _test_connection_steps_and_raise( + steps: List[TestConnectionStep], +) -> TestConnectionResult: + """ + Run the test connection as part of the ingestion workflow + Raise an exception if something fails + """ + test_connection_result = _test_connection_steps_during_ingestion(steps) + + for step in test_connection_result.steps: + if not step.passed and step.mandatory: + raise SourceConnectionException( + f"Failed to run the test connection step: {step.name}" + ) + if not step.passed: + logger.warning( + f"You might be missing metadata in: {step.name} due to {step.message}" + ) + + return test_connection_result def test_connection_steps( @@ -232,8 +258,8 @@ def test_connection_steps( service_type: str, test_fn: dict, automation_workflow: Optional[AutomationWorkflow] = None, - timeout_seconds: int = 3 * 60, -) -> None: + timeout_seconds: Optional[int] = THREE_MIN, +) -> TestConnectionResult: """ Test the connection steps with a given timeout @@ -268,9 +294,12 @@ def test_connection_steps( for step in test_connection_definition.steps ] - return timeout(timeout_seconds)(_test_connection_steps)( - metadata, steps, automation_workflow - ) + if timeout_seconds: + return timeout(timeout_seconds)(_test_connection_steps)( + metadata, steps, automation_workflow + ) + + return _test_connection_steps(metadata, steps, automation_workflow) def test_connection_engine_step(connection: Engine) -> None: @@ -289,7 +318,7 @@ def test_connection_db_common( service_connection, automation_workflow: Optional[AutomationWorkflow] = None, queries: dict = None, - timeout_seconds: int = 3 * 60, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -339,6 +368,7 @@ def test_connection_db_schema_sources( service_connection, automation_workflow: Optional[AutomationWorkflow] = None, queries: dict = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -393,6 +423,7 @@ def custom_executor(engine_: Engine, inspector_fn_str: str): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) kill_active_connections(engine) diff --git a/ingestion/src/metadata/ingestion/source/api/rest/connection.py b/ingestion/src/metadata/ingestion/source/api/rest/connection.py index e48c28fc5148..676a053f8069 100644 --- a/ingestion/src/metadata/ingestion/source/api/rest/connection.py +++ b/ingestion/src/metadata/ingestion/source/api/rest/connection.py @@ -25,6 +25,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN class SchemaURLError(Exception): @@ -54,6 +55,7 @@ def test_connection( client: Response, service_connection: RestConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -84,4 +86,5 @@ def custom_schema_exec(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/connection.py index 0b6cf700026a..ee60049c5d87 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/connection.py @@ -12,7 +12,6 @@ """ Source connection handler """ - from typing import Optional from pydomo import Domo @@ -29,6 +28,7 @@ test_connection_steps, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: DomoDashboardConnection) -> OMPyDomoClient: @@ -57,6 +57,7 @@ def test_connection( client: OMPyDomoClient, service_connection: DomoDashboardConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -77,4 +78,5 @@ def custom_test_page_list(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/lightdash/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/lightdash/connection.py index db653b1b02b3..aa8b0554e630 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/lightdash/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/lightdash/connection.py @@ -12,7 +12,6 @@ """ Source connection handler """ - from typing import Optional from metadata.generated.schema.entity.automations.workflow import ( @@ -27,6 +26,7 @@ ) from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.lightdash.client import LightdashApiClient +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -49,6 +49,7 @@ def test_connection( client: LightdashApiClient, service_connection: LightdashConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -65,4 +66,5 @@ def custom_executor(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/connection.py index c5b235ec4cd0..4803d544cca2 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/looker/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/connection.py @@ -26,6 +26,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: LookerConnection) -> Looker40SDK: @@ -49,6 +50,7 @@ def test_connection( client: Looker40SDK, service_connection: LookerConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -81,4 +83,5 @@ def validate_api_version(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/connection.py index 1d8000ea439e..6ac232c2a3fc 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.metabase.client import MetabaseClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: MetabaseConnection) -> MetabaseClient: @@ -37,6 +38,7 @@ def test_connection( client: MetabaseClient, service_connection: MetabaseConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -54,4 +56,5 @@ def custom_executor(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/mode/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/mode/connection.py index 339931b4c840..ece0bcd97d23 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/mode/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/mode/connection.py @@ -24,6 +24,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.mode.client import ModeApiClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: ModeConnection) -> ModeApiClient: @@ -38,6 +39,7 @@ def test_connection( client: ModeApiClient, service_connection: ModeConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -55,4 +57,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/mstr/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/mstr/connection.py index df6efbde2a67..fc193ddf6734 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/mstr/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/mstr/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.mstr.client import MSTRClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: MstrConnection) -> MSTRClient: @@ -37,6 +38,7 @@ def test_connection( client: MSTRClient, service_connection: MstrConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -50,4 +52,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/connection.py index c9bae539f9f0..2ac8f064866f 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/connection.py @@ -27,6 +27,7 @@ PowerBiClient, ) from metadata.ingestion.source.dashboard.powerbi.file_client import PowerBiFileClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: PowerBIConnection) -> PowerBiApiClient: @@ -46,6 +47,7 @@ def test_connection( client: PowerBiClient, service_connection: PowerBIConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -58,4 +60,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py index 38eef1660863..257c58623a9f 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.qlikcloud.client import QlikCloudClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: QlikCloudConnection) -> QlikCloudClient: @@ -37,6 +38,7 @@ def test_connection( client: QlikCloudClient, service_connection: QlikCloudConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -50,4 +52,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py index 94ed95e10d51..2cbd25bdb7cc 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.qliksense.client import QlikSenseClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: QlikSenseConnection) -> QlikSenseClient: @@ -37,6 +38,7 @@ def test_connection( client: QlikSenseClient, service_connection: QlikSenseConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -50,4 +52,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/quicksight/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/quicksight/connection.py index 4d2d70dd8ba2..22786be835ee 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/quicksight/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/quicksight/connection.py @@ -24,6 +24,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: QuickSightConnection): @@ -43,6 +44,7 @@ def test_connection( client: AWSClient, service_connection: QuickSightConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -60,4 +62,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/redash/connection.py index ae015da7e868..cfb43fcc1b04 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/redash/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/redash/connection.py @@ -27,6 +27,7 @@ ) from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.redash.client import RedashApiClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: RedashConnection) -> RedashApiClient: @@ -45,6 +46,7 @@ def test_connection( client: RedashApiClient, service_connection: RedashConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -58,4 +60,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/sigma/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/sigma/connection.py index b2e2204157ec..0cb9c78306d5 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/sigma/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/sigma/connection.py @@ -27,6 +27,7 @@ ) from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.dashboard.sigma.client import SigmaApiClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: SigmaConnection) -> SigmaApiClient: @@ -45,6 +46,7 @@ def test_connection( client: SigmaApiClient, service_connection: SigmaConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -58,4 +60,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/connection.py index a6e5f5aee81e..e4aac1aa13ba 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/connection.py @@ -49,6 +49,7 @@ from metadata.ingestion.source.database.postgres.connection import ( get_connection as pg_get_connection, ) +from metadata.utils.constants import THREE_MIN def get_connection(connection: SupersetConnection) -> SupersetAPIClient: @@ -69,6 +70,7 @@ def test_connection( client: Union[SupersetAPIClient, Engine], service_connection: SupersetConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -96,4 +98,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py index 8feef66c0e0e..191cf83334d1 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py @@ -38,6 +38,7 @@ TABLEAU_GET_WORKBOOKS_PARAM_DICT, ) from metadata.ingestion.source.dashboard.tableau.client import TableauClient +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger from metadata.utils.ssl_registry import get_verify_ssl_fn @@ -70,6 +71,7 @@ def test_connection( client: TableauClient, service_connection: TableauConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -106,6 +108,7 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/athena/connection.py b/ingestion/src/metadata/ingestion/source/database/athena/connection.py index c151be6fd69f..67d422df14fe 100644 --- a/ingestion/src/metadata/ingestion/source/database/athena/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/athena/connection.py @@ -37,6 +37,7 @@ test_connection_steps, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: AthenaConnection) -> str: @@ -94,6 +95,7 @@ def test_connection( engine: Engine, service_connection: AthenaConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -126,4 +128,5 @@ def custom_executor_for_view(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py b/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py index 7256e8507916..e2a47dbe409f 100644 --- a/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py @@ -35,6 +35,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -> str: @@ -105,6 +106,7 @@ def test_connection( engine: Engine, service_connection: AzureSQLConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -115,4 +117,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py b/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py index 06bc25a031aa..ac4d4031e4c9 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py @@ -47,6 +47,7 @@ ) from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.bigquery.queries import BIGQUERY_TEST_STATEMENT +from metadata.utils.constants import THREE_MIN from metadata.utils.credentials import set_google_credentials from metadata.utils.logger import ingestion_logger @@ -109,6 +110,7 @@ def test_connection( engine: Engine, service_connection: BigQueryConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -168,6 +170,7 @@ def test_connection_inner(engine): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) test_connection_inner(engine) diff --git a/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py b/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py index 990842604c52..303453c129d2 100644 --- a/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py @@ -29,6 +29,7 @@ ) from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.bigtable.client import MultiProjectClient +from metadata.utils.constants import THREE_MIN from metadata.utils.credentials import set_google_credentials from metadata.utils.logger import ingestion_logger @@ -95,6 +96,7 @@ def test_connection( client: MultiProjectClient, service_connection: BigTableConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -113,4 +115,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py b/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py index b0c17fded9d6..07357ef071c0 100644 --- a/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py @@ -35,6 +35,7 @@ from metadata.ingestion.source.database.clickhouse.queries import ( CLICKHOUSE_SQL_STATEMENT_TEST, ) +from metadata.utils.constants import THREE_MIN HTTPS_PROTOCOL = "https" @@ -67,6 +68,7 @@ def test_connection( engine: Engine, service_connection: ClickhouseConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -81,4 +83,5 @@ def test_connection( service_connection=service_connection, automation_workflow=automation_workflow, queries=queries, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/couchbase/connection.py b/ingestion/src/metadata/ingestion/source/database/couchbase/connection.py index 7f988a920707..537c1896ccc2 100644 --- a/ingestion/src/metadata/ingestion/source/database/couchbase/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/couchbase/connection.py @@ -25,6 +25,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: CouchbaseConnection): @@ -49,6 +50,7 @@ def test_connection( client: Any, service_connection: CouchbaseConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -85,4 +87,5 @@ def test_get_collections(client: Cluster, holder: SchemaHolder): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/connection.py b/ingestion/src/metadata/ingestion/source/database/databricks/connection.py index 7d7854bebf9f..e737122b55e9 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/connection.py @@ -39,6 +39,7 @@ from metadata.ingestion.source.database.databricks.queries import ( DATABRICKS_GET_CATALOGS, ) +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -71,6 +72,7 @@ def test_connection( connection: Engine, service_connection: DatabricksConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -109,5 +111,5 @@ def test_database_query(engine: Engine, statement: str): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, - timeout_seconds=service_connection.connectionTimeout, + timeout_seconds=service_connection.connectionTimeout or timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/datalake/connection.py b/ingestion/src/metadata/ingestion/source/database/datalake/connection.py index 9ec0a8625287..fed07e769625 100644 --- a/ingestion/src/metadata/ingestion/source/database/datalake/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/datalake/connection.py @@ -38,6 +38,7 @@ ) from metadata.ingestion.source.database.datalake.clients.gcs import DatalakeGcsClient from metadata.ingestion.source.database.datalake.clients.s3 import DatalakeS3Client +from metadata.utils.constants import THREE_MIN # Only import specific datalake dependencies if necessary @@ -91,6 +92,7 @@ def test_connection( connection: DatalakeClient, service_connection: DatalakeConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -107,4 +109,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/db2/connection.py b/ingestion/src/metadata/ingestion/source/database/db2/connection.py index 6ea82c758c20..1b974804a828 100644 --- a/ingestion/src/metadata/ingestion/source/database/db2/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/db2/connection.py @@ -29,6 +29,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: Db2Connection) -> Engine: @@ -47,6 +48,7 @@ def test_connection( engine: Engine, service_connection: Db2Connection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -57,4 +59,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/deltalake/connection.py b/ingestion/src/metadata/ingestion/source/database/deltalake/connection.py index 5219e89595a0..41f7c79621cd 100644 --- a/ingestion/src/metadata/ingestion/source/database/deltalake/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/deltalake/connection.py @@ -33,6 +33,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN @dataclass @@ -83,6 +84,7 @@ def test_connection( connection: DeltalakeClient, service_connection: DeltaLakeConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -102,4 +104,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/domodatabase/connection.py b/ingestion/src/metadata/ingestion/source/database/domodatabase/connection.py index 897c9dd767c5..e822002145c0 100644 --- a/ingestion/src/metadata/ingestion/source/database/domodatabase/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/domodatabase/connection.py @@ -28,6 +28,7 @@ test_connection_steps, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: DomoDatabaseConnection) -> Domo: @@ -51,6 +52,7 @@ def test_connection( domo: Domo, service_connection: DomoDatabaseConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -70,4 +72,5 @@ def custom_executor(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/doris/connection.py b/ingestion/src/metadata/ingestion/source/database/doris/connection.py index d4eff8b228c1..c77c1f701cd2 100644 --- a/ingestion/src/metadata/ingestion/source/database/doris/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/doris/connection.py @@ -31,6 +31,7 @@ test_connection_db_schema_sources, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: DorisConnection) -> Engine: @@ -49,6 +50,7 @@ def test_connection( engine: Engine, service_connection: DorisConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -59,4 +61,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/druid/connection.py b/ingestion/src/metadata/ingestion/source/database/druid/connection.py index 4c81c12d0d66..840add585b52 100644 --- a/ingestion/src/metadata/ingestion/source/database/druid/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/druid/connection.py @@ -29,6 +29,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: DruidConnection) -> str: @@ -52,6 +53,7 @@ def test_connection( engine: Engine, service_connection: DruidConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -62,4 +64,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py b/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py index 06dd5a338e25..b6aee09810be 100644 --- a/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py @@ -24,6 +24,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: DynamoDBConnection): @@ -48,6 +49,7 @@ def test_connection( client: AWSClient, service_connection: DynamoDBConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -63,4 +65,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/glue/connection.py b/ingestion/src/metadata/ingestion/source/database/glue/connection.py index b718faa94083..90ebbc8aa2e2 100644 --- a/ingestion/src/metadata/ingestion/source/database/glue/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/glue/connection.py @@ -25,6 +25,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: GlueConnection) -> Engine: @@ -39,6 +40,7 @@ def test_connection( client: AWSClient, service_connection: GlueConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -69,4 +71,5 @@ def custom_executor_for_table(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/greenplum/connection.py b/ingestion/src/metadata/ingestion/source/database/greenplum/connection.py index 2614bf4d2967..976bb94ca727 100644 --- a/ingestion/src/metadata/ingestion/source/database/greenplum/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/greenplum/connection.py @@ -31,6 +31,7 @@ from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.greenplum.queries import GREENPLUM_GET_DATABASE +from metadata.utils.constants import THREE_MIN def get_connection(connection: GreenplumConnection) -> Engine: @@ -49,6 +50,7 @@ def test_connection( engine: Engine, service_connection: GreenplumConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -63,4 +65,5 @@ def test_connection( service_connection=service_connection, automation_workflow=automation_workflow, queries=queries, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/hive/connection.py b/ingestion/src/metadata/ingestion/source/database/hive/connection.py index 58ff134f488d..8569e02642b5 100644 --- a/ingestion/src/metadata/ingestion/source/database/hive/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/hive/connection.py @@ -45,6 +45,7 @@ test_connection_db_schema_sources, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN HIVE_POSTGRES_SCHEME = "hive+postgres" HIVE_MYSQL_SCHEME = "hive+mysql" @@ -181,6 +182,7 @@ def test_connection( engine: Engine, service_connection: HiveConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -210,4 +212,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/iceberg/connection.py b/ingestion/src/metadata/ingestion/source/database/iceberg/connection.py index 6f315b807a24..352b83004437 100644 --- a/ingestion/src/metadata/ingestion/source/database/iceberg/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/iceberg/connection.py @@ -26,6 +26,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.iceberg.catalog import IcebergCatalogFactory +from metadata.utils.constants import THREE_MIN def get_connection(connection: IcebergConnection) -> Catalog: @@ -42,6 +43,7 @@ def test_connection( catalog: Catalog, service_connection: IcebergConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -65,4 +67,5 @@ def custom_executor_for_tables(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/impala/connection.py b/ingestion/src/metadata/ingestion/source/database/impala/connection.py index b28262d69469..62920aa20259 100644 --- a/ingestion/src/metadata/ingestion/source/database/impala/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/impala/connection.py @@ -34,6 +34,7 @@ test_connection_db_schema_sources, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: ImpalaConnection) -> str: @@ -108,6 +109,7 @@ def test_connection( engine: Engine, service_connection: ImpalaConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -118,4 +120,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/mariadb/connection.py b/ingestion/src/metadata/ingestion/source/database/mariadb/connection.py index 87942ac961e3..c4e15423a072 100644 --- a/ingestion/src/metadata/ingestion/source/database/mariadb/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/mariadb/connection.py @@ -31,6 +31,7 @@ test_connection_db_schema_sources, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: MariaDBConnection) -> Engine: @@ -49,6 +50,7 @@ def test_connection( engine: Engine, service_connection: MariaDBConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -59,4 +61,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py b/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py index 56611eb40c61..9f9697a274a6 100644 --- a/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py @@ -29,6 +29,7 @@ from metadata.ingestion.connections.builders import get_connection_url_common from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: MongoDBConnection): @@ -44,6 +45,7 @@ def test_connection( client: MongoClient, service_connection: MongoDBConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -75,4 +77,5 @@ def test_get_collections(client_: MongoClient, holder_: SchemaHolder): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/mssql/connection.py b/ingestion/src/metadata/ingestion/source/database/mssql/connection.py index 9979e01fa632..d1664cc2f5ae 100644 --- a/ingestion/src/metadata/ingestion/source/database/mssql/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/mssql/connection.py @@ -36,6 +36,7 @@ MSSQL_GET_DATABASE, MSSQL_TEST_GET_QUERIES, ) +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: MssqlConnection) -> str: @@ -60,6 +61,7 @@ def test_connection( engine: Engine, service_connection: MssqlConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -75,4 +77,5 @@ def test_connection( service_connection=service_connection, automation_workflow=automation_workflow, queries=queries, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/mysql/connection.py b/ingestion/src/metadata/ingestion/source/database/mysql/connection.py index d4a8b2d0c946..a8bd34f77863 100644 --- a/ingestion/src/metadata/ingestion/source/database/mysql/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/mysql/connection.py @@ -35,6 +35,7 @@ test_connection_db_schema_sources, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: MysqlConnection) -> Engine: @@ -63,6 +64,7 @@ def test_connection( engine: Engine, service_connection: MysqlConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -73,4 +75,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/oracle/connection.py b/ingestion/src/metadata/ingestion/source/database/oracle/connection.py index ade912396452..2323c6875a07 100644 --- a/ingestion/src/metadata/ingestion/source/database/oracle/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/oracle/connection.py @@ -39,6 +39,7 @@ from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.oracle.queries import CHECK_ACCESS_TO_ALL +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger CX_ORACLE_LIB_VERSION = "8.3.0" @@ -132,6 +133,7 @@ def test_connection( engine: Engine, service_connection: OracleConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -146,4 +148,5 @@ def test_connection( service_connection=service_connection, automation_workflow=automation_workflow, queries=test_conn_queries, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/pinotdb/connection.py b/ingestion/src/metadata/ingestion/source/database/pinotdb/connection.py index 63d25adcbd6f..4051da4fd1e3 100644 --- a/ingestion/src/metadata/ingestion/source/database/pinotdb/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/pinotdb/connection.py @@ -31,6 +31,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: PinotDBConnection) -> str: @@ -63,6 +64,7 @@ def test_connection( engine: Engine, service_connection: PinotDBConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -73,4 +75,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/connection.py b/ingestion/src/metadata/ingestion/source/database/postgres/connection.py index 4ea066a8e8bb..fab91e4e9215 100644 --- a/ingestion/src/metadata/ingestion/source/database/postgres/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/postgres/connection.py @@ -42,6 +42,7 @@ from metadata.ingestion.source.database.postgres.utils import ( get_postgres_time_column_name, ) +from metadata.utils.constants import THREE_MIN def get_connection(connection: PostgresConnection) -> Engine: @@ -71,6 +72,7 @@ def test_connection( engine: Engine, service_connection: PostgresConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -90,4 +92,5 @@ def test_connection( service_connection=service_connection, automation_workflow=automation_workflow, queries=queries, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/presto/connection.py b/ingestion/src/metadata/ingestion/source/database/presto/connection.py index 1847c3207c40..00222d50f503 100644 --- a/ingestion/src/metadata/ingestion/source/database/presto/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/presto/connection.py @@ -38,6 +38,7 @@ ) from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.presto.queries import PRESTO_SHOW_CATALOGS +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: PrestoConnection) -> str: @@ -84,6 +85,7 @@ def test_connection( engine: Engine, service_connection: PrestoConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -113,4 +115,5 @@ def custom_executor_for_table(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/connection.py b/ingestion/src/metadata/ingestion/source/database/redshift/connection.py index a9193d5be298..4a7db793a633 100644 --- a/ingestion/src/metadata/ingestion/source/database/redshift/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/redshift/connection.py @@ -44,6 +44,7 @@ REDSHIFT_TEST_GET_QUERIES, REDSHIFT_TEST_PARTITION_DETAILS, ) +from metadata.utils.constants import THREE_MIN def get_connection(connection: RedshiftConnection) -> Engine: @@ -62,6 +63,7 @@ def test_connection( engine: Engine, service_connection: RedshiftConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -101,6 +103,7 @@ def test_get_queries_permissions(engine_: Engine): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) kill_active_connections(engine) diff --git a/ingestion/src/metadata/ingestion/source/database/salesforce/connection.py b/ingestion/src/metadata/ingestion/source/database/salesforce/connection.py index 9cfd79e08846..d3cac4f4722e 100644 --- a/ingestion/src/metadata/ingestion/source/database/salesforce/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/salesforce/connection.py @@ -24,6 +24,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: SalesforceConnection) -> Salesforce: @@ -48,6 +49,7 @@ def test_connection( client: Salesforce, service_connection: SalesforceConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -60,4 +62,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/saperp/connection.py b/ingestion/src/metadata/ingestion/source/database/saperp/connection.py index ee2b2bff902e..2b4ae95e0295 100644 --- a/ingestion/src/metadata/ingestion/source/database/saperp/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/saperp/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.saperp.client import SapErpClient +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -37,6 +38,7 @@ def test_connection( client: SapErpClient, service_connection: SapErpConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: test_fn = { "GetTables": client.test_table_api, @@ -47,4 +49,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/saphana/connection.py b/ingestion/src/metadata/ingestion/source/database/saphana/connection.py index 18567eedb5ad..b3a4739acb17 100644 --- a/ingestion/src/metadata/ingestion/source/database/saphana/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/saphana/connection.py @@ -41,6 +41,7 @@ test_connection_steps, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_database_connection_url(connection: SapHanaConnection) -> str: @@ -156,6 +157,7 @@ def test_connection( engine: Engine, service_connection: SapHanaConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -167,4 +169,5 @@ def test_connection( test_fn=_build_test_fn_dict(engine, service_connection), service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/sas/connection.py b/ingestion/src/metadata/ingestion/source/database/sas/connection.py index a11bd1204cd1..9acbf4d140cb 100644 --- a/ingestion/src/metadata/ingestion/source/database/sas/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/sas/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.sas.client import SASClient +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -37,6 +38,7 @@ def test_connection( client: SASClient, service_connection: SASConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: test_fn = {"CheckAccess": client.check_connection} test_connection_steps( @@ -44,4 +46,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/singlestore/connection.py b/ingestion/src/metadata/ingestion/source/database/singlestore/connection.py index fdd371ff9f86..0513a641cde1 100644 --- a/ingestion/src/metadata/ingestion/source/database/singlestore/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/singlestore/connection.py @@ -32,6 +32,7 @@ test_connection_db_schema_sources, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: SingleStoreConnection) -> Engine: @@ -50,6 +51,7 @@ def test_connection( engine: Engine, service_connection: SingleStoreConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -60,4 +62,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py index 599a3f3815e3..e85522c5ba36 100644 --- a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py @@ -47,6 +47,7 @@ SNOWFLAKE_TEST_GET_TABLES, SNOWFLAKE_TEST_GET_VIEWS, ) +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -145,6 +146,7 @@ def test_connection( engine: Engine, service_connection: SnowflakeConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -195,6 +197,7 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/sqlite/connection.py b/ingestion/src/metadata/ingestion/source/database/sqlite/connection.py index 03b0e4b685b4..ff4cbdaaa88d 100644 --- a/ingestion/src/metadata/ingestion/source/database/sqlite/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/sqlite/connection.py @@ -28,6 +28,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: SQLiteConnection) -> str: @@ -52,6 +53,7 @@ def test_connection( engine: Engine, service_connection: SQLiteConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -62,4 +64,5 @@ def test_connection( engine=engine, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/teradata/connection.py b/ingestion/src/metadata/ingestion/source/database/teradata/connection.py index 014af8e288b3..f1c371b4255f 100644 --- a/ingestion/src/metadata/ingestion/source/database/teradata/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/teradata/connection.py @@ -32,6 +32,7 @@ from metadata.ingestion.connections.test_connections import test_connection_db_common from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.teradata.queries import TERADATA_GET_DATABASE +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: TeradataConnection) -> str: @@ -82,6 +83,7 @@ def test_connection( engine: Engine, service_connection: TeradataConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -95,4 +97,5 @@ def test_connection( service_connection=service_connection, automation_workflow=automation_workflow, queries=queries, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/trino/connection.py b/ingestion/src/metadata/ingestion/source/database/trino/connection.py index 1a7a3a5ebd59..2a987a7a7d52 100644 --- a/ingestion/src/metadata/ingestion/source/database/trino/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/trino/connection.py @@ -41,6 +41,7 @@ ) from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.trino.queries import TRINO_GET_DATABASE +from metadata.utils.constants import THREE_MIN def get_connection_url(connection: TrinoConnection) -> str: @@ -124,6 +125,7 @@ def test_connection( engine: Engine, service_connection: TrinoConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -139,4 +141,5 @@ def test_connection( service_connection=service_connection, automation_workflow=automation_workflow, queries=queries, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py index 2ba24d3ac328..3a37052dea90 100644 --- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py @@ -27,6 +27,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.unitycatalog.client import UnityCatalogClient from metadata.ingestion.source.database.unitycatalog.models import DatabricksTable +from metadata.utils.constants import THREE_MIN from metadata.utils.db_utils import get_host_from_host_port from metadata.utils.logger import ingestion_logger @@ -54,6 +55,7 @@ def test_connection( connection: WorkspaceClient, service_connection: UnityCatalogConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -97,5 +99,5 @@ def get_tables(connection: WorkspaceClient, table_obj: DatabricksTable): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, - timeout_seconds=service_connection.connectionTimeout, + timeout_seconds=service_connection.connectionTimeout or timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/database/vertica/connection.py b/ingestion/src/metadata/ingestion/source/database/vertica/connection.py index 93d515850ed3..39f3b41e607d 100644 --- a/ingestion/src/metadata/ingestion/source/database/vertica/connection.py +++ b/ingestion/src/metadata/ingestion/source/database/vertica/connection.py @@ -33,6 +33,7 @@ VERTICA_LIST_DATABASES, VERTICA_TEST_GET_QUERIES, ) +from metadata.utils.constants import THREE_MIN def get_connection(connection: VerticaConnection) -> Engine: @@ -51,6 +52,7 @@ def test_connection( engine: Engine, service_connection: VerticaConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -66,4 +68,5 @@ def test_connection( service_connection=service_connection, automation_workflow=automation_workflow, queries=queries, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py b/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py index ef8c4b0689ad..519844534bca 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py +++ b/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py @@ -31,6 +31,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -120,6 +121,7 @@ def test_connection( client: KafkaClient, service_connection: Union[KafkaConnection, RedpandaConnection], automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -154,4 +156,5 @@ def schema_registry_test(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/messaging/kinesis/connection.py b/ingestion/src/metadata/ingestion/source/messaging/kinesis/connection.py index dc5b3cc69a82..b9bc119c1dab 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/kinesis/connection.py +++ b/ingestion/src/metadata/ingestion/source/messaging/kinesis/connection.py @@ -24,6 +24,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -41,6 +42,7 @@ def test_connection( client, service_connection: KinesisConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -54,4 +56,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/messaging/redpanda/connection.py b/ingestion/src/metadata/ingestion/source/messaging/redpanda/connection.py index def0edd58848..b923c5c8a025 100644 --- a/ingestion/src/metadata/ingestion/source/messaging/redpanda/connection.py +++ b/ingestion/src/metadata/ingestion/source/messaging/redpanda/connection.py @@ -28,6 +28,7 @@ from metadata.ingestion.source.messaging.kafka.connection import ( test_connection as test_kafka_connection, ) +from metadata.utils.constants import THREE_MIN from metadata.utils.logger import ingestion_logger logger = ingestion_logger() @@ -45,6 +46,7 @@ def test_connection( client: KafkaClient, service_connection: RedpandaConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -56,4 +58,5 @@ def test_connection( client=client, service_connection=service_connection, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/metadata/alationsink/connection.py b/ingestion/src/metadata/ingestion/source/metadata/alationsink/connection.py index d3f6bfa4b743..a3f58a4ef027 100644 --- a/ingestion/src/metadata/ingestion/source/metadata/alationsink/connection.py +++ b/ingestion/src/metadata/ingestion/source/metadata/alationsink/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.metadata.alationsink.client import AlationSinkClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: AlationSinkConnection) -> AlationSinkClient: @@ -37,6 +38,7 @@ def test_connection( client: AlationSinkClient, service_connection: AlationSinkConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -50,4 +52,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/metadata/amundsen/connection.py b/ingestion/src/metadata/ingestion/source/metadata/amundsen/connection.py index 2862421b8efe..394834f4928a 100644 --- a/ingestion/src/metadata/ingestion/source/metadata/amundsen/connection.py +++ b/ingestion/src/metadata/ingestion/source/metadata/amundsen/connection.py @@ -30,6 +30,7 @@ from metadata.ingestion.source.metadata.amundsen.queries import ( NEO4J_AMUNDSEN_USER_QUERY, ) +from metadata.utils.constants import THREE_MIN def get_connection(connection: AmundsenConnection) -> Neo4jHelper: @@ -56,6 +57,7 @@ def test_connection( client: Neo4jHelper, service_connection: AmundsenConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -71,4 +73,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/metadata/atlas/connection.py b/ingestion/src/metadata/ingestion/source/metadata/atlas/connection.py index 66b0d259b78c..a5218c666715 100644 --- a/ingestion/src/metadata/ingestion/source/metadata/atlas/connection.py +++ b/ingestion/src/metadata/ingestion/source/metadata/atlas/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.metadata.atlas.client import AtlasClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: AtlasConnection) -> AtlasClient: @@ -37,6 +38,7 @@ def test_connection( client: AtlasClient, service_connection: AtlasConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -50,4 +52,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/mlmodel/mlflow/connection.py b/ingestion/src/metadata/ingestion/source/mlmodel/mlflow/connection.py index b12c3a221767..335d4e4e4144 100644 --- a/ingestion/src/metadata/ingestion/source/mlmodel/mlflow/connection.py +++ b/ingestion/src/metadata/ingestion/source/mlmodel/mlflow/connection.py @@ -24,6 +24,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: MlflowConnection) -> MlflowClient: @@ -41,6 +42,7 @@ def test_connection( client: MlflowClient, service_connection: MlflowConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -54,4 +56,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/mlmodel/sagemaker/connection.py b/ingestion/src/metadata/ingestion/source/mlmodel/sagemaker/connection.py index f2dd565494c8..9c67e8f67ffb 100644 --- a/ingestion/src/metadata/ingestion/source/mlmodel/sagemaker/connection.py +++ b/ingestion/src/metadata/ingestion/source/mlmodel/sagemaker/connection.py @@ -23,6 +23,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: SageMakerConnection): @@ -37,6 +38,7 @@ def test_connection( client, service_connection: SageMakerConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -50,4 +52,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airbyte/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/airbyte/connection.py index 099c84fa7557..d4387b78b931 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/airbyte/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/airbyte/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.pipeline.airbyte.client import AirbyteClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: AirbyteConnection) -> AirbyteClient: @@ -37,6 +38,7 @@ def test_connection( client: AirbyteClient, service_connection: AirbyteConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -50,4 +52,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py index 705030cb9de7..2bf2a6786d61 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py @@ -42,6 +42,7 @@ test_connection_steps, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN # Only import when needed @@ -103,6 +104,7 @@ def test_connection( engine: Engine, service_connection: AirflowConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -116,4 +118,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py index 021d14b935d9..9884e97060e8 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py @@ -24,6 +24,7 @@ from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.pipeline.dagster.client import DagsterClient from metadata.ingestion.source.pipeline.dagster.queries import TEST_QUERY_GRAPHQL +from metadata.utils.constants import THREE_MIN def get_connection(connection: DagsterConnection) -> DagsterClient: @@ -38,6 +39,7 @@ def test_connection( client: DagsterClient, service_connection: DagsterConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -54,4 +56,5 @@ def custom_executor_for_pipeline(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/connection.py index 0ba178943ff9..6faf3e976a75 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/connection.py @@ -24,6 +24,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.database.databricks.client import DatabricksClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: DatabricksPipelineConnection) -> DatabricksClient: @@ -38,6 +39,7 @@ def test_connection( client: DatabricksClient, service_connection: DatabricksPipelineConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -51,4 +53,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/connection.py index 04e3ff56d8b5..992089be64f2 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/connection.py @@ -25,6 +25,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.pipeline.dbtcloud.client import DBTCloudClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: DBTCloudConnection) -> DBTCloudClient: @@ -39,6 +40,7 @@ def test_connection( client: DBTCloudClient, service_connection: DBTCloudConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -57,4 +59,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/connection.py index e25a10cc767e..1ff928dd59c4 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/connection.py @@ -29,6 +29,7 @@ test_connection_steps, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: DomoPipelineConnection) -> Domo: @@ -47,6 +48,7 @@ def test_connection( connection: Domo, service_connection: DomoPipelineConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -64,4 +66,5 @@ def custom_executor(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/fivetran/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/fivetran/connection.py index 644e19377004..b60828feab48 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/fivetran/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/fivetran/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.pipeline.fivetran.client import FivetranClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: FivetranConnection) -> FivetranClient: @@ -37,6 +38,7 @@ def test_connection( client: FivetranClient, service_connection: FivetranConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -50,4 +52,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/flink/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/flink/connection.py index db5cc77b2b55..a868ae01ddf5 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/flink/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/flink/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.pipeline.flink.client import FlinkClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: FlinkConnection) -> FlinkClient: @@ -37,6 +38,7 @@ def test_connection( client: FlinkClient, service_connection: FlinkConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -49,4 +51,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/connection.py index 725277bbc514..222ed9a84ab0 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/connection.py @@ -24,6 +24,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: GluePipelineConnection): @@ -38,6 +39,7 @@ def test_connection( client, service_connection: GluePipelineConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -51,4 +53,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/connection.py index 27fda4bf3de8..c8a3f4e87d94 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/connection.py @@ -24,6 +24,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.pipeline.kafkaconnect.client import KafkaConnectClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: KafkaConnectConnection) -> KafkaConnectClient: @@ -38,6 +39,7 @@ def test_connection( client: KafkaConnectClient, service_connection: KafkaConnectConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -55,4 +57,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py index dae1f25a524a..1b4b86e8cef0 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py @@ -24,6 +24,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.pipeline.nifi.client import NifiClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: NifiConnection) -> NifiClient: @@ -53,6 +54,7 @@ def test_connection( client: NifiClient, service_connection: NifiConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -69,4 +71,5 @@ def custom_executor(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/openlineage/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/openlineage/connection.py index 74d488d77430..f578531672f5 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/openlineage/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/openlineage/connection.py @@ -31,6 +31,7 @@ test_connection_steps, ) from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN def get_connection(connection: OpenLineageConnection) -> KafkaConsumer: @@ -76,6 +77,7 @@ def test_connection( client: KafkaConsumer, service_connection: OpenLineageConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -94,4 +96,5 @@ def custom_executor(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/pipeline/spline/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/spline/connection.py index aeb1b81e67da..2b4d0dcfe8bd 100644 --- a/ingestion/src/metadata/ingestion/source/pipeline/spline/connection.py +++ b/ingestion/src/metadata/ingestion/source/pipeline/spline/connection.py @@ -23,6 +23,7 @@ from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.pipeline.spline.client import SplineClient +from metadata.utils.constants import THREE_MIN def get_connection(connection: SplineConnection) -> SplineClient: @@ -38,6 +39,7 @@ def test_connection( client: SplineClient, service_connection: SplineConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -51,4 +53,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py b/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py index 52df2e9e3d29..dbd726756985 100644 --- a/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py +++ b/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py @@ -43,7 +43,7 @@ from metadata.ingestion.connections.builders import init_empty_connection_arguments from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.utils.constants import UTF_8 +from metadata.utils.constants import THREE_MIN, UTF_8 from metadata.utils.helpers import init_staging_dir CA_CERT_FILE_NAME = "root.pem" @@ -185,6 +185,7 @@ def test_connection( client: Elasticsearch, service_connection: ElasticsearchConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -204,4 +205,5 @@ def test_get_search_indexes(): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/storage/gcs/connection.py b/ingestion/src/metadata/ingestion/source/storage/gcs/connection.py index a11d156c8383..cafb53cdc159 100644 --- a/ingestion/src/metadata/ingestion/source/storage/gcs/connection.py +++ b/ingestion/src/metadata/ingestion/source/storage/gcs/connection.py @@ -32,6 +32,7 @@ ) from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.ingestion.source.storage.gcs.client import MultiProjectClient +from metadata.utils.constants import THREE_MIN from metadata.utils.credentials import set_google_credentials from metadata.utils.logger import ingestion_logger @@ -136,6 +137,7 @@ def test_connection( client: GcsObjectStoreClient, service_connection: GcsConnection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -156,4 +158,5 @@ def test_connection( test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/ingestion/source/storage/s3/connection.py b/ingestion/src/metadata/ingestion/source/storage/s3/connection.py index 2df0b9a8ff44..3c745dda162c 100644 --- a/ingestion/src/metadata/ingestion/source/storage/s3/connection.py +++ b/ingestion/src/metadata/ingestion/source/storage/s3/connection.py @@ -29,6 +29,7 @@ ) from metadata.ingestion.connections.test_connections import test_connection_steps from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.utils.constants import THREE_MIN @dataclass @@ -53,6 +54,7 @@ def test_connection( client: S3ObjectStoreClient, service_connection: S3Connection, automation_workflow: Optional[AutomationWorkflow] = None, + timeout_seconds: Optional[int] = THREE_MIN, ) -> None: """ Test connection. This can be executed either as part @@ -80,4 +82,5 @@ def test_buckets(connection: S3Connection, client: S3ObjectStoreClient): test_fn=test_fn, service_type=service_connection.type.value, automation_workflow=automation_workflow, + timeout_seconds=timeout_seconds, ) diff --git a/ingestion/src/metadata/utils/constants.py b/ingestion/src/metadata/utils/constants.py index cbb1b7b5f3e9..2f1c921eb4d1 100644 --- a/ingestion/src/metadata/utils/constants.py +++ b/ingestion/src/metadata/utils/constants.py @@ -40,6 +40,7 @@ DOT = "_DOT_" TEN_MIN = 10 * 60 +THREE_MIN = 3 * 60 UTF_8 = "utf-8" CHUNKSIZE = 200000 DEFAULT_DATABASE = "default" diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py index 15bf98e132a0..17ff06540705 100644 --- a/ingestion/src/metadata/workflow/application.py +++ b/ingestion/src/metadata/workflow/application.py @@ -14,10 +14,6 @@ from abc import ABC, abstractmethod from typing import List, Optional -from metadata.config.common import WorkflowExecutionError -from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( - OpenMetadataConnection, -) from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -25,13 +21,11 @@ from metadata.generated.schema.metadataIngestion.application import ( OpenMetadataApplicationConfig, ) -from metadata.generated.schema.metadataIngestion.workflow import LogLevels -from metadata.ingestion.api.step import Step, Summary +from metadata.ingestion.api.step import Step from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.utils.importer import import_from_module from metadata.utils.logger import ingestion_logger from metadata.workflow.base import BaseWorkflow -from metadata.workflow.workflow_status_mixin import SUCCESS_THRESHOLD_VALUE logger = ingestion_logger() @@ -92,15 +86,9 @@ def __init__(self, config_dict: dict): # Applications are associated to the OpenMetadata Service self.service_type: ServiceType = ServiceType.Metadata - metadata_config: OpenMetadataConnection = ( - self.config.workflowConfig.openMetadataServerConfig - ) - log_level: LogLevels = self.config.workflowConfig.loggerLevel - super().__init__( config=self.config, - log_level=log_level, - metadata_config=metadata_config, + workflow_config=self.workflow_config, service_type=self.service_type, ) @@ -134,26 +122,8 @@ def execute_internal(self) -> None: """Workflow-specific logic to execute safely""" self.runner.run() - def calculate_success(self) -> float: - return self.runner.get_status().calculate_success() - def get_failures(self) -> List[StackTraceError]: return self.workflow_steps()[0].get_status().failures def workflow_steps(self) -> List[Step]: return [self.runner] - - def raise_from_status_internal(self, raise_warnings=False): - """Check failed status in the runner""" - if ( - self.runner.get_status().failures - and self.calculate_success() < SUCCESS_THRESHOLD_VALUE - ): - raise WorkflowExecutionError( - f"{self.runner.name} reported errors: {Summary.from_step(self.runner)}" - ) - - if raise_warnings and self.runner.get_status().warnings: - raise WorkflowExecutionError( - f"{self.runner.name} reported warning: {Summary.from_step(self.runner)}" - ) diff --git a/ingestion/src/metadata/workflow/base.py b/ingestion/src/metadata/workflow/base.py index 87bca9565a22..e777d6731ec1 100644 --- a/ingestion/src/metadata/workflow/base.py +++ b/ingestion/src/metadata/workflow/base.py @@ -16,8 +16,10 @@ import uuid from abc import ABC, abstractmethod from datetime import datetime +from statistics import mean from typing import Any, Dict, List, Optional, TypeVar, Union +from metadata.config.common import WorkflowExecutionError from metadata.generated.schema.api.services.ingestionPipelines.createIngestionPipeline import ( CreateIngestionPipelineRequest, ) @@ -32,10 +34,13 @@ from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) -from metadata.generated.schema.metadataIngestion.workflow import LogLevels +from metadata.generated.schema.metadataIngestion.workflow import ( + LogLevels, + WorkflowConfig, +) from metadata.generated.schema.tests.testSuite import ServiceType from metadata.generated.schema.type.entityReference import EntityReference -from metadata.ingestion.api.step import Step +from metadata.ingestion.api.step import Step, Summary from metadata.ingestion.ometa.client_utils import create_ometa_client from metadata.ingestion.ometa.ometa_api import OpenMetadata from metadata.timer.repeated_timer import RepeatedTimer @@ -49,10 +54,7 @@ from metadata.utils.helpers import datetime_to_ts from metadata.utils.logger import ingestion_logger, set_loggers_level from metadata.workflow.workflow_output_handler import WorkflowOutputHandler -from metadata.workflow.workflow_status_mixin import ( - SUCCESS_THRESHOLD_VALUE, - WorkflowStatusMixin, -) +from metadata.workflow.workflow_status_mixin import WorkflowStatusMixin logger = ingestion_logger() @@ -82,8 +84,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin): def __init__( self, config: Union[Any, Dict], - log_level: LogLevels, - metadata_config: OpenMetadataConnection, + workflow_config: WorkflowConfig, service_type: ServiceType, output_handler: WorkflowOutputHandler = WorkflowOutputHandler(), ): @@ -92,19 +93,22 @@ def __init__( """ self.output_handler = output_handler self.config = config + self.workflow_config = workflow_config self.service_type = service_type self._timer: Optional[RepeatedTimer] = None self._ingestion_pipeline: Optional[IngestionPipeline] = None self._start_ts = datetime_to_ts(datetime.now()) + self._execution_time_tracker = ExecutionTimeTracker( - log_level == LogLevels.DEBUG + self.workflow_config.loggerLevel == LogLevels.DEBUG ) - set_loggers_level(log_level.value) + set_loggers_level(self.workflow_config.loggerLevel.value) # We create the ometa client at the workflow level and pass it to the steps - self.metadata_config = metadata_config - self.metadata = create_ometa_client(metadata_config) + self.metadata = create_ometa_client( + self.workflow_config.openMetadataServerConfig + ) self.set_ingestion_pipeline_status(state=PipelineState.running) self.post_init() @@ -157,9 +161,22 @@ def post_init(self) -> None: def execute_internal(self) -> None: """Workflow-specific logic to execute safely""" - @abstractmethod - def calculate_success(self) -> float: - """Get the success % of the internal execution""" + def calculate_success(self) -> Optional[float]: + """ + Get the success % of the internal execution. + Since we'll use this to get a single success % from multiple steps, we'll take + the minimum success % from all the steps. This way, we can have a proper + workflow status. + E.g., if we have no errors on the source but a bunch of them on the sink, + we still want the flow to be marked as a failure or partial success. + """ + if not self.workflow_steps(): + logger.warning("No steps to calculate success") + return None + + return mean( + [step.get_status().calculate_success() for step in self.workflow_steps()] + ) @abstractmethod def get_failures(self) -> List[StackTraceError]: @@ -169,9 +186,22 @@ def get_failures(self) -> List[StackTraceError]: def workflow_steps(self) -> List[Step]: """Steps to report status from""" - @abstractmethod def raise_from_status_internal(self, raise_warnings=False) -> None: """Based on the internal workflow status, raise a WorkflowExecutionError""" + for step in self.workflow_steps(): + if ( + step.get_status().failures + and step.get_status().calculate_success() + < self.workflow_config.successThreshold + ): + raise WorkflowExecutionError( + f"{step.name} reported errors: {Summary.from_step(step)}" + ) + + if raise_warnings and step.status.warnings: + raise WorkflowExecutionError( + f"{step.name} reported warning: {Summary.from_step(step)}" + ) def execute(self) -> None: """ @@ -186,7 +216,7 @@ def execute(self) -> None: try: self.execute_internal() - if SUCCESS_THRESHOLD_VALUE <= self.calculate_success() < 100: + if self.workflow_config.successThreshold <= self.calculate_success() < 100: pipeline_state = PipelineState.partialSuccess # Any unhandled exception breaking the workflow should update the status diff --git a/ingestion/src/metadata/workflow/ingestion.py b/ingestion/src/metadata/workflow/ingestion.py index cfa78c1259ed..1e28f5013178 100644 --- a/ingestion/src/metadata/workflow/ingestion.py +++ b/ingestion/src/metadata/workflow/ingestion.py @@ -24,9 +24,6 @@ from typing import List, Tuple, Type, cast from metadata.config.common import WorkflowExecutionError -from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( - OpenMetadataConnection, -) from metadata.generated.schema.entity.services.connections.serviceConnection import ( ServiceConnection, ) @@ -38,7 +35,7 @@ OpenMetadataWorkflowConfig, ) from metadata.ingestion.api.parser import parse_workflow_config_gracefully -from metadata.ingestion.api.step import Step, Summary +from metadata.ingestion.api.step import Step from metadata.ingestion.api.steps import BulkSink, Processor, Sink, Source, Stage from metadata.ingestion.models.custom_types import ServiceWithConnectionType from metadata.profiler.api.models import ProfilerProcessorConfig @@ -55,14 +52,15 @@ ) from metadata.utils.logger import ingestion_logger from metadata.workflow.base import BaseWorkflow, InvalidWorkflowJSONException -from metadata.workflow.workflow_status_mixin import SUCCESS_THRESHOLD_VALUE logger = ingestion_logger() class IngestionWorkflow(BaseWorkflow, ABC): """ - Base Ingestion Workflow implementation + Base Ingestion Workflow implementation. This is used for all + workflows minus the application one, which directly inherits the + BaseWorkflow. """ config: OpenMetadataWorkflowConfig @@ -79,14 +77,9 @@ def __init__(self, config: OpenMetadataWorkflowConfig): self.config.source.type ) - metadata_config: OpenMetadataConnection = ( - self.config.workflowConfig.openMetadataServerConfig - ) - super().__init__( config=config, - log_level=config.workflowConfig.loggerLevel, - metadata_config=metadata_config, + workflow_config=config.workflowConfig, service_type=self.service_type, ) @@ -137,37 +130,12 @@ def execute_internal(self): if bulk_sink: bulk_sink.run() - def calculate_success(self) -> float: - return self.source.get_status().calculate_success() - def get_failures(self) -> List[StackTraceError]: return self.source.get_status().failures def workflow_steps(self) -> List[Step]: return [self.source] + list(self.steps) - def raise_from_status_internal(self, raise_warnings=False): - """ - Check the status of all steps - """ - if ( - self.source.get_status().failures - and self.calculate_success() < SUCCESS_THRESHOLD_VALUE - ): - raise WorkflowExecutionError( - f"{self.source.name} reported errors: {Summary.from_step(self.source)}" - ) - - for step in self.steps: - if step.status.failures: - raise WorkflowExecutionError( - f"{step.name} reported errors: {Summary.from_step(step)}" - ) - if raise_warnings and step.status.warnings: - raise WorkflowExecutionError( - f"{step.name} reported warnings: {Summary.from_step(step)}" - ) - def _retrieve_service_connection_if_needed(self, service_type: ServiceType) -> None: """ We override the current `serviceConnection` source config object if source workflow service already exists diff --git a/ingestion/src/metadata/workflow/workflow_output_handler.py b/ingestion/src/metadata/workflow/workflow_output_handler.py index d1a2070e2a7e..06df1f0dd54b 100644 --- a/ingestion/src/metadata/workflow/workflow_output_handler.py +++ b/ingestion/src/metadata/workflow/workflow_output_handler.py @@ -14,6 +14,7 @@ """ import time +from statistics import mean from typing import Any, Dict, List, Optional, Type, Union from pydantic import BaseModel @@ -114,16 +115,15 @@ def print_summary(self, steps: List[Step], debug: bool = False): self._print_summary(steps) - def _print_summary(self, steps: List[Step]): + def _print_summary(self, steps: List[Step]) -> None: failures: List[Failure] = [] - total_records: int = 0 - total_errors: int = 0 + if not steps: + log_ansi_encoded_string(message="No steps to process.") + return for step in steps: step_summary = Summary.from_step(step) - total_records += step_summary.records or 0 - total_errors += step_summary.errors or 0 failures.append( Failure(name=step.name, failures=step.get_status().failures) ) @@ -141,15 +141,18 @@ def _print_summary(self, steps: List[Step]): log_ansi_encoded_string(message=f"Filtered: {step_summary.filtered}") log_ansi_encoded_string(message=f"Errors: {step_summary.errors}") + log_ansi_encoded_string( + message=f"Success %: {step.get_status().calculate_success()}" + ) self._print_failures_if_apply(failures) - total_success = max(total_records, 1) + # If nothing is processed, we'll have a success of 100% + success_pct = mean([step.get_status().calculate_success() for step in steps]) log_ansi_encoded_string( color=ANSI.BRIGHT_CYAN, bold=True, - message="Success %: " - + f"{round(total_success * 100 / (total_success + total_errors), 2)}", + message="Workflow Success %: " + f"{round(success_pct, 2)}", ) def _print_debug_summary(self, steps: List[Step]): diff --git a/ingestion/src/metadata/workflow/workflow_status_mixin.py b/ingestion/src/metadata/workflow/workflow_status_mixin.py index e648ed00d439..fe8d99715c21 100644 --- a/ingestion/src/metadata/workflow/workflow_status_mixin.py +++ b/ingestion/src/metadata/workflow/workflow_status_mixin.py @@ -37,8 +37,6 @@ logger = ometa_logger() -SUCCESS_THRESHOLD_VALUE = 90 - class WorkflowResultStatus(Enum): SUCCESS = 0 diff --git a/ingestion/tests/unit/profiler/test_workflow.py b/ingestion/tests/unit/profiler/test_workflow.py index 8cc358c6f9d8..aeeb3df971d1 100644 --- a/ingestion/tests/unit/profiler/test_workflow.py +++ b/ingestion/tests/unit/profiler/test_workflow.py @@ -26,9 +26,6 @@ Table, TableProfilerConfig, ) -from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( - OpenMetadataConnection, -) from metadata.generated.schema.entity.services.databaseService import ( DatabaseService, DatabaseServiceType, @@ -36,6 +33,7 @@ from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import ( DatabaseServiceProfilerPipeline, ) +from metadata.generated.schema.metadataIngestion.workflow import WorkflowConfig from metadata.generated.schema.type.entityReference import EntityReference from metadata.profiler.api.models import ProfilerProcessorConfig from metadata.profiler.interface.sqlalchemy.profiler_interface import ( @@ -122,7 +120,7 @@ def test_init_workflow(mocked_method, mocked_orm): # pylint: disable=unused-arg mocked_method.assert_called() assert isinstance(workflow.source.source_config, DatabaseServiceProfilerPipeline) - assert isinstance(workflow.metadata_config, OpenMetadataConnection) + assert isinstance(workflow.workflow_config, WorkflowConfig) profiler_processor_step = workflow.steps[0] assert isinstance(profiler_processor_step.profiler_config, ProfilerProcessorConfig) diff --git a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md index 09a6f8e63ed0..24bc4ebc30df 100644 --- a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md +++ b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md @@ -113,75 +113,14 @@ We believe this update will bring greater consistency and clarity to our version # Backward Incompatible Changes -## 1.5.0 +## 1.6.0 -### Multi Owners -OpenMetadata allows a single user or a team to be tagged as owners for any data assets. In Release 1.5.0, we allow users to tag multiple individual owners or a single team. This will allow organizations to add ownership to multiple individuals without necessarily needing to create a team around them like previously. +### Ingestion Workflow Status -This is a backward incompatible change, if you are using APIs, please make sure the owner field is now changed to “owners” +We are updating how we compute the success percentage. Previously, we took into account for partial success the results +of the Source (e.g., the tables we were able to properly retrieve from Snowflake, Redshift, etc.). This means that we had +an error threshold in there were if up to 90% of the tables were successfully ingested, we would still consider the +workflow as successful. However, any errors when sending the information to OpenMetadata would be considered as a failure. -### Import/Export Format -To support the multi-owner format, we have now changed how we export and import the CSV file in glossary, services, database, schema, table, etc. The new format will be -user:userName;team:TeamName - -If you are importing an older file, please make sure to make this change. - -### Pydantic V2 -The core of OpenMetadata are the JSON Schemas that define the metadata standard. These schemas are automatically translated into Java, Typescript, and Python code with Pydantic classes. - -In this release, we have [migrated](https://docs.pydantic.dev/latest/migration/) the codebase from Pydantic V1 to Pydantic V2. - -### Deployment Related Changes (OSS only) - -`./bootstrap/bootstrap_storage.sh` **removed** - -OpenMetadata community has built rolling upgrades to database schema and the data to make upgrades easier. This tool is now called as ./bootstrap/openmetadata-ops.sh and has been part of our releases since 1.3. The `bootstrap_storage.sh` doesn’t support new native schemas in OpenMetadata. Hence, we have deleted this tool from this release. - -While upgrading, please refer to our Upgrade Notes in the documentation. Always follow the best practices provided there. - -### Database Connection Pooling - -OpenMetadata uses Jdbi to handle database-related operations such as read/write/delete. In this release, we introduced additional configs to help with connection pooling, allowing the efficient use of a database with low resources. - -Please update the defaults if your cluster is running at a large scale to scale up the connections efficiently. - -For the new configuration, please refer to the [doc](https://docs.open-metadata.org/latest/deployment/database-connection-pooling) here - -### Data Insights - -The Data Insights application is meant to give you a quick glance at your data's state and allow you to take action based on the information you receive. To continue pursuing this objective, the application was completely refactored to allow customizability. - -Part of this refactor was making Data Insights an internal application, no longer relying on an external pipeline. This means triggering Data Insights from the Python SDK will no longer be possible. - -With this change you will need to run a backfill on the Data Insights for the last couple of days since the Data Assets data changed. - -### UI Changes - -#### New Explore Page - -Explore page displays hierarchically organized data assets by grouping them into `services > database > schema > tables/stored procedures`. This helps users organically find the data asset they are looking for based on a known database or schema they were using. This is a new feature and changes the way the Explore page was built in previous releases. - -#### Connector Schema Changes - -In the latest release, several updates and enhancements have been made to the JSON schema across various connectors. These changes aim to improve security, configurability, and expand integration capabilities. Here's a detailed breakdown of the updates: - -- **KafkaConnect**: Added `schemaRegistryTopicSuffixName` to enhance topic configuration flexibility for schema registries. -- **GCS Datalake**: Introduced `bucketNames` field, allowing users to specify targeted storage buckets within the Google Cloud Storage environment. -- **OpenLineage**: Added `saslConfig` to enhance security by enabling SASL (Simple Authentication and Security Layer) configuration. -- **Salesforce**: Added sslConfig to strengthen the security layer for Salesforce connections by supporting SSL. -- **DeltaLake**: Updated schema by moving metastoreConnection to a newly created `metastoreConfig.json` file. Additionally, introduced `configSource` to better define source configurations, with new support for `metastoreConfig.json` and `storageConfig.json`. -- **Iceberg RestCatalog**: Removed clientId and `clientSecret` as mandatory fields, making the schema more flexible for different authentication methods. -- **DBT Cloud Pipelines**: Added as a new connector to support cloud-native data transformation workflows using DBT. -- **Looker**: Expanded support to include connections using GitLab integration, offering more flexible and secure version control. -- **Tableau**: Enhanced support by adding capabilities for connecting with `TableauPublishedDatasource` and `TableauEmbeddedDatasource`, providing more granular control over data visualization and reporting. - -### Include DDL -During the Database Metadata ingestion, we can optionally pick up the DDL for both tables and views. During the metadata ingestion, we use the view DDLs to generate the View Lineage. - -To reduce the processing time for out-of-the-box workflows, we are disabling the include DDL by default, whereas before, it was enabled, which potentially led to long-running workflows. - -### Secrets Manager -Starting with the release 1.5.0, the JWT Token for the bots will be sent to the Secrets Manager if you configured one. It won't appear anymore in your dag_generated_configs in Airflow. - -### Python SDK -The `metadata insight` command has been removed. Since Data Insights application was moved to be an internal system application instead of relying on external pipelines the SDK command to run the pipeline was removed. +Now, we're changing this behavior to consider the success rate of all the steps involved in the workflow. The UI will +then show more `Partial Success` statuses rather than `Failed`, properly reflecting the real state of the workflow. diff --git a/openmetadata-docs/content/v1.5.x/connectors/ingestion/lineage/spark-lineage.md b/openmetadata-docs/content/v1.5.x/connectors/ingestion/lineage/spark-lineage.md index 3e48cfc5eaa0..43a7ebf8cc44 100644 --- a/openmetadata-docs/content/v1.5.x/connectors/ingestion/lineage/spark-lineage.md +++ b/openmetadata-docs/content/v1.5.x/connectors/ingestion/lineage/spark-lineage.md @@ -343,3 +343,40 @@ spark.openmetadata.transport.timeout 30 ``` After all these steps are completed you can start/restart your compute instance and you are ready to extract the lineage from spark to OpenMetadata. + + +## Using Spark Agent with Glue + +Follow the below steps in order to use OpenMetadata Spark Agent with glue. + +### 1. Specify the OpenMetadata Spark Agent JAR URL + +1. Upload the OpenMetadata Spark Agent Jar to S3 +2. Navigate to the glue job,In the Job details tab, navigate to Advanced properties → Libraries → Dependent Jars path +3. Add the S3 url of OpenMetadata Spark Agent Jar in the Dependent Jars path. + +{% image + src="/images/v1.5/connectors/spark/glue-job-jar.png" + alt="Glue Job Configure Jar" + caption="Glue Job Configure Jar" + /%} + + +### 2. Add Spark configuration in Job Parameters + +In the same Job details tab, add a new property under Job parameters: + +1. Add the `--conf` property with following value, make sure to customize this configuration as described in the above documentation. + +``` +spark.extraListeners=org.openmetadata.spark.agent.OpenMetadataSparkListener --conf spark.openmetadata.transport.hostPort=https://your-org.host:port --conf spark.openmetadata.transport.type=openmetadata --conf spark.openmetadata.transport.jwtToken= --conf spark.openmetadata.transport.pipelineServiceName=glue_spark_pipeline_service --conf spark.openmetadata.transport.pipelineName=glue_pipeline_name --conf spark.openmetadata.transport.timeout=30 +``` + +2. Add the `--user-jars-first` parameter and set its value to `true` + +{% image + src="/images/v1.5/connectors/spark/glue-job-params.png" + alt="Glue Job Configure Params" + caption="Glue Job Configure Params" + /%} + diff --git a/openmetadata-docs/content/v1.5.x/connectors/metadata/alation/index.md b/openmetadata-docs/content/v1.5.x/connectors/metadata/alation/index.md index e732fe1d71ab..a7ff96c95ded 100644 --- a/openmetadata-docs/content/v1.5.x/connectors/metadata/alation/index.md +++ b/openmetadata-docs/content/v1.5.x/connectors/metadata/alation/index.md @@ -122,9 +122,9 @@ Choose either postgres or mysql connection depending on the db: **ingestDatasources**: Specifies if databases, schemas and tables should be included while ingesting. By default is set to `true`. -**ingestDomains**: Specifies if hidden domains and subdomains should be included while ingesting. By default is set to `true`. +**ingestDomains**: Specifies if domains and subdomains should be included while ingesting. By default is set to `true`. -**ingestDashboards**: Specifies if hidden BI sources and dashboards should be included while ingesting. By default is set to `true`. +**ingestDashboards**: Specifies if BI sources and dashboards should be included while ingesting. By default is set to `true`. **alationTagClassificationName**: Specify the classification name under which the tags from alation will be created in OpenMetadata. By default it is set to `alationTags`. diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/lineage/spark-lineage.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/lineage/spark-lineage.md index 3e48cfc5eaa0..3bc39c4340e9 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/lineage/spark-lineage.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/lineage/spark-lineage.md @@ -343,3 +343,39 @@ spark.openmetadata.transport.timeout 30 ``` After all these steps are completed you can start/restart your compute instance and you are ready to extract the lineage from spark to OpenMetadata. + + +## Using Spark Agent with Glue + +Follow the below steps in order to use OpenMetadata Spark Agent with glue. + +### 1. Specify the OpenMetadata Spark Agent JAR URL + +1. Upload the OpenMetadata Spark Agent Jar to S3 +2. Navigate to the glue job,In the Job details tab, navigate to Advanced properties → Libraries → Dependent Jars path +3. Add the S3 url of OpenMetadata Spark Agent Jar in the Dependent Jars path. + +{% image + src="/images/v1.6/connectors/spark/glue-job-jar.png" + alt="Glue Job Configure Jar" + caption="Glue Job Configure Jar" + /%} + + +### 2. Add Spark configuration in Job Parameters + +In the same Job details tab, add a new property under Job parameters: + +1. Add the `--conf` property with following value, make sure to customize this configuration as described in the above documentation. + +``` +spark.extraListeners=org.openmetadata.spark.agent.OpenMetadataSparkListener --conf spark.openmetadata.transport.hostPort=https://your-org.host:port --conf spark.openmetadata.transport.type=openmetadata --conf spark.openmetadata.transport.jwtToken= --conf spark.openmetadata.transport.pipelineServiceName=glue_spark_pipeline_service --conf spark.openmetadata.transport.pipelineName=glue_pipeline_name --conf spark.openmetadata.transport.timeout=30 +``` + +2. Add the `--user-jars-first` parameter and set its value to `true` + +{% image + src="/images/v1.6/connectors/spark/glue-job-params.png" + alt="Glue Job Configure Params" + caption="Glue Job Configure Params" + /%} diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/metadata/alation/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/metadata/alation/index.md index e732fe1d71ab..a7ff96c95ded 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/metadata/alation/index.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/metadata/alation/index.md @@ -122,9 +122,9 @@ Choose either postgres or mysql connection depending on the db: **ingestDatasources**: Specifies if databases, schemas and tables should be included while ingesting. By default is set to `true`. -**ingestDomains**: Specifies if hidden domains and subdomains should be included while ingesting. By default is set to `true`. +**ingestDomains**: Specifies if domains and subdomains should be included while ingesting. By default is set to `true`. -**ingestDashboards**: Specifies if hidden BI sources and dashboards should be included while ingesting. By default is set to `true`. +**ingestDashboards**: Specifies if BI sources and dashboards should be included while ingesting. By default is set to `true`. **alationTagClassificationName**: Specify the classification name under which the tags from alation will be created in OpenMetadata. By default it is set to `alationTags`. diff --git a/openmetadata-docs/images/v1.5/connectors/spark/glue-job-jar.png b/openmetadata-docs/images/v1.5/connectors/spark/glue-job-jar.png new file mode 100644 index 000000000000..5ce7b558770c Binary files /dev/null and b/openmetadata-docs/images/v1.5/connectors/spark/glue-job-jar.png differ diff --git a/openmetadata-docs/images/v1.5/connectors/spark/glue-job-params.png b/openmetadata-docs/images/v1.5/connectors/spark/glue-job-params.png new file mode 100644 index 000000000000..3cef8e39272a Binary files /dev/null and b/openmetadata-docs/images/v1.5/connectors/spark/glue-job-params.png differ diff --git a/openmetadata-docs/images/v1.6/connectors/spark/glue-job-jar.png b/openmetadata-docs/images/v1.6/connectors/spark/glue-job-jar.png new file mode 100644 index 000000000000..5ce7b558770c Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/spark/glue-job-jar.png differ diff --git a/openmetadata-docs/images/v1.6/connectors/spark/glue-job-params.png b/openmetadata-docs/images/v1.6/connectors/spark/glue-job-params.png new file mode 100644 index 000000000000..3cef8e39272a Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/spark/glue-job-params.png differ diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 34d540749a1d..0b2e0ccb5e93 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -69,7 +69,7 @@ import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; -import org.openmetadata.schema.type.customproperties.TableConfig; +import org.openmetadata.schema.type.customProperties.TableConfig; import org.openmetadata.service.Entity; import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.jdbi3.EntityRepository; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java index 998b1fae172a..748413f379e5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java @@ -218,7 +218,7 @@ private DataInsightTotalAssetTemplate createTotalAssetTemplate( dateWithCount.forEach((key, value) -> dateMap.put(key, value.intValue())); processDateMapToNormalize(dateMap); - int changeInTotalAssets = (int) (currentCount - previousCount); + int changeInTotalAssets = (int) Math.abs(currentCount - previousCount); if (previousCount == 0D) { // it should be undefined @@ -276,7 +276,7 @@ private DataInsightDescriptionAndOwnerTemplate createDescriptionTemplate( currentPercentCompleted = (currentCompletedDescription / currentTotalAssetCount) * 100; } - int changeCount = (int) (currentCompletedDescription - previousCompletedDescription); + int changeCount = (int) Math.abs(currentCompletedDescription - previousCompletedDescription); return getTemplate( DataInsightDescriptionAndOwnerTemplate.MetricType.DESCRIPTION, @@ -327,7 +327,7 @@ private DataInsightDescriptionAndOwnerTemplate createOwnershipTemplate( currentPercentCompleted = (currentHasOwner / currentTotalAssetCount) * 100; } - int changeCount = (int) (currentHasOwner - previousHasOwner); + int changeCount = (int) Math.abs(currentHasOwner - previousHasOwner); return getTemplate( DataInsightDescriptionAndOwnerTemplate.MetricType.OWNER, @@ -376,7 +376,7 @@ private DataInsightDescriptionAndOwnerTemplate createTierTemplate( currentPercentCompleted = (currentHasTier / currentTotalAssetCount) * 100; } - int changeCount = (int) (currentHasTier - previousHasTier); + int changeCount = (int) Math.abs(currentHasTier - previousHasTier); // TODO: Understand if we actually use this tierData for anything. Map tierData = new HashMap<>(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index adb45d380433..244ea2b33ec5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -1,6 +1,8 @@ package org.openmetadata.service.apps.bundles.searchIndex; import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER; +import static org.openmetadata.service.Entity.TEST_CASE_RESOLUTION_STATUS; +import static org.openmetadata.service.Entity.TEST_CASE_RESULT; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS; import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY; @@ -90,6 +92,7 @@ public class SearchIndexApp extends AbstractNativeApplication { "storedProcedure", "storageService", "testCaseResolutionStatus", + "testCaseResult", "apiService", "apiEndpoint", "apiCollection", @@ -101,7 +104,8 @@ public class SearchIndexApp extends AbstractNativeApplication { ReportData.ReportDataType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA.value(), ReportData.ReportDataType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA.value(), ReportData.ReportDataType.AGGREGATED_COST_ANALYSIS_REPORT_DATA.value(), - "testCaseResolutionStatus"); + TEST_CASE_RESOLUTION_STATUS, + TEST_CASE_RESULT); private final List paginatedSources = new ArrayList<>(); private Processor entityProcessor; private Processor entityTimeSeriesProcessor; @@ -174,6 +178,8 @@ private void cleanUpStaleJobsFromRuns() { } private void initializeJob() { + List paginatedEntityTimeSeriesSources = new ArrayList<>(); + // Remove any Stale Jobs cleanUpStaleJobsFromRuns(); @@ -205,9 +211,11 @@ private void initializeJob() { if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) { source.setCursor(jobData.getAfterCursor()); } - paginatedSources.add(source); + paginatedEntityTimeSeriesSources.add(source); } }); + // Add Time Series Sources at the End of the List to Process them last + paginatedSources.addAll(paginatedEntityTimeSeriesSources); if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) { this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords); this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index bd18e717eb70..a5a471df3667 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -4582,6 +4582,30 @@ void insert( @Bind("json") String json, @Bind("incidentStateId") String incidentStateId); + @SqlQuery( + """ + SELECT dqdts1.json FROM + data_quality_data_time_series dqdts1 + INNER JOIN ( + SELECT tc.fqnHash + FROM entity_relationship er + INNER JOIN test_case tc ON er.toId = tc.id + where fromEntity = 'testSuite' AND toEntity = 'testCase' and fromId = :testSuiteId + ) ts ON dqdts1.entityFQNHash = ts.fqnHash + LEFT JOIN data_quality_data_time_series dqdts2 ON + (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp) + WHERE dqdts2.entityFQNHash IS NULL""") + List listLastTestCaseResultsForTestSuite(@BindMap Map params); + + @SqlQuery( + """ + SELECT dqdts1.json FROM + data_quality_data_time_series dqdts1 + LEFT JOIN data_quality_data_time_series dqdts2 ON + (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp) + WHERE dqdts2.entityFQNHash IS NULL AND dqdts1.entityFQNHash = :testCaseFQN""") + String listLastTestCaseResult(@BindFQN("testCaseFQN") String testCaseFQN); + default void insert( String testCaseFQN, String extension, @@ -4597,6 +4621,10 @@ default void insert( json, incidentStateId != null ? incidentStateId.toString() : null); } + + default List listLastTestCaseResultsForTestSuite(UUID testSuiteId) { + return listLastTestCaseResultsForTestSuite(Map.of("testSuiteId", testSuiteId.toString())); + } } class EntitiesCountRowMapper implements RowMapper { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index b0e60c337799..b6783c922b06 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -144,7 +144,7 @@ import org.openmetadata.schema.type.api.BulkOperationResult; import org.openmetadata.schema.type.api.BulkResponse; import org.openmetadata.schema.type.csv.CsvImportResult; -import org.openmetadata.schema.type.customproperties.TableConfig; +import org.openmetadata.schema.type.customProperties.TableConfig; import org.openmetadata.schema.utils.EntityInterfaceUtil; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; @@ -1495,15 +1495,15 @@ private String getFormattedDateTimeField( private void validateTableType(JsonNode fieldValue, String propertyConfig, String fieldName) { TableConfig tableConfig = JsonUtils.convertValue(JsonUtils.readTree(propertyConfig), TableConfig.class); - org.openmetadata.schema.type.customproperties.Table tableValue = + org.openmetadata.schema.type.customProperties.Table tableValue = JsonUtils.convertValue( JsonUtils.readTree(String.valueOf(fieldValue)), - org.openmetadata.schema.type.customproperties.Table.class); + org.openmetadata.schema.type.customProperties.Table.class); Set configColumns = tableConfig.getColumns(); try { JsonUtils.validateJsonSchema( - tableValue, org.openmetadata.schema.type.customproperties.Table.class); + tableValue, org.openmetadata.schema.type.customProperties.Table.class); Set fieldColumns = new HashSet<>(); fieldValue.get("columns").forEach(column -> fieldColumns.add(column.asText())); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index 64e0228cdf13..40d15e09a1a5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -12,6 +12,7 @@ import static org.openmetadata.service.Entity.TEST_DEFINITION; import static org.openmetadata.service.Entity.TEST_SUITE; import static org.openmetadata.service.Entity.getEntityByName; +import static org.openmetadata.service.Entity.getEntityTimeSeriesRepository; import static org.openmetadata.service.Entity.populateEntityFieldTags; import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; import static org.openmetadata.service.security.mask.PIIMasker.maskSampleData; @@ -379,6 +380,7 @@ private ResultSummary getResultSummary( @SneakyThrows private TestCaseResult getTestCaseResult(TestCase testCase) { + TestCaseResult testCaseResult; if (testCase.getTestCaseResult() != null) { // we'll return the saved state if it exists otherwise we'll fetch it from the database // Should be the case if listing from the search repo. as the test case result @@ -387,10 +389,21 @@ private TestCaseResult getTestCaseResult(TestCase testCase) { } SearchListFilter searchListFilter = new SearchListFilter(); searchListFilter.addQueryParam("testCaseFQN", testCase.getFullyQualifiedName()); - EntityTimeSeriesRepository timeSeriesRepository = - Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT); - return (TestCaseResult) - timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null); + TestCaseResultRepository timeSeriesRepository = + (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT); + try { + testCaseResult = + timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null); + } catch (Exception e) { + // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back + // to database + LOG.debug( + "Error fetching test case result from search. Fetching from test case results from database", + e); + testCaseResult = + timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName()); + } + return testCaseResult; } public ResultList getTestCaseResults(String fqn, Long startTs, Long endTs) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java index d9c937fd8073..7af2339fb0f7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java @@ -96,6 +96,21 @@ public Response addTestCaseResult( return Response.created(uriInfo.getRequestUri()).entity(testCaseResult).build(); } + public ResultList listLastTestCaseResultsForTestSuite(UUID testSuiteId) { + List json = + ((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao) + .listLastTestCaseResultsForTestSuite(testSuiteId); + List testCaseResults = JsonUtils.readObjects(json, TestCaseResult.class); + return new ResultList<>(testCaseResults, null, null, testCaseResults.size()); + } + + public TestCaseResult listLastTestCaseResult(String testCaseFQN) { + String json = + ((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao) + .listLastTestCaseResult(testCaseFQN); + return JsonUtils.readValue(json, TestCaseResult.class); + } + @Override protected void postCreate(TestCaseResult entity) { super.postCreate(entity); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index 32656809143e..61654fb453fa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -51,38 +51,6 @@ public class TestSuiteRepository extends EntityRepository { private static final String UPDATE_FIELDS = "tests"; private static final String PATCH_FIELDS = "tests"; - private static final String EXECUTION_SUMMARY_AGGS = - """ - { - "aggregations": { - "status_counts": { - "terms": { - "field": "testCaseResult.testCaseStatus" - } - } - } - } - """; - - private static final String ENTITY_EXECUTION_SUMMARY_AGGS = - """ - { - "aggregations": { - "entityLinks": { - "terms": { - "field": "entityLink.nonNormalized" - }, - "aggs": { - "status_counts": { - "terms": { - "field": "testCaseResult.testCaseStatus" - } - } - } - } - } - }"""; - private static final String ENTITY_EXECUTION_SUMMARY_FILTER = """ { @@ -275,14 +243,25 @@ public TestSummary getTestSummary(UUID testSuiteId) { @SneakyThrows private List getResultSummary(UUID testSuiteId) { List resultSummaries = new ArrayList<>(); + ResultList latestTestCaseResultResults; String groupBy = "testCaseFQN.keyword"; SearchListFilter searchListFilter = new SearchListFilter(); searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString()); - EntityTimeSeriesRepository entityTimeSeriesRepository = + TestCaseResultRepository entityTimeSeriesRepository = (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT); - ResultList latestTestCaseResultResults = - entityTimeSeriesRepository.listLatestFromSearch( - EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null); + try { + latestTestCaseResultResults = + entityTimeSeriesRepository.listLatestFromSearch( + EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null); + } catch (Exception e) { + // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back + // to database + LOG.debug( + "Error fetching test case result from search. Fetching from test case results from database", + e); + latestTestCaseResultResults = + entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId); + } latestTestCaseResultResults .getData() diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java index f82772dc36f6..c835589fad57 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java @@ -42,8 +42,8 @@ import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.Relationship; -import org.openmetadata.schema.type.customproperties.EnumConfig; -import org.openmetadata.schema.type.customproperties.TableConfig; +import org.openmetadata.schema.type.customProperties.EnumConfig; +import org.openmetadata.schema.type.customProperties.TableConfig; import org.openmetadata.service.Entity; import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.exception.CatalogExceptionMessage; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcess.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcess.java index 2400d6799766..1eb4ee0b5adc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcess.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcess.java @@ -3,6 +3,7 @@ import java.util.List; import java.util.Map; import org.jdbi.v3.core.Handle; +import org.jdbi.v3.core.Jdbi; import org.openmetadata.service.migration.QueryStatus; import org.openmetadata.service.migration.context.MigrationOps; @@ -15,7 +16,7 @@ interface MigrationProcessCallback { // Ex: if the server is 1.0.0 the migration version for that server is 1.0.0 // This version is used to sort all the upgrade migrations and apply them in order - void initialize(Handle handle); + void initialize(Handle handle, Jdbi jdbi); List getMigrationOps(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcessImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcessImpl.java index 75897e005539..d3317e477005 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcessImpl.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcessImpl.java @@ -9,6 +9,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.core.Handle; +import org.jdbi.v3.core.Jdbi; import org.openmetadata.schema.api.security.AuthenticationConfiguration; import org.openmetadata.sdk.PipelineServiceClientInterface; import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory; @@ -23,6 +24,7 @@ public class MigrationProcessImpl implements MigrationProcess { protected MigrationDAO migrationDAO; protected CollectionDAO collectionDAO; + protected Jdbi jdbi; protected Handle handle; protected PipelineServiceClientInterface pipelineServiceClient; protected AuthenticationConfiguration authenticationConfiguration; @@ -35,8 +37,9 @@ public MigrationProcessImpl(MigrationFile migrationFile) { } @Override - public void initialize(Handle handle) { + public void initialize(Handle handle, Jdbi jdbi) { this.handle = handle; + this.jdbi = jdbi; this.collectionDAO = handle.attach(CollectionDAO.class); this.migrationDAO = handle.attach(MigrationDAO.class); this.pipelineServiceClient = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java index d40083b32da4..7f6cb2be0d88 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java @@ -159,11 +159,22 @@ private List filterAndGetMigrationsToRun( try { for (MigrationFile file : applyMigrations) { file.parseSQLFiles(); - String clazzName = file.getMigrationProcessClassName(); - MigrationProcess process = - (MigrationProcess) - Class.forName(clazzName).getConstructor(MigrationFile.class).newInstance(file); - processes.add(process); + String extClazzName = null; + if (file.version.contains("collate")) { + extClazzName = file.getMigrationProcessExtClassName(); + } + if (extClazzName != null) { + MigrationProcess collateProcess = + (MigrationProcess) + Class.forName(extClazzName).getConstructor(MigrationFile.class).newInstance(file); + processes.add(collateProcess); + } else { + String clazzName = file.getMigrationProcessClassName(); + MigrationProcess openMetadataProcess = + (MigrationProcess) + Class.forName(clazzName).getConstructor(MigrationFile.class).newInstance(file); + processes.add(openMetadataProcess); + } } } catch (Exception e) { LOG.error("Failed to list and add migrations to run due to ", e); @@ -254,7 +265,7 @@ public void runMigrationWorkflows() { row.add(process.getVersion()); try { // Initialize - runStepAndAddStatus(row, () -> process.initialize(transactionHandler)); + runStepAndAddStatus(row, () -> process.initialize(transactionHandler, jdbi)); // Schema Changes runSchemaChanges(row, process); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/MigrationFile.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/MigrationFile.java index 66d834e94e61..b93fbdb2d4eb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/MigrationFile.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/MigrationFile.java @@ -113,6 +113,18 @@ public String getMigrationProcessClassName() { return clazzName; } + public String getMigrationProcessExtClassName() { + String clazzName = + String.format( + "io.collate.service.migration.%s.%s.Migration", dbPackageName, getVersionPackageName()); + try { + Class.forName(clazzName); + } catch (ClassNotFoundException e) { + return null; + } + return clazzName; + } + public String getMigrationsFilePath() { if (connectionType == ConnectionType.MYSQL) { return Paths.get(dir.getAbsolutePath(), "mysql").toString(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index 60b0e55aa3b2..6941d0e86dc0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -122,6 +122,7 @@ protected List getEntitySpecificOperations() { MetadataOperation.VIEW_DATA_PROFILE, MetadataOperation.VIEW_SAMPLE_DATA, MetadataOperation.VIEW_USAGE, + MetadataOperation.VIEW_PROFILER_GLOBAL_CONFIGURATION, MetadataOperation.EDIT_TESTS, MetadataOperation.EDIT_QUERIES, MetadataOperation.EDIT_DATA_PROFILE, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java index 743632c80256..c463fac3fb6e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java @@ -34,6 +34,7 @@ import org.openmetadata.schema.settings.SettingsType; import org.openmetadata.schema.system.ValidationResponse; import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.util.EntitiesCount; import org.openmetadata.schema.util.ServicesCount; import org.openmetadata.sdk.PipelineServiceClientInterface; @@ -46,6 +47,8 @@ import org.openmetadata.service.resources.Collection; import org.openmetadata.service.security.Authorizer; import org.openmetadata.service.security.JwtFilter; +import org.openmetadata.service.security.policyevaluator.OperationContext; +import org.openmetadata.service.security.policyevaluator.ResourceContext; import org.openmetadata.service.util.ResultList; import org.openmetadata.service.util.email.EmailUtil; @@ -145,8 +148,18 @@ public Settings getSettingByName( schema = @Schema(implementation = Settings.class))) }) public Settings getProfilerConfigurationSetting( - @Context UriInfo uriInfo, @Context SecurityContext securityContext) { - authorizer.authorizeAdminOrBot(securityContext); + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Entity type for which to get the global profiler configuration", + schema = @Schema(type = "string")) + @QueryParam("entityType") + @DefaultValue("table") + String entityType) { + ResourceContext resourceContext = new ResourceContext(entityType); + OperationContext operationContext = + new OperationContext(entityType, MetadataOperation.VIEW_PROFILER_GLOBAL_CONFIGURATION); + authorizer.authorize(securityContext, operationContext, resourceContext); return systemRepository.getConfigWithKey(SettingsType.PROFILER_CONFIGURATION.value()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java index b9bbdc7d10c9..5815f984d7cf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java @@ -26,7 +26,7 @@ public class EntityBuilderConstant { public static final String FIELD_DISPLAY_NAME_NGRAM = "displayName.ngram"; public static final String PRE_TAG = ""; public static final String POST_TAG = ""; - public static final Integer MAX_AGGREGATE_SIZE = 50; + public static final Integer MAX_AGGREGATE_SIZE = 10000; public static final Integer MAX_RESULT_HITS = 10000; public static final String QUERY = "query"; public static final String QUERY_NGRAM = "query.ngram"; diff --git a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json index cf47f916fd05..e35afbf1dcda 100644 --- a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json +++ b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json @@ -101,7 +101,7 @@ "indexName": "test_case_result_search_index", "indexMappingFile": "/elasticsearch/%s/test_case_result_index_mapping.json", "alias": "testCaseResult", - "parentAliases": ["testSuite", "testCase", "all"], + "parentAliases": ["testSuite", "testCase"], "childAliases": [] }, "testSuite": { diff --git a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json index 3169c3634a42..d853ea037dc9 100644 --- a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json @@ -39,6 +39,7 @@ "storedProcedure", "dataProduct", "testCaseResolutionStatus", + "testCaseResult", "apiService", "apiEndpoint", "apiCollection" diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index 383aec3d954a..248bc9b51fbc 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -89,7 +89,7 @@ import org.openmetadata.schema.type.TagLabel.TagSource; import org.openmetadata.schema.type.TaskStatus; import org.openmetadata.schema.type.csv.CsvImportResult; -import org.openmetadata.schema.type.customproperties.TableConfig; +import org.openmetadata.schema.type.customProperties.TableConfig; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.jdbi3.EntityRepository.EntityUpdater; diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java index 591e3ee5e757..1cd3e7d8264a 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java @@ -48,8 +48,8 @@ import org.openmetadata.schema.type.ChangeDescription; import org.openmetadata.schema.type.CustomPropertyConfig; import org.openmetadata.schema.type.EntityReference; -import org.openmetadata.schema.type.customproperties.EnumConfig; -import org.openmetadata.schema.type.customproperties.TableConfig; +import org.openmetadata.schema.type.customProperties.EnumConfig; +import org.openmetadata.schema.type.customProperties.TableConfig; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.EntityResourceTest; import org.openmetadata.service.resources.types.TypeResource; diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json b/openmetadata-spec/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json index 76c7e415c94f..315348886eef 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json @@ -21,6 +21,7 @@ "ViewTests", "ViewQueries", "ViewDataProfile", + "ViewProfilerGlobalConfiguration", "ViewSampleData", "ViewTestCaseFailedRowsSample", "EditAll", diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json index 1a1b17787e6e..1703f4b5b805 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json @@ -163,6 +163,14 @@ "$ref": "#/definitions/logLevels", "default": "INFO" }, + "successThreshold": { + "title": "Success Threshold", + "description": "The percentage of successfully processed records that must be achieved for the pipeline to be considered successful. Otherwise, the pipeline will be marked as failed.", + "type": "integer", + "default": 90, + "minimum": 0, + "maximum": 100 + }, "openMetadataServerConfig": { "$ref": "../entity/services/connections/metadata/openMetadataConnection.json" }, diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json index a96aa49df4d5..4c2528391daf 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json @@ -105,22 +105,6 @@ "type": "string" } }, - "enumWithDescriptions": { - "$comment": "@om-field-type", - "description": "List of values in Enum with description for each key.", - "type": "array", - "items": { - "type": "object", - "properties": { - "key": { - "type": "string" - }, - "description": { - "type": "string" - } - } - } - }, "timezone": { "description": "Timezone of the user in the format `America/Los_Angeles`, `Brazil/East`, etc.", "type": "string", diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/complexTypes.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/complexTypes.json index ff9034ac95ce..6bb1023e16d4 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/complexTypes.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/complexTypes.json @@ -104,7 +104,7 @@ "title": "Table", "description": "A table-type custom property having rows and columns where all column data types are strings.", "type": "object", - "javaType": "org.openmetadata.schema.type.customproperties.Table", + "javaType": "org.openmetadata.schema.type.customProperties.Table", "properties": { "columns": { "type": "array", diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json index 459dbac8aab6..e9ffe0df6b9a 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json @@ -3,7 +3,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "EnumConfig", "type": "object", - "javaType": "org.openmetadata.schema.type.customproperties.EnumConfig", + "javaType": "org.openmetadata.schema.type.customProperties.EnumConfig", "description": "Applies to Enum type, this config is used to define list of enum values", "properties": { "multiSelect": { diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/tableConfig.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/tableConfig.json index d45cb45a9f9c..55a1bdaf78a7 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/tableConfig.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/tableConfig.json @@ -4,7 +4,7 @@ "title": "TableConfig", "description": "Custom property configuration for table-type property where all column data types are strings.", "type": "object", - "javaType": "org.openmetadata.schema.type.customproperties.TableConfig", + "javaType": "org.openmetadata.schema.type.customProperties.TableConfig", "properties": { "columns": { "type": "array", diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/Alation.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/Alation.md new file mode 100644 index 000000000000..584dc3089780 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/Alation.md @@ -0,0 +1,374 @@ +# Alation + +In this section, we provide guides and references to use the Alation connector. + +## Requirements + +Follow the official documentation to generate a API Access Token from [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis#create-an-api-access-token) + +## Connection Details + +$$section +### Host Port $(id="hostPort") + +Host and port of the Alation service. +$$ + +$$section +### Authentication Type $(id="authType") + +Following authentication types are supported: + +1. Basic Authentication: We'll use the user credentials to generate the access token required to authenticate Alation APIs. +- username: Username of the user. +- password: Password of the user. + +2. Access Token Authentication: The access token created using the steps mentioned [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis#create-via-ui) can directly be entered. We'll use that directly to authenticate the Alation APIs +- accessToken: Generated access token +$$ + +## Alation backend database Connection details +Alation APIs do not provide us with some of the metadata. This metadata we extract directly from the alation's backend database by query the tables directly. +Note that this is a optional config and if it is not provided primary metadata will still be ingested. +Below is the metadata fetched from alation database: +`1. User and Group Relationships` + +Choose either postgres or mysql connection depending on the db: +## Postgres Connection + +You can use Postgres Connection when you have SSO enabled and your Superset is backed by Postgres database. + +### Connection Scheme + +SQLAlchemy driver scheme options. + + +### Username + +Username to connect to Postgres. + + +$$note + +Make sure the user has select privileges on `dashboards`, `tables` & `slices` tables of superset schema. + +$$ + +$$section +### Auth Config $(id="authType") +There are 2 types of auth configs: +- Basic Auth. +- IAM based Auth. + +User can authenticate the Postgres Instance with auth type as `Basic Authentication` i.e. Password **or** by using `IAM based Authentication` to connect to AWS related services. +$$ + +## Basic Auth + +### Password + +Password to connect to Postgres. + + +## IAM Auth Config + +$$section +### AWS Access Key ID $(id="awsAccessKeyId") + +When you interact with AWS, you specify your AWS security credentials to verify who you are and whether you have permission to access the resources that you are requesting. AWS uses the security credentials to authenticate and authorize your requests ([docs](https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html)). + +Access keys consist of two parts: +1. An access key ID (for example, `AKIAIOSFODNN7EXAMPLE`), +2. And a secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`). + +You must use both the access key ID and secret access key together to authenticate your requests. + +You can find further information on how to manage your access keys [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html) +$$ + +$$section +### AWS Secret Access Key $(id="awsSecretAccessKey") + +Secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`). +$$ + +$$section +### AWS Region $(id="awsRegion") + +Each AWS Region is a separate geographic area in which AWS clusters data centers ([docs](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html)). + +As AWS can have instances in multiple regions, we need to know the region the service you want reach belongs to. + +Note that the AWS Region is the only required parameter when configuring a connection. When connecting to the services programmatically, there are different ways in which we can extract and use the rest of AWS configurations. You can find further information about configuring your credentials [here](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials). +$$ + +$$section +### AWS Session Token $(id="awsSessionToken") + +If you are using temporary credentials to access your services, you will need to inform the AWS Access Key ID and AWS Secrets Access Key. Also, these will include an AWS Session Token. + +You can find more information on [Using temporary credentials with AWS resources](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). +$$ + +$$section +### Endpoint URL $(id="endPointURL") + +To connect programmatically to an AWS service, you use an endpoint. An *endpoint* is the URL of the entry point for an AWS web service. The AWS SDKs and the AWS Command Line Interface (AWS CLI) automatically use the default endpoint for each service in an AWS Region. But you can specify an alternate endpoint for your API requests. + +Find more information on [AWS service endpoints](https://docs.aws.amazon.com/general/latest/gr/rande.html). +$$ + +$$section +### Profile Name $(id="profileName") + +A named profile is a collection of settings and credentials that you can apply to an AWS CLI command. When you specify a profile to run a command, the settings and credentials are used to run that command. Multiple named profiles can be stored in the config and credentials files. + +You can inform this field if you'd like to use a profile other than `default`. + +Find here more information about [Named profiles for the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html). +$$ + +$$section +### Assume Role ARN $(id="assumeRoleArn") + +Typically, you use `AssumeRole` within your account or for cross-account access. In this field you'll set the `ARN` (Amazon Resource Name) of the policy of the other account. + +A user who wants to access a role in a different account must also have permissions that are delegated from the account administrator. The administrator must attach a policy that allows the user to call `AssumeRole` for the `ARN` of the role in the other account. + +This is a required field if you'd like to `AssumeRole`. + +Find more information on [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html). +$$ + +$$section +### Assume Role Session Name $(id="assumeRoleSessionName") + +An identifier for the assumed role session. Use the role session name to uniquely identify a session when the same role is assumed by different principals or for different reasons. + +By default, we'll use the name `OpenMetadataSession`. + +Find more information about the [Role Session Name](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=An%20identifier%20for%20the%20assumed%20role%20session.). +$$ + +$$section +### Assume Role Source Identity $(id="assumeRoleSourceIdentity") + +The source identity specified by the principal that is calling the `AssumeRole` operation. You can use source identity information in AWS CloudTrail logs to determine who took actions with a role. + +Find more information about [Source Identity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=Required%3A%20No-,SourceIdentity,-The%20source%20identity). +$$ + +### Host and Port + +Host and port of the Postgres service. + +Example: `localhost:5432` + +$$section +### Database $(id="database") + +Initial Postgres database to connect to. If you want to ingest all databases, set `ingestAllDatabases` to true. +$$ + +### SSL Mode + +SSL Mode to connect to postgres database. + +$$section +### Classification Name $(id="classificationName") + +By default, the Postgres policy tags in OpenMetadata are classified under the name `PostgresPolicyTags`. However, you can create a custom classification name of your choice for these tags. Once you have ingested Postgres data, the custom classification name will be visible in the Classifications list on the Tags page. +$$ + +$$section +### Ingest All Databases $(id="ingestAllDatabases") + +If ticked, the workflow will be able to ingest all database in the cluster. If not ticked, the workflow will only ingest tables from the database set above. +$$ + +$$section +### SSL Mode $(id="sslMode") + +SSL Mode to connect to postgres database. E.g, `prefer`, `verify-ca`, `allow` etc. +$$ +$$note +if you are using `IAM auth`, select either `allow` (recommended) or other option based on your use case. +$$ + + +### SSL CA +The CA certificate used for SSL validation (`sslrootcert`). + +$$note +Postgres only needs CA Certificate +$$ + +$$section +### Connection Arguments $(id="connectionArguments") + +Additional connection arguments such as security or protocol configs that can be sent to service during connection. +$$ + +$$section +### Connection Options $(id="connectionOptions") + +Additional connection options to build the URL that can be sent to service during the connection. +$$ +-------- + +## Mysql Connection + +You can use Mysql Connection when you have SSO enabled and your Superset is backed by Mysql database. + +### Scheme +SQLAlchemy driver scheme options. + +### Username +Username to connect to Mysql. + +$$note + +Make sure the user has select privileges on `dashboards`, `tables` & `slices` tables of superset schema. + +$$ + +### Password +Password to connect to Mysql. + +### Host Port +Host and port of the Mysql service. This should be specified as a string in the format `hostname:port`. + +**Example**: `localhost:3306`, `host.docker.internal:3306` + +$$section +### Database Name $(id="databaseName") +In OpenMetadata, the Database Service hierarchy works as follows: +``` +Database Service > Database > Schema > Table +``` +In the case of MySQL, we won't have a Database as such. If you'd like to see your data in a database named something other than `default`, you can specify the name in this field. +$$ + +$$section +### Database Schema $(id="databaseSchema") +This is an optional parameter. When set, the value will be used to restrict the metadata reading to a single database (corresponding to the value passed in this field). When left blank, OpenMetadata will scan all the databases. +$$ + +$$section +### SSL CA $(id="caCertificate") +The CA certificate used for SSL validation (`ssl_ca`) +$$ + +$$section +### SSL Certificate $(id="sslCertificate") +The SSL certificate used for client authentication (`ssl_cert`) +$$ + +$$section +### SSL Key $(id="sslKey") +The private key associated with the SSL certificate (`ssl_key`) +$$ + +### Connection Options +Additional connection options to build the URL that can be sent to the service during the connection. + +### Connection Arguments +Additional connection arguments such as security or protocol configs that can be sent to the service during connection. + + +$$section +### Project Name $(id="projectName") + +Project name to create the refreshToken. Can be anything. +$$ + + +$$section +### Pagination Limit $(id="paginationLimit") + +Pagination limit used for Alation APIs pagination +$$ + +$$section +### Include Undeployed Datasources $(id="includeUndeployedDatasources") + +Specifies if undeployed datasources should be included while ingesting. By default is set to `false`. +$$ + +$$section +### Include Hidden Datasources $(id="includeHiddenDatasources") + +Specifies if hidden datasources should be included while ingesting. By default is set to `false`. +$$ + +$$section +### Ingest Datasources $(id="ingestDatasources") + +Specifies if databases, schemas and tables should be included while ingesting. By default is set to `true`. +$$ + +$$section +### Ingest Users And Groups $(id="ingestUsersAndGroups") + +Specifies if users and groups should be included while ingesting. By default is set to `true`. +$$ + +$$section +### Ingest Domains $(id="ingestDomains") + +Specifies if domains and subdomains should be included while ingesting. By default is set to `true`. +$$ + +$$section +### Ingest KnowledgeArticles $(id="ingestKnowledgeArticles") + +Specifies if knowledge articles should be included while ingesting. By default is set to `true`. +$$ + +$$section +### Ingest Dashboards $(id="ingestDashboards") + +Specifies if BI sources and dashboards should be included while ingesting. By default is set to `true`. +$$ + +$$section +### Alation Tag Classification Name $(id="alationTagClassificationName") + +Specifies if hidden datasources should be included while ingesting. By default is set to `false`. +$$ + +$$section +### Ingest Datasources $(id="ingestDatasources") + +Specifies if hidden datasources should be included while ingesting. By default is set to `false`. +$$ + +$$section +### Ingest Datasources $(id="ingestDatasources") + +Specifies if hidden datasources should be included while ingesting. By default is set to `false`. +$$ + + +$$section +### Connection Options $(id="connectionOptions") + +Additional connection options to build the URL that can be sent to service during the connection. +$$ + +$$section +### Connection Arguments $(id="connectionArguments") + +Additional connection arguments such as security or protocol configs that can be sent to service during connection. + +The following arguments are intended to be used in conjunction and are specifically for Alation DataSource APIs: +- skip: This parameter determines the count of records to bypass at the start of the dataset. When set to 0, as in this case, it means that no records will be bypassed. If set to 10, it will bypass the first 10 records. + +- limit: This argument specifies the maximum number of records to return. Here, it's set to 10, meaning only the first 10 records will be returned. + +To perform incremental ingestion, these arguments should be used together. For instance, if there are a total of 30 datasources in Alation, the ingestion can be configured to execute three times, with each execution ingesting 10 datasources. +- 1st execution: {"skip": 0, "limit": 10} +- 2nd execution: {"skip": 10, "limit": 10} +- 3rd execution: {"skip": 20, "limit": 10} + +$$ diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts index 41ce4bffd6c3..aeb7dfcc8195 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts @@ -67,6 +67,7 @@ export enum EntityType { WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData', WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData', TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index', + TEST_CASE_RESULT = 'test_case_result_search_index', EVENT_SUBSCRIPTION = 'eventsubscription', LINEAGE_EDGE = 'lineageEdge', API_SERVICE = 'apiService', diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts index b4cb53d84b53..4dc9971edb48 100644 --- a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts @@ -45,7 +45,7 @@ import { PipelineService } from '../generated/entity/services/pipelineService'; import { SearchService } from '../generated/entity/services/searchService'; import { Team } from '../generated/entity/teams/team'; import { User } from '../generated/entity/teams/user'; -import { TestCase } from '../generated/tests/testCase'; +import { TestCase, TestCaseResult } from '../generated/tests/testCase'; import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus'; import { TestSuite } from '../generated/tests/testSuite'; import { TagLabel } from '../generated/type/tagLabel'; @@ -145,6 +145,15 @@ export interface TestCaseResolutionStatusSearchSource serviceType: string; description: string; } +export interface TestCaseResultSearchSource + extends SearchSourceBase, + TestCaseResult { + name: string; + displayName: string; + fullyQualifiedName: string; + serviceType: string; + description: string; +} export interface IngestionPipelineSearchSource extends SearchSourceBase, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json index 69e218615f5a..928d0c1a2707 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json @@ -59,6 +59,7 @@ "storedProcedure", "dataProduct", "testCaseResolutionStatus", + "testCaseResult", "apiService", "apiEndpoint", "apiCollection", diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts index 359f7c8cf87c..ffbcefbaad73 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts @@ -193,6 +193,11 @@ export const MOCK_APPLICATION_ENTITY_STATS = { failedRecords: 0, successRecords: 4, }, + [EntityType.TEST_CASE_RESULT]: { + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, }; export const MOCK_APPLICATION_ENTITY_STATS_DATA = [ @@ -412,4 +417,10 @@ export const MOCK_APPLICATION_ENTITY_STATS_DATA = [ failedRecords: 0, successRecords: 4, }, + { + name: EntityType.TEST_CASE_RESULT, + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, ];