Add roles to tags and classification

.github/workflows/maven-sonar-build.yml

@@ -16,7 +16,7 @@ on:
   push:
     branches:
       - main
-      - '0.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+'
     paths:
       - "openmetadata-service/**"
       - "openmetadata-ui/**"
@@ -116,4 +116,4 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           fail_on_test_failures: true
-          report_paths: 'openmetadata-service/target/surefire-reports/TEST-*.xml'
+          report_paths: 'openmetadata-service/target/surefire-reports/TEST-*.xml'

.github/workflows/sync-docs-v1.yml

@@ -39,7 +39,7 @@ jobs:
         destination-repository-name: 'docs-v1'
         user-email: [email protected]
         commit-message: See ORIGIN_COMMIT from $GITHUB_REF
-        target-branch: publish
+        target-branch: main
 
     - name: Push images
       id: push_images
@@ -54,7 +54,7 @@ jobs:
         destination-repository-name: 'docs-v1'
         user-email: [email protected]
         commit-message: See ORIGIN_COMMIT from $GITHUB_REF
-        target-branch: publish
+        target-branch: main
 
     - name: Prepare Collate content and partials
       id: prepare_collate

.run/Template JUnit.run.xml

@@ -5,7 +5,7 @@
     <option name="MAIN_CLASS_NAME" value="" />
     <option name="METHOD_NAME" value="" />
     <option name="TEST_OBJECT" value="package" />
-    <option name="VM_PARAMETERS" value="-ea -DjdbcContainerClassName=org.testcontainers.containers.MySQLContainer -DjdbcContainerImage=mysql:8 -DelasticSearchContainerClassName=docker.elastic.co/elasticsearch/elasticsearch:8.10.2 -DopenSearchContainerClassName=opensearchproject/opensearch:1.3.0 -DrunESTestCases=false"/>
+    <option name="VM_PARAMETERS" value="-ea -DjdbcContainerClassName=org.testcontainers.containers.MySQLContainer -DjdbcContainerImage=mysql:8 -DelasticSearchContainerClassName=docker.elastic.co/elasticsearch/elasticsearch:8.11.4 -DopenSearchContainerClassName=opensearchproject/opensearch:1.3.0 -DrunESTestCases=false"/>
     <method v="2">
       <option name="Make" enabled="true" />
     </method>

SECURITY.md

@@ -7,8 +7,8 @@ currently being supported with security updates.
 
 | Version | Supported          |
 | ------- | ------------------ |
+| 1.5.x   | :white_check_mark: |
 | 1.4.x   | :white_check_mark: |
-| 1.3.x   | :white_check_mark: |
 
 ## Reporting a Vulnerability
 

bootstrap/sql/migrations/native/1.5.7/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,17 @@
+UPDATE test_definition
+SET json = JSON_MERGE_PRESERVE(
+        json,
+        JSON_OBJECT(
+            'parameterDefinition',
+            JSON_ARRAY(
+                JSON_OBJECT(
+                    'name', 'caseSensitiveColumns',
+                    'dataType', 'BOOLEAN',
+                    'required', false,
+                    'description', 'Use case sensitivity when comparing the columns.',
+                    'displayName', 'Case sensitive columns'
+                )
+            )
+        )
+    )
+WHERE name = 'tableDiff';

bootstrap/sql/migrations/native/1.5.7/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,8 @@
+UPDATE test_definition
+SET json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        (json->'parameterDefinition')::jsonb || 
+        '{"name": "caseSensitiveColumns", "dataType": "BOOLEAN", "required": false, "description": "Use case sensitivity when comparing the columns.", "displayName": "Case sensitive columns"}'::jsonb
+    )
+WHERE name = 'tableDiff';

bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql

@@ -43,3 +43,8 @@ UPDATE installed_apps SET json = JSON_SET(json, '$.supportsInterrupt', true) whe
 UPDATE apps_marketplace  SET json = JSON_SET(json, '$.supportsInterrupt', true) where name = 'SearchIndexingApplication';
 
 ALTER TABLE apps_extension_time_series ADD COLUMN appName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.appName') STORED NOT NULL;
+
+-- Add supportsDataDiff for Athena, BigQuery, Mssql, Mysql, Oracle, Postgres, Redshift, SapHana, Snowflake, Trino
+UPDATE dbservice_entity
+SET json = JSON_SET(json, '$.connection.config.supportsDataDiff', 'true')
+WHERE serviceType IN ('Athena','BigQuery','Mssql','Mysql','Oracle','Postgres','Redshift','SapHana','Snowflake','Trino');

bootstrap/sql/migrations/native/1.6.0/mysql/schemaChanges.sql

@@ -5,4 +5,11 @@ ALTER TABLE apps_extension_time_series MODIFY COLUMN extension VARCHAR(255) NOT
 CREATE INDEX apps_extension_time_series_extension ON apps_extension_time_series(extension);
 
 -- Clean dangling workflows not removed after test connection
-truncate automations_workflow;
+truncate automations_workflow;
+
+-- App Data Store
+CREATE TABLE IF NOT EXISTS apps_data_store (
+    identifier VARCHAR(256) NOT NULL,      
+    type VARCHAR(256) NOT NULL,   
+    json JSON NOT NULL
+);

bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql

@@ -60,3 +60,8 @@ SET json = jsonb_set(
 where name = 'SearchIndexingApplication';
 
 ALTER TABLE apps_extension_time_series ADD COLUMN appName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'appName') STORED NOT NULL;
+
+-- Add supportsDataDiff for Athena, BigQuery, Mssql, Mysql, Oracle, Postgres, Redshift, SapHana, Snowflake, Trino
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb, '{connection,config,supportsDataDiff}', 'true'::jsonb)
+WHERE serviceType IN ('Athena','BigQuery','Mssql','Mysql','Oracle','Postgres','Redshift','SapHana','Snowflake','Trino');

bootstrap/sql/migrations/native/1.6.0/postgres/schemaChanges.sql

@@ -5,4 +5,11 @@ ALTER TABLE apps_extension_time_series ALTER COLUMN extension SET NOT NULL;
 CREATE INDEX IF NOT EXISTS apps_extension_time_series_extension ON apps_extension_time_series(extension);
 
 -- Clean dangling workflows not removed after test connection
-truncate automations_workflow;
+truncate automations_workflow;
+
+-- App Data Store
+CREATE TABLE IF NOT EXISTS apps_data_store (
+    identifier VARCHAR(256) NOT NULL,      
+    type VARCHAR(256) NOT NULL,   
+    json JSON NOT NULL
+);

docker/development/docker-compose-gcp.yml

@@ -46,7 +46,7 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node

docker/development/docker-compose.yml

@@ -42,7 +42,7 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node

docker/docker-compose-quickstart/docker-compose-postgres.yml

@@ -41,7 +41,7 @@ services:
 
   elasticsearch:
     container_name: openmetadata_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m

docker/docker-compose-quickstart/docker-compose.yml

@@ -39,7 +39,7 @@ services:
 
   elasticsearch:
     container_name: openmetadata_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m

docker/run_local_docker.sh

@@ -50,8 +50,8 @@ echo "Running local docker using mode [$mode] database [$database] and skipping
 cd ../
 
 echo "Stopping any previous Local Docker Containers"
-docker compose -f docker/development/docker-compose-postgres.yml down
-docker compose -f docker/development/docker-compose.yml down
+docker compose -f docker/development/docker-compose-postgres.yml down --remove-orphans
+docker compose -f docker/development/docker-compose.yml down --remove-orphans
 
 if [[ $skipMaven == "false" ]]; then
     if [[ $mode == "no-ui" ]]; then

ingestion/examples/sample_data/api_service/service.json

@@ -1,9 +1,9 @@
 {
-	"type": "REST",
+	"type": "rest",
 	"serviceName": "sample_api_service", 
 	"serviceConnection": {
 		"config": {
-			"type": "REST",
+			"type": "Rest",
 			"openAPISchemaURL": "https://petstore3.swagger.io/",
     		"token":"mock_token"
 		}

ingestion/examples/sample_data/ometa_api_service/service.json

@@ -1,9 +1,9 @@
 {
-	"type": "REST",
+	"type": "rest",
 	"serviceName": "ometa_api_service", 
 	"serviceConnection": {
 		"config": {
-			"type": "REST",
+			"type": "Rest",
 			"openAPISchemaURL": "https://docs.open-metadata.org/swagger.html",
     		"token":"token"
 		}

ingestion/pyproject.toml

@@ -73,6 +73,9 @@ markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')"
 ]
 
+[project.entry-points.pytest11]
+pytest_openmetadata = "_openmetadata_testutils.pytest_openmetadata.plugin"
+
 [tool.pylint.BASIC]
 # W1203: logging-fstring-interpolation - f-string brings better readability and unifies style
 # W1202: logging-format-interpolation - lazy formatting in logging functions

ingestion/setup.py

@@ -46,7 +46,7 @@
     "databricks-sdk": "databricks-sdk>=0.18.0,<0.20.0",
     "trino": "trino[sqlalchemy]",
     "spacy": "spacy<3.8",
-    "looker-sdk": "looker-sdk>=22.20.0",
+    "looker-sdk": "looker-sdk>=22.20.0,!=24.18.0",
     "lkml": "lkml~=1.3",
     "tableau": "tableau-api-lib~=0.1",
     "pyhive": "pyhive[hive_pure_sasl]~=0.7",
@@ -183,7 +183,7 @@
         VERSIONS["azure-storage-blob"],
         VERSIONS["azure-identity"],
     },
-    "db2": {"ibm-db-sa~=0.3"},
+    "db2": {"ibm-db-sa~=0.4.1", "ibm-db>=2.0.0"},
     "db2-ibmi": {"sqlalchemy-ibmi~=0.9.3"},
     "databricks": {
         VERSIONS["sqlalchemy-databricks"],
@@ -236,7 +236,7 @@
         "impyla~=0.18.0",
     },
     "iceberg": {
-        "pyiceberg>=0.5",
+        "pyiceberg==0.5.1",
         # Forcing the version of a few packages so it plays nicely with other requirements.
         VERSIONS["pydantic"],
         VERSIONS["adlfs"],

ingestion/src/_openmetadata_testutils/pytest_openmetadata/plugin.py

@@ -0,0 +1,86 @@
+from typing import Type
+
+import pytest
+
+from _openmetadata_testutils.ometa import int_admin_ometa
+from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User
+from metadata.generated.schema.metadataIngestion.workflow import LogLevels
+from metadata.ingestion.api.common import Entity
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.workflow.ingestion import IngestionWorkflow
+
+
+@pytest.fixture(scope="session")
+def metadata():
+    return int_admin_ometa()
+
+
+@pytest.fixture(scope="session")
+def sink_config(metadata):
+    return {
+        "type": "metadata-rest",
+        "config": {},
+    }
+
+
+@pytest.fixture(scope="session")
+def workflow_config(metadata):
+    return {
+        "loggerLevel": LogLevels.DEBUG.value,
+        "openMetadataServerConfig": metadata.config.model_dump(),
+    }
+
+
+@pytest.fixture(scope="session")
+def ingestion_bot_wokflow_config(metadata, workflow_config):
+    ingestion_bot: User = metadata.get_by_name(
+        entity=User, fqn="ingestion-bot", nullable=False
+    )
+    ingestion_bot_auth: AuthenticationMechanism = metadata.get_by_id(
+        entity=AuthenticationMechanism, entity_id=ingestion_bot.id, nullable=False
+    )
+    workflow_config = workflow_config.copy()
+    workflow_config["openMetadataServerConfig"]["securityConfig"][
+        "jwtToken"
+    ] = ingestion_bot_auth.config.JWTToken
+    return workflow_config
+
+
+@pytest.fixture()
+def clean_up_fqn(metadata):
+    fqns = []
+
+    def inner(entity_type: type[Entity], fqn: str):
+        fqns.append((entity_type, fqn))
+
+    yield inner
+    for entity_type, fqn in fqns:
+        entity = metadata.get_by_name(entity_type, fqn, fields=["*"])
+        if entity:
+            metadata.delete(entity_type, entity.id, recursive=True, hard_delete=True)
+
+
+@pytest.fixture(scope="session")
+def ingestion_bot_workflow_config(metadata: OpenMetadata):
+    ingestion_bot: User = metadata.get_by_name(entity=User, fqn="ingestion-bot")
+    ingestion_bot_auth: AuthenticationMechanism = metadata.get_by_id(
+        entity=AuthenticationMechanism, entity_id=ingestion_bot.id
+    )
+    config = metadata.config.model_dump()
+    config["securityConfig"]["jwtToken"] = ingestion_bot_auth.config.JWTToken
+    return {
+        "loggerLevel": LogLevels.DEBUG.value,
+        "openMetadataServerConfig": config,
+    }
+
+
+@pytest.fixture(scope="module")
+def run_workflow():
+    def _run(workflow_type: Type[IngestionWorkflow], config, raise_from_status=True):
+        workflow: IngestionWorkflow = workflow_type.create(config)
+        workflow.execute()
+        if raise_from_status:
+            workflow.raise_from_status()
+        return workflow
+
+    return _run