diff --git a/.github/workflows/maven-sonar-build.yml b/.github/workflows/maven-sonar-build.yml
index e0295d0cb1c6..5e5a03e86159 100644
--- a/.github/workflows/maven-sonar-build.yml
+++ b/.github/workflows/maven-sonar-build.yml
@@ -16,7 +16,7 @@ on:
   push:
     branches:
       - main
-      - '0.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+'
     paths:
       - "openmetadata-service/**"
       - "openmetadata-ui/**"
@@ -116,4 +116,4 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           fail_on_test_failures: true
-          report_paths: 'openmetadata-service/target/surefire-reports/TEST-*.xml'
\ No newline at end of file
+          report_paths: 'openmetadata-service/target/surefire-reports/TEST-*.xml'
diff --git a/.github/workflows/sync-docs-v1.yml b/.github/workflows/sync-docs-v1.yml
index cde9663804e1..15e7d76f514e 100644
--- a/.github/workflows/sync-docs-v1.yml
+++ b/.github/workflows/sync-docs-v1.yml
@@ -39,7 +39,7 @@ jobs:
         destination-repository-name: 'docs-v1'
         user-email: openmetadata@getcollate.io
         commit-message: See ORIGIN_COMMIT from $GITHUB_REF
-        target-branch: publish
+        target-branch: main
 
     - name: Push images
       id: push_images
@@ -54,7 +54,7 @@ jobs:
         destination-repository-name: 'docs-v1'
         user-email: openmetadata@getcollate.io
         commit-message: See ORIGIN_COMMIT from $GITHUB_REF
-        target-branch: publish
+        target-branch: main
 
     - name: Prepare Collate content and partials
       id: prepare_collate
diff --git a/.run/Template JUnit.run.xml b/.run/Template JUnit.run.xml
index e9df4c41e253..a76cd50c6bd1 100644
--- a/.run/Template JUnit.run.xml	
+++ b/.run/Template JUnit.run.xml	
@@ -5,7 +5,7 @@
     <option name="MAIN_CLASS_NAME" value="" />
     <option name="METHOD_NAME" value="" />
     <option name="TEST_OBJECT" value="package" />
-    <option name="VM_PARAMETERS" value="-ea -DjdbcContainerClassName=org.testcontainers.containers.MySQLContainer -DjdbcContainerImage=mysql:8 -DelasticSearchContainerClassName=docker.elastic.co/elasticsearch/elasticsearch:8.10.2 -DopenSearchContainerClassName=opensearchproject/opensearch:1.3.0 -DrunESTestCases=false"/>
+    <option name="VM_PARAMETERS" value="-ea -DjdbcContainerClassName=org.testcontainers.containers.MySQLContainer -DjdbcContainerImage=mysql:8 -DelasticSearchContainerClassName=docker.elastic.co/elasticsearch/elasticsearch:8.11.4 -DopenSearchContainerClassName=opensearchproject/opensearch:1.3.0 -DrunESTestCases=false"/>
     <method v="2">
       <option name="Make" enabled="true" />
     </method>
diff --git a/SECURITY.md b/SECURITY.md
index 943cb78397d0..d2ecf1e9121b 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -7,8 +7,8 @@ currently being supported with security updates.
 
 | Version | Supported          |
 | ------- | ------------------ |
+| 1.5.x   | :white_check_mark: |
 | 1.4.x   | :white_check_mark: |
-| 1.3.x   | :white_check_mark: |
 
 ## Reporting a Vulnerability
 
diff --git a/bootstrap/sql/migrations/native/1.5.7/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.7/mysql/postDataMigrationSQLScript.sql
new file mode 100644
index 000000000000..0aaa75e38767
--- /dev/null
+++ b/bootstrap/sql/migrations/native/1.5.7/mysql/postDataMigrationSQLScript.sql
@@ -0,0 +1,17 @@
+UPDATE test_definition
+SET json = JSON_MERGE_PRESERVE(
+        json,
+        JSON_OBJECT(
+            'parameterDefinition',
+            JSON_ARRAY(
+                JSON_OBJECT(
+                    'name', 'caseSensitiveColumns',
+                    'dataType', 'BOOLEAN',
+                    'required', false,
+                    'description', 'Use case sensitivity when comparing the columns.',
+                    'displayName', 'Case sensitive columns'
+                )
+            )
+        )
+    )
+WHERE name = 'tableDiff';
diff --git a/bootstrap/sql/migrations/native/1.5.7/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.7/postgres/postDataMigrationSQLScript.sql
new file mode 100644
index 000000000000..32afd14731f4
--- /dev/null
+++ b/bootstrap/sql/migrations/native/1.5.7/postgres/postDataMigrationSQLScript.sql
@@ -0,0 +1,8 @@
+UPDATE test_definition
+SET json = jsonb_set(
+        json,
+        '{parameterDefinition}',
+        (json->'parameterDefinition')::jsonb || 
+        '{"name": "caseSensitiveColumns", "dataType": "BOOLEAN", "required": false, "description": "Use case sensitivity when comparing the columns.", "displayName": "Case sensitive columns"}'::jsonb
+    )
+WHERE name = 'tableDiff';
diff --git a/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql
index 04bf3b4dd173..5445e9b31c96 100644
--- a/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql
+++ b/bootstrap/sql/migrations/native/1.6.0/mysql/postDataMigrationSQLScript.sql
@@ -43,3 +43,8 @@ UPDATE installed_apps SET json = JSON_SET(json, '$.supportsInterrupt', true) whe
 UPDATE apps_marketplace  SET json = JSON_SET(json, '$.supportsInterrupt', true) where name = 'SearchIndexingApplication';
 
 ALTER TABLE apps_extension_time_series ADD COLUMN appName VARCHAR(256) GENERATED ALWAYS AS (json ->> '$.appName') STORED NOT NULL;
+
+-- Add supportsDataDiff for Athena, BigQuery, Mssql, Mysql, Oracle, Postgres, Redshift, SapHana, Snowflake, Trino
+UPDATE dbservice_entity
+SET json = JSON_SET(json, '$.connection.config.supportsDataDiff', 'true')
+WHERE serviceType IN ('Athena','BigQuery','Mssql','Mysql','Oracle','Postgres','Redshift','SapHana','Snowflake','Trino');
diff --git a/bootstrap/sql/migrations/native/1.6.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.0/mysql/schemaChanges.sql
index 6ddfd9369954..2f737915fad0 100644
--- a/bootstrap/sql/migrations/native/1.6.0/mysql/schemaChanges.sql
+++ b/bootstrap/sql/migrations/native/1.6.0/mysql/schemaChanges.sql
@@ -5,4 +5,11 @@ ALTER TABLE apps_extension_time_series MODIFY COLUMN extension VARCHAR(255) NOT
 CREATE INDEX apps_extension_time_series_extension ON apps_extension_time_series(extension);
 
 -- Clean dangling workflows not removed after test connection
-truncate automations_workflow;
\ No newline at end of file
+truncate automations_workflow;
+
+-- App Data Store
+CREATE TABLE IF NOT EXISTS apps_data_store (
+    identifier VARCHAR(256) NOT NULL,      
+    type VARCHAR(256) NOT NULL,   
+    json JSON NOT NULL
+);
\ No newline at end of file
diff --git a/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql
index e7de73cc09b6..799659bd1403 100644
--- a/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql
+++ b/bootstrap/sql/migrations/native/1.6.0/postgres/postDataMigrationSQLScript.sql
@@ -60,3 +60,8 @@ SET json = jsonb_set(
 where name = 'SearchIndexingApplication';
 
 ALTER TABLE apps_extension_time_series ADD COLUMN appName VARCHAR(256) GENERATED ALWAYS AS (json ->> 'appName') STORED NOT NULL;
+
+-- Add supportsDataDiff for Athena, BigQuery, Mssql, Mysql, Oracle, Postgres, Redshift, SapHana, Snowflake, Trino
+UPDATE dbservice_entity
+SET json = jsonb_set(json::jsonb, '{connection,config,supportsDataDiff}', 'true'::jsonb)
+WHERE serviceType IN ('Athena','BigQuery','Mssql','Mysql','Oracle','Postgres','Redshift','SapHana','Snowflake','Trino');
diff --git a/bootstrap/sql/migrations/native/1.6.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.0/postgres/schemaChanges.sql
index 6cd2d24b6dd6..6987056dcc48 100644
--- a/bootstrap/sql/migrations/native/1.6.0/postgres/schemaChanges.sql
+++ b/bootstrap/sql/migrations/native/1.6.0/postgres/schemaChanges.sql
@@ -5,4 +5,11 @@ ALTER TABLE apps_extension_time_series ALTER COLUMN extension SET NOT NULL;
 CREATE INDEX IF NOT EXISTS apps_extension_time_series_extension ON apps_extension_time_series(extension);
 
 -- Clean dangling workflows not removed after test connection
-truncate automations_workflow;
\ No newline at end of file
+truncate automations_workflow;
+
+-- App Data Store
+CREATE TABLE IF NOT EXISTS apps_data_store (
+    identifier VARCHAR(256) NOT NULL,      
+    type VARCHAR(256) NOT NULL,   
+    json JSON NOT NULL
+);
\ No newline at end of file
diff --git a/docker/development/docker-compose-gcp.yml b/docker/development/docker-compose-gcp.yml
index 169cee1a5914..2e8c31421e21 100644
--- a/docker/development/docker-compose-gcp.yml
+++ b/docker/development/docker-compose-gcp.yml
@@ -46,7 +46,7 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node
diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml
index ba05ead6d895..b5509137a521 100644
--- a/docker/development/docker-compose.yml
+++ b/docker/development/docker-compose.yml
@@ -42,7 +42,7 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node
diff --git a/docker/docker-compose-quickstart/docker-compose-postgres.yml b/docker/docker-compose-quickstart/docker-compose-postgres.yml
index 66ac68bb7660..4b279df6cb28 100644
--- a/docker/docker-compose-quickstart/docker-compose-postgres.yml
+++ b/docker/docker-compose-quickstart/docker-compose-postgres.yml
@@ -41,7 +41,7 @@ services:
 
   elasticsearch:
     container_name: openmetadata_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
diff --git a/docker/docker-compose-quickstart/docker-compose.yml b/docker/docker-compose-quickstart/docker-compose.yml
index 7ce4f1479b7e..a66b57ddfb63 100644
--- a/docker/docker-compose-quickstart/docker-compose.yml
+++ b/docker/docker-compose-quickstart/docker-compose.yml
@@ -39,7 +39,7 @@ services:
 
   elasticsearch:
     container_name: openmetadata_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m
diff --git a/docker/run_local_docker.sh b/docker/run_local_docker.sh
index f1f2a948e378..183b2f491e3d 100755
--- a/docker/run_local_docker.sh
+++ b/docker/run_local_docker.sh
@@ -50,8 +50,8 @@ echo "Running local docker using mode [$mode] database [$database] and skipping
 cd ../
 
 echo "Stopping any previous Local Docker Containers"
-docker compose -f docker/development/docker-compose-postgres.yml down
-docker compose -f docker/development/docker-compose.yml down
+docker compose -f docker/development/docker-compose-postgres.yml down --remove-orphans
+docker compose -f docker/development/docker-compose.yml down --remove-orphans
 
 if [[ $skipMaven == "false" ]]; then
     if [[ $mode == "no-ui" ]]; then
diff --git a/ingestion/examples/sample_data/api_service/service.json b/ingestion/examples/sample_data/api_service/service.json
index ded28abd509d..22cda4eba763 100644
--- a/ingestion/examples/sample_data/api_service/service.json
+++ b/ingestion/examples/sample_data/api_service/service.json
@@ -1,9 +1,9 @@
 {
-	"type": "REST",
+	"type": "rest",
 	"serviceName": "sample_api_service", 
 	"serviceConnection": {
 		"config": {
-			"type": "REST",
+			"type": "Rest",
 			"openAPISchemaURL": "https://petstore3.swagger.io/",
     		"token":"mock_token"
 		}
diff --git a/ingestion/examples/sample_data/ometa_api_service/service.json b/ingestion/examples/sample_data/ometa_api_service/service.json
index 6e3c672e9601..9552e9d04eae 100644
--- a/ingestion/examples/sample_data/ometa_api_service/service.json
+++ b/ingestion/examples/sample_data/ometa_api_service/service.json
@@ -1,9 +1,9 @@
 {
-	"type": "REST",
+	"type": "rest",
 	"serviceName": "ometa_api_service", 
 	"serviceConnection": {
 		"config": {
-			"type": "REST",
+			"type": "Rest",
 			"openAPISchemaURL": "https://docs.open-metadata.org/swagger.html",
     		"token":"token"
 		}
diff --git a/ingestion/pyproject.toml b/ingestion/pyproject.toml
index 65555975686a..3df3755aff9a 100644
--- a/ingestion/pyproject.toml
+++ b/ingestion/pyproject.toml
@@ -73,6 +73,9 @@ markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')"
 ]
 
+[project.entry-points.pytest11]
+pytest_openmetadata = "_openmetadata_testutils.pytest_openmetadata.plugin"
+
 [tool.pylint.BASIC]
 # W1203: logging-fstring-interpolation - f-string brings better readability and unifies style
 # W1202: logging-format-interpolation - lazy formatting in logging functions
diff --git a/ingestion/setup.py b/ingestion/setup.py
index 560c62625419..65b9fd521cc2 100644
--- a/ingestion/setup.py
+++ b/ingestion/setup.py
@@ -46,7 +46,7 @@
     "databricks-sdk": "databricks-sdk>=0.18.0,<0.20.0",
     "trino": "trino[sqlalchemy]",
     "spacy": "spacy<3.8",
-    "looker-sdk": "looker-sdk>=22.20.0",
+    "looker-sdk": "looker-sdk>=22.20.0,!=24.18.0",
     "lkml": "lkml~=1.3",
     "tableau": "tableau-api-lib~=0.1",
     "pyhive": "pyhive[hive_pure_sasl]~=0.7",
@@ -183,7 +183,7 @@
         VERSIONS["azure-storage-blob"],
         VERSIONS["azure-identity"],
     },
-    "db2": {"ibm-db-sa~=0.3"},
+    "db2": {"ibm-db-sa~=0.4.1", "ibm-db>=2.0.0"},
     "db2-ibmi": {"sqlalchemy-ibmi~=0.9.3"},
     "databricks": {
         VERSIONS["sqlalchemy-databricks"],
@@ -236,7 +236,7 @@
         "impyla~=0.18.0",
     },
     "iceberg": {
-        "pyiceberg>=0.5",
+        "pyiceberg==0.5.1",
         # Forcing the version of a few packages so it plays nicely with other requirements.
         VERSIONS["pydantic"],
         VERSIONS["adlfs"],
diff --git a/ingestion/src/_openmetadata_testutils/pytest_openmetadata/__init__.py b/ingestion/src/_openmetadata_testutils/pytest_openmetadata/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/ingestion/src/_openmetadata_testutils/pytest_openmetadata/plugin.py b/ingestion/src/_openmetadata_testutils/pytest_openmetadata/plugin.py
new file mode 100644
index 000000000000..eb021de0ce55
--- /dev/null
+++ b/ingestion/src/_openmetadata_testutils/pytest_openmetadata/plugin.py
@@ -0,0 +1,86 @@
+from typing import Type
+
+import pytest
+
+from _openmetadata_testutils.ometa import int_admin_ometa
+from metadata.generated.schema.entity.teams.user import AuthenticationMechanism, User
+from metadata.generated.schema.metadataIngestion.workflow import LogLevels
+from metadata.ingestion.api.common import Entity
+from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.workflow.ingestion import IngestionWorkflow
+
+
+@pytest.fixture(scope="session")
+def metadata():
+    return int_admin_ometa()
+
+
+@pytest.fixture(scope="session")
+def sink_config(metadata):
+    return {
+        "type": "metadata-rest",
+        "config": {},
+    }
+
+
+@pytest.fixture(scope="session")
+def workflow_config(metadata):
+    return {
+        "loggerLevel": LogLevels.DEBUG.value,
+        "openMetadataServerConfig": metadata.config.model_dump(),
+    }
+
+
+@pytest.fixture(scope="session")
+def ingestion_bot_wokflow_config(metadata, workflow_config):
+    ingestion_bot: User = metadata.get_by_name(
+        entity=User, fqn="ingestion-bot", nullable=False
+    )
+    ingestion_bot_auth: AuthenticationMechanism = metadata.get_by_id(
+        entity=AuthenticationMechanism, entity_id=ingestion_bot.id, nullable=False
+    )
+    workflow_config = workflow_config.copy()
+    workflow_config["openMetadataServerConfig"]["securityConfig"][
+        "jwtToken"
+    ] = ingestion_bot_auth.config.JWTToken
+    return workflow_config
+
+
+@pytest.fixture()
+def clean_up_fqn(metadata):
+    fqns = []
+
+    def inner(entity_type: type[Entity], fqn: str):
+        fqns.append((entity_type, fqn))
+
+    yield inner
+    for entity_type, fqn in fqns:
+        entity = metadata.get_by_name(entity_type, fqn, fields=["*"])
+        if entity:
+            metadata.delete(entity_type, entity.id, recursive=True, hard_delete=True)
+
+
+@pytest.fixture(scope="session")
+def ingestion_bot_workflow_config(metadata: OpenMetadata):
+    ingestion_bot: User = metadata.get_by_name(entity=User, fqn="ingestion-bot")
+    ingestion_bot_auth: AuthenticationMechanism = metadata.get_by_id(
+        entity=AuthenticationMechanism, entity_id=ingestion_bot.id
+    )
+    config = metadata.config.model_dump()
+    config["securityConfig"]["jwtToken"] = ingestion_bot_auth.config.JWTToken
+    return {
+        "loggerLevel": LogLevels.DEBUG.value,
+        "openMetadataServerConfig": config,
+    }
+
+
+@pytest.fixture(scope="module")
+def run_workflow():
+    def _run(workflow_type: Type[IngestionWorkflow], config, raise_from_status=True):
+        workflow: IngestionWorkflow = workflow_type.create(config)
+        workflow.execute()
+        if raise_from_status:
+            workflow.raise_from_status()
+        return workflow
+
+    return _run
diff --git a/ingestion/src/_openmetadata_testutils/pytest_openmetadata/test_utils.py b/ingestion/src/_openmetadata_testutils/pytest_openmetadata/test_utils.py
new file mode 100644
index 000000000000..386488955af9
--- /dev/null
+++ b/ingestion/src/_openmetadata_testutils/pytest_openmetadata/test_utils.py
@@ -0,0 +1,99 @@
+from collections import deque
+from typing import List, Union
+
+from pydantic import BaseModel
+
+
+def assert_equal_pydantic_objects(
+    expected: Union[BaseModel, List[BaseModel]],
+    actual: Union[BaseModel, List[BaseModel]],
+    ignore_none=True,
+):
+    """Compare 2 pydantic objects recursively and raise an AssertionError if they are not equal along with all
+    the differences by field. If `ignore_none` is set to True, expected None values will be ignored. This can be
+    useful when comparing objects that are partially filled.
+
+    Example:
+        >>> from pydantic import BaseModel
+        >>> class A(BaseModel):
+        ...     a: int
+        >>> class B(BaseModel):
+        ...     b: A
+        >>> a1 = A(a=1)
+        >>> a2 = A(a=2)
+        >>> b1 = B(b=a1)
+        >>> b2 = B(b=a2)
+        >>> assert_equal_pydantic_objects(a1, b1)
+        Traceback (most recent call last):
+        ```
+        AssertionError: objects mismatched on type at : expected: [A], actual: [B]
+        >>> assert_equal_pydantic_objects(a1, a2)
+        Traceback (most recent call last):
+        ```
+        AssertionError: objects mismatched on field: [a], expected: [1], actual: [2]
+        >>> assert_equal_pydantic_objects(b1, b2)
+        Traceback (most recent call last):
+        ```
+        AssertionError: objects mismatched on field: [b.a], expected: [1], actual: [2]
+        >>> assert_equal_pydantic_objects([a1, a2], [a2, a1])
+        Traceback (most recent call last):
+        ```
+        AssertionError: objects mismatched on field: [0].a, expected: [1], actual: [2]
+
+    Args:
+        expected (BaseModel): The expected pydantic object.
+        actual (BaseModel): The actual pydantic object.
+        ignore_none (bool, optional): Whether to ignore None values. Defaults to True.
+
+    Raises:
+        AssertionError: If the objects are not equal. The error message will contain all the differences.
+    """
+    errors = []
+    queue = deque([(expected, actual, "")])
+    while queue:
+        expected, actual, current_key_prefix = queue.popleft()
+        if not isinstance(expected, actual.__class__):
+            errors.append(
+                f"objects mismatched on type at {current_key_prefix}: "
+                f"expected: [{type(expected).__name__}], actual: [{type(actual).__name__}]"
+            )
+            continue
+        if issubclass(expected.__class__, BaseModel) and isinstance(
+            expected.model_dump(), dict
+        ):
+            for key, expected_value in expected.model_dump().items():
+                if expected_value is None and ignore_none:
+                    continue
+                actual_value = actual.model_dump().get(key)
+                new_key_prefix = (
+                    f"{current_key_prefix}.{key}" if current_key_prefix else key
+                )
+                if issubclass(getattr(expected, key).__class__, BaseModel):
+                    queue.append(
+                        (getattr(expected, key), getattr(actual, key), new_key_prefix)
+                    )
+                elif expected_value != actual_value:
+                    errors.append(
+                        f"objects mismatched on field: [{new_key_prefix}], expected: [{expected_value}], actual: [{actual_value}]"
+                    )
+        elif isinstance(expected, list):
+            if not isinstance(actual, list):
+                errors.append(
+                    f"validation error on field: [{current_key_prefix}], expected: [list], actual: [{type(actual).__name__}]"
+                )
+            elif len(expected) != len(actual):
+                errors.append(
+                    f"mismatch length at {current_key_prefix}: expected: [{len(expected)}], actual: [{len(actual)}]"
+                )
+            else:
+                for i, (expected_item, actual_item) in enumerate(zip(expected, actual)):
+                    queue.append(
+                        (expected_item, actual_item, f"{current_key_prefix}[{i}]")
+                    )
+        else:
+            if expected != actual:
+                errors.append(
+                    f"mismatch at {current_key_prefix}: expected: [{expected}], actual: [{actual}]"
+                )
+    if errors:
+        raise AssertionError("\n".join(errors))
diff --git a/ingestion/src/metadata/data_quality/builders/i_validator_builder.py b/ingestion/src/metadata/data_quality/builders/i_validator_builder.py
index 22466873ce03..8cfabb96d5d4 100644
--- a/ingestion/src/metadata/data_quality/builders/i_validator_builder.py
+++ b/ingestion/src/metadata/data_quality/builders/i_validator_builder.py
@@ -15,7 +15,7 @@
 """
 
 from abc import ABC, abstractmethod
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Optional, Type, Union
 
 from metadata.data_quality.validations.base_test_handler import BaseTestValidator
@@ -23,6 +23,7 @@
     RuntimeParameterSetter,
 )
 from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
+from metadata.generated.schema.type.basic import Timestamp
 from metadata.profiler.processor.runner import QueryRunner
 from metadata.utils.importer import import_test_case_class
 
@@ -91,7 +92,9 @@ def reset(self):
         self._validator = self.validator_cls(
             self.runner,
             test_case=self.test_case,
-            execution_date=int(datetime.now().timestamp() * 1000),
+            execution_date=Timestamp(
+                int(datetime.now(tz=timezone.utc).timestamp() * 1000)
+            ),
         )
 
     @abstractmethod
diff --git a/ingestion/src/metadata/data_quality/validations/base_test_handler.py b/ingestion/src/metadata/data_quality/validations/base_test_handler.py
index be93fdc3832b..3e1363c93864 100644
--- a/ingestion/src/metadata/data_quality/validations/base_test_handler.py
+++ b/ingestion/src/metadata/data_quality/validations/base_test_handler.py
@@ -17,9 +17,9 @@
 
 import reprlib
 from abc import ABC, abstractmethod
-from datetime import datetime
 from typing import TYPE_CHECKING, Callable, List, Optional, Type, TypeVar, Union
 
+from metadata.data_quality.validations import utils
 from metadata.data_quality.validations.runtime_param_setter.param_setter import (
     RuntimeParameterSetter,
 )
@@ -29,6 +29,7 @@
     TestResultValue,
 )
 from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue
+from metadata.generated.schema.type.basic import Timestamp
 from metadata.profiler.processor.runner import QueryRunner
 
 if TYPE_CHECKING:
@@ -50,7 +51,7 @@ def __init__(
         self,
         runner: Union[QueryRunner, List["DataFrame"]],
         test_case: TestCase,
-        execution_date: Union[datetime, float],
+        execution_date: Timestamp,
     ) -> None:
         self.runner = runner
         self.test_case = test_case
@@ -65,40 +66,21 @@ def run_validation(self) -> TestCaseResult:
         """
         raise NotImplementedError
 
+    @staticmethod
     def get_test_case_param_value(
-        self,
-        test_case_param_vals: list[TestCaseParameterValue],
+        test_case_param_vals: List[TestCaseParameterValue],
         name: str,
         type_: T,
         default: Optional[R] = None,
         pre_processor: Optional[Callable] = None,
     ) -> Optional[Union[R, T]]:
-        """Give a column and a type return the value with the appropriate type casting for the
-        test case definition.
-
-        Args:
-            test_case: the test case
-            type_ (Union[float, int, str]): type for the value
-            name (str): column name
-            default (_type_, optional): Default value to return if column is not found
-            pre_processor: pre processor function/type to use against the value before casting to type_
-        """
-        value = next(
-            (param.value for param in test_case_param_vals if param.name == name), None
+        return utils.get_test_case_param_value(
+            test_case_param_vals, name, type_, default, pre_processor
         )
 
-        if not value:
-            return default if default is not None else None
-
-        if not pre_processor:
-            return type_(value)
-
-        pre_processed_value = pre_processor(value)
-        return type_(pre_processed_value)
-
     def get_test_case_result_object(  # pylint: disable=too-many-arguments
         self,
-        execution_date: Union[datetime, float],
+        execution_date: Timestamp,
         status: TestCaseStatus,
         result: str,
         test_result_value: List[TestResultValue],
diff --git a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py
index 6b8dfa207224..a587bb986983 100644
--- a/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py
+++ b/ingestion/src/metadata/data_quality/validations/column/base/columnValuesToBeInSet.py
@@ -20,6 +20,7 @@
 
 from sqlalchemy import Column
 
+from metadata.data_quality.validations import utils
 from metadata.data_quality.validations.base_test_handler import BaseTestValidator
 from metadata.generated.schema.tests.basic import (
     TestCaseResult,
@@ -50,11 +51,8 @@ def run_validation(self) -> TestCaseResult:
             literal_eval,
         )
 
-        match_enum = self.get_test_case_param_value(
-            self.test_case.parameterValues,  # type: ignore
-            "matchEnum",
-            bool,
-            default=False,
+        match_enum = utils.get_bool_test_case_param(
+            self.test_case.parameterValues, "matchEnum"
         )
 
         try:
diff --git a/ingestion/src/metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py b/ingestion/src/metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py
index 9ca775e39d74..ffadb0e0c6bd 100644
--- a/ingestion/src/metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py
+++ b/ingestion/src/metadata/data_quality/validations/runtime_param_setter/table_diff_params_setter.py
@@ -13,6 +13,7 @@
 from typing import List, Optional
 from urllib.parse import urlparse
 
+from metadata.data_quality.validations import utils
 from metadata.data_quality.validations.models import (
     Column,
     TableDiffRuntimeParameters,
@@ -27,6 +28,7 @@
 from metadata.ingestion.source.connections import get_connection
 from metadata.profiler.orm.registry import Dialects
 from metadata.utils import fqn
+from metadata.utils.collections import CaseInsensitiveList
 
 
 class TableDiffParamsSetter(RuntimeParameterSetter):
@@ -49,15 +51,26 @@ def __init__(self, *args, **kwargs):
         }
 
     def get_parameters(self, test_case) -> TableDiffRuntimeParameters:
+        service1_url = (
+            str(get_connection(self.service_connection_config).url)
+            if self.service_connection_config
+            else None
+        )
         service1: DatabaseService = self.ometa_client.get_by_id(
             DatabaseService, self.table_entity.service.id, nullable=False
         )
         table2_fqn = self.get_parameter(test_case, "table2")
+        case_sensitive_columns: bool = utils.get_bool_test_case_param(
+            test_case.parameterValues, "caseSensitiveColumns"
+        )
         if table2_fqn is None:
             raise ValueError("table2 not set")
         table2: Table = self.ometa_client.get_by_name(
             Table, fqn=table2_fqn, nullable=False
         )
+        service2_url = (
+            service1_url if table2.service == self.table_entity.service else None
+        )
         service2: DatabaseService = self.ometa_client.get_by_id(
             DatabaseService, table2.service.id, nullable=False
         )
@@ -69,10 +82,15 @@ def get_parameters(self, test_case) -> TableDiffRuntimeParameters:
                     self.table_entity.fullyQualifiedName.root
                 ),
                 serviceUrl=self.get_data_diff_url(
-                    service1, self.table_entity.fullyQualifiedName.root
+                    service1,
+                    self.table_entity.fullyQualifiedName.root,
+                    override_url=service1_url,
                 ),
                 columns=self.filter_relevant_columns(
-                    self.table_entity.columns, key_columns, extra_columns
+                    self.table_entity.columns,
+                    key_columns,
+                    extra_columns,
+                    case_sensitive=case_sensitive_columns,
                 ),
             ),
             table2=TableParameter(
@@ -80,10 +98,14 @@ def get_parameters(self, test_case) -> TableDiffRuntimeParameters:
                 serviceUrl=self.get_data_diff_url(
                     service2,
                     table2_fqn,
-                    override_url=self.get_parameter(test_case, "service2Url"),
+                    override_url=self.get_parameter(test_case, "service2Url")
+                    or service2_url,
                 ),
                 columns=self.filter_relevant_columns(
-                    table2.columns, key_columns, extra_columns
+                    table2.columns,
+                    key_columns,
+                    extra_columns,
+                    case_sensitive=case_sensitive_columns,
                 ),
             ),
             keyColumns=key_columns,
@@ -145,9 +167,17 @@ def get_key_columns(self, test_case) -> List[str]:
 
     @staticmethod
     def filter_relevant_columns(
-        columns: List[Column], key_columns: List[str], extra_columns: List[str]
+        columns: List[Column],
+        key_columns: List[str],
+        extra_columns: List[str],
+        case_sensitive: bool,
     ) -> List[Column]:
-        return [c for c in columns if c.name.root in [*key_columns, *extra_columns]]
+        validated_columns = (
+            [*key_columns, *extra_columns]
+            if case_sensitive
+            else CaseInsensitiveList([*key_columns, *extra_columns])
+        )
+        return [c for c in columns if c.name.root in validated_columns]
 
     @staticmethod
     def get_parameter(test_case: TestCase, key: str, default=None):
@@ -184,7 +214,7 @@ def get_data_diff_url(
         if hasattr(db_service.connection.config, "supportsDatabase"):
             kwargs["path"] = f"/{database}"
         if kwargs["scheme"] in {Dialects.MSSQL, Dialects.Snowflake}:
-            kwargs["path"] += f"/{schema}"
+            kwargs["path"] = f"/{database}/{schema}"
         return url._replace(**kwargs).geturl()
 
     @staticmethod
diff --git a/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py
index 3b59c59da7a2..39a04474a40d 100644
--- a/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py
+++ b/ingestion/src/metadata/data_quality/validations/table/sqlalchemy/tableDiff.py
@@ -11,17 +11,19 @@
 # pylint: disable=missing-module-docstring
 import logging
 import traceback
+from decimal import Decimal
 from itertools import islice
-from typing import Dict, Iterable, List, Optional, Tuple
+from typing import Dict, Iterable, List, Optional, Tuple, cast
 from urllib.parse import urlparse
 
 import data_diff
 import sqlalchemy.types
 from data_diff.diff_tables import DiffResultWrapper
 from data_diff.errors import DataDiffMismatchingKeyTypesError
-from data_diff.utils import ArithAlphanumeric
+from data_diff.utils import ArithAlphanumeric, CaseInsensitiveDict
 from sqlalchemy import Column as SAColumn
 
+from metadata.data_quality.validations import utils
 from metadata.data_quality.validations.base_test_handler import BaseTestValidator
 from metadata.data_quality.validations.mixins.sqa_validator_mixin import (
     SQAValidatorMixin,
@@ -75,6 +77,18 @@ def masked(s: str, mask: bool = True) -> str:
     return "***" if mask else s
 
 
+def is_numeric(t: type) -> bool:
+    """Check if a type is numeric.
+
+    Args:
+        t: type to check
+
+    Returns:
+        True if the type is numeric otherwise False
+    """
+    return t in [int, float, Decimal]
+
+
 class TableDiffValidator(BaseTestValidator, SQAValidatorMixin):
     """
     Compare two tables and fail if the number of differences exceeds a threshold
@@ -167,12 +181,14 @@ def get_incomparable_columns(self) -> List[str]:
             self.runtime_params.table1.path,
             self.runtime_params.keyColumns,
             extra_columns=self.runtime_params.extraColumns,
+            case_sensitive=self.get_case_sensitive(),
         ).with_schema()
         table2 = data_diff.connect_to_table(
             self.runtime_params.table2.serviceUrl,
             self.runtime_params.table2.path,
             self.runtime_params.keyColumns,
             extra_columns=self.runtime_params.extraColumns,
+            case_sensitive=self.get_case_sensitive(),
         ).with_schema()
         result = []
         for column in table1.key_columns + table1.extra_columns:
@@ -185,7 +201,8 @@ def get_incomparable_columns(self) -> List[str]:
             col2_type = self._get_column_python_type(
                 table2._schema[column]  # pylint: disable=protected-access
             )
-
+            if is_numeric(col1_type) and is_numeric(col2_type):
+                continue
             if col1_type != col2_type:
                 result.append(column)
         return result
@@ -228,11 +245,13 @@ def get_table_diff(self) -> DiffResultWrapper:
             self.runtime_params.table1.serviceUrl,
             self.runtime_params.table1.path,
             self.runtime_params.keyColumns,  # type: ignore
+            case_sensitive=self.get_case_sensitive(),
         )
         table2 = data_diff.connect_to_table(
             self.runtime_params.table2.serviceUrl,
             self.runtime_params.table2.path,
             self.runtime_params.keyColumns,  # type: ignore
+            case_sensitive=self.get_case_sensitive(),
         )
         data_diff_kwargs = {
             "key_columns": self.runtime_params.keyColumns,
@@ -308,7 +327,9 @@ def _validate_dialects(self):
     def get_column_diff(self) -> Optional[TestCaseResult]:
         """Get the column diff between the two tables. If there are no differences, return None."""
         removed, added = self.get_changed_added_columns(
-            self.runtime_params.table1.columns, self.runtime_params.table2.columns
+            self.runtime_params.table1.columns,
+            self.runtime_params.table2.columns,
+            self.get_case_sensitive(),
         )
         changed = self.get_incomparable_columns()
         if removed or added or changed:
@@ -321,7 +342,7 @@ def get_column_diff(self) -> Optional[TestCaseResult]:
 
     @staticmethod
     def get_changed_added_columns(
-        left: List[Column], right: List[Column]
+        left: List[Column], right: List[Column], case_sensitive: bool
     ) -> Optional[Tuple[List[str], List[str]]]:
         """Given a list of columns from two tables, return the columns that are removed and added.
 
@@ -335,6 +356,10 @@ def get_changed_added_columns(
         removed: List[str] = []
         added: List[str] = []
         right_columns_dict: Dict[str, Column] = {c.name.root: c for c in right}
+        if not case_sensitive:
+            right_columns_dict = cast(
+                Dict[str, Column], CaseInsensitiveDict(right_columns_dict)
+            )
         for column in left:
             table2_column = right_columns_dict.get(column.name.root)
             if table2_column is None:
@@ -345,7 +370,10 @@ def get_changed_added_columns(
         return removed, added
 
     def column_validation_result(
-        self, removed: List[str], added: List[str], changed: List[str]
+        self,
+        removed: List[str],
+        added: List[str],
+        changed: List[str],
     ) -> TestCaseResult:
         """Build the result for a column validation result. Messages will only be added
         for non-empty categories. Values will be populated reported for all categories.
@@ -367,13 +395,18 @@ def column_validation_result(
             message += f"\n  Added columns: {','.join(added)}\n"
         if changed:
             message += "\n  Changed columns:"
+            table1_columns = {
+                c.name.root: c for c in self.runtime_params.table1.columns
+            }
+            table2_columns = {
+                c.name.root: c for c in self.runtime_params.table2.columns
+            }
+            if not self.get_case_sensitive():
+                table1_columns = CaseInsensitiveDict(table1_columns)
+                table2_columns = CaseInsensitiveDict(table2_columns)
             for col in changed:
-                col1 = next(
-                    c for c in self.runtime_params.table1.columns if c.name.root == col
-                )
-                col2 = next(
-                    c for c in self.runtime_params.table2.columns if c.name.root == col
-                )
+                col1 = table1_columns[col]
+                col2 = table2_columns[col]
                 message += (
                     f"\n    {col}: {col1.dataType.value} -> {col2.dataType.value}"
                 )
@@ -432,3 +465,8 @@ def safe_table_diff_iterator(self) -> DiffResultWrapper:
                 if str(ex) == "2":
                     # This is a known issue in data_diff where the diff object is closed
                     pass
+
+    def get_case_sensitive(self):
+        return utils.get_bool_test_case_param(
+            self.test_case.parameterValues, "caseSensitiveColumns"
+        )
diff --git a/ingestion/src/metadata/data_quality/validations/utils.py b/ingestion/src/metadata/data_quality/validations/utils.py
new file mode 100644
index 000000000000..77f2037731eb
--- /dev/null
+++ b/ingestion/src/metadata/data_quality/validations/utils.py
@@ -0,0 +1,56 @@
+"""
+Data quality validation utility functions.
+"""
+
+from typing import Callable, List, Optional, TypeVar, Union
+
+from metadata.generated.schema.tests.testCase import TestCaseParameterValue
+
+T = TypeVar("T", bound=Callable)
+R = TypeVar("R")
+
+
+def get_test_case_param_value(
+    test_case_param_vals: List[TestCaseParameterValue],
+    name: str,
+    type_: T,
+    default: Optional[R] = None,
+    pre_processor: Optional[Callable] = None,
+) -> Optional[Union[R, T]]:
+    """Return a test case parameter value with the appropriate type casting for the test case definition.
+
+    Args:
+        test_case_param_vals: list of test case parameter values
+        type_ (Union[float, int, str]): type for the value
+        name (str): column name
+        default (_type_, optional): Default value to return if column is not found
+        pre_processor: pre processor function/type to use against the value before casting to type_
+    """
+    value = next(
+        (param.value for param in test_case_param_vals if param.name == name), None
+    )
+
+    if not value:
+        return default if default is not None else None
+
+    if not pre_processor:
+        return type_(value)
+
+    pre_processed_value = pre_processor(value)
+    return type_(pre_processed_value)
+
+
+def get_bool_test_case_param(
+    test_case_param_vals: List[TestCaseParameterValue],
+    name: str,
+) -> Optional[Union[R, T]]:
+    """Return a test case parameter value as a boolean. Boolean values are always False by default.
+
+    Args:
+        test_case_param_vals: list of test case parameter values
+        name (str): column name
+    """
+    str_val: str = get_test_case_param_value(test_case_param_vals, name, str, None)
+    if str_val is None:
+        return False
+    return str_val.lower() == "true"
diff --git a/ingestion/src/metadata/examples/workflows/rest.yaml b/ingestion/src/metadata/examples/workflows/rest.yaml
index d155eaea5dbc..3469a02ba58f 100644
--- a/ingestion/src/metadata/examples/workflows/rest.yaml
+++ b/ingestion/src/metadata/examples/workflows/rest.yaml
@@ -3,9 +3,9 @@ source:
   serviceName: openapi_rest
   serviceConnection:
     config:
-      type: REST
+      type: Rest
       openAPISchemaURL: https://docs.open-metadata.org/swagger.json
-      token: <optional_jwt_token>
+      # token: <jwt_token>
   sourceConfig:
     config:
       type: ApiMetadata
diff --git a/ingestion/src/metadata/ingestion/api/parser.py b/ingestion/src/metadata/ingestion/api/parser.py
index 84aa898844cd..08db6f2bf228 100644
--- a/ingestion/src/metadata/ingestion/api/parser.py
+++ b/ingestion/src/metadata/ingestion/api/parser.py
@@ -19,8 +19,8 @@
     Workflow as AutomationWorkflow,
 )
 from metadata.generated.schema.entity.services.apiService import (
-    ApiServiceConnection,
-    APIServiceType,
+    ApiConnection,
+    ApiServiceType,
 )
 from metadata.generated.schema.entity.services.dashboardService import (
     DashboardConnection,
@@ -135,7 +135,7 @@
 # Build a service type map dynamically from JSON Schema covered types
 SERVICE_TYPE_MAP = {
     "Backend": PipelineConnection,  # For Airflow backend
-    **{service: ApiServiceConnection for service in APIServiceType.__members__},
+    **{service: ApiConnection for service in ApiServiceType.__members__},
     **{service: DatabaseConnection for service in DatabaseServiceType.__members__},
     **{service: DashboardConnection for service in DashboardServiceType.__members__},
     **{service: MessagingConnection for service in MessagingServiceType.__members__},
@@ -183,7 +183,7 @@ class InvalidWorkflowException(Exception):
 def get_service_type(
     source_type: str,
 ) -> Union[
-    Type[ApiServiceConnection],
+    Type[ApiConnection],
     Type[DashboardConnection],
     Type[DatabaseConnection],
     Type[MessagingConnection],
@@ -233,7 +233,7 @@ def get_source_config_class(
 def get_connection_class(
     source_type: str,
     service_type: Union[
-        Type[ApiServiceConnection],
+        Type[ApiConnection],
         Type[DashboardConnection],
         Type[DatabaseConnection],
         Type[MessagingConnection],
@@ -557,6 +557,7 @@ def parse_automation_workflow_gracefully(
             message="Error parsing the service connection",
         )
 
+    #
     raise ParsingConfigurationError(
         "Uncaught error when parsing the Ingestion Pipeline!"
     )
diff --git a/ingestion/src/metadata/ingestion/connections/test_connections.py b/ingestion/src/metadata/ingestion/connections/test_connections.py
index e98b67dbe32b..c71b04a89d4e 100644
--- a/ingestion/src/metadata/ingestion/connections/test_connections.py
+++ b/ingestion/src/metadata/ingestion/connections/test_connections.py
@@ -40,6 +40,7 @@
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import kill_active_connections
 from metadata.profiler.orm.functions.conn_test import ConnTestFn
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import cli_logger
 from metadata.utils.timeout import timeout
 
@@ -92,25 +93,24 @@ def _test_connection_steps(
     metadata: OpenMetadata,
     steps: List[TestConnectionStep],
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+) -> TestConnectionResult:
     """
     Run all the function steps and raise any errors
     """
 
     if automation_workflow:
-        _test_connection_steps_automation_workflow(
+        return _test_connection_steps_automation_workflow(
             metadata=metadata, steps=steps, automation_workflow=automation_workflow
         )
 
-    else:
-        _test_connection_steps_during_ingestion(steps=steps)
+    return _test_connection_steps_during_ingestion(steps=steps)
 
 
 def _test_connection_steps_automation_workflow(
     metadata: OpenMetadata,
     steps: List[TestConnectionStep],
     automation_workflow: AutomationWorkflow,
-) -> None:
+) -> TestConnectionResult:
     """
     Run the test connection as part of the automation workflow
     We need to update the automation workflow in each step
@@ -187,33 +187,40 @@ def _test_connection_steps_automation_workflow(
             )
         )
 
+    return test_connection_result
 
-def _test_connection_steps_during_ingestion(steps: List[TestConnectionStep]) -> None:
-    """
-    Run the test connection as part of the ingestion workflow
-    Raise an exception if something fails
-    """
-    test_connection_result = TestConnectionIngestionResult()
+
+def _test_connection_steps_during_ingestion(
+    steps: List[TestConnectionStep],
+) -> TestConnectionResult:
+    """Run the test connection steps during ingestion"""
+    test_connection_result = TestConnectionResult(
+        status=StatusType.Running,
+        steps=[],
+    )
     for step in steps:
         try:
+            logger.info(f"Running {step.name}...")
             step.function()
-            test_connection_result.success.append(f"'{step.name}': Pass")
-
-        except Exception as exc:
-            logger.debug(traceback.format_exc())
-            logger.warning(f"{step.name}-{exc}")
-            if step.mandatory:
-                test_connection_result.failed.append(
-                    f"'{step.name}': This is a mandatory step and we won't be able to extract"
-                    f" necessary metadata. Failed due to: {exc}"
+            test_connection_result.steps.append(
+                TestConnectionStepResult(
+                    name=step.name,
+                    mandatory=step.mandatory,
+                    passed=True,
                 )
-
-            else:
-                test_connection_result.warning.append(
-                    f"'{step.name}': This is a optional and the ingestion will continue to work as expected."
-                    f"Failed due to: {exc}"
+            )
+        except Exception as err:
+            logger.debug(traceback.format_exc())
+            logger.error(f"{step.name}-{err}")
+            test_connection_result.steps.append(
+                TestConnectionStepResult(
+                    name=step.name,
+                    mandatory=step.mandatory,
+                    passed=False,
+                    message=step.error_message,
+                    errorLog=str(err),
                 )
-
+            )
             if step.short_circuit:
                 # break the workflow if the step is a short circuit step
                 break
@@ -221,10 +228,20 @@ def _test_connection_steps_during_ingestion(steps: List[TestConnectionStep]) ->
     logger.info("Test connection results:")
     logger.info(test_connection_result)
 
-    if test_connection_result.failed:
-        raise SourceConnectionException(
-            f"Some steps failed when testing the connection: [{test_connection_result}]"
-        )
+    return test_connection_result
+
+
+def raise_test_connection_exception(result: TestConnectionResult) -> None:
+    """Raise if needed an exception for the test connection"""
+    for step in result.steps:
+        if not step.passed and step.mandatory:
+            raise SourceConnectionException(
+                f"Failed to run the test connection step: {step.name}"
+            )
+        if not step.passed:
+            logger.warning(
+                f"You might be missing metadata in: {step.name} due to {step.message}"
+            )
 
 
 def test_connection_steps(
@@ -232,8 +249,8 @@ def test_connection_steps(
     service_type: str,
     test_fn: dict,
     automation_workflow: Optional[AutomationWorkflow] = None,
-    timeout_seconds: int = 3 * 60,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test the connection steps with a given timeout
 
@@ -268,9 +285,12 @@ def test_connection_steps(
         for step in test_connection_definition.steps
     ]
 
-    return timeout(timeout_seconds)(_test_connection_steps)(
-        metadata, steps, automation_workflow
-    )
+    if timeout_seconds:
+        return timeout(timeout_seconds)(_test_connection_steps)(
+            metadata, steps, automation_workflow
+        )
+
+    return _test_connection_steps(metadata, steps, automation_workflow)
 
 
 def test_connection_engine_step(connection: Engine) -> None:
@@ -289,8 +309,8 @@ def test_connection_db_common(
     service_connection,
     automation_workflow: Optional[AutomationWorkflow] = None,
     queries: dict = None,
-    timeout_seconds: int = 3 * 60,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -322,7 +342,7 @@ def test_connection_db_common(
     for key, query in queries.items():
         test_fn[key] = partial(test_query, statement=query, engine=engine)
 
-    test_connection_steps(
+    result = test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
@@ -332,6 +352,8 @@ def test_connection_db_common(
 
     kill_active_connections(engine)
 
+    return result
+
 
 def test_connection_db_schema_sources(
     metadata: OpenMetadata,
@@ -339,7 +361,8 @@ def test_connection_db_schema_sources(
     service_connection,
     automation_workflow: Optional[AutomationWorkflow] = None,
     queries: dict = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -388,15 +411,18 @@ def custom_executor(engine_: Engine, inspector_fn_str: str):
     for key, query in queries.items():
         test_fn[key] = partial(test_query, statement=query, engine=engine)
 
-    test_connection_steps(
+    result = test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
 
     kill_active_connections(engine)
 
+    return result
+
 
 def test_query(engine: Engine, statement: str):
     """
diff --git a/ingestion/src/metadata/ingestion/source/api/api_service.py b/ingestion/src/metadata/ingestion/source/api/api_service.py
index 38bddd260829..6b5e7380c785 100644
--- a/ingestion/src/metadata/ingestion/source/api/api_service.py
+++ b/ingestion/src/metadata/ingestion/source/api/api_service.py
@@ -26,8 +26,8 @@
 from metadata.generated.schema.entity.data.apiCollection import APICollection
 from metadata.generated.schema.entity.data.apiEndpoint import APIEndpoint
 from metadata.generated.schema.entity.services.apiService import (
+    ApiConnection,
     ApiService,
-    ApiServiceConnection,
 )
 from metadata.generated.schema.metadataIngestion.apiServiceMetadataPipeline import (
     ApiServiceMetadataPipeline,
@@ -39,6 +39,9 @@
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.topology import (
     NodeStage,
@@ -112,7 +115,7 @@ class ApiServiceSource(TopologyRunnerMixin, Source, ABC):
     source_config: ApiServiceMetadataPipeline
     config: WorkflowSource
     # Big union of types we want to fetch dynamically
-    service_connection: ApiServiceConnection.model_fields["config"].annotation
+    service_connection: ApiConnection.model_fields["config"].annotation
 
     topology = ApiServiceTopology()
     context = TopologyContextManager(topology)
@@ -175,7 +178,10 @@ def close(self):
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
 
     def mark_api_collections_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
         """Method to mark the api collection as deleted"""
diff --git a/ingestion/src/metadata/ingestion/source/api/rest/connection.py b/ingestion/src/metadata/ingestion/source/api/rest/connection.py
index 055ecbb0c0ab..2aa2f09c165c 100644
--- a/ingestion/src/metadata/ingestion/source/api/rest/connection.py
+++ b/ingestion/src/metadata/ingestion/source/api/rest/connection.py
@@ -20,11 +20,15 @@
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
-from metadata.generated.schema.entity.services.connections.apiService.restConnection import (
-    RESTConnection,
+from metadata.generated.schema.entity.services.connections.api.restConnection import (
+    RestConnection,
+)
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
 )
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 class SchemaURLError(Exception):
@@ -39,7 +43,7 @@ class InvalidOpenAPISchemaError(Exception):
     """
 
 
-def get_connection(connection: RESTConnection) -> Response:
+def get_connection(connection: RestConnection) -> Response:
     """
     Create connection
     """
@@ -52,9 +56,10 @@ def get_connection(connection: RESTConnection) -> Response:
 def test_connection(
     metadata: OpenMetadata,
     client: Response,
-    service_connection: RESTConnection,
+    service_connection: RestConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -79,9 +84,10 @@ def custom_schema_exec():
 
     test_fn = {"CheckURL": custom_url_exec, "CheckSchema": custom_schema_exec}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/api/rest/metadata.py b/ingestion/src/metadata/ingestion/source/api/rest/metadata.py
index bd2776258fae..925c0f1e943f 100644
--- a/ingestion/src/metadata/ingestion/source/api/rest/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/api/rest/metadata.py
@@ -23,8 +23,8 @@
 )
 from metadata.generated.schema.entity.data.apiCollection import APICollection
 from metadata.generated.schema.entity.data.apiEndpoint import ApiRequestMethod
-from metadata.generated.schema.entity.services.connections.apiService.restConnection import (
-    RESTConnection,
+from metadata.generated.schema.entity.services.connections.api.restConnection import (
+    RestConnection,
 )
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
@@ -62,10 +62,10 @@ def create(
         cls, config_dict, metadata: OpenMetadata, pipeline_name: Optional[str] = None
     ):
         config: WorkflowSource = WorkflowSource.model_validate(config_dict)
-        connection: RESTConnection = config.serviceConnection.root.config
-        if not isinstance(connection, RESTConnection):
+        connection: RestConnection = config.serviceConnection.root.config
+        if not isinstance(connection, RestConnection):
             raise InvalidSourceException(
-                f"Expected RESTConnection, but got {connection}"
+                f"Expected RestConnection, but got {connection}"
             )
         return cls(config, metadata)
 
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py
index 5a30430c4eec..96df9f912f78 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py
@@ -55,6 +55,9 @@
 from metadata.ingestion.api.models import Either, Entity
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import C, TopologyRunnerMixin
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.lineage.sql_lineage import get_column_fqn
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
@@ -554,7 +557,10 @@ def get_dashboard(self) -> Any:
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
 
     def prepare(self):
         """By default, nothing to prepare"""
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/connection.py
index 0b6cf700026a..7bc32cdf63d0 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/connection.py
@@ -12,7 +12,6 @@
 """
 Source connection handler
 """
-
 from typing import Optional
 
 from pydomo import Domo
@@ -24,11 +23,15 @@
 from metadata.generated.schema.entity.services.connections.dashboard.domoDashboardConnection import (
     DomoDashboardConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: DomoDashboardConnection) -> OMPyDomoClient:
@@ -57,7 +60,8 @@ def test_connection(
     client: OMPyDomoClient,
     service_connection: DomoDashboardConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -72,9 +76,10 @@ def custom_test_page_list():
         "GetCharts": client.custom.test_list_cards,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/lightdash/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/lightdash/connection.py
index db653b1b02b3..1d0cb137eb0c 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/lightdash/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/lightdash/connection.py
@@ -12,7 +12,6 @@
 """
 Source connection handler
 """
-
 from typing import Optional
 
 from metadata.generated.schema.entity.automations.workflow import (
@@ -21,12 +20,16 @@
 from metadata.generated.schema.entity.services.connections.dashboard.lightdashConnection import (
     LightdashConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.lightdash.client import LightdashApiClient
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -49,7 +52,8 @@ def test_connection(
     client: LightdashApiClient,
     service_connection: LightdashConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -60,9 +64,10 @@ def custom_executor():
 
     test_fn = {"GetDashboards": custom_executor}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/looker/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/looker/connection.py
index c5b235ec4cd0..cee7c9f2d68a 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/looker/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/looker/connection.py
@@ -24,8 +24,12 @@
 from metadata.generated.schema.entity.services.connections.dashboard.lookerConnection import (
     LookerConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: LookerConnection) -> Looker40SDK:
@@ -49,7 +53,8 @@ def test_connection(
     client: Looker40SDK,
     service_connection: LookerConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -76,9 +81,10 @@ def validate_api_version():
         "ListLookMLModels": list_datamodels_test,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/connection.py
index 1d8000ea439e..f8c09bc85992 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.dashboard.metabaseConnection import (
     MetabaseConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.metabase.client import MetabaseClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: MetabaseConnection) -> MetabaseClient:
@@ -37,7 +41,8 @@ def test_connection(
     client: MetabaseClient,
     service_connection: MetabaseConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -49,9 +54,10 @@ def custom_executor():
 
     test_fn = {"GetDashboards": custom_executor}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/mode/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/mode/connection.py
index 339931b4c840..96a670538a4c 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/mode/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/mode/connection.py
@@ -21,9 +21,13 @@
 from metadata.generated.schema.entity.services.connections.dashboard.modeConnection import (
     ModeConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.mode.client import ModeApiClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: ModeConnection) -> ModeApiClient:
@@ -38,7 +42,8 @@ def test_connection(
     client: ModeApiClient,
     service_connection: ModeConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -50,9 +55,10 @@ def test_connection(
         )
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/mstr/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/mstr/connection.py
index df6efbde2a67..156022a8aaca 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/mstr/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/mstr/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.dashboard.mstrConnection import (
     MstrConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.mstr.client import MSTRClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: MstrConnection) -> MSTRClient:
@@ -37,7 +41,8 @@ def test_connection(
     client: MSTRClient,
     service_connection: MstrConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -45,9 +50,10 @@ def test_connection(
 
     test_fn = {"GetProjects": client.get_projects_list}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/connection.py
index c9bae539f9f0..a7a73e7f406a 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/connection.py
@@ -20,6 +20,9 @@
 from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import (
     PowerBIConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.powerbi.client import (
@@ -27,6 +30,7 @@
     PowerBiClient,
 )
 from metadata.ingestion.source.dashboard.powerbi.file_client import PowerBiFileClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: PowerBIConnection) -> PowerBiApiClient:
@@ -46,16 +50,18 @@ def test_connection(
     client: PowerBiClient,
     service_connection: PowerBIConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
     test_fn = {"GetDashboards": client.api_client.fetch_dashboards}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py
index 38eef1660863..784d5088f2a0 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/qlikcloud/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.dashboard.qlikCloudConnection import (
     QlikCloudConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.qlikcloud.client import QlikCloudClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: QlikCloudConnection) -> QlikCloudClient:
@@ -37,7 +41,8 @@ def test_connection(
     client: QlikCloudClient,
     service_connection: QlikCloudConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -45,9 +50,10 @@ def test_connection(
 
     test_fn = {"GetDashboards": client.get_dashboards_list_test_conn}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py
index 94ed95e10d51..6ef88f4066a7 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/qliksense/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.dashboard.qlikSenseConnection import (
     QlikSenseConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.qliksense.client import QlikSenseClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: QlikSenseConnection) -> QlikSenseClient:
@@ -37,7 +41,8 @@ def test_connection(
     client: QlikSenseClient,
     service_connection: QlikSenseConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -45,9 +50,10 @@ def test_connection(
 
     test_fn = {"GetDashboards": client.get_dashboard_for_test_connection}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/quicksight/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/quicksight/connection.py
index 4d2d70dd8ba2..6926a140eb8e 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/quicksight/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/quicksight/connection.py
@@ -22,8 +22,12 @@
 from metadata.generated.schema.entity.services.connections.dashboard.quickSightConnection import (
     QuickSightConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: QuickSightConnection):
@@ -43,7 +47,8 @@ def test_connection(
     client: AWSClient,
     service_connection: QuickSightConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -55,9 +60,10 @@ def test_connection(
         )
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/quicksight/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/quicksight/metadata.py
index 11d16866200d..fe091527d083 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/quicksight/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/quicksight/metadata.py
@@ -247,6 +247,7 @@ def yield_dashboard_lineage_details(  # pylint: disable=too-many-locals
                             **data_source["RelationalTable"]
                         )
                     except (KeyError, ValidationError) as err:
+                        data_source_resp = None
                         yield Either(
                             left=StackTraceError(
                                 name="Lineage",
@@ -257,62 +258,70 @@ def yield_dashboard_lineage_details(  # pylint: disable=too-many-locals
                                 stackTrace=traceback.format_exc(),
                             )
                         )
+                    if data_source_resp:
+                        schema_name = data_source_resp.schema_name
+                        table_name = data_source_resp.table_name
 
-                    schema_name = data_source_resp.schema_name
-                    table_name = data_source_resp.table_name
-
-                    list_data_source_func = lambda kwargs: self.client.list_data_sources(  # pylint: disable=unnecessary-lambda-assignment
-                        **kwargs
-                    )
+                        list_data_source_func = lambda kwargs: self.client.list_data_sources(  # pylint: disable=unnecessary-lambda-assignment
+                            **kwargs
+                        )
 
-                    data_source_summary_list = self._check_pagination(
-                        listing_method=list_data_source_func,
-                        entity_key="DataSources",
-                    )
+                        data_source_summary_list = self._check_pagination(
+                            listing_method=list_data_source_func,
+                            entity_key="DataSources",
+                        )
 
-                    data_source_ids = [
-                        data_source_arn["DataSourceId"]
-                        for data_source_arn in data_source_summary_list or []
-                        if data_source_arn["Arn"] in data_source_resp.datasource_arn
-                    ]
+                        data_source_ids = [
+                            data_source_arn["DataSourceId"]
+                            for data_source_arn in data_source_summary_list or []
+                            if data_source_arn["Arn"] in data_source_resp.datasource_arn
+                        ]
 
-                    for data_source_id in data_source_ids or []:
-                        data_source_resp = DescribeDataSourceResponse(
-                            **self.client.describe_data_source(
-                                AwsAccountId=self.aws_account_id,
-                                DataSourceId=data_source_id,
-                            )
-                        ).DataSource
-                        if data_source_resp and data_source_resp.DataSourceParameters:
-                            data_source_dict = data_source_resp.DataSourceParameters
-                            for db in data_source_dict.keys() or []:
-                                from_fqn = fqn.build(
-                                    self.metadata,
-                                    entity_type=Table,
-                                    service_name=db_service_name,
-                                    database_name=data_source_dict[db].get("Database"),
-                                    schema_name=schema_name,
-                                    table_name=table_name,
-                                    skip_es_search=True,
+                        for data_source_id in data_source_ids or []:
+                            data_source_resp = DescribeDataSourceResponse(
+                                **self.client.describe_data_source(
+                                    AwsAccountId=self.aws_account_id,
+                                    DataSourceId=data_source_id,
                                 )
-                                from_entity = self.metadata.get_by_name(
-                                    entity=Table,
-                                    fqn=from_fqn,
-                                )
-                                to_fqn = fqn.build(
-                                    self.metadata,
-                                    entity_type=Dashboard,
-                                    service_name=self.config.serviceName,
-                                    dashboard_name=dashboard_details.DashboardId,
-                                )
-                                to_entity = self.metadata.get_by_name(
-                                    entity=Dashboard,
-                                    fqn=to_fqn,
-                                )
-                                if from_entity is not None and to_entity is not None:
-                                    yield self._get_add_lineage_request(
-                                        to_entity=to_entity, from_entity=from_entity
+                            ).DataSource
+                            if (
+                                data_source_resp
+                                and data_source_resp.DataSourceParameters
+                            ):
+                                data_source_dict = data_source_resp.DataSourceParameters
+                                for db in data_source_dict.keys() or []:
+                                    from_fqn = fqn.build(
+                                        self.metadata,
+                                        entity_type=Table,
+                                        service_name=db_service_name,
+                                        database_name=data_source_dict[db].get(
+                                            "Database"
+                                        ),
+                                        schema_name=schema_name,
+                                        table_name=table_name,
+                                        skip_es_search=True,
+                                    )
+                                    from_entity = self.metadata.get_by_name(
+                                        entity=Table,
+                                        fqn=from_fqn,
+                                    )
+                                    to_fqn = fqn.build(
+                                        self.metadata,
+                                        entity_type=Dashboard,
+                                        service_name=self.config.serviceName,
+                                        dashboard_name=dashboard_details.DashboardId,
+                                    )
+                                    to_entity = self.metadata.get_by_name(
+                                        entity=Dashboard,
+                                        fqn=to_fqn,
                                     )
+                                    if (
+                                        from_entity is not None
+                                        and to_entity is not None
+                                    ):
+                                        yield self._get_add_lineage_request(
+                                            to_entity=to_entity, from_entity=from_entity
+                                        )
         except Exception as exc:  # pylint: disable=broad-except
             yield Either(
                 left=StackTraceError(
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/quicksight/models.py b/ingestion/src/metadata/ingestion/source/dashboard/quicksight/models.py
index 47a6661820cb..fa279af2dc57 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/quicksight/models.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/quicksight/models.py
@@ -50,12 +50,12 @@ class DashboardResp(BaseModel):
     RequestId: Optional[str] = None
 
 
-class DataSource(BaseModel):
+class DataSourceModel(BaseModel):
     DataSourceId: str
     DataSourceParameters: Optional[dict] = None
 
 
 class DescribeDataSourceResponse(BaseModel):
-    DataSource: Optional[DataSource] = None
+    DataSource: Optional[DataSourceModel] = None
     RequestId: Optional[str] = None
     Status: Optional[int] = None
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/redash/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/redash/connection.py
index ae015da7e868..a0aa013aa01d 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/redash/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/redash/connection.py
@@ -21,12 +21,16 @@
 from metadata.generated.schema.entity.services.connections.dashboard.redashConnection import (
     RedashConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.redash.client import RedashApiClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: RedashConnection) -> RedashApiClient:
@@ -45,7 +49,8 @@ def test_connection(
     client: RedashApiClient,
     service_connection: RedashConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -53,9 +58,10 @@ def test_connection(
 
     test_fn = {"GetDashboards": client.dashboards}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/sigma/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/sigma/connection.py
index b2e2204157ec..a6d33c67d930 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/sigma/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/sigma/connection.py
@@ -21,12 +21,16 @@
 from metadata.generated.schema.entity.services.connections.dashboard.sigmaConnection import (
     SigmaConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.dashboard.sigma.client import SigmaApiClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: SigmaConnection) -> SigmaApiClient:
@@ -45,7 +49,8 @@ def test_connection(
     client: SigmaApiClient,
     service_connection: SigmaConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -53,9 +58,10 @@ def test_connection(
 
     test_fn = {"GetToken": client.get_auth_token, "GetWorkbooks": client.get_dashboards}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/connection.py
index d1a3a5fa3b59..1aa87e343831 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/superset/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/connection.py
@@ -29,6 +29,9 @@
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
     PostgresConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.generated.schema.entity.utils.supersetApiConnection import (
     SupersetApiConnection,
 )
@@ -49,6 +52,7 @@
 from metadata.ingestion.source.database.postgres.connection import (
     get_connection as pg_get_connection,
 )
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: SupersetConnection) -> SupersetAPIClient:
@@ -69,7 +73,8 @@ def test_connection(
     client: Union[SupersetAPIClient, Engine],
     service_connection: SupersetConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -84,11 +89,17 @@ def test_connection(
     else:
         test_fn["CheckAccess"] = partial(test_connection_engine_step, client)
         test_fn["GetDashboards"] = partial(test_query, client, FETCH_DASHBOARDS_TEST)
-        test_fn["GetCharts"] = partial(test_query, client, FETCH_ALL_CHARTS_TEST)
+        if isinstance(service_connection.connection, MysqlConnection):
+            test_fn["GetCharts"] = partial(
+                test_query, client, FETCH_ALL_CHARTS_TEST.replace('"', "`")
+            )
+        else:
+            test_fn["GetCharts"] = partial(test_query, client, FETCH_ALL_CHARTS_TEST)
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py
index 8be179f0b114..636641b61f8e 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py
@@ -26,6 +26,9 @@
 from metadata.generated.schema.entity.data.chart import Chart
 from metadata.generated.schema.entity.data.dashboardDataModel import DataModelType
 from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
+    MysqlConnection,
+)
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
@@ -81,7 +84,10 @@ def prepare(self):
         the required information which is not available in fetch_charts_with_id api
         """
         try:
-            charts = self.engine.execute(FETCH_ALL_CHARTS)
+            if isinstance(self.service_connection.connection, MysqlConnection):
+                charts = self.engine.execute(FETCH_ALL_CHARTS.replace('"', "`"))
+            else:
+                charts = self.engine.execute(FETCH_ALL_CHARTS)
             for chart in charts:
                 chart_detail = FetchChart(**chart)
                 self.all_charts[chart_detail.id] = chart_detail
diff --git a/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py b/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py
index 8feef66c0e0e..e3e976e62b8d 100644
--- a/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/tableau/connection.py
@@ -24,6 +24,9 @@
 from metadata.generated.schema.entity.services.connections.dashboard.tableauConnection import (
     TableauConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.generated.schema.security.credentials.accessTokenAuth import (
     AccessTokenAuth,
 )
@@ -38,6 +41,7 @@
     TABLEAU_GET_WORKBOOKS_PARAM_DICT,
 )
 from metadata.ingestion.source.dashboard.tableau.client import TableauClient
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.ssl_registry import get_verify_ssl_fn
 
@@ -70,7 +74,8 @@ def test_connection(
     client: TableauClient,
     service_connection: TableauConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -101,11 +106,12 @@ def test_connection(
         "GetDataModels": client.test_get_datamodels,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
 
 
diff --git a/ingestion/src/metadata/ingestion/source/database/athena/connection.py b/ingestion/src/metadata/ingestion/source/database/athena/connection.py
index c151be6fd69f..59baec43250e 100644
--- a/ingestion/src/metadata/ingestion/source/database/athena/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/athena/connection.py
@@ -27,6 +27,9 @@
 from metadata.generated.schema.entity.services.connections.database.athenaConnection import (
     AthenaConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -37,6 +40,7 @@
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: AthenaConnection) -> str:
@@ -94,7 +98,8 @@ def test_connection(
     engine: Engine,
     service_connection: AthenaConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -121,9 +126,10 @@ def custom_executor_for_view():
         "GetViews": custom_executor_for_view,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py b/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py
index 7256e8507916..fe01c321b157 100644
--- a/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/azuresql/connection.py
@@ -28,6 +28,9 @@
 from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
     MssqlConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -35,6 +38,7 @@
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: Union[AzureSQLConnection, MssqlConnection]) -> str:
@@ -105,14 +109,16 @@ def test_connection(
     engine: Engine,
     service_connection: AzureSQLConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py b/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py
index 06bc25a031aa..132f0671953d 100644
--- a/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/bigquery/connection.py
@@ -27,6 +27,9 @@
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.generated.schema.security.credentials.gcpCredentials import (
     GcpCredentialsPath,
 )
@@ -47,6 +50,7 @@
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.bigquery.queries import BIGQUERY_TEST_STATEMENT
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.credentials import set_google_credentials
 from metadata.utils.logger import ingestion_logger
 
@@ -109,7 +113,8 @@ def test_connection(
     engine: Engine,
     service_connection: BigQueryConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -163,14 +168,15 @@ def test_connection_inner(engine):
             ),
         }
 
-        test_connection_steps(
+        return test_connection_steps(
             metadata=metadata,
             test_fn=test_fn,
             service_type=service_connection.type.value,
             automation_workflow=automation_workflow,
+            timeout_seconds=timeout_seconds,
         )
 
-    test_connection_inner(engine)
+    return test_connection_inner(engine)
 
 
 def get_table_view_names(connection, schema=None):
diff --git a/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py b/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py
index 990842604c52..4665e1004124 100644
--- a/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/bigtable/connection.py
@@ -19,6 +19,9 @@
 from metadata.generated.schema.entity.services.connections.database.bigTableConnection import (
     BigTableConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.generated.schema.security.credentials.gcpValues import (
     GcpCredentialsValues,
     SingleProjectId,
@@ -29,6 +32,7 @@
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.bigtable.client import MultiProjectClient
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.credentials import set_google_credentials
 from metadata.utils.logger import ingestion_logger
 
@@ -95,7 +99,8 @@ def test_connection(
     client: MultiProjectClient,
     service_connection: BigTableConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -108,9 +113,10 @@ def test_connection(
         "GetRows": tester.get_row,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py b/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py
index b0c17fded9d6..020f8ab3df0b 100644
--- a/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/clickhouse/connection.py
@@ -23,6 +23,9 @@
 from metadata.generated.schema.entity.services.connections.database.clickhouseConnection import (
     ClickhouseConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -35,6 +38,7 @@
 from metadata.ingestion.source.database.clickhouse.queries import (
     CLICKHOUSE_SQL_STATEMENT_TEST,
 )
+from metadata.utils.constants import THREE_MIN
 
 HTTPS_PROTOCOL = "https"
 
@@ -67,7 +71,8 @@ def test_connection(
     engine: Engine,
     service_connection: ClickhouseConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -75,10 +80,11 @@ def test_connection(
 
     queries = {"GetQueries": CLICKHOUSE_SQL_STATEMENT_TEST}
 
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
         queries=queries,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/couchbase/connection.py b/ingestion/src/metadata/ingestion/source/database/couchbase/connection.py
index 7f988a920707..8a6a8f67547e 100644
--- a/ingestion/src/metadata/ingestion/source/database/couchbase/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/couchbase/connection.py
@@ -23,8 +23,12 @@
 from metadata.generated.schema.entity.services.connections.database.couchbaseConnection import (
     CouchbaseConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: CouchbaseConnection):
@@ -49,7 +53,8 @@ def test_connection(
     client: Any,
     service_connection: CouchbaseConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -80,9 +85,10 @@ def test_get_collections(client: Cluster, holder: SchemaHolder):
         "GetCollections": partial(test_get_collections, client, holder),
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/couchbase/metadata.py b/ingestion/src/metadata/ingestion/source/database/couchbase/metadata.py
index 7201971da504..67f90c472e68 100644
--- a/ingestion/src/metadata/ingestion/source/database/couchbase/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/database/couchbase/metadata.py
@@ -27,10 +27,7 @@
     SAMPLE_SIZE,
     CommonNoSQLSource,
 )
-from metadata.ingestion.source.database.couchbase.queries import (
-    COUCHBASE_GET_DATA,
-    COUCHBASE_SQL_STATEMENT,
-)
+from metadata.ingestion.source.database.couchbase.queries import COUCHBASE_GET_DATA
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -111,18 +108,14 @@ def get_table_columns_dict(self, schema_name: str, table_name: str) -> List[Dict
         """
         try:
             database_name = self.context.get().database
-            query = COUCHBASE_SQL_STATEMENT.format(table_name=table_name)
-            result = self.couchbase.query(query)
-            for row in result.rows():
-                if len(row) > 0:
-                    query_coln = COUCHBASE_GET_DATA.format(
-                        database_name=database_name,
-                        schema_name=schema_name,
-                        table_name=table_name,
-                        sample_size=SAMPLE_SIZE,
-                    )
-                    query_iter = self.couchbase.query(query_coln)
-                    return list(query_iter.rows())
+            query_coln = COUCHBASE_GET_DATA.format(
+                database_name=database_name,
+                schema_name=schema_name,
+                table_name=table_name,
+                sample_size=SAMPLE_SIZE,
+            )
+            query_iter = self.couchbase.query(query_coln)
+            return list(query_iter.rows())
         except Exception as exp:
             logger.debug(f"Failed to list column names for table [{table_name}]: {exp}")
             logger.debug(traceback.format_exc())
diff --git a/ingestion/src/metadata/ingestion/source/database/couchbase/queries.py b/ingestion/src/metadata/ingestion/source/database/couchbase/queries.py
index 40a783eb4c12..3a32d74d66f9 100644
--- a/ingestion/src/metadata/ingestion/source/database/couchbase/queries.py
+++ b/ingestion/src/metadata/ingestion/source/database/couchbase/queries.py
@@ -14,10 +14,6 @@
 
 import textwrap
 
-COUCHBASE_SQL_STATEMENT = textwrap.dedent(
-    """ SELECT * FROM system:indexes WHERE keyspace_id = '{table_name}' AND is_primary = TRUE """
-)
-
 COUCHBASE_GET_DATA = textwrap.dedent(
     """ select crc.* from `{database_name}`.`{schema_name}`.`{table_name}` crc limit {sample_size} """
 )
diff --git a/ingestion/src/metadata/ingestion/source/database/database_service.py b/ingestion/src/metadata/ingestion/source/database/database_service.py
index aee239c55efe..600f211170fb 100644
--- a/ingestion/src/metadata/ingestion/source/database/database_service.py
+++ b/ingestion/src/metadata/ingestion/source/database/database_service.py
@@ -58,6 +58,9 @@
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.life_cycle import OMetaLifeCycleData
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.models.topology import (
@@ -609,4 +612,7 @@ def yield_external_table_lineage(self) -> Iterable[Either[AddLineageRequest]]:
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/connection.py b/ingestion/src/metadata/ingestion/source/database/databricks/connection.py
index 7d7854bebf9f..14d3d3e392e9 100644
--- a/ingestion/src/metadata/ingestion/source/database/databricks/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/databricks/connection.py
@@ -25,6 +25,9 @@
 from metadata.generated.schema.entity.services.connections.database.databricksConnection import (
     DatabricksConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -39,6 +42,7 @@
 from metadata.ingestion.source.database.databricks.queries import (
     DATABRICKS_GET_CATALOGS,
 )
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -71,7 +75,8 @@ def test_connection(
     connection: Engine,
     service_connection: DatabricksConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -104,10 +109,10 @@ def test_database_query(engine: Engine, statement: str):
         "GetQueries": client.test_query_api_access,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
-        timeout_seconds=service_connection.connectionTimeout,
+        timeout_seconds=service_connection.connectionTimeout or timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/datalake/connection.py b/ingestion/src/metadata/ingestion/source/database/datalake/connection.py
index 9ec0a8625287..fdc11f6021d0 100644
--- a/ingestion/src/metadata/ingestion/source/database/datalake/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/datalake/connection.py
@@ -31,6 +31,9 @@
 from metadata.generated.schema.entity.services.connections.database.datalakeConnection import (
     DatalakeConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.datalake.clients.azure_blob import (
@@ -38,6 +41,7 @@
 )
 from metadata.ingestion.source.database.datalake.clients.gcs import DatalakeGcsClient
 from metadata.ingestion.source.database.datalake.clients.s3 import DatalakeS3Client
+from metadata.utils.constants import THREE_MIN
 
 
 # Only import specific datalake dependencies if necessary
@@ -91,7 +95,8 @@ def test_connection(
     connection: DatalakeClient,
     service_connection: DatalakeConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -102,9 +107,10 @@ def test_connection(
         ),
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/db2/connection.py b/ingestion/src/metadata/ingestion/source/database/db2/connection.py
index 6ea82c758c20..cadde6775f38 100644
--- a/ingestion/src/metadata/ingestion/source/database/db2/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/db2/connection.py
@@ -22,6 +22,9 @@
 from metadata.generated.schema.entity.services.connections.database.db2Connection import (
     Db2Connection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -29,6 +32,7 @@
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: Db2Connection) -> Engine:
@@ -47,14 +51,16 @@ def test_connection(
     engine: Engine,
     service_connection: Db2Connection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/constants.py b/ingestion/src/metadata/ingestion/source/database/dbt/constants.py
index 63731473b72d..83c49c0724a4 100644
--- a/ingestion/src/metadata/ingestion/source/database/dbt/constants.py
+++ b/ingestion/src/metadata/ingestion/source/database/dbt/constants.py
@@ -22,6 +22,61 @@
 # Based on https://schemas.getdbt.com/dbt/catalog/v1.json
 REQUIRED_CATALOG_KEYS = ["name", "type", "index"]
 
+REQUIRED_RESULTS_KEYS = {
+    "status",
+    "timing",
+    "thread_id",
+    "execution_time",
+    "message",
+    "adapter_response",
+    "unique_id",
+}
+
+REQUIRED_NODE_KEYS = {
+    "schema_",
+    "schema",
+    "freshness",
+    "name",
+    "resource_type",
+    "path",
+    "unique_id",
+    "source_name",
+    "source_description",
+    "source_meta",
+    "loader",
+    "identifier",
+    "relation_name",
+    "fqn",
+    "alias",
+    "checksum",
+    "config",
+    "column_name",
+    "test_metadata",
+    "original_file_path",
+    "root_path",
+    "database",
+    "tags",
+    "description",
+    "columns",
+    "meta",
+    "owner",
+    "created_at",
+    "group",
+    "sources",
+    "compiled",
+    "docs",
+    "version",
+    "latest_version",
+    "package_name",
+    "depends_on",
+    "compiled_code",
+    "compiled_sql",
+    "raw_code",
+    "raw_sql",
+    "language",
+}
+
+
 NONE_KEYWORDS_LIST = ["none", "null"]
 
 DBT_CATALOG_FILE_NAME = "catalog.json"
diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_service.py b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_service.py
index 44307b6df755..aa2d65f4e2cf 100644
--- a/ingestion/src/metadata/ingestion/source/database/dbt/dbt_service.py
+++ b/ingestion/src/metadata/ingestion/source/database/dbt/dbt_service.py
@@ -13,7 +13,7 @@
 """
 
 from abc import ABC, abstractmethod
-from typing import Iterable
+from typing import Iterable, List
 
 from dbt_artifacts_parser.parser import parse_catalog, parse_manifest, parse_run_results
 from pydantic import Field
@@ -37,6 +37,10 @@
     TopologyNode,
 )
 from metadata.ingestion.source.database.database_service import DataModelLink
+from metadata.ingestion.source.database.dbt.constants import (
+    REQUIRED_NODE_KEYS,
+    REQUIRED_RESULTS_KEYS,
+)
 from metadata.ingestion.source.database.dbt.dbt_config import get_dbt_details
 from metadata.ingestion.source.database.dbt.models import (
     DbtFiles,
@@ -169,51 +173,27 @@ def remove_manifest_non_required_keys(self, manifest_dict: dict):
             }
         )
 
-        required_nodes_keys = {
-            "schema_",
-            "schema",
-            "name",
-            "resource_type",
-            "path",
-            "unique_id",
-            "fqn",
-            "alias",
-            "checksum",
-            "config",
-            "column_name",
-            "test_metadata",
-            "original_file_path",
-            "root_path",
-            "database",
-            "tags",
-            "description",
-            "columns",
-            "meta",
-            "owner",
-            "created_at",
-            "group",
-            "sources",
-            "compiled",
-            "docs",
-            "version",
-            "latest_version",
-            "package_name",
-            "depends_on",
-            "compiled_code",
-            "compiled_sql",
-            "raw_code",
-            "raw_sql",
-            "language",
-        }
+        for field in ["nodes", "sources"]:
+            for node, value in manifest_dict.get(  # pylint: disable=unused-variable
+                field
+            ).items():
+                keys_to_delete = [
+                    key for key in value if key.lower() not in REQUIRED_NODE_KEYS
+                ]
+                for key in keys_to_delete:
+                    del value[key]
 
-        for node, value in manifest_dict.get(  # pylint: disable=unused-variable
-            "nodes"
-        ).items():
-            keys_to_delete = [
-                key for key in value if key.lower() not in required_nodes_keys
-            ]
-            for key in keys_to_delete:
-                del value[key]
+    def remove_run_result_non_required_keys(self, run_results: List[dict]):
+        """
+        Method to remove the non required keys from run results file
+        """
+        for run_result in run_results:
+            for result in run_result.get("results"):
+                keys_to_delete = [
+                    key for key in result if key.lower() not in REQUIRED_RESULTS_KEYS
+                ]
+                for key in keys_to_delete:
+                    del result[key]
 
     def get_dbt_files(self) -> Iterable[DbtFiles]:
         dbt_files = get_dbt_details(self.source_config.dbtConfigSource)
@@ -225,6 +205,10 @@ def get_dbt_objects(self) -> Iterable[DbtObjects]:
         self.remove_manifest_non_required_keys(
             manifest_dict=self.context.get().dbt_file.dbt_manifest
         )
+        if self.context.get().dbt_file.dbt_run_results:
+            self.remove_run_result_non_required_keys(
+                run_results=self.context.get().dbt_file.dbt_run_results
+            )
         dbt_objects = DbtObjects(
             dbt_catalog=parse_catalog(self.context.get().dbt_file.dbt_catalog)
             if self.context.get().dbt_file.dbt_catalog
diff --git a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py
index 8337f0f47bce..d312f1a186b0 100644
--- a/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/database/dbt/metadata.py
@@ -288,10 +288,7 @@ def yield_dbt_tags(
                     for tag_name in dbt_tags_list
                 ]
                 yield from get_ometa_tag_and_classification(
-                    tags=[
-                        tag_label.split(fqn.FQN_SEPARATOR)[1]
-                        for tag_label in dbt_tag_labels
-                    ],
+                    tags=[fqn.split(tag_label)[1] for tag_label in dbt_tag_labels],
                     classification_name=self.tag_classification_name,
                     tag_description="dbt Tags",
                     classification_description="dbt classification",
diff --git a/ingestion/src/metadata/ingestion/source/database/deltalake/connection.py b/ingestion/src/metadata/ingestion/source/database/deltalake/connection.py
index 5219e89595a0..20d799007a1e 100644
--- a/ingestion/src/metadata/ingestion/source/database/deltalake/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/deltalake/connection.py
@@ -31,8 +31,12 @@
 from metadata.generated.schema.entity.services.connections.database.deltaLakeConnection import (
     DeltaLakeConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 @dataclass
@@ -83,7 +87,8 @@ def test_connection(
     connection: DeltalakeClient,
     service_connection: DeltaLakeConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -97,9 +102,10 @@ def test_connection(
         ),
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/domodatabase/connection.py b/ingestion/src/metadata/ingestion/source/database/domodatabase/connection.py
index 897c9dd767c5..3dbe0729e52f 100644
--- a/ingestion/src/metadata/ingestion/source/database/domodatabase/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/domodatabase/connection.py
@@ -23,11 +23,15 @@
 from metadata.generated.schema.entity.services.connections.database.domoDatabaseConnection import (
     DomoDatabaseConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: DomoDatabaseConnection) -> Domo:
@@ -51,7 +55,8 @@ def test_connection(
     domo: Domo,
     service_connection: DomoDatabaseConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -65,9 +70,10 @@ def custom_executor():
         "GetTables": custom_executor,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/doris/connection.py b/ingestion/src/metadata/ingestion/source/database/doris/connection.py
index d4eff8b228c1..4707ea926637 100644
--- a/ingestion/src/metadata/ingestion/source/database/doris/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/doris/connection.py
@@ -22,6 +22,9 @@
 from metadata.generated.schema.entity.services.connections.database.dorisConnection import (
     DorisConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -31,6 +34,7 @@
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: DorisConnection) -> Engine:
@@ -49,14 +53,16 @@ def test_connection(
     engine: Engine,
     service_connection: DorisConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_schema_sources(
+    return test_connection_db_schema_sources(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/druid/connection.py b/ingestion/src/metadata/ingestion/source/database/druid/connection.py
index 4c81c12d0d66..a43b76df34c7 100644
--- a/ingestion/src/metadata/ingestion/source/database/druid/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/druid/connection.py
@@ -22,6 +22,9 @@
 from metadata.generated.schema.entity.services.connections.database.druidConnection import (
     DruidConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -29,6 +32,7 @@
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: DruidConnection) -> str:
@@ -52,14 +56,16 @@ def test_connection(
     engine: Engine,
     service_connection: DruidConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py b/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py
index 06dd5a338e25..085b37e366a0 100644
--- a/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/dynamodb/connection.py
@@ -22,8 +22,12 @@
 from metadata.generated.schema.entity.services.connections.database.dynamoDBConnection import (
     DynamoDBConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: DynamoDBConnection):
@@ -48,7 +52,8 @@ def test_connection(
     client: AWSClient,
     service_connection: DynamoDBConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -58,9 +63,10 @@ def test_connection(
         "ListTables": partial(check_list_tables, client),
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/glue/connection.py b/ingestion/src/metadata/ingestion/source/database/glue/connection.py
index b718faa94083..d1f8f4649bb7 100644
--- a/ingestion/src/metadata/ingestion/source/database/glue/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/glue/connection.py
@@ -23,8 +23,12 @@
 from metadata.generated.schema.entity.services.connections.database.glueConnection import (
     GlueConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: GlueConnection) -> Engine:
@@ -39,7 +43,8 @@ def test_connection(
     client: AWSClient,
     service_connection: GlueConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -64,9 +69,10 @@ def custom_executor_for_table():
         "GetTables": custom_executor_for_table,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/greenplum/connection.py b/ingestion/src/metadata/ingestion/source/database/greenplum/connection.py
index 2614bf4d2967..b9ad6eff9d01 100644
--- a/ingestion/src/metadata/ingestion/source/database/greenplum/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/greenplum/connection.py
@@ -23,6 +23,9 @@
 from metadata.generated.schema.entity.services.connections.database.greenplumConnection import (
     GreenplumConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -31,6 +34,7 @@
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.greenplum.queries import GREENPLUM_GET_DATABASE
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: GreenplumConnection) -> Engine:
@@ -49,7 +53,8 @@ def test_connection(
     engine: Engine,
     service_connection: GreenplumConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -57,10 +62,11 @@ def test_connection(
     queries = {
         "GetDatabases": GREENPLUM_GET_DATABASE,
     }
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
         queries=queries,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/hive/connection.py b/ingestion/src/metadata/ingestion/source/database/hive/connection.py
index 1dd20e513905..0681f72c20ca 100644
--- a/ingestion/src/metadata/ingestion/source/database/hive/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/hive/connection.py
@@ -18,7 +18,7 @@
 from typing import Any, Optional
 from urllib.parse import quote_plus
 
-from pydantic import SecretStr
+from pydantic import SecretStr, ValidationError
 from sqlalchemy.engine import Engine
 
 from metadata.generated.schema.entity.automations.workflow import (
@@ -34,6 +34,9 @@
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
     PostgresConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -45,6 +48,7 @@
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 HIVE_POSTGRES_SCHEME = "hive+postgres"
 HIVE_MYSQL_SCHEME = "hive+mysql"
@@ -181,18 +185,35 @@ def test_connection(
     engine: Engine,
     service_connection: HiveConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
 
-    if service_connection.metastoreConnection:
+    if service_connection.metastoreConnection and isinstance(
+        service_connection.metastoreConnection, dict
+    ):
+        try:
+            service_connection.metastoreConnection = MysqlConnection.model_validate(
+                service_connection.metastoreConnection
+            )
+        except ValidationError:
+            try:
+                service_connection.metastoreConnection = (
+                    PostgresConnection.model_validate(
+                        service_connection.metastoreConnection
+                    )
+                )
+            except ValidationError:
+                raise ValueError("Invalid metastore connection")
         engine = get_metastore_connection(service_connection.metastoreConnection)
 
-    test_connection_db_schema_sources(
+    return test_connection_db_schema_sources(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/iceberg/connection.py b/ingestion/src/metadata/ingestion/source/database/iceberg/connection.py
index 6f315b807a24..6b8b804dc80b 100644
--- a/ingestion/src/metadata/ingestion/source/database/iceberg/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/iceberg/connection.py
@@ -23,9 +23,13 @@
 from metadata.generated.schema.entity.services.connections.database.icebergConnection import (
     IcebergConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.iceberg.catalog import IcebergCatalogFactory
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: IcebergConnection) -> Catalog:
@@ -42,7 +46,8 @@ def test_connection(
     catalog: Catalog,
     service_connection: IcebergConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -60,9 +65,10 @@ def custom_executor_for_tables():
         "GetTables": custom_executor_for_tables,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/impala/connection.py b/ingestion/src/metadata/ingestion/source/database/impala/connection.py
index b28262d69469..4a984214ae27 100644
--- a/ingestion/src/metadata/ingestion/source/database/impala/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/impala/connection.py
@@ -24,6 +24,9 @@
 from metadata.generated.schema.entity.services.connections.database.impalaConnection import (
     ImpalaConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -34,6 +37,7 @@
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: ImpalaConnection) -> str:
@@ -108,14 +112,16 @@ def test_connection(
     engine: Engine,
     service_connection: ImpalaConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_schema_sources(
+    return test_connection_db_schema_sources(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/mariadb/connection.py b/ingestion/src/metadata/ingestion/source/database/mariadb/connection.py
index 87942ac961e3..1af77fab582e 100644
--- a/ingestion/src/metadata/ingestion/source/database/mariadb/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/mariadb/connection.py
@@ -22,6 +22,9 @@
 from metadata.generated.schema.entity.services.connections.database.mariaDBConnection import (
     MariaDBConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -31,6 +34,7 @@
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: MariaDBConnection) -> Engine:
@@ -49,14 +53,16 @@ def test_connection(
     engine: Engine,
     service_connection: MariaDBConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_schema_sources(
+    return test_connection_db_schema_sources(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py b/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py
index 56611eb40c61..0d77e2c16568 100644
--- a/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/mongodb/connection.py
@@ -26,9 +26,13 @@
 from metadata.generated.schema.entity.services.connections.database.mongoDBConnection import (
     MongoDBConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import get_connection_url_common
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: MongoDBConnection):
@@ -44,7 +48,8 @@ def test_connection(
     client: MongoClient,
     service_connection: MongoDBConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -70,9 +75,10 @@ def test_get_collections(client_: MongoClient, holder_: SchemaHolder):
         "GetCollections": partial(test_get_collections, client, holder),
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/mssql/connection.py b/ingestion/src/metadata/ingestion/source/database/mssql/connection.py
index 9979e01fa632..92d8b1281226 100644
--- a/ingestion/src/metadata/ingestion/source/database/mssql/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/mssql/connection.py
@@ -22,6 +22,9 @@
 from metadata.generated.schema.entity.services.connections.database.mssqlConnection import (
     MssqlConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -36,6 +39,7 @@
     MSSQL_GET_DATABASE,
     MSSQL_TEST_GET_QUERIES,
 )
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: MssqlConnection) -> str:
@@ -60,7 +64,8 @@ def test_connection(
     engine: Engine,
     service_connection: MssqlConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -69,10 +74,11 @@ def test_connection(
         "GetQueries": MSSQL_TEST_GET_QUERIES,
         "GetDatabases": MSSQL_GET_DATABASE,
     }
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
         queries=queries,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/mysql/connection.py b/ingestion/src/metadata/ingestion/source/database/mysql/connection.py
index d4a8b2d0c946..86dc523ef8b2 100644
--- a/ingestion/src/metadata/ingestion/source/database/mysql/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/mysql/connection.py
@@ -26,6 +26,9 @@
 from metadata.generated.schema.entity.services.connections.database.mysqlConnection import (
     MysqlConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -35,6 +38,7 @@
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: MysqlConnection) -> Engine:
@@ -63,14 +67,16 @@ def test_connection(
     engine: Engine,
     service_connection: MysqlConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_schema_sources(
+    return test_connection_db_schema_sources(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/oracle/connection.py b/ingestion/src/metadata/ingestion/source/database/oracle/connection.py
index ade912396452..83665684cbc2 100644
--- a/ingestion/src/metadata/ingestion/source/database/oracle/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/oracle/connection.py
@@ -31,6 +31,9 @@
     OracleServiceName,
     OracleTNSConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -39,6 +42,7 @@
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.oracle.queries import CHECK_ACCESS_TO_ALL
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 CX_ORACLE_LIB_VERSION = "8.3.0"
@@ -132,7 +136,8 @@ def test_connection(
     engine: Engine,
     service_connection: OracleConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -140,10 +145,11 @@ def test_connection(
 
     test_conn_queries = {"CheckAccess": CHECK_ACCESS_TO_ALL}
 
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
         queries=test_conn_queries,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/pinotdb/connection.py b/ingestion/src/metadata/ingestion/source/database/pinotdb/connection.py
index 2f25281d670f..8f148f3718a3 100644
--- a/ingestion/src/metadata/ingestion/source/database/pinotdb/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/pinotdb/connection.py
@@ -12,6 +12,7 @@
 """
 Source connection handler
 """
+from copy import deepcopy
 from typing import Optional
 
 from sqlalchemy.engine import Engine
@@ -22,13 +23,18 @@
 from metadata.generated.schema.entity.services.connections.database.pinotDBConnection import (
     PinotDBConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
     get_connection_url_common,
+    init_empty_connection_arguments,
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: PinotDBConnection) -> str:
@@ -41,8 +47,16 @@ def get_connection(connection: PinotDBConnection) -> Engine:
     """
     Create connection
     """
+    # TODO: Rename database field to DatabaseSchema
+    # Pinot does not support multi database concept
+    if connection.database is not None:
+        if not connection.connectionArguments:
+            connection.connectionArguments = init_empty_connection_arguments()
+        connection.connectionArguments.root["database"] = connection.database
+    connection_copy = deepcopy(connection)
+    connection_copy.database = None
     return create_generic_db_connection(
-        connection=connection,
+        connection=connection_copy,
         get_connection_url_fn=get_connection_url,
         get_connection_args_fn=get_connection_args_common,
     )
@@ -53,14 +67,16 @@ def test_connection(
     engine: Engine,
     service_connection: PinotDBConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/pinotdb/metadata.py b/ingestion/src/metadata/ingestion/source/database/pinotdb/metadata.py
index fed715470986..b0767679b2dd 100644
--- a/ingestion/src/metadata/ingestion/source/database/pinotdb/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/database/pinotdb/metadata.py
@@ -9,7 +9,10 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """PinotDb source module"""
-from typing import Optional
+from typing import Iterable, Optional
+
+from pinotdb import sqlalchemy as pinot_sqlalchemy
+from sqlalchemy import types
 
 from metadata.generated.schema.entity.services.connections.database.pinotDBConnection import (
     PinotDBConnection,
@@ -22,6 +25,31 @@
 from metadata.ingestion.source.database.common_db_source import CommonDbSourceService
 
 
+def get_type_custom(data_type, field_size):
+    type_map = {
+        "int": types.BigInteger,
+        "long": types.BigInteger,
+        "float": types.Float,
+        "double": types.Numeric,
+        # BOOLEAN, is added after release 0.7.1.
+        # In release 0.7.1 and older releases, BOOLEAN is equivalent to STRING.
+        "boolean": types.Boolean,
+        "timestamp": types.TIMESTAMP,
+        "string": types.String,
+        "json": types.JSON,
+        "bytes": types.LargeBinary,
+        "big_decimal": types.DECIMAL,
+        # Complex types
+        "struct": types.BLOB,
+        "map": types.BLOB,
+        "array": types.ARRAY,
+    }
+    return type_map.get(data_type.lower())
+
+
+pinot_sqlalchemy.get_type = get_type_custom
+
+
 class PinotdbSource(CommonDbSourceService):
     """
     Implements the necessary methods to extract
@@ -39,3 +67,15 @@ def create(
                 f"Expected PinotdbConnection, but got {connection}"
             )
         return cls(config, metadata)
+
+    def get_database_names(self) -> Iterable[str]:
+        """
+        Default case with a single database.
+
+        It might come informed - or not - from the source.
+
+        Sources with multiple databases should overwrite this and
+        apply the necessary filters.
+        """
+        # TODO: Add databaseDisplayName field in PinotDBConnection
+        yield "default"
diff --git a/ingestion/src/metadata/ingestion/source/database/postgres/connection.py b/ingestion/src/metadata/ingestion/source/database/postgres/connection.py
index 4ea066a8e8bb..f447f13dfb9e 100644
--- a/ingestion/src/metadata/ingestion/source/database/postgres/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/postgres/connection.py
@@ -27,6 +27,9 @@
 from metadata.generated.schema.entity.services.connections.database.postgresConnection import (
     PostgresConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -42,6 +45,7 @@
 from metadata.ingestion.source.database.postgres.utils import (
     get_postgres_time_column_name,
 )
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: PostgresConnection) -> Engine:
@@ -71,7 +75,8 @@ def test_connection(
     engine: Engine,
     service_connection: PostgresConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -84,10 +89,11 @@ def test_connection(
         "GetDatabases": POSTGRES_GET_DATABASE,
         "GetTags": POSTGRES_TEST_GET_TAGS,
     }
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
         queries=queries,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/presto/connection.py b/ingestion/src/metadata/ingestion/source/database/presto/connection.py
index 1847c3207c40..b45499700912 100644
--- a/ingestion/src/metadata/ingestion/source/database/presto/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/presto/connection.py
@@ -25,6 +25,9 @@
 from metadata.generated.schema.entity.services.connections.database.prestoConnection import (
     PrestoConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -38,6 +41,7 @@
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.presto.queries import PRESTO_SHOW_CATALOGS
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: PrestoConnection) -> str:
@@ -84,7 +88,8 @@ def test_connection(
     engine: Engine,
     service_connection: PrestoConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -108,9 +113,10 @@ def custom_executor_for_table():
         "GetTables": custom_executor_for_table,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/query_parser_source.py b/ingestion/src/metadata/ingestion/source/database/query_parser_source.py
index f502c576a3e3..20d8fbaec0dc 100644
--- a/ingestion/src/metadata/ingestion/source/database/query_parser_source.py
+++ b/ingestion/src/metadata/ingestion/source/database/query_parser_source.py
@@ -20,6 +20,9 @@
 )
 from metadata.generated.schema.type.tableQuery import TableQuery
 from metadata.ingestion.api.steps import Source
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_test_connection_fn
 from metadata.utils.helpers import get_start_and_end
@@ -121,4 +124,5 @@ def close(self):
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.engine)
+        result = test_connection_fn(self.engine)
+        raise_test_connection_exception(result)
diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/connection.py b/ingestion/src/metadata/ingestion/source/database/redshift/connection.py
index a9193d5be298..e904ba8a95db 100644
--- a/ingestion/src/metadata/ingestion/source/database/redshift/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/redshift/connection.py
@@ -24,6 +24,9 @@
 from metadata.generated.schema.entity.services.connections.database.redshiftConnection import (
     RedshiftConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -44,6 +47,7 @@
     REDSHIFT_TEST_GET_QUERIES,
     REDSHIFT_TEST_PARTITION_DETAILS,
 )
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: RedshiftConnection) -> Engine:
@@ -62,7 +66,8 @@ def test_connection(
     engine: Engine,
     service_connection: RedshiftConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -96,11 +101,14 @@ def test_get_queries_permissions(engine_: Engine):
         ),
     }
 
-    test_connection_steps(
+    result = test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
 
     kill_active_connections(engine)
+
+    return result
diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py b/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py
index 270eae010920..92267757dd91 100644
--- a/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/database/redshift/metadata.py
@@ -96,6 +96,7 @@
 )
 from metadata.utils.filters import filter_by_database
 from metadata.utils.helpers import get_start_and_end
+from metadata.utils.importer import import_side_effects
 from metadata.utils.logger import ingestion_logger
 from metadata.utils.sqlalchemy_utils import (
     get_all_table_comments,
@@ -105,6 +106,9 @@
 
 logger = ingestion_logger()
 
+import_side_effects(
+    "metadata.ingestion.source.database.redshift.profiler.system",
+)
 
 STANDARD_TABLE_TYPES = {
     "r": TableType.Regular,
diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/profiler/__init__.py b/ingestion/src/metadata/ingestion/source/database/redshift/profiler/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/profiler/system.py b/ingestion/src/metadata/ingestion/source/database/redshift/profiler/system.py
new file mode 100644
index 000000000000..8ea963e8ed4c
--- /dev/null
+++ b/ingestion/src/metadata/ingestion/source/database/redshift/profiler/system.py
@@ -0,0 +1,120 @@
+from typing import Dict, List
+
+from pydantic import TypeAdapter
+from sqlalchemy.orm import DeclarativeMeta, Session
+
+from metadata.generated.schema.entity.data.table import SystemProfile
+from metadata.ingestion.source.database.redshift.queries import (
+    STL_QUERY,
+    get_metric_result,
+    get_query_results,
+)
+from metadata.profiler.metrics.system.dml_operation import DatabaseDMLOperations
+from metadata.profiler.metrics.system.system import (
+    SYSTEM_QUERY_RESULT_CACHE,
+    get_system_metrics_for_dialect,
+)
+from metadata.profiler.orm.registry import Dialects
+from metadata.utils.logger import profiler_logger
+from metadata.utils.profiler_utils import get_value_from_cache, set_cache
+
+logger = profiler_logger()
+
+
+@get_system_metrics_for_dialect.register(Dialects.Redshift)
+def _(
+    dialect: str,
+    session: Session,
+    table: DeclarativeMeta,
+    *args,
+    **kwargs,
+) -> List[SystemProfile]:
+    """List all the DML operations for reshifts tables
+
+    Args:
+        dialect (str): redshift
+        session (Session): session object
+        table (DeclarativeMeta): orm table
+
+    Returns:
+        List[Dict]:
+    """
+    logger.debug(f"Fetching system metrics for {dialect}")
+    database = session.get_bind().url.database
+    schema = table.__table_args__["schema"]  # type: ignore
+
+    metric_results: List[Dict] = []
+
+    # get inserts ddl queries
+    inserts = get_value_from_cache(
+        SYSTEM_QUERY_RESULT_CACHE, f"{Dialects.Redshift}.{database}.{schema}.inserts"
+    )
+    if not inserts:
+        insert_query = STL_QUERY.format(
+            alias="si",
+            join_type="LEFT",
+            condition="sd.query is null",
+            database=database,
+            schema=schema,
+        )
+        inserts = get_query_results(
+            session,
+            insert_query,
+            DatabaseDMLOperations.INSERT.value,
+        )
+        set_cache(
+            SYSTEM_QUERY_RESULT_CACHE,
+            f"{Dialects.Redshift}.{database}.{schema}.inserts",
+            inserts,
+        )
+    metric_results.extend(get_metric_result(inserts, table.__tablename__))
+
+    # get deletes ddl queries
+    deletes = get_value_from_cache(
+        SYSTEM_QUERY_RESULT_CACHE, f"{Dialects.Redshift}.{database}.{schema}.deletes"
+    )
+    if not deletes:
+        delete_query = STL_QUERY.format(
+            alias="sd",
+            join_type="RIGHT",
+            condition="si.query is null",
+            database=database,
+            schema=schema,
+        )
+        deletes = get_query_results(
+            session,
+            delete_query,
+            DatabaseDMLOperations.DELETE.value,
+        )
+        set_cache(
+            SYSTEM_QUERY_RESULT_CACHE,
+            f"{Dialects.Redshift}.{database}.{schema}.deletes",
+            deletes,
+        )
+    metric_results.extend(get_metric_result(deletes, table.__tablename__))  # type: ignore
+
+    # get updates ddl queries
+    updates = get_value_from_cache(
+        SYSTEM_QUERY_RESULT_CACHE, f"{Dialects.Redshift}.{database}.{schema}.updates"
+    )
+    if not updates:
+        update_query = STL_QUERY.format(
+            alias="si",
+            join_type="INNER",
+            condition="sd.query is not null",
+            database=database,
+            schema=schema,
+        )
+        updates = get_query_results(
+            session,
+            update_query,
+            DatabaseDMLOperations.UPDATE.value,
+        )
+        set_cache(
+            SYSTEM_QUERY_RESULT_CACHE,
+            f"{Dialects.Redshift}.{database}.{schema}.updates",
+            updates,
+        )
+    metric_results.extend(get_metric_result(updates, table.__tablename__))  # type: ignore
+
+    return TypeAdapter(List[SystemProfile]).validate_python(metric_results)
diff --git a/ingestion/src/metadata/ingestion/source/database/redshift/queries.py b/ingestion/src/metadata/ingestion/source/database/redshift/queries.py
index 6a2210626ea8..65b11a400fe6 100644
--- a/ingestion/src/metadata/ingestion/source/database/redshift/queries.py
+++ b/ingestion/src/metadata/ingestion/source/database/redshift/queries.py
@@ -13,6 +13,13 @@
 """
 
 import textwrap
+from typing import List
+
+from sqlalchemy import text
+from sqlalchemy.orm.session import Session
+
+from metadata.utils.profiler_utils import QueryResult
+from metadata.utils.time_utils import datetime_to_timestamp
 
 # Not able to use SYS_QUERY_HISTORY here. Few users not getting any results
 REDSHIFT_SQL_STATEMENT = textwrap.dedent(
@@ -375,3 +382,85 @@
   and end_time >= '{start_date}'
 ORDER BY end_time DESC
 """
+
+
+STL_QUERY = """
+    with data as (
+        select
+            {alias}.*
+        from 
+            pg_catalog.stl_insert si
+            {join_type} join pg_catalog.stl_delete sd on si.query = sd.query
+        where 
+            {condition}
+    )
+	SELECT
+        SUM(data."rows") AS "rows",
+        sti."database",
+        sti."schema",
+        sti."table",
+        DATE_TRUNC('second', data.starttime) AS starttime
+    FROM
+        data
+        INNER JOIN  pg_catalog.svv_table_info sti ON data.tbl = sti.table_id
+    where
+        sti."database" = '{database}' AND
+       	sti."schema" = '{schema}' AND
+        "rows" != 0 AND
+        DATE(data.starttime) >= CURRENT_DATE - 1
+    GROUP BY 2,3,4,5
+    ORDER BY 5 DESC
+"""
+
+
+def get_query_results(
+    session: Session,
+    query,
+    operation,
+) -> List[QueryResult]:
+    """get query results either from cache or from the database
+
+    Args:
+        session (Session): session
+        query (_type_): query
+        operation (_type_): operation
+
+    Returns:
+        List[QueryResult]:
+    """
+    cursor = session.execute(text(query))
+    results = [
+        QueryResult(
+            database_name=row.database,
+            schema_name=row.schema,
+            table_name=row.table,
+            query_text=None,
+            query_type=operation,
+            start_time=row.starttime,
+            rows=row.rows,
+        )
+        for row in cursor
+    ]
+
+    return results
+
+
+def get_metric_result(ddls: List[QueryResult], table_name: str) -> List:
+    """Given query results, retur the metric result
+
+    Args:
+        ddls (List[QueryResult]): list of query results
+        table_name (str): table name
+
+    Returns:
+        List:
+    """
+    return [
+        {
+            "timestamp": datetime_to_timestamp(ddl.start_time, milliseconds=True),
+            "operation": ddl.query_type,
+            "rowsAffected": ddl.rows,
+        }
+        for ddl in ddls
+        if ddl.table_name == table_name
+    ]
diff --git a/ingestion/src/metadata/ingestion/source/database/salesforce/connection.py b/ingestion/src/metadata/ingestion/source/database/salesforce/connection.py
index 9cfd79e08846..6a41e4389c47 100644
--- a/ingestion/src/metadata/ingestion/source/database/salesforce/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/salesforce/connection.py
@@ -22,8 +22,12 @@
 from metadata.generated.schema.entity.services.connections.database.salesforceConnection import (
     SalesforceConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: SalesforceConnection) -> Salesforce:
@@ -48,16 +52,18 @@ def test_connection(
     client: Salesforce,
     service_connection: SalesforceConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
     test_fn = {"CheckAccess": client.describe}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py b/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py
index 5e7e203fabdb..a40ac76fa694 100644
--- a/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/database/salesforce/metadata.py
@@ -48,6 +48,9 @@
 from metadata.generated.schema.type.basic import EntityName, FullyQualifiedEntityName
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
@@ -203,15 +206,18 @@ def get_tables_name_and_type(self) -> Optional[Iterable[Tuple[str, str]]]:
                 )
             )
 
-    def get_table_description(self, table_name: str) -> Optional[str]:
+    def get_table_description(
+        self, table_name: str, object_label: Optional[str]
+    ) -> Optional[str]:
         """
         Method to get the table description for salesforce with Tooling API
         """
+        table_description = None
         try:
             result = self.client.toolingexecute(
                 f"query/?q=SELECT+Description+FROM+EntityDefinition+WHERE+QualifiedApiName='{table_name}'"
             )
-            return result["records"][0]["Description"]
+            table_description = result["records"][0]["Description"]
         except KeyError as err:
             logger.warning(
                 f"Unable to get required key from Tooling API response for table [{table_name}]: {err}"
@@ -225,7 +231,7 @@ def get_table_description(self, table_name: str) -> Optional[str]:
             logger.warning(
                 f"Unable to get description with Tooling API for table [{table_name}]: {exc}"
             )
-        return None
+        return table_description if table_description else object_label
 
     def yield_table(
         self, table_name_and_type: Tuple[str, TableType]
@@ -241,11 +247,13 @@ def yield_table(
                 f"sobjects/{table_name}/describe/",
                 params=None,
             )
-            columns = self.get_columns(salesforce_objects["fields"])
+            columns = self.get_columns(salesforce_objects.get("fields", []))
             table_request = CreateTableRequest(
                 name=EntityName(table_name),
                 tableType=table_type,
-                description=self.get_table_description(table_name),
+                description=self.get_table_description(
+                    table_name, salesforce_objects.get("label")
+                ),
                 columns=columns,
                 tableConstraints=table_constraints,
                 databaseSchema=FullyQualifiedEntityName(
@@ -370,4 +378,5 @@ def close(self):
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.client, self.service_connection)
+        result = test_connection_fn(self.client, self.service_connection)
+        raise_test_connection_exception(result)
diff --git a/ingestion/src/metadata/ingestion/source/database/saperp/connection.py b/ingestion/src/metadata/ingestion/source/database/saperp/connection.py
index ee2b2bff902e..ba5bac40dbb1 100644
--- a/ingestion/src/metadata/ingestion/source/database/saperp/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/saperp/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.database.sapErpConnection import (
     SapErpConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.saperp.client import SapErpClient
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -37,14 +41,16 @@ def test_connection(
     client: SapErpClient,
     service_connection: SapErpConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     test_fn = {
         "GetTables": client.test_table_api,
         "GetColumns": client.test_column_api,
     }
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/saphana/connection.py b/ingestion/src/metadata/ingestion/source/database/saphana/connection.py
index 18567eedb5ad..ae8fe0ce333c 100644
--- a/ingestion/src/metadata/ingestion/source/database/saphana/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/saphana/connection.py
@@ -30,6 +30,9 @@
 from metadata.generated.schema.entity.services.connections.database.sapHanaConnection import (
     SapHanaConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -41,6 +44,7 @@
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_database_connection_url(connection: SapHanaConnection) -> str:
@@ -156,15 +160,17 @@ def test_connection(
     engine: Engine,
     service_connection: SapHanaConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=_build_test_fn_dict(engine, service_connection),
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/saphana/lineage.py b/ingestion/src/metadata/ingestion/source/database/saphana/lineage.py
index 0c2bdf824adc..c474c4bfea6f 100644
--- a/ingestion/src/metadata/ingestion/source/database/saphana/lineage.py
+++ b/ingestion/src/metadata/ingestion/source/database/saphana/lineage.py
@@ -29,6 +29,9 @@
 )
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException, Source
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_test_connection_fn
 from metadata.ingestion.source.database.saphana.cdata_parser import (
@@ -156,4 +159,5 @@ def parse_cdata(
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.engine)
+        result = test_connection_fn(self.engine)
+        raise_test_connection_exception(result)
diff --git a/ingestion/src/metadata/ingestion/source/database/sas/connection.py b/ingestion/src/metadata/ingestion/source/database/sas/connection.py
index a11bd1204cd1..014c1286dd67 100644
--- a/ingestion/src/metadata/ingestion/source/database/sas/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/sas/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.database.sasConnection import (
     SASConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.sas.client import SASClient
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -37,11 +41,13 @@ def test_connection(
     client: SASClient,
     service_connection: SASConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     test_fn = {"CheckAccess": client.check_connection}
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/sas/metadata.py b/ingestion/src/metadata/ingestion/source/database/sas/metadata.py
index aadfc8c3c9fd..c6fbc8fb0c29 100644
--- a/ingestion/src/metadata/ingestion/source/database/sas/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/database/sas/metadata.py
@@ -71,6 +71,9 @@
 from metadata.ingestion.api.common import Entity
 from metadata.ingestion.api.models import Either, StackTraceError
 from metadata.ingestion.api.steps import InvalidSourceException
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
@@ -907,4 +910,7 @@ def close(self) -> None:
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
diff --git a/ingestion/src/metadata/ingestion/source/database/singlestore/connection.py b/ingestion/src/metadata/ingestion/source/database/singlestore/connection.py
index fdd371ff9f86..c3aa666f4ad0 100644
--- a/ingestion/src/metadata/ingestion/source/database/singlestore/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/singlestore/connection.py
@@ -23,6 +23,9 @@
 from metadata.generated.schema.entity.services.connections.database.singleStoreConnection import (
     SingleStoreConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -32,6 +35,7 @@
     test_connection_db_schema_sources,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: SingleStoreConnection) -> Engine:
@@ -50,14 +54,16 @@ def test_connection(
     engine: Engine,
     service_connection: SingleStoreConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_schema_sources(
+    return test_connection_db_schema_sources(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py
index 599a3f3815e3..5cbc9479fbf8 100644
--- a/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/snowflake/connection.py
@@ -28,6 +28,9 @@
 from metadata.generated.schema.entity.services.connections.database.snowflakeConnection import (
     SnowflakeConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -47,6 +50,7 @@
     SNOWFLAKE_TEST_GET_TABLES,
     SNOWFLAKE_TEST_GET_VIEWS,
 )
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -145,7 +149,8 @@ def test_connection(
     engine: Engine,
     service_connection: SnowflakeConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow.
@@ -190,11 +195,12 @@ def test_connection(
         ),
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
 
 
diff --git a/ingestion/src/metadata/ingestion/source/database/snowflake/profiler/system_metrics.py b/ingestion/src/metadata/ingestion/source/database/snowflake/profiler/system.py
similarity index 100%
rename from ingestion/src/metadata/ingestion/source/database/snowflake/profiler/system_metrics.py
rename to ingestion/src/metadata/ingestion/source/database/snowflake/profiler/system.py
diff --git a/ingestion/src/metadata/ingestion/source/database/sqlite/connection.py b/ingestion/src/metadata/ingestion/source/database/sqlite/connection.py
index 03b0e4b685b4..f7215cca7c6b 100644
--- a/ingestion/src/metadata/ingestion/source/database/sqlite/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/sqlite/connection.py
@@ -22,12 +22,16 @@
 from metadata.generated.schema.entity.services.connections.database.sqliteConnection import (
     SQLiteConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
 )
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: SQLiteConnection) -> str:
@@ -52,14 +56,16 @@ def test_connection(
     engine: Engine,
     service_connection: SQLiteConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/teradata/connection.py b/ingestion/src/metadata/ingestion/source/database/teradata/connection.py
index 014af8e288b3..be8d1e856bd9 100644
--- a/ingestion/src/metadata/ingestion/source/database/teradata/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/teradata/connection.py
@@ -24,6 +24,9 @@
 from metadata.generated.schema.entity.services.connections.database.teradataConnection import (
     TeradataConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -32,6 +35,7 @@
 from metadata.ingestion.connections.test_connections import test_connection_db_common
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.teradata.queries import TERADATA_GET_DATABASE
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: TeradataConnection) -> str:
@@ -82,17 +86,19 @@ def test_connection(
     engine: Engine,
     service_connection: TeradataConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
     queries = {"GetDatabases": TERADATA_GET_DATABASE}
 
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
         queries=queries,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/trino/connection.py b/ingestion/src/metadata/ingestion/source/database/trino/connection.py
index 1a7a3a5ebd59..6ee18950850a 100644
--- a/ingestion/src/metadata/ingestion/source/database/trino/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/trino/connection.py
@@ -29,6 +29,9 @@
 from metadata.generated.schema.entity.services.connections.database.trinoConnection import (
     TrinoConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -41,6 +44,7 @@
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.trino.queries import TRINO_GET_DATABASE
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection_url(connection: TrinoConnection) -> str:
@@ -124,7 +128,8 @@ def test_connection(
     engine: Engine,
     service_connection: TrinoConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -133,10 +138,11 @@ def test_connection(
         "GetDatabases": TRINO_GET_DATABASE,
     }
 
-    test_connection_db_schema_sources(
+    return test_connection_db_schema_sources(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
         queries=queries,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py
index 2ba24d3ac328..b17be4dd0222 100644
--- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/connection.py
@@ -23,10 +23,14 @@
 from metadata.generated.schema.entity.services.connections.database.unityCatalogConnection import (
     UnityCatalogConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.unitycatalog.client import UnityCatalogClient
 from metadata.ingestion.source.database.unitycatalog.models import DatabricksTable
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.db_utils import get_host_from_host_port
 from metadata.utils.logger import ingestion_logger
 
@@ -54,7 +58,8 @@ def test_connection(
     connection: WorkspaceClient,
     service_connection: UnityCatalogConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -92,10 +97,10 @@ def get_tables(connection: WorkspaceClient, table_obj: DatabricksTable):
         "GetQueries": client.test_query_api_access,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
-        timeout_seconds=service_connection.connectionTimeout,
+        timeout_seconds=service_connection.connectionTimeout or timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py b/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py
index cafde0b443f5..3189102eec3e 100644
--- a/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py
+++ b/ingestion/src/metadata/ingestion/source/database/unitycatalog/lineage.py
@@ -30,6 +30,9 @@
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException, Source
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.lineage.sql_lineage import get_column_fqn
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_test_connection_fn
@@ -159,4 +162,7 @@ def _iter(self, *_, **__) -> Iterable[Either[AddLineageRequest]]:
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
diff --git a/ingestion/src/metadata/ingestion/source/database/vertica/connection.py b/ingestion/src/metadata/ingestion/source/database/vertica/connection.py
index 93d515850ed3..945564d3ac7e 100644
--- a/ingestion/src/metadata/ingestion/source/database/vertica/connection.py
+++ b/ingestion/src/metadata/ingestion/source/database/vertica/connection.py
@@ -22,6 +22,9 @@
 from metadata.generated.schema.entity.services.connections.database.verticaConnection import (
     VerticaConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import (
     create_generic_db_connection,
     get_connection_args_common,
@@ -33,6 +36,7 @@
     VERTICA_LIST_DATABASES,
     VERTICA_TEST_GET_QUERIES,
 )
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: VerticaConnection) -> Engine:
@@ -51,7 +55,8 @@ def test_connection(
     engine: Engine,
     service_connection: VerticaConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -60,10 +65,11 @@ def test_connection(
         "GetQueries": VERTICA_TEST_GET_QUERIES,
         "GetDatabases": VERTICA_LIST_DATABASES,
     }
-    test_connection_db_common(
+    return test_connection_db_common(
         metadata=metadata,
         engine=engine,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
         queries=queries,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py b/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py
index ef8c4b0689ad..66c05a2d368e 100644
--- a/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py
+++ b/ingestion/src/metadata/ingestion/source/messaging/kafka/connection.py
@@ -29,8 +29,12 @@
 from metadata.generated.schema.entity.services.connections.messaging.redpandaConnection import (
     RedpandaConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -120,7 +124,8 @@ def test_connection(
     client: KafkaClient,
     service_connection: Union[KafkaConnection, RedpandaConnection],
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -149,9 +154,10 @@ def schema_registry_test():
         "CheckSchemaRegistry": schema_registry_test,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/messaging/kinesis/connection.py b/ingestion/src/metadata/ingestion/source/messaging/kinesis/connection.py
index dc5b3cc69a82..573dd986a65b 100644
--- a/ingestion/src/metadata/ingestion/source/messaging/kinesis/connection.py
+++ b/ingestion/src/metadata/ingestion/source/messaging/kinesis/connection.py
@@ -22,8 +22,12 @@
 from metadata.generated.schema.entity.services.connections.messaging.kinesisConnection import (
     KinesisConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -41,7 +45,8 @@ def test_connection(
     client,
     service_connection: KinesisConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -49,9 +54,10 @@ def test_connection(
 
     test_fn = {"GetTopics": client.list_streams}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py b/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py
index faa033784a0f..b6d5f0d8eb1f 100644
--- a/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py
+++ b/ingestion/src/metadata/ingestion/source/messaging/messaging_service.py
@@ -34,6 +34,9 @@
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.topology import (
     NodeStage,
@@ -198,7 +201,10 @@ def prepare(self):
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
 
     def mark_topics_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
         """Method to mark the topics as deleted"""
diff --git a/ingestion/src/metadata/ingestion/source/messaging/redpanda/connection.py b/ingestion/src/metadata/ingestion/source/messaging/redpanda/connection.py
index def0edd58848..34c01ceb9dcc 100644
--- a/ingestion/src/metadata/ingestion/source/messaging/redpanda/connection.py
+++ b/ingestion/src/metadata/ingestion/source/messaging/redpanda/connection.py
@@ -20,6 +20,9 @@
 from metadata.generated.schema.entity.services.connections.messaging.redpandaConnection import (
     RedpandaConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.messaging.kafka.connection import KafkaClient
 from metadata.ingestion.source.messaging.kafka.connection import (
@@ -28,6 +31,7 @@
 from metadata.ingestion.source.messaging.kafka.connection import (
     test_connection as test_kafka_connection,
 )
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
@@ -45,15 +49,17 @@ def test_connection(
     client: KafkaClient,
     service_connection: RedpandaConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
 
-    test_kafka_connection(
+    return test_kafka_connection(
         metadata=metadata,
         client=client,
         service_connection=service_connection,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/metadata/alationsink/connection.py b/ingestion/src/metadata/ingestion/source/metadata/alationsink/connection.py
index d3f6bfa4b743..64a6b1998925 100644
--- a/ingestion/src/metadata/ingestion/source/metadata/alationsink/connection.py
+++ b/ingestion/src/metadata/ingestion/source/metadata/alationsink/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.metadata.alationSinkConnection import (
     AlationSinkConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.metadata.alationsink.client import AlationSinkClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: AlationSinkConnection) -> AlationSinkClient:
@@ -37,7 +41,8 @@ def test_connection(
     client: AlationSinkClient,
     service_connection: AlationSinkConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -45,9 +50,10 @@ def test_connection(
 
     test_fn = {"CheckAccess": client.list_native_datasources}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/metadata/alationsink/metadata.py b/ingestion/src/metadata/ingestion/source/metadata/alationsink/metadata.py
index fa83c71ddd52..c29b81f7173b 100644
--- a/ingestion/src/metadata/ingestion/source/metadata/alationsink/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/metadata/alationsink/metadata.py
@@ -33,6 +33,9 @@
 )
 from metadata.ingestion.api.models import Either, Entity
 from metadata.ingestion.api.steps import InvalidSourceException, Source
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.ometa.utils import model_str
 from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
@@ -459,6 +462,7 @@ def close(self):
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(
+        result = test_connection_fn(
             self.metadata, self.alation_sink_client, self.service_connection
         )
+        raise_test_connection_exception(result)
diff --git a/ingestion/src/metadata/ingestion/source/metadata/amundsen/connection.py b/ingestion/src/metadata/ingestion/source/metadata/amundsen/connection.py
index 2862421b8efe..8eabc792c96e 100644
--- a/ingestion/src/metadata/ingestion/source/metadata/amundsen/connection.py
+++ b/ingestion/src/metadata/ingestion/source/metadata/amundsen/connection.py
@@ -21,6 +21,9 @@
 from metadata.generated.schema.entity.services.connections.metadata.amundsenConnection import (
     AmundsenConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
@@ -30,6 +33,7 @@
 from metadata.ingestion.source.metadata.amundsen.queries import (
     NEO4J_AMUNDSEN_USER_QUERY,
 )
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: AmundsenConnection) -> Neo4jHelper:
@@ -56,7 +60,8 @@ def test_connection(
     client: Neo4jHelper,
     service_connection: AmundsenConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -66,9 +71,10 @@ def test_connection(
         "CheckAccess": partial(client.execute_query, query=NEO4J_AMUNDSEN_USER_QUERY)
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/metadata/amundsen/metadata.py b/ingestion/src/metadata/ingestion/source/metadata/amundsen/metadata.py
index a561aa7ab136..8c9d077e434d 100644
--- a/ingestion/src/metadata/ingestion/source/metadata/amundsen/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/metadata/amundsen/metadata.py
@@ -56,6 +56,9 @@
 from metadata.ingestion.api.common import Entity
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException, Source
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.user import OMetaUserProfile
 from metadata.ingestion.ometa.client_utils import get_chart_entities_from_id
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
@@ -460,4 +463,7 @@ def get_database_service(self, service_name: str) -> DatabaseService:
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
diff --git a/ingestion/src/metadata/ingestion/source/metadata/atlas/connection.py b/ingestion/src/metadata/ingestion/source/metadata/atlas/connection.py
index 66b0d259b78c..4a5f934a051b 100644
--- a/ingestion/src/metadata/ingestion/source/metadata/atlas/connection.py
+++ b/ingestion/src/metadata/ingestion/source/metadata/atlas/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.metadata.atlasConnection import (
     AtlasConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.metadata.atlas.client import AtlasClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: AtlasConnection) -> AtlasClient:
@@ -37,7 +41,8 @@ def test_connection(
     client: AtlasClient,
     service_connection: AtlasConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -45,9 +50,10 @@ def test_connection(
 
     test_fn = {"CheckAccess": client.list_entities}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/metadata/atlas/metadata.py b/ingestion/src/metadata/ingestion/source/metadata/atlas/metadata.py
index 1f967e35a921..2711c5ca1469 100644
--- a/ingestion/src/metadata/ingestion/source/metadata/atlas/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/metadata/atlas/metadata.py
@@ -41,6 +41,9 @@
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either, Entity, StackTraceError
 from metadata.ingestion.api.steps import InvalidSourceException, Source
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.connections import get_connection, get_test_connection_fn
@@ -477,4 +480,7 @@ def get_lineage_entity_ref(
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
diff --git a/ingestion/src/metadata/ingestion/source/mlmodel/mlflow/connection.py b/ingestion/src/metadata/ingestion/source/mlmodel/mlflow/connection.py
index b12c3a221767..3957227936ad 100644
--- a/ingestion/src/metadata/ingestion/source/mlmodel/mlflow/connection.py
+++ b/ingestion/src/metadata/ingestion/source/mlmodel/mlflow/connection.py
@@ -22,8 +22,12 @@
 from metadata.generated.schema.entity.services.connections.mlmodel.mlflowConnection import (
     MlflowConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: MlflowConnection) -> MlflowClient:
@@ -41,7 +45,8 @@ def test_connection(
     client: MlflowClient,
     service_connection: MlflowConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -49,9 +54,10 @@ def test_connection(
 
     test_fn = {"GetModels": client.search_registered_models}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/mlmodel/mlmodel_service.py b/ingestion/src/metadata/ingestion/source/mlmodel/mlmodel_service.py
index f49baae8cf97..e9517f819895 100644
--- a/ingestion/src/metadata/ingestion/source/mlmodel/mlmodel_service.py
+++ b/ingestion/src/metadata/ingestion/source/mlmodel/mlmodel_service.py
@@ -38,6 +38,9 @@
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.topology import (
     NodeStage,
@@ -176,7 +179,10 @@ def close(self):
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
 
     def mark_mlmodels_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
         """Method to mark the mlmodels as deleted"""
diff --git a/ingestion/src/metadata/ingestion/source/mlmodel/sagemaker/connection.py b/ingestion/src/metadata/ingestion/source/mlmodel/sagemaker/connection.py
index f2dd565494c8..ddf5f308e5e8 100644
--- a/ingestion/src/metadata/ingestion/source/mlmodel/sagemaker/connection.py
+++ b/ingestion/src/metadata/ingestion/source/mlmodel/sagemaker/connection.py
@@ -21,8 +21,12 @@
 from metadata.generated.schema.entity.services.connections.mlmodel.sageMakerConnection import (
     SageMakerConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: SageMakerConnection):
@@ -37,7 +41,8 @@ def test_connection(
     client,
     service_connection: SageMakerConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -45,9 +50,10 @@ def test_connection(
 
     test_fn = {"GetModels": client.list_models}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airbyte/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/airbyte/connection.py
index 099c84fa7557..03393d90fea1 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/airbyte/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/airbyte/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.pipeline.airbyteConnection import (
     AirbyteConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.airbyte.client import AirbyteClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: AirbyteConnection) -> AirbyteClient:
@@ -37,7 +41,8 @@ def test_connection(
     client: AirbyteClient,
     service_connection: AirbyteConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -45,9 +50,10 @@ def test_connection(
 
     test_fn = {"GetPipelines": client.list_workspaces}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py
index 705030cb9de7..70dacd993d54 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/airflow/connection.py
@@ -36,12 +36,16 @@
 from metadata.generated.schema.entity.services.connections.pipeline.backendConnection import (
     BackendConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_engine_step,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 # Only import when needed
@@ -103,7 +107,8 @@ def test_connection(
     engine: Engine,
     service_connection: AirflowConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -111,9 +116,10 @@ def test_connection(
 
     test_fn = {"CheckAccess": partial(test_connection_engine_step, engine)}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airflow/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/airflow/metadata.py
index 3d20858099c7..9e3bd5d7541b 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/airflow/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/airflow/metadata.py
@@ -15,7 +15,7 @@
 from collections import Counter
 from datetime import datetime
 from enum import Enum
-from typing import Iterable, List, Optional, cast
+from typing import Dict, Iterable, List, Optional, cast
 
 from airflow.models import BaseOperator, DagRun, TaskInstance
 from airflow.models.dag import DagModel
@@ -142,6 +142,16 @@ def session(self) -> Session:
 
         return self._session
 
+    @staticmethod
+    def _extract_serialized_task(task: Dict) -> Dict:
+        """
+        Given the serialization changes introduced in Airflow 2.10,
+        ensure compatibility with all versions.
+        """
+        if task.keys() == {"__var", "__type"}:
+            return task["__var"]
+        return task
+
     def get_pipeline_status(self, dag_id: str) -> List[DagRun]:
         """
         Return the DagRuns of given dag
@@ -328,7 +338,9 @@ def get_pipelines_list(self) -> Iterable[AirflowDagDetails]:
                     max_active_runs=data.get("max_active_runs", None),
                     description=data.get("_description", None),
                     start_date=data.get("start_date", None),
-                    tasks=data.get("tasks", []),
+                    tasks=list(
+                        map(self._extract_serialized_task, data.get("tasks", []))
+                    ),
                     schedule_interval=get_schedule_interval(data),
                     owner=self.fetch_dag_owners(data),
                 )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py
index 021d14b935d9..b17564bd2fa1 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/dagster/connection.py
@@ -20,10 +20,14 @@
 from metadata.generated.schema.entity.services.connections.pipeline.dagsterConnection import (
     DagsterConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.dagster.client import DagsterClient
 from metadata.ingestion.source.pipeline.dagster.queries import TEST_QUERY_GRAPHQL
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: DagsterConnection) -> DagsterClient:
@@ -38,7 +42,8 @@ def test_connection(
     client: DagsterClient,
     service_connection: DagsterConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -49,9 +54,10 @@ def custom_executor_for_pipeline():
 
     test_fn = {"GetPipelines": custom_executor_for_pipeline}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/connection.py
index 0ba178943ff9..adbb9707ddcf 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/databrickspipeline/connection.py
@@ -21,9 +21,13 @@
 from metadata.generated.schema.entity.services.connections.pipeline.databricksPipelineConnection import (
     DatabricksPipelineConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.database.databricks.client import DatabricksClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: DatabricksPipelineConnection) -> DatabricksClient:
@@ -38,7 +42,8 @@ def test_connection(
     client: DatabricksClient,
     service_connection: DatabricksPipelineConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -46,9 +51,10 @@ def test_connection(
 
     test_fn = {"GetPipelines": client.list_jobs_test_connection}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/connection.py
index 04e3ff56d8b5..0c98d03bd84e 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/dbtcloud/connection.py
@@ -22,9 +22,13 @@
 from metadata.generated.schema.entity.services.connections.pipeline.dbtCloudConnection import (
     DBTCloudConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.dbtcloud.client import DBTCloudClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: DBTCloudConnection) -> DBTCloudClient:
@@ -39,7 +43,8 @@ def test_connection(
     client: DBTCloudClient,
     service_connection: DBTCloudConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -52,9 +57,10 @@ def test_connection(
         "GetRuns": partial(client.get_runs, job_id=job_id),
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/connection.py
index e25a10cc767e..eed706734aeb 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/domopipeline/connection.py
@@ -24,11 +24,15 @@
 from metadata.generated.schema.entity.services.connections.pipeline.domoPipelineConnection import (
     DomoPipelineConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: DomoPipelineConnection) -> Domo:
@@ -47,7 +51,8 @@ def test_connection(
     connection: Domo,
     service_connection: DomoPipelineConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -59,9 +64,10 @@ def custom_executor():
 
     test_fn = {"GetPipelines": custom_executor}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/fivetran/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/fivetran/connection.py
index 644e19377004..5096b62f0268 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/fivetran/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/fivetran/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.pipeline.fivetranConnection import (
     FivetranConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.fivetran.client import FivetranClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: FivetranConnection) -> FivetranClient:
@@ -37,7 +41,8 @@ def test_connection(
     client: FivetranClient,
     service_connection: FivetranConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -45,9 +50,10 @@ def test_connection(
 
     test_fn = {"GetPipelines": client.list_groups}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/flink/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/flink/connection.py
index db5cc77b2b55..a633b3096571 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/flink/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/flink/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.pipeline.flinkConnection import (
     FlinkConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.flink.client import FlinkClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: FlinkConnection) -> FlinkClient:
@@ -37,16 +41,18 @@ def test_connection(
     client: FlinkClient,
     service_connection: FlinkConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
     """
     test_fn = {"GetPipelines": client.get_jobs}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/connection.py
index 725277bbc514..b953934147b8 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/connection.py
@@ -22,8 +22,12 @@
 from metadata.generated.schema.entity.services.connections.pipeline.gluePipelineConnection import (
     GluePipelineConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: GluePipelineConnection):
@@ -38,7 +42,8 @@ def test_connection(
     client,
     service_connection: GluePipelineConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -46,9 +51,10 @@ def test_connection(
 
     test_fn = {"GetPipelines": client.list_workflows}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/metadata.py
index 94dc58f7c563..66ef26f27211 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/metadata.py
@@ -25,6 +25,7 @@
     Task,
     TaskStatus,
 )
+from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.connections.pipeline.gluePipelineConnection import (
     GluePipelineConnection,
 )
@@ -40,14 +41,25 @@
     SourceUrl,
     Timestamp,
 )
+from metadata.generated.schema.type.entityLineage import EntitiesEdge, LineageDetails
+from metadata.generated.schema.type.entityLineage import Source as LineageSource
+from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import InvalidSourceException
 from metadata.ingestion.models.pipeline_status import OMetaPipelineStatus
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.ingestion.source.pipeline.gluepipeline.models import (
+    AmazonRedshift,
+    CatalogSource,
+    JDBCSource,
+    JobNodeResponse,
+    S3Source,
+    S3Target,
+)
 from metadata.ingestion.source.pipeline.pipeline_service import PipelineServiceSource
 from metadata.utils import fqn
 from metadata.utils.logger import ingestion_logger
-from metadata.utils.time_utils import convert_timestamp_to_milliseconds
+from metadata.utils.time_utils import datetime_to_timestamp
 
 logger = ingestion_logger()
 
@@ -63,6 +75,28 @@
     "incomplete": StatusType.Failed,
     "pending": StatusType.Pending,
 }
+TABLE_MODEL_MAP = {
+    "AmazonRedshiftSource": AmazonRedshift,
+    "AmazonRedshiftTarget": AmazonRedshift,
+    "AthenaConnectorSource": JDBCSource,
+    "JDBCConnectorSource": JDBCSource,
+    "JDBCConnectorTarget": JDBCSource,
+    "DirectJDBCSource": CatalogSource,
+    "RedshiftSource": CatalogSource,
+    "RedshiftTarget": CatalogSource,
+    "DirectJDBC": CatalogSource,
+}
+STORAGE_MODEL_MAP = {
+    "S3CsvSource": S3Source,
+    "S3JsonSource": S3Source,
+    "S3ParquetSource": S3Source,
+    "S3HudiSource": S3Source,
+    "S3DeltaSource": S3Source,
+    "S3DirectTarget": S3Target,
+    "S3DeltaDirectTarget": S3Target,
+    "S3GlueParquetTarget": S3Target,
+    "S3HudiDirectTarget": S3Target,
+}
 
 
 class GluepipelineSource(PipelineServiceSource):
@@ -145,9 +179,88 @@ def get_downstream_tasks(self, task_unique_id, tasks):
                 downstream_tasks.append(self.task_id_mapping[edges["DestinationId"]])
         return downstream_tasks
 
+    def get_lineage_details(self, job) -> Optional[dict]:
+        """
+        Get the Lineage Details of the pipeline
+        """
+        lineage_details = {"sources": [], "targets": []}
+        try:
+            job_details = JobNodeResponse.model_validate(
+                self.glue.get_job(JobName=job)
+            ).Job
+            if job_details and job_details.config_nodes:
+                nodes = job_details.config_nodes
+                for _, node in nodes.items():
+                    for key, entity in node.items():
+                        table_model, storage_model = None, None
+                        if key in TABLE_MODEL_MAP:
+                            table_model = TABLE_MODEL_MAP[key].model_validate(entity)
+                        elif "Catalog" in key:
+                            table_model = CatalogSource.model_validate(entity)
+                        elif key in STORAGE_MODEL_MAP:
+                            storage_model = STORAGE_MODEL_MAP[key].model_validate(
+                                entity
+                            )
+                        if table_model:
+                            for db_service_name in self.get_db_service_names():
+                                table_entity = self.metadata.get_entity_reference(
+                                    entity=Table,
+                                    fqn=fqn.build(
+                                        metadata=self.metadata,
+                                        entity_type=Table,
+                                        table_name=table_model.table_name,
+                                        database_name=table_model.database_name,
+                                        schema_name=table_model.schema_name,
+                                        service_name=db_service_name,
+                                    ),
+                                )
+                                if table_entity:
+                                    if key.endswith("Source"):
+                                        lineage_details["sources"].append(table_entity)
+                                    else:
+                                        lineage_details["targets"].append(table_entity)
+                                    break
+                        if storage_model:
+                            for path in storage_model.Paths or [storage_model.Path]:
+                                container = self.metadata.es_search_container_by_path(
+                                    full_path=path
+                                )
+                                if container and container[0]:
+                                    storage_entity = EntityReference(
+                                        id=container[0].id,
+                                        type="container",
+                                        name=container[0].name.root,
+                                        fullyQualifiedName=container[
+                                            0
+                                        ].fullyQualifiedName.root,
+                                    )
+                                    if storage_entity:
+                                        if key.endswith("Source"):
+                                            lineage_details["sources"].append(
+                                                storage_entity
+                                            )
+                                        else:
+                                            lineage_details["targets"].append(
+                                                storage_entity
+                                            )
+                                        break
+
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Failed to get lineage details for job : {job} due to : {exc}"
+            )
+        return lineage_details
+
     def yield_pipeline_status(
         self, pipeline_details: Any
     ) -> Iterable[Either[OMetaPipelineStatus]]:
+        pipeline_fqn = fqn.build(
+            metadata=self.metadata,
+            entity_type=Pipeline,
+            service_name=self.context.get().pipeline_service,
+            pipeline_name=self.context.get().pipeline,
+        )
         for job in self.job_name_list:
             try:
                 runs = self.glue.get_job_runs(JobName=job)
@@ -161,13 +274,13 @@ def yield_pipeline_status(
                                 attempt["JobRunState"].lower(), StatusType.Pending
                             ).value,
                             startTime=Timestamp(
-                                convert_timestamp_to_milliseconds(
-                                    attempt["StartedOn"].timestamp()
+                                datetime_to_timestamp(
+                                    attempt["StartedOn"], milliseconds=True
                                 )
                             ),
                             endTime=Timestamp(
-                                convert_timestamp_to_milliseconds(
-                                    attempt["CompletedOn"].timestamp()
+                                datetime_to_timestamp(
+                                    attempt["CompletedOn"], milliseconds=True
                                 )
                             ),
                         )
@@ -175,20 +288,14 @@ def yield_pipeline_status(
                     pipeline_status = PipelineStatus(
                         taskStatus=task_status,
                         timestamp=Timestamp(
-                            convert_timestamp_to_milliseconds(
-                                attempt["StartedOn"].timestamp()
+                            datetime_to_timestamp(
+                                attempt["StartedOn"], milliseconds=True
                             )
                         ),
                         executionStatus=STATUS_MAP.get(
                             attempt["JobRunState"].lower(), StatusType.Pending
                         ).value,
                     )
-                    pipeline_fqn = fqn.build(
-                        metadata=self.metadata,
-                        entity_type=Pipeline,
-                        service_name=self.context.get().pipeline_service,
-                        pipeline_name=self.context.get().pipeline,
-                    )
                     yield Either(
                         right=OMetaPipelineStatus(
                             pipeline_fqn=pipeline_fqn,
@@ -199,7 +306,7 @@ def yield_pipeline_status(
                 yield Either(
                     left=StackTraceError(
                         name=pipeline_fqn,
-                        error=f"Failed to yield pipeline status: {exc}",
+                        error=f"Failed to yield pipeline status for job {job}: {exc}",
                         stackTrace=traceback.format_exc(),
                     )
                 )
@@ -210,3 +317,42 @@ def yield_pipeline_lineage_details(
         """
         Get lineage between pipeline and data sources
         """
+        try:
+            pipeline_fqn = fqn.build(
+                metadata=self.metadata,
+                entity_type=Pipeline,
+                service_name=self.context.get().pipeline_service,
+                pipeline_name=self.context.get().pipeline,
+            )
+
+            pipeline_entity = self.metadata.get_by_name(
+                entity=Pipeline, fqn=pipeline_fqn
+            )
+
+            lineage_details = LineageDetails(
+                pipeline=EntityReference(id=pipeline_entity.id.root, type="pipeline"),
+                source=LineageSource.PipelineLineage,
+            )
+
+            for job in self.job_name_list:
+                lineage_enities = self.get_lineage_details(job)
+                for source in lineage_enities.get("sources"):
+                    for target in lineage_enities.get("targets"):
+                        yield Either(
+                            right=AddLineageRequest(
+                                edge=EntitiesEdge(
+                                    fromEntity=source,
+                                    toEntity=target,
+                                    lineageDetails=lineage_details,
+                                )
+                            )
+                        )
+
+        except Exception as exc:
+            yield Either(
+                left=StackTraceError(
+                    name=pipeline_details.get(NAME),
+                    error=f"Wild error ingesting pipeline lineage {pipeline_details} - {exc}",
+                    stackTrace=traceback.format_exc(),
+                )
+            )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/models.py b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/models.py
new file mode 100644
index 000000000000..84090b1febe8
--- /dev/null
+++ b/ingestion/src/metadata/ingestion/source/pipeline/gluepipeline/models.py
@@ -0,0 +1,78 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Glue Pipeline Source Model module
+"""
+
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class EntityDetails(BaseModel):
+    Value: str
+
+
+class SourceDetails(BaseModel):
+    schema_details: EntityDetails = Field(alias="Schema")
+    table_details: EntityDetails = Field(alias="Table")
+
+
+class AmazonRedshift(BaseModel):
+    Name: str
+    Data: SourceDetails
+    database_name: Optional[str] = None
+
+    @property
+    def table_name(self):
+        if self.Data:
+            return self.Data.table_details.Value
+        return None
+
+    @property
+    def schema_name(self):
+        if self.Data:
+            return self.Data.schema_details.Value
+        return None
+
+
+class CatalogSource(BaseModel):
+    Name: str
+    database_name: str = Field(alias="Database")
+    schema_name: Optional[str] = None
+    table_name: str = Field(alias="Table")
+
+
+class JDBCSource(BaseModel):
+    Name: str
+    schema_name: Optional[str] = Field(default=None, alias="SchemaName")
+    database_name: Optional[str] = None
+    table_name: str = Field(alias="ConnectionTable")
+
+
+class S3Source(BaseModel):
+    Name: str
+    Paths: List[str]
+
+
+class S3Target(BaseModel):
+    Name: str
+    Path: str
+    Paths: Optional[str] = None
+
+
+class JobNodes(BaseModel):
+    config_nodes: Optional[dict] = Field(alias="CodeGenConfigurationNodes")
+
+
+class JobNodeResponse(BaseModel):
+    Job: Optional[JobNodes] = None
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/connection.py
index 27fda4bf3de8..38ea63920f0b 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/kafkaconnect/connection.py
@@ -21,9 +21,13 @@
 from metadata.generated.schema.entity.services.connections.pipeline.kafkaConnectConnection import (
     KafkaConnectConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.kafkaconnect.client import KafkaConnectClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: KafkaConnectConnection) -> KafkaConnectClient:
@@ -38,7 +42,8 @@ def test_connection(
     client: KafkaConnectClient,
     service_connection: KafkaConnectConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -50,9 +55,10 @@ def test_connection(
         "GetPlugins": client.get_connector_plugins,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py
index dae1f25a524a..097b8cc1dbb4 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/nifi/connection.py
@@ -21,9 +21,13 @@
     BasicAuthentication,
     NifiConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.nifi.client import NifiClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: NifiConnection) -> NifiClient:
@@ -53,7 +57,8 @@ def test_connection(
     client: NifiClient,
     service_connection: NifiConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -64,9 +69,10 @@ def custom_executor():
 
     test_fn = {"GetPipelines": custom_executor}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/openlineage/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/openlineage/connection.py
index 74d488d77430..ec7abe8b80a1 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/openlineage/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/openlineage/connection.py
@@ -26,11 +26,15 @@
 from metadata.generated.schema.entity.services.connections.pipeline.openLineageConnection import (
     SecurityProtocol as KafkaSecProtocol,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import (
     SourceConnectionException,
     test_connection_steps,
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: OpenLineageConnection) -> KafkaConsumer:
@@ -76,7 +80,8 @@ def test_connection(
     client: KafkaConsumer,
     service_connection: OpenLineageConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -89,9 +94,10 @@ def custom_executor():
 
     test_fn = {"GetWatermarkOffsets": custom_executor}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/pipeline_service.py b/ingestion/src/metadata/ingestion/source/pipeline/pipeline_service.py
index 40449f08bc1d..d9e8129aac91 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/pipeline_service.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/pipeline_service.py
@@ -35,6 +35,9 @@
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.models.ometa_lineage import OMetaLineageRequest
@@ -251,7 +254,10 @@ def get_pipeline(self) -> Any:
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
 
     def register_record(self, pipeline_request: CreatePipelineRequest) -> None:
         """Mark the pipeline record as scanned and update the pipeline_source_state"""
@@ -285,6 +291,16 @@ def get_db_service_names(self) -> List[str]:
             else []
         )
 
+    def get_storage_service_names(self) -> List[str]:
+        """
+        Get the list of storage service names
+        """
+        return (
+            self.source_config.lineageInformation.storageServiceNames or []
+            if self.source_config.lineageInformation
+            else []
+        )
+
     def prepare(self):
         """
         Method to implement any required logic before starting the ingestion process
diff --git a/ingestion/src/metadata/ingestion/source/pipeline/spline/connection.py b/ingestion/src/metadata/ingestion/source/pipeline/spline/connection.py
index aeb1b81e67da..acee709f941f 100644
--- a/ingestion/src/metadata/ingestion/source/pipeline/spline/connection.py
+++ b/ingestion/src/metadata/ingestion/source/pipeline/spline/connection.py
@@ -20,9 +20,13 @@
 from metadata.generated.schema.entity.services.connections.pipeline.splineConnection import (
     SplineConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.pipeline.spline.client import SplineClient
+from metadata.utils.constants import THREE_MIN
 
 
 def get_connection(connection: SplineConnection) -> SplineClient:
@@ -38,7 +42,8 @@ def test_connection(
     client: SplineClient,
     service_connection: SplineConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -46,9 +51,10 @@ def test_connection(
 
     test_fn = {"GetPipelines": client.get_pipelines_test_connection}
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py b/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py
index 52df2e9e3d29..19c1b2e2c15a 100644
--- a/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py
+++ b/ingestion/src/metadata/ingestion/source/search/elasticsearch/connection.py
@@ -40,10 +40,13 @@
 from metadata.generated.schema.entity.services.connections.search.elasticSearchConnection import (
     ElasticsearchConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.builders import init_empty_connection_arguments
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
-from metadata.utils.constants import UTF_8
+from metadata.utils.constants import THREE_MIN, UTF_8
 from metadata.utils.helpers import init_staging_dir
 
 CA_CERT_FILE_NAME = "root.pem"
@@ -185,7 +188,8 @@ def test_connection(
     client: Elasticsearch,
     service_connection: ElasticsearchConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -199,9 +203,10 @@ def test_get_search_indexes():
         "GetSearchIndexes": test_get_search_indexes,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/search/search_service.py b/ingestion/src/metadata/ingestion/source/search/search_service.py
index ce49c3245a1f..6bf2eaadd374 100644
--- a/ingestion/src/metadata/ingestion/source/search/search_service.py
+++ b/ingestion/src/metadata/ingestion/source/search/search_service.py
@@ -41,6 +41,9 @@
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.search_index_data import OMetaIndexSampleData
 from metadata.ingestion.models.topology import (
@@ -192,7 +195,10 @@ def prepare(self):
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
 
     def mark_search_indexes_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
         """Method to mark the search index as deleted"""
diff --git a/ingestion/src/metadata/ingestion/source/storage/gcs/connection.py b/ingestion/src/metadata/ingestion/source/storage/gcs/connection.py
index a11d156c8383..5d1f78ade555 100644
--- a/ingestion/src/metadata/ingestion/source/storage/gcs/connection.py
+++ b/ingestion/src/metadata/ingestion/source/storage/gcs/connection.py
@@ -22,6 +22,9 @@
 from metadata.generated.schema.entity.services.connections.storage.gcsConnection import (
     GcsConnection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.generated.schema.security.credentials.gcpValues import (
     GcpCredentialsValues,
     SingleProjectId,
@@ -32,6 +35,7 @@
 )
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.ingestion.source.storage.gcs.client import MultiProjectClient
+from metadata.utils.constants import THREE_MIN
 from metadata.utils.credentials import set_google_credentials
 from metadata.utils.logger import ingestion_logger
 
@@ -136,7 +140,8 @@ def test_connection(
     client: GcsObjectStoreClient,
     service_connection: GcsConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -151,9 +156,10 @@ def test_connection(
         "GetMetrics": tester.get_metrics,
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/storage/s3/connection.py b/ingestion/src/metadata/ingestion/source/storage/s3/connection.py
index 2df0b9a8ff44..d2ba001b6ea9 100644
--- a/ingestion/src/metadata/ingestion/source/storage/s3/connection.py
+++ b/ingestion/src/metadata/ingestion/source/storage/s3/connection.py
@@ -27,8 +27,12 @@
 from metadata.generated.schema.entity.services.connections.storage.s3Connection import (
     S3Connection,
 )
+from metadata.generated.schema.entity.services.connections.testConnectionResult import (
+    TestConnectionResult,
+)
 from metadata.ingestion.connections.test_connections import test_connection_steps
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils.constants import THREE_MIN
 
 
 @dataclass
@@ -53,7 +57,8 @@ def test_connection(
     client: S3ObjectStoreClient,
     service_connection: S3Connection,
     automation_workflow: Optional[AutomationWorkflow] = None,
-) -> None:
+    timeout_seconds: Optional[int] = THREE_MIN,
+) -> TestConnectionResult:
     """
     Test connection. This can be executed either as part
     of a metadata workflow or during an Automation Workflow
@@ -75,9 +80,10 @@ def test_buckets(connection: S3Connection, client: S3ObjectStoreClient):
         ),
     }
 
-    test_connection_steps(
+    return test_connection_steps(
         metadata=metadata,
         test_fn=test_fn,
         service_type=service_connection.type.value,
         automation_workflow=automation_workflow,
+        timeout_seconds=timeout_seconds,
     )
diff --git a/ingestion/src/metadata/ingestion/source/storage/storage_service.py b/ingestion/src/metadata/ingestion/source/storage/storage_service.py
index 5afb3b19b1d7..9a57e5231253 100644
--- a/ingestion/src/metadata/ingestion/source/storage/storage_service.py
+++ b/ingestion/src/metadata/ingestion/source/storage/storage_service.py
@@ -40,6 +40,9 @@
 from metadata.ingestion.api.models import Either
 from metadata.ingestion.api.steps import Source
 from metadata.ingestion.api.topology_runner import TopologyRunnerMixin
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.models.delete_entity import DeleteEntity
 from metadata.ingestion.models.ometa_classification import OMetaTagAndClassification
 from metadata.ingestion.models.topology import (
@@ -236,7 +239,10 @@ def register_record(self, container_request: CreateContainerRequest) -> None:
 
     def test_connection(self) -> None:
         test_connection_fn = get_test_connection_fn(self.service_connection)
-        test_connection_fn(self.metadata, self.connection_obj, self.service_connection)
+        result = test_connection_fn(
+            self.metadata, self.connection_obj, self.service_connection
+        )
+        raise_test_connection_exception(result)
 
     def mark_containers_as_deleted(self) -> Iterable[Either[DeleteEntity]]:
         """Method to mark the containers as deleted"""
diff --git a/ingestion/src/metadata/parsers/json_schema_parser.py b/ingestion/src/metadata/parsers/json_schema_parser.py
index 412d3be013b1..716fd237984e 100644
--- a/ingestion/src/metadata/parsers/json_schema_parser.py
+++ b/ingestion/src/metadata/parsers/json_schema_parser.py
@@ -16,7 +16,7 @@
 import json
 import traceback
 from enum import Enum
-from typing import List, Optional, Type
+from typing import List, Optional, Tuple, Type
 
 from pydantic import BaseModel
 
@@ -66,6 +66,49 @@ def parse_json_schema(
     return None
 
 
+def get_child_models(key, value, field_models, cls: Type[BaseModel] = FieldModel):
+    """
+    Method to parse the child objects in the json schema
+    """
+    try:
+        cls_obj = cls(
+            name=key,
+            displayName=value.get("title"),
+            dataType=JsonSchemaDataTypes(value.get("type", "unknown")).name,
+            description=value.get("description"),
+        )
+        children = None
+        if value.get("type") == JsonSchemaDataTypes.RECORD.value:
+            children = get_json_schema_fields(value.get("properties"), cls=cls)
+        if value.get("type") == JsonSchemaDataTypes.ARRAY.value:
+            datatype_display, children = get_json_schema_array_fields(
+                value.get("items"), cls=cls
+            )
+            cls_obj.dataTypeDisplay = f"ARRAY<{datatype_display}>"
+        cls_obj.children = children
+        field_models.append(cls_obj)
+    except Exception as exc:  # pylint: disable=broad-except
+        logger.debug(traceback.format_exc())
+        logger.warning(f"Unable to parse the json schema into models: {exc}")
+
+
+def get_json_schema_array_fields(
+    array_items, cls: Type[BaseModel] = FieldModel
+) -> Optional[Tuple[str, List[FieldModel]]]:
+    """
+    Recursively convert the parsed array schema into required models
+    """
+    field_models = []
+    if array_items.get("type") == JsonSchemaDataTypes.RECORD.value:
+        for key, value in array_items.get("properties", {}).items():
+            get_child_models(key, value, field_models, cls)
+
+    return (
+        JsonSchemaDataTypes(array_items.get("type", "unknown")).name,
+        field_models or None,
+    )
+
+
 def get_json_schema_fields(
     properties, cls: Type[BaseModel] = FieldModel
 ) -> Optional[List[FieldModel]]:
@@ -74,20 +117,6 @@ def get_json_schema_fields(
     """
     field_models = []
     for key, value in properties.items():
-        try:
-            field_models.append(
-                cls(
-                    name=key,
-                    displayName=value.get("title"),
-                    dataType=JsonSchemaDataTypes(value.get("type", "unknown")).name,
-                    description=value.get("description"),
-                    children=get_json_schema_fields(value.get("properties"), cls=cls)
-                    if value.get("type") == "object"
-                    else None,
-                )
-            )
-        except Exception as exc:  # pylint: disable=broad-except
-            logger.debug(traceback.format_exc())
-            logger.warning(f"Unable to parse the json schema into models: {exc}")
+        get_child_models(key, value, field_models, cls)
 
     return field_models
diff --git a/ingestion/src/metadata/profiler/metrics/system/queries/redshift.py b/ingestion/src/metadata/profiler/metrics/system/queries/redshift.py
deleted file mode 100644
index 37712e4f1b76..000000000000
--- a/ingestion/src/metadata/profiler/metrics/system/queries/redshift.py
+++ /dev/null
@@ -1,104 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""
-Redshift System Metric Queries and queries operations
-"""
-
-from typing import List
-
-from sqlalchemy import text
-from sqlalchemy.orm import Session
-
-from metadata.utils.profiler_utils import QueryResult
-from metadata.utils.time_utils import datetime_to_timestamp
-
-STL_QUERY = """
-    with data as (
-        select
-            {alias}.*
-        from 
-            pg_catalog.stl_insert si
-            {join_type} join pg_catalog.stl_delete sd on si.query = sd.query
-        where 
-            {condition}
-    )
-	SELECT
-        SUM(data."rows") AS "rows",
-        sti."database",
-        sti."schema",
-        sti."table",
-        DATE_TRUNC('second', data.starttime) AS starttime
-    FROM
-        data
-        INNER JOIN  pg_catalog.svv_table_info sti ON data.tbl = sti.table_id
-    where
-        sti."database" = '{database}' AND
-       	sti."schema" = '{schema}' AND
-        "rows" != 0 AND
-        DATE(data.starttime) >= CURRENT_DATE - 1
-    GROUP BY 2,3,4,5
-    ORDER BY 5 DESC
-"""
-
-
-def get_query_results(
-    session: Session,
-    query,
-    operation,
-) -> List[QueryResult]:
-    """get query results either from cache or from the database
-
-    Args:
-        cache (Optional[List[QueryResult]]): cache results
-        session (Session): session
-        query (_type_): query
-        operation (_type_): operation
-
-    Returns:
-        List[QueryResult]:
-    """
-    cursor = session.execute(text(query))
-    results = [
-        QueryResult(
-            database_name=row.database,
-            schema_name=row.schema,
-            table_name=row.table,
-            query_text=None,
-            query_type=operation,
-            start_time=row.starttime,
-            rows=row.rows,
-        )
-        for row in cursor
-    ]
-
-    return results
-
-
-def get_metric_result(ddls: List[QueryResult], table_name: str) -> List:
-    """Given query results, retur the metric result
-
-    Args:
-        ddls (List[QueryResult]): list of query results
-        table_name (str): table name
-
-    Returns:
-        List:
-    """
-    return [
-        {
-            "timestamp": datetime_to_timestamp(ddl.start_time, milliseconds=True),
-            "operation": ddl.query_type,
-            "rowsAffected": ddl.rows,
-        }
-        for ddl in ddls
-        if ddl.table_name == table_name
-    ]
diff --git a/ingestion/src/metadata/profiler/metrics/system/system.py b/ingestion/src/metadata/profiler/metrics/system/system.py
index a981f2803de7..47524c430a2d 100644
--- a/ingestion/src/metadata/profiler/metrics/system/system.py
+++ b/ingestion/src/metadata/profiler/metrics/system/system.py
@@ -26,7 +26,7 @@
 from metadata.generated.schema.entity.services.connections.database.bigQueryConnection import (
     BigQueryConnection,
 )
-from metadata.ingestion.source.database.snowflake.profiler.system_metrics import (
+from metadata.ingestion.source.database.snowflake.profiler.system import (
     build_snowflake_query_results,
 )
 from metadata.profiler.metrics.core import SystemMetric
@@ -39,11 +39,6 @@
     JOBS,
     BigQueryQueryResult,
 )
-from metadata.profiler.metrics.system.queries.redshift import (
-    STL_QUERY,
-    get_metric_result,
-    get_query_results,
-)
 from metadata.profiler.orm.registry import Dialects
 from metadata.utils.dispatch import valuedispatch
 from metadata.utils.helpers import deep_size_of_dict
@@ -87,7 +82,6 @@ def get_system_metrics_for_dialect(
             } else returns None
     """
     logger.debug(f"System metrics not support for {dialect}. Skipping processing.")
-    return None
 
 
 @get_system_metrics_for_dialect.register(Dialects.BigQuery)
@@ -190,105 +184,6 @@ def _(
     return TypeAdapter(List[SystemProfile]).validate_python(metric_results)
 
 
-@get_system_metrics_for_dialect.register(Dialects.Redshift)
-def _(
-    dialect: str,
-    session: Session,
-    table: DeclarativeMeta,
-    *args,
-    **kwargs,
-) -> List[SystemProfile]:
-    """List all the DML operations for reshifts tables
-
-    Args:
-        dialect (str): redshift
-        session (Session): session object
-        table (DeclarativeMeta): orm table
-
-    Returns:
-        List[Dict]:
-    """
-    logger.debug(f"Fetching system metrics for {dialect}")
-    database = session.get_bind().url.database
-    schema = table.__table_args__["schema"]  # type: ignore
-
-    metric_results: List[Dict] = []
-
-    # get inserts ddl queries
-    inserts = get_value_from_cache(
-        SYSTEM_QUERY_RESULT_CACHE, f"{Dialects.Redshift}.{database}.{schema}.inserts"
-    )
-    if not inserts:
-        insert_query = STL_QUERY.format(
-            alias="si",
-            join_type="LEFT",
-            condition="sd.query is null",
-            database=database,
-            schema=schema,
-        )
-        inserts = get_query_results(
-            session,
-            insert_query,
-            DatabaseDMLOperations.INSERT.value,
-        )
-        set_cache(
-            SYSTEM_QUERY_RESULT_CACHE,
-            f"{Dialects.Redshift}.{database}.{schema}.inserts",
-            inserts,
-        )
-    metric_results.extend(get_metric_result(inserts, table.__tablename__))  # type: ignore
-
-    # get deletes ddl queries
-    deletes = get_value_from_cache(
-        SYSTEM_QUERY_RESULT_CACHE, f"{Dialects.Redshift}.{database}.{schema}.deletes"
-    )
-    if not deletes:
-        delete_query = STL_QUERY.format(
-            alias="sd",
-            join_type="RIGHT",
-            condition="si.query is null",
-            database=database,
-            schema=schema,
-        )
-        deletes = get_query_results(
-            session,
-            delete_query,
-            DatabaseDMLOperations.DELETE.value,
-        )
-        set_cache(
-            SYSTEM_QUERY_RESULT_CACHE,
-            f"{Dialects.Redshift}.{database}.{schema}.deletes",
-            deletes,
-        )
-    metric_results.extend(get_metric_result(deletes, table.__tablename__))  # type: ignore
-
-    # get updates ddl queries
-    updates = get_value_from_cache(
-        SYSTEM_QUERY_RESULT_CACHE, f"{Dialects.Redshift}.{database}.{schema}.updates"
-    )
-    if not updates:
-        update_query = STL_QUERY.format(
-            alias="si",
-            join_type="INNER",
-            condition="sd.query is not null",
-            database=database,
-            schema=schema,
-        )
-        updates = get_query_results(
-            session,
-            update_query,
-            DatabaseDMLOperations.UPDATE.value,
-        )
-        set_cache(
-            SYSTEM_QUERY_RESULT_CACHE,
-            f"{Dialects.Redshift}.{database}.{schema}.updates",
-            updates,
-        )
-    metric_results.extend(get_metric_result(updates, table.__tablename__))  # type: ignore
-
-    return TypeAdapter(List[SystemProfile]).validate_python(metric_results)
-
-
 @get_system_metrics_for_dialect.register(Dialects.Snowflake)
 def _(
     dialect: str,
diff --git a/ingestion/src/metadata/profiler/orm/functions/table_metric_computer.py b/ingestion/src/metadata/profiler/orm/functions/table_metric_computer.py
index 5566600c61d6..9284b7264243 100644
--- a/ingestion/src/metadata/profiler/orm/functions/table_metric_computer.py
+++ b/ingestion/src/metadata/profiler/orm/functions/table_metric_computer.py
@@ -412,7 +412,7 @@ def compute(self):
         )
         res = self.runner._session.execute(query).first()
         if not res:
-            return None
+            return super().compute()
         if res.rowCount is None or (
             res.rowCount == 0 and self._entity.tableType == TableType.View
         ):
diff --git a/ingestion/src/metadata/utils/class_helper.py b/ingestion/src/metadata/utils/class_helper.py
index 281afb854db6..ce2da2454037 100644
--- a/ingestion/src/metadata/utils/class_helper.py
+++ b/ingestion/src/metadata/utils/class_helper.py
@@ -68,7 +68,7 @@
 from metadata.generated.schema.metadataIngestion.workflow import SourceConfig
 
 SERVICE_TYPE_REF = {
-    ServiceType.API.value: "apiService",
+    ServiceType.Api.value: "apiService",
     ServiceType.Database.value: "databaseService",
     ServiceType.Dashboard.value: "dashboardService",
     ServiceType.Pipeline.value: "pipelineService",
diff --git a/ingestion/src/metadata/utils/collections.py b/ingestion/src/metadata/utils/collections.py
new file mode 100644
index 000000000000..16895ed0366d
--- /dev/null
+++ b/ingestion/src/metadata/utils/collections.py
@@ -0,0 +1,27 @@
+"""
+Uility classes for collections
+"""
+
+
+class CaseInsensitiveString(str):
+    """
+    A case-insensitive string. Useful for case-insensitive comparisons like SQL.
+    """
+
+    def __eq__(self, other):
+        return self.casefold() == other.casefold()
+
+    def __hash__(self):
+        return hash(self.casefold())
+
+
+class CaseInsensitiveList(list):
+    """A case-insensitive list that treats all its string elements as case-insensitive.
+    Non-string elements are treated with default behavior."""
+
+    def __contains__(self, item):
+        return (
+            any(CaseInsensitiveString(x) == item for x in self)
+            if isinstance(item, str)
+            else any(x == item for x in self)
+        )
diff --git a/ingestion/src/metadata/utils/constants.py b/ingestion/src/metadata/utils/constants.py
index cbb1b7b5f3e9..2f1c921eb4d1 100644
--- a/ingestion/src/metadata/utils/constants.py
+++ b/ingestion/src/metadata/utils/constants.py
@@ -40,6 +40,7 @@
 
 DOT = "_DOT_"
 TEN_MIN = 10 * 60
+THREE_MIN = 3 * 60
 UTF_8 = "utf-8"
 CHUNKSIZE = 200000
 DEFAULT_DATABASE = "default"
diff --git a/ingestion/src/metadata/utils/test_suite.py b/ingestion/src/metadata/utils/test_suite.py
deleted file mode 100644
index fd4e90e1b88b..000000000000
--- a/ingestion/src/metadata/utils/test_suite.py
+++ /dev/null
@@ -1,84 +0,0 @@
-#  Copyright 2021 Collate
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#  http://www.apache.org/licenses/LICENSE-2.0
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-"""
-Helper module for test suite functions
-"""
-
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Callable, List, Optional
-
-from metadata.generated.schema.tests.basic import (
-    TestCaseResult,
-    TestCaseStatus,
-    TestResultValue,
-)
-from metadata.generated.schema.tests.testCase import TestCaseParameterValue
-
-
-def get_test_case_param_value(
-    test_case_param_vals: list[TestCaseParameterValue],
-    name: str,
-    type_,
-    default=None,
-    pre_processor: Optional[Callable] = None,
-):
-    """Give a column and a type return the value with the appropriate type casting for the
-    test case definition.
-
-    Args:
-        test_case: the test case
-        type_ (Union[float, int, str]): type for the value
-        name (str): column name
-        default (_type_, optional): Default value to return if column is not found
-        pre_processor: pre processor function/type to use against the value before casting to type_
-    """
-    value = next(
-        (param.value for param in test_case_param_vals if param.name == name), None
-    )
-
-    if not value:
-        return default
-
-    if not pre_processor:
-        return type_(value)
-
-    pre_processed_value = pre_processor(value)
-    return type_(pre_processed_value)
-
-
-def build_test_case_result(
-    execution_datetime: datetime,
-    status: TestCaseStatus,
-    result: str,
-    test_result_value: List[TestResultValue],
-    sample_data: Optional[str] = None,
-) -> TestCaseResult:
-    """create a test case result object
-
-    Args:
-        execution_datetime (datetime): execution datetime of the test
-        status (TestCaseStatus): failed, succeed, aborted
-        result (str): message to display
-        testResultValue (List[TestResultValue]): values for the test result
-
-    Returns:
-        TestCaseResult:
-    """
-    return TestCaseResult(
-        timestamp=execution_datetime,
-        testCaseStatus=status,
-        result=result,
-        testResultValue=test_result_value,
-        sampleData=sample_data,
-    )
diff --git a/ingestion/src/metadata/utils/time_utils.py b/ingestion/src/metadata/utils/time_utils.py
index b2c4196dab38..0bd54f9a3ca9 100644
--- a/ingestion/src/metadata/utils/time_utils.py
+++ b/ingestion/src/metadata/utils/time_utils.py
@@ -17,13 +17,19 @@
 from math import floor
 from typing import Union
 
+from metadata.generated.schema.type.basic import Timestamp
 from metadata.utils.deprecation import deprecated
 from metadata.utils.helpers import datetime_to_ts
+from metadata.utils.logger import utils_logger
+
+logger = utils_logger()
 
 
 def datetime_to_timestamp(datetime_value: datetime, milliseconds=False) -> int:
-    """Convert a datetime object to timestamp integer. Datetime can be timezone aware or naive. Result
-    will always be in UTC.
+    """Convert a datetime object to timestamp integer. If datetime is timezone aware, it will be converted to UTC.
+    If it is naive it will be assumed to be in UTC.
+
+    # TODO: not sure the milliseconds flag is useful. Maybe this should return a 'Timestamp' object instead.
 
     Args:
         datetime_value (_type_): datetime object
@@ -37,12 +43,29 @@ def datetime_to_timestamp(datetime_value: datetime, milliseconds=False) -> int:
             f"Object of type {type(datetime_value).__name__} has not method `timestamp()`"
         )
 
+    datetime_value = (
+        datetime_value.replace(tzinfo=timezone.utc)
+        if datetime_value.tzinfo is None
+        else datetime_value.astimezone(timezone.utc)
+    )
     tmsap = datetime_value.timestamp()
     if milliseconds:
         return int(tmsap * 1000)
     return int(tmsap)
 
 
+def timestamp_to_datetime(ts: Timestamp) -> datetime:
+    """Convert a timestamp to datetime object in UTC.
+
+    Args:
+        ts (Timestamp): timestamp
+
+    Returns:
+        datetime: datetime object
+    """
+    return datetime.fromtimestamp(ts.root / 1000, tz=timezone.utc)
+
+
 def get_beginning_of_day_timestamp_mill(
     days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0
 ) -> int:
diff --git a/ingestion/src/metadata/workflow/application.py b/ingestion/src/metadata/workflow/application.py
index 15bf98e132a0..b22d4b1bd789 100644
--- a/ingestion/src/metadata/workflow/application.py
+++ b/ingestion/src/metadata/workflow/application.py
@@ -14,10 +14,6 @@
 from abc import ABC, abstractmethod
 from typing import List, Optional
 
-from metadata.config.common import WorkflowExecutionError
-from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
-    OpenMetadataConnection,
-)
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
@@ -25,13 +21,11 @@
 from metadata.generated.schema.metadataIngestion.application import (
     OpenMetadataApplicationConfig,
 )
-from metadata.generated.schema.metadataIngestion.workflow import LogLevels
-from metadata.ingestion.api.step import Step, Summary
+from metadata.ingestion.api.step import Step
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils.importer import import_from_module
 from metadata.utils.logger import ingestion_logger
 from metadata.workflow.base import BaseWorkflow
-from metadata.workflow.workflow_status_mixin import SUCCESS_THRESHOLD_VALUE
 
 logger = ingestion_logger()
 
@@ -84,29 +78,24 @@ class ApplicationWorkflow(BaseWorkflow, ABC):
     config: OpenMetadataApplicationConfig
     runner: Optional[AppRunner]
 
-    def __init__(self, config_dict: dict):
+    def __init__(self, config: OpenMetadataApplicationConfig):
         self.runner = None  # Will be passed in post-init
-        # TODO: Create a parse_gracefully method
-        self.config = OpenMetadataApplicationConfig.model_validate(config_dict)
+        self.config = config
 
         # Applications are associated to the OpenMetadata Service
         self.service_type: ServiceType = ServiceType.Metadata
 
-        metadata_config: OpenMetadataConnection = (
-            self.config.workflowConfig.openMetadataServerConfig
-        )
-        log_level: LogLevels = self.config.workflowConfig.loggerLevel
-
         super().__init__(
             config=self.config,
-            log_level=log_level,
-            metadata_config=metadata_config,
+            workflow_config=config.workflowConfig,
             service_type=self.service_type,
         )
 
     @classmethod
     def create(cls, config_dict: dict):
-        return cls(config_dict)
+        # TODO: Create a parse_gracefully method
+        config = OpenMetadataApplicationConfig.model_validate(config_dict)
+        return cls(config)
 
     def post_init(self) -> None:
         """
@@ -134,26 +123,8 @@ def execute_internal(self) -> None:
         """Workflow-specific logic to execute safely"""
         self.runner.run()
 
-    def calculate_success(self) -> float:
-        return self.runner.get_status().calculate_success()
-
     def get_failures(self) -> List[StackTraceError]:
         return self.workflow_steps()[0].get_status().failures
 
     def workflow_steps(self) -> List[Step]:
         return [self.runner]
-
-    def raise_from_status_internal(self, raise_warnings=False):
-        """Check failed status in the runner"""
-        if (
-            self.runner.get_status().failures
-            and self.calculate_success() < SUCCESS_THRESHOLD_VALUE
-        ):
-            raise WorkflowExecutionError(
-                f"{self.runner.name} reported errors: {Summary.from_step(self.runner)}"
-            )
-
-        if raise_warnings and self.runner.get_status().warnings:
-            raise WorkflowExecutionError(
-                f"{self.runner.name} reported warning: {Summary.from_step(self.runner)}"
-            )
diff --git a/ingestion/src/metadata/workflow/base.py b/ingestion/src/metadata/workflow/base.py
index 87bca9565a22..e777d6731ec1 100644
--- a/ingestion/src/metadata/workflow/base.py
+++ b/ingestion/src/metadata/workflow/base.py
@@ -16,8 +16,10 @@
 import uuid
 from abc import ABC, abstractmethod
 from datetime import datetime
+from statistics import mean
 from typing import Any, Dict, List, Optional, TypeVar, Union
 
+from metadata.config.common import WorkflowExecutionError
 from metadata.generated.schema.api.services.ingestionPipelines.createIngestionPipeline import (
     CreateIngestionPipelineRequest,
 )
@@ -32,10 +34,13 @@
 from metadata.generated.schema.entity.services.ingestionPipelines.status import (
     StackTraceError,
 )
-from metadata.generated.schema.metadataIngestion.workflow import LogLevels
+from metadata.generated.schema.metadataIngestion.workflow import (
+    LogLevels,
+    WorkflowConfig,
+)
 from metadata.generated.schema.tests.testSuite import ServiceType
 from metadata.generated.schema.type.entityReference import EntityReference
-from metadata.ingestion.api.step import Step
+from metadata.ingestion.api.step import Step, Summary
 from metadata.ingestion.ometa.client_utils import create_ometa_client
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.timer.repeated_timer import RepeatedTimer
@@ -49,10 +54,7 @@
 from metadata.utils.helpers import datetime_to_ts
 from metadata.utils.logger import ingestion_logger, set_loggers_level
 from metadata.workflow.workflow_output_handler import WorkflowOutputHandler
-from metadata.workflow.workflow_status_mixin import (
-    SUCCESS_THRESHOLD_VALUE,
-    WorkflowStatusMixin,
-)
+from metadata.workflow.workflow_status_mixin import WorkflowStatusMixin
 
 logger = ingestion_logger()
 
@@ -82,8 +84,7 @@ class BaseWorkflow(ABC, WorkflowStatusMixin):
     def __init__(
         self,
         config: Union[Any, Dict],
-        log_level: LogLevels,
-        metadata_config: OpenMetadataConnection,
+        workflow_config: WorkflowConfig,
         service_type: ServiceType,
         output_handler: WorkflowOutputHandler = WorkflowOutputHandler(),
     ):
@@ -92,19 +93,22 @@ def __init__(
         """
         self.output_handler = output_handler
         self.config = config
+        self.workflow_config = workflow_config
         self.service_type = service_type
         self._timer: Optional[RepeatedTimer] = None
         self._ingestion_pipeline: Optional[IngestionPipeline] = None
         self._start_ts = datetime_to_ts(datetime.now())
+
         self._execution_time_tracker = ExecutionTimeTracker(
-            log_level == LogLevels.DEBUG
+            self.workflow_config.loggerLevel == LogLevels.DEBUG
         )
 
-        set_loggers_level(log_level.value)
+        set_loggers_level(self.workflow_config.loggerLevel.value)
 
         # We create the ometa client at the workflow level and pass it to the steps
-        self.metadata_config = metadata_config
-        self.metadata = create_ometa_client(metadata_config)
+        self.metadata = create_ometa_client(
+            self.workflow_config.openMetadataServerConfig
+        )
         self.set_ingestion_pipeline_status(state=PipelineState.running)
 
         self.post_init()
@@ -157,9 +161,22 @@ def post_init(self) -> None:
     def execute_internal(self) -> None:
         """Workflow-specific logic to execute safely"""
 
-    @abstractmethod
-    def calculate_success(self) -> float:
-        """Get the success % of the internal execution"""
+    def calculate_success(self) -> Optional[float]:
+        """
+        Get the success % of the internal execution.
+        Since we'll use this to get a single success % from multiple steps, we'll take
+        the minimum success % from all the steps. This way, we can have a proper
+        workflow status.
+        E.g., if we have no errors on the source but a bunch of them on the sink,
+        we still want the flow to be marked as a failure or partial success.
+        """
+        if not self.workflow_steps():
+            logger.warning("No steps to calculate success")
+            return None
+
+        return mean(
+            [step.get_status().calculate_success() for step in self.workflow_steps()]
+        )
 
     @abstractmethod
     def get_failures(self) -> List[StackTraceError]:
@@ -169,9 +186,22 @@ def get_failures(self) -> List[StackTraceError]:
     def workflow_steps(self) -> List[Step]:
         """Steps to report status from"""
 
-    @abstractmethod
     def raise_from_status_internal(self, raise_warnings=False) -> None:
         """Based on the internal workflow status, raise a WorkflowExecutionError"""
+        for step in self.workflow_steps():
+            if (
+                step.get_status().failures
+                and step.get_status().calculate_success()
+                < self.workflow_config.successThreshold
+            ):
+                raise WorkflowExecutionError(
+                    f"{step.name} reported errors: {Summary.from_step(step)}"
+                )
+
+            if raise_warnings and step.status.warnings:
+                raise WorkflowExecutionError(
+                    f"{step.name} reported warning: {Summary.from_step(step)}"
+                )
 
     def execute(self) -> None:
         """
@@ -186,7 +216,7 @@ def execute(self) -> None:
         try:
             self.execute_internal()
 
-            if SUCCESS_THRESHOLD_VALUE <= self.calculate_success() < 100:
+            if self.workflow_config.successThreshold <= self.calculate_success() < 100:
                 pipeline_state = PipelineState.partialSuccess
 
         # Any unhandled exception breaking the workflow should update the status
diff --git a/ingestion/src/metadata/workflow/ingestion.py b/ingestion/src/metadata/workflow/ingestion.py
index cfa78c1259ed..1e28f5013178 100644
--- a/ingestion/src/metadata/workflow/ingestion.py
+++ b/ingestion/src/metadata/workflow/ingestion.py
@@ -24,9 +24,6 @@
 from typing import List, Tuple, Type, cast
 
 from metadata.config.common import WorkflowExecutionError
-from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
-    OpenMetadataConnection,
-)
 from metadata.generated.schema.entity.services.connections.serviceConnection import (
     ServiceConnection,
 )
@@ -38,7 +35,7 @@
     OpenMetadataWorkflowConfig,
 )
 from metadata.ingestion.api.parser import parse_workflow_config_gracefully
-from metadata.ingestion.api.step import Step, Summary
+from metadata.ingestion.api.step import Step
 from metadata.ingestion.api.steps import BulkSink, Processor, Sink, Source, Stage
 from metadata.ingestion.models.custom_types import ServiceWithConnectionType
 from metadata.profiler.api.models import ProfilerProcessorConfig
@@ -55,14 +52,15 @@
 )
 from metadata.utils.logger import ingestion_logger
 from metadata.workflow.base import BaseWorkflow, InvalidWorkflowJSONException
-from metadata.workflow.workflow_status_mixin import SUCCESS_THRESHOLD_VALUE
 
 logger = ingestion_logger()
 
 
 class IngestionWorkflow(BaseWorkflow, ABC):
     """
-    Base Ingestion Workflow implementation
+    Base Ingestion Workflow implementation. This is used for all
+    workflows minus the application one, which directly inherits the
+    BaseWorkflow.
     """
 
     config: OpenMetadataWorkflowConfig
@@ -79,14 +77,9 @@ def __init__(self, config: OpenMetadataWorkflowConfig):
             self.config.source.type
         )
 
-        metadata_config: OpenMetadataConnection = (
-            self.config.workflowConfig.openMetadataServerConfig
-        )
-
         super().__init__(
             config=config,
-            log_level=config.workflowConfig.loggerLevel,
-            metadata_config=metadata_config,
+            workflow_config=config.workflowConfig,
             service_type=self.service_type,
         )
 
@@ -137,37 +130,12 @@ def execute_internal(self):
         if bulk_sink:
             bulk_sink.run()
 
-    def calculate_success(self) -> float:
-        return self.source.get_status().calculate_success()
-
     def get_failures(self) -> List[StackTraceError]:
         return self.source.get_status().failures
 
     def workflow_steps(self) -> List[Step]:
         return [self.source] + list(self.steps)
 
-    def raise_from_status_internal(self, raise_warnings=False):
-        """
-        Check the status of all steps
-        """
-        if (
-            self.source.get_status().failures
-            and self.calculate_success() < SUCCESS_THRESHOLD_VALUE
-        ):
-            raise WorkflowExecutionError(
-                f"{self.source.name} reported errors: {Summary.from_step(self.source)}"
-            )
-
-        for step in self.steps:
-            if step.status.failures:
-                raise WorkflowExecutionError(
-                    f"{step.name} reported errors: {Summary.from_step(step)}"
-                )
-            if raise_warnings and step.status.warnings:
-                raise WorkflowExecutionError(
-                    f"{step.name} reported warnings: {Summary.from_step(step)}"
-                )
-
     def _retrieve_service_connection_if_needed(self, service_type: ServiceType) -> None:
         """
         We override the current `serviceConnection` source config object if source workflow service already exists
diff --git a/ingestion/src/metadata/workflow/profiler.py b/ingestion/src/metadata/workflow/profiler.py
index 4354b8c96bc7..5f8d692d2f20 100644
--- a/ingestion/src/metadata/workflow/profiler.py
+++ b/ingestion/src/metadata/workflow/profiler.py
@@ -20,6 +20,9 @@
     OpenMetadataWorkflowConfig,
 )
 from metadata.ingestion.api.steps import Processor, Sink
+from metadata.ingestion.connections.test_connections import (
+    raise_test_connection_exception,
+)
 from metadata.ingestion.source.connections import get_test_connection_fn
 from metadata.pii.processor import PIIProcessor
 from metadata.profiler.processor.processor import ProfilerProcessor
@@ -75,12 +78,13 @@ def set_steps(self):
         else:
             self.steps = (profiler_processor, sink)
 
-    def test_connection(self):
+    def test_connection(self) -> None:
         service_config = self.config.source.serviceConnection.root.config
         conn = get_ssl_connection(service_config)
 
         test_connection_fn = get_test_connection_fn(service_config)
-        test_connection_fn(self.metadata, conn, service_config)
+        result = test_connection_fn(self.metadata, conn, service_config)
+        raise_test_connection_exception(result)
 
     def _get_sink(self) -> Sink:
         sink_type = self.config.sink.type
diff --git a/ingestion/src/metadata/workflow/workflow_output_handler.py b/ingestion/src/metadata/workflow/workflow_output_handler.py
index d1a2070e2a7e..06df1f0dd54b 100644
--- a/ingestion/src/metadata/workflow/workflow_output_handler.py
+++ b/ingestion/src/metadata/workflow/workflow_output_handler.py
@@ -14,6 +14,7 @@
 """
 
 import time
+from statistics import mean
 from typing import Any, Dict, List, Optional, Type, Union
 
 from pydantic import BaseModel
@@ -114,16 +115,15 @@ def print_summary(self, steps: List[Step], debug: bool = False):
 
         self._print_summary(steps)
 
-    def _print_summary(self, steps: List[Step]):
+    def _print_summary(self, steps: List[Step]) -> None:
         failures: List[Failure] = []
-        total_records: int = 0
-        total_errors: int = 0
+        if not steps:
+            log_ansi_encoded_string(message="No steps to process.")
+            return
 
         for step in steps:
             step_summary = Summary.from_step(step)
 
-            total_records += step_summary.records or 0
-            total_errors += step_summary.errors or 0
             failures.append(
                 Failure(name=step.name, failures=step.get_status().failures)
             )
@@ -141,15 +141,18 @@ def _print_summary(self, steps: List[Step]):
                 log_ansi_encoded_string(message=f"Filtered: {step_summary.filtered}")
 
             log_ansi_encoded_string(message=f"Errors: {step_summary.errors}")
+            log_ansi_encoded_string(
+                message=f"Success %: {step.get_status().calculate_success()}"
+            )
 
         self._print_failures_if_apply(failures)
 
-        total_success = max(total_records, 1)
+        # If nothing is processed, we'll have a success of 100%
+        success_pct = mean([step.get_status().calculate_success() for step in steps])
         log_ansi_encoded_string(
             color=ANSI.BRIGHT_CYAN,
             bold=True,
-            message="Success %: "
-            + f"{round(total_success * 100 / (total_success + total_errors), 2)}",
+            message="Workflow Success %: " + f"{round(success_pct, 2)}",
         )
 
     def _print_debug_summary(self, steps: List[Step]):
diff --git a/ingestion/src/metadata/workflow/workflow_status_mixin.py b/ingestion/src/metadata/workflow/workflow_status_mixin.py
index e648ed00d439..fe8d99715c21 100644
--- a/ingestion/src/metadata/workflow/workflow_status_mixin.py
+++ b/ingestion/src/metadata/workflow/workflow_status_mixin.py
@@ -37,8 +37,6 @@
 
 logger = ometa_logger()
 
-SUCCESS_THRESHOLD_VALUE = 90
-
 
 class WorkflowResultStatus(Enum):
     SUCCESS = 0
diff --git a/ingestion/tests/cli_e2e/test_cli_mysql.py b/ingestion/tests/cli_e2e/test_cli_mysql.py
index 04c8f9041cac..41b4ac1c5297 100644
--- a/ingestion/tests/cli_e2e/test_cli_mysql.py
+++ b/ingestion/tests/cli_e2e/test_cli_mysql.py
@@ -117,7 +117,7 @@ def expected_filtered_schema_excludes() -> int:
 
     @staticmethod
     def expected_filtered_table_includes() -> int:
-        return 82
+        return 83
 
     @staticmethod
     def expected_filtered_table_excludes() -> int:
@@ -125,4 +125,4 @@ def expected_filtered_table_excludes() -> int:
 
     @staticmethod
     def expected_filtered_mix() -> int:
-        return 82
+        return 83
diff --git a/ingestion/tests/integration/data_quality/conftest.py b/ingestion/tests/integration/data_quality/conftest.py
index 5fac9ade6519..a5ff6df9b264 100644
--- a/ingestion/tests/integration/data_quality/conftest.py
+++ b/ingestion/tests/integration/data_quality/conftest.py
@@ -113,11 +113,17 @@ def ingest_postgres_metadata(
             "serviceConnection": postgres_service.connection.model_copy(
                 update={
                     "config": postgres_service.connection.config.model_copy(
-                        update={"ingestAllDatabases": True}
+                        update={
+                            "ingestAllDatabases": True,
+                        }
                     )
                 }
             ),
-            "sourceConfig": {"config": {}},
+            "sourceConfig": {
+                "config": {
+                    "schemaFilterPattern": {"excludes": ["information_schema"]},
+                }
+            },
         },
         "sink": sink_config,
         "workflowConfig": workflow_config,
diff --git a/ingestion/tests/integration/data_quality/test_data_diff.py b/ingestion/tests/integration/data_quality/test_table_diff.py
similarity index 90%
rename from ingestion/tests/integration/data_quality/test_data_diff.py
rename to ingestion/tests/integration/data_quality/test_table_diff.py
index 6df481341e85..62d69ca6bebf 100644
--- a/ingestion/tests/integration/data_quality/test_data_diff.py
+++ b/ingestion/tests/integration/data_quality/test_table_diff.py
@@ -234,6 +234,45 @@ def __init__(self, *args, **kwargs):
                     testCaseStatus=TestCaseStatus.Failed,
                 ),
             ),
+            (
+                TestCaseDefinition(
+                    name="postgres_different_case_columns_fail",
+                    testDefinitionName="tableDiff",
+                    computePassedFailedRowCount=True,
+                    parameterValues=[
+                        TestCaseParameterValue(
+                            name="caseSensitiveColumns", value="true"
+                        )
+                    ],
+                ),
+                "POSTGRES_SERVICE.dvdrental.public.customer_different_case_columns",
+                TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
+                    testCaseStatus=TestCaseStatus.Failed,
+                    testResultValue=[
+                        TestResultValue(name="removedColumns", value="1"),
+                        TestResultValue(name="addedColumns", value="0"),
+                        TestResultValue(name="changedColumns", value="0"),
+                    ],
+                ),
+            ),
+            (
+                TestCaseDefinition(
+                    name="postgres_different_case_columns_success",
+                    testDefinitionName="tableDiff",
+                    computePassedFailedRowCount=True,
+                    parameterValues=[
+                        TestCaseParameterValue(
+                            name="caseSensitiveColumns", value="false"
+                        )
+                    ],
+                ),
+                "POSTGRES_SERVICE.dvdrental.public.customer_different_case_columns",
+                TestCaseResult(
+                    timestamp=int(datetime.now().timestamp() * 1000),
+                    testCaseStatus=TestCaseStatus.Success,
+                ),
+            ),
             (
                 TestCaseDefinition(
                     name="table_from_another_db",
@@ -302,7 +341,7 @@ def test_happy_paths(
         },
         "processor": {
             "type": "orm-test-runner",
-            "config": {"testCases": [parameters.test_case_defintion.dict()]},
+            "config": {"testCases": [parameters.test_case_defintion.model_dump()]},
         },
         "sink": sink_config,
         "workflowConfig": workflow_config,
@@ -442,6 +481,16 @@ def test_error_paths(
 
 def add_changed_tables(connection: Connection):
     connection.execute("CREATE TABLE customer_200 AS SELECT * FROM customer LIMIT 200;")
+    connection.execute(
+        "CREATE TABLE customer_different_case_columns AS SELECT * FROM customer;"
+    )
+    connection.execute(
+        'ALTER TABLE customer_different_case_columns RENAME COLUMN first_name TO "First_Name";'
+    )
+    # TODO: this appears to be unsupported by data diff. Cross data type comparison is flaky.
+    # connection.execute(
+    #     "ALTER TABLE customer_different_case_columns ALTER COLUMN store_id TYPE decimal"
+    # )
     connection.execute("CREATE TABLE changed_customer AS SELECT * FROM customer;")
     connection.execute(
         "UPDATE changed_customer SET first_name = 'John' WHERE MOD(customer_id, 2) = 0;"
diff --git a/ingestion/tests/integration/integration_base.py b/ingestion/tests/integration/integration_base.py
index d54c9e19fff7..6cb82487792b 100644
--- a/ingestion/tests/integration/integration_base.py
+++ b/ingestion/tests/integration/integration_base.py
@@ -99,9 +99,24 @@
     Markdown,
     TestCaseEntityName,
 )
+from metadata.generated.schema.type.tagLabel import (
+    LabelType,
+    State,
+    TagFQN,
+    TagLabel,
+    TagSource,
+)
 from metadata.ingestion.ometa.ometa_api import C, T
 from metadata.utils.dispatch import class_register
 
+TIER1_TAG: TagLabel = TagLabel(
+    tagFQN=TagFQN(f"Tier.Tier1"),
+    name="Tier1",
+    source=TagSource.Classification,
+    labelType=LabelType.Automated,
+    state=State.Suggested,
+)
+
 COLUMNS = [
     Column(name="id", dataType=DataType.BIGINT),
     Column(name="another", dataType=DataType.BIGINT),
diff --git a/ingestion/tests/integration/ometa/test_ometa_es_api.py b/ingestion/tests/integration/ometa/test_ometa_es_api.py
index a66d57c0a9a2..23eee87c10a2 100644
--- a/ingestion/tests/integration/ometa/test_ometa_es_api.py
+++ b/ingestion/tests/integration/ometa/test_ometa_es_api.py
@@ -14,6 +14,7 @@
 import logging
 import time
 import uuid
+from copy import deepcopy
 from unittest import TestCase
 from unittest.mock import patch
 
@@ -52,7 +53,7 @@
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 from metadata.utils import fqn
 
-from ..integration_base import get_create_entity
+from ..integration_base import TIER1_TAG, get_create_entity
 
 
 class OMetaESTest(TestCase):
@@ -363,3 +364,30 @@ def side_effect(path: str, data=None):
                 )
             )
             assert len(assets) == 10
+
+    def test_paginate_with_filters(self):
+        """We can paginate only tier 1 tables"""
+        # prepare some tables with tier 1 tags
+        for idx, name in enumerate([f"filtered_{i}" for i in range(10)]):
+            table = self.metadata.create_or_update(
+                data=get_create_entity(
+                    entity=Table,
+                    name=EntityName(name),
+                    reference=self.create_schema_entity.fullyQualifiedName,
+                )
+            )
+            if idx % 2 == 0:
+                dest = deepcopy(table)
+                dest.tags = [TIER1_TAG]
+                self.metadata.patch(entity=Table, source=table, destination=dest)
+
+        query_filter = (
+            '{"query":{"bool":{"must":[{"bool":{"must":['
+            '{"term":{"tier.tagFQN":"Tier.Tier1"}},'
+            f'{{"term":{{"service.displayName.keyword":"{self.service_entity.name.root}"}}}}'
+            "]}}]}}}"
+        )
+        assets = list(
+            self.metadata.paginate_es(entity=Table, query_filter=query_filter, size=2)
+        )
+        assert len(assets) == 5
diff --git a/ingestion/tests/integration/postgres/test_data_quality.py b/ingestion/tests/integration/postgres/test_data_quality.py
index 0e27e7d03cf1..0612dff82312 100644
--- a/ingestion/tests/integration/postgres/test_data_quality.py
+++ b/ingestion/tests/integration/postgres/test_data_quality.py
@@ -1,3 +1,6 @@
+import glob
+import json
+import os.path
 import sys
 from dataclasses import dataclass
 from typing import List
@@ -6,6 +9,7 @@
 
 from _openmetadata_testutils.pydantic.test_utils import assert_equal_pydantic_objects
 from metadata.data_quality.api.models import TestCaseDefinition
+from metadata.generated.schema.entity.data.table import Table
 from metadata.generated.schema.entity.services.databaseService import DatabaseService
 from metadata.generated.schema.metadataIngestion.testSuitePipeline import (
     TestSuiteConfigType,
@@ -19,12 +23,17 @@
     SourceConfig,
     WorkflowConfig,
 )
-from metadata.generated.schema.tests.basic import TestCaseStatus
+from metadata.generated.schema.tests.basic import (
+    TestCaseResult,
+    TestCaseStatus,
+    TestResultValue,
+)
 from metadata.generated.schema.tests.testCase import TestCase
 from metadata.generated.schema.tests.testSuite import TestSuite
 from metadata.generated.schema.type.basic import ComponentConfig
 from metadata.ingestion.api.status import TruncatedStackTraceError
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
+from metadata.utils import entity_link
 from metadata.workflow.data_quality import TestSuiteWorkflow
 from metadata.workflow.metadata import MetadataWorkflow
 
@@ -32,7 +41,7 @@
     pytest.skip("requires python 3.9+", allow_module_level=True)
 
 
-@pytest.fixture()
+@pytest.fixture(scope="module")
 def run_data_quality_workflow(
     run_workflow,
     ingestion_config,
@@ -66,6 +75,7 @@ def run_data_quality_workflow(
                             "parameterValues": [
                                 {"name": "allowedValues", "value": "['Tom', 'Jerry']"}
                             ],
+                            "computePassedFailedRowCount": True,
                         },
                         {
                             "name": "first_name_includes_tom_and_jerry",
@@ -73,7 +83,7 @@ def run_data_quality_workflow(
                             "columnName": "first_name",
                             "parameterValues": [
                                 {"name": "allowedValues", "value": "['Tom', 'Jerry']"},
-                                {"name": "matchEnum", "value": ""},
+                                {"name": "matchEnum", "value": "false"},
                             ],
                         },
                         {
@@ -82,7 +92,7 @@ def run_data_quality_workflow(
                             "columnName": "first_name",
                             "parameterValues": [
                                 {"name": "allowedValues", "value": "['Tom', 'Jerry']"},
-                                {"name": "matchEnum", "value": "True"},
+                                {"name": "matchEnum", "value": "true"},
                             ],
                         },
                         {
@@ -91,6 +101,104 @@ def run_data_quality_workflow(
                             "columnName": "customer_id",
                             "parameterValues": [],
                         },
+                        {
+                            "name": "column_values_not_match_regex",
+                            "testDefinitionName": "columnValuesToNotMatchRegex",
+                            "columnName": "email",
+                            "parameterValues": [
+                                {"name": "forbiddenRegex", "value": ".*@example\\.com$"}
+                            ],
+                        },
+                        {
+                            "name": "table_column_count_between",
+                            "testDefinitionName": "tableColumnCountToBeBetween",
+                            "parameterValues": [
+                                {"name": "minColValue", "value": "8"},
+                                {"name": "maxColValue", "value": "12"},
+                            ],
+                        },
+                        {
+                            "name": "table_column_count_equal",
+                            "testDefinitionName": "tableColumnCountToEqual",
+                            "parameterValues": [{"name": "columnCount", "value": "11"}],
+                        },
+                        {
+                            "name": "table_column_name_exists",
+                            "testDefinitionName": "tableColumnNameToExist",
+                            "parameterValues": [
+                                {"name": "columnName", "value": "customer_id"}
+                            ],
+                        },
+                        {
+                            "name": "table_column_names_match_set",
+                            "testDefinitionName": "tableColumnToMatchSet",
+                            "parameterValues": [
+                                {
+                                    "name": "columnNames",
+                                    "value": "customer_id, store_id, first_name, last_name, email, address_id, activebool, create_date, last_update, active, json_field",
+                                },
+                                {"name": "ordered", "value": "false"},
+                            ],
+                        },
+                        {
+                            "name": "custom_sql_query_count",
+                            "testDefinitionName": "tableCustomSQLQuery",
+                            "parameterValues": [
+                                {
+                                    "name": "sqlExpression",
+                                    "value": "SELECT CASE WHEN COUNT(*) > 0 THEN 0 ELSE 1 END FROM customer WHERE active = 1",
+                                },
+                                {"name": "strategy", "value": "COUNT"},
+                                {"name": "threshold", "value": "0"},
+                            ],
+                        },
+                        {
+                            "name": "custom_sql_query_rows",
+                            "testDefinitionName": "tableCustomSQLQuery",
+                            "parameterValues": [
+                                {
+                                    "name": "sqlExpression",
+                                    "value": "SELECT * FROM customer WHERE active = 1",
+                                },
+                                {"name": "strategy", "value": "ROWS"},
+                                {"name": "threshold", "value": "10"},
+                            ],
+                        },
+                        {
+                            "name": "table_row_count_between",
+                            "testDefinitionName": "tableRowCountToBeBetween",
+                            "parameterValues": [
+                                {"name": "minValue", "value": "100"},
+                                {"name": "maxValue", "value": "1000"},
+                            ],
+                        },
+                        {
+                            "name": "table_row_count_equal",
+                            "testDefinitionName": "tableRowCountToEqual",
+                            "parameterValues": [{"name": "value", "value": "599"}],
+                        },
+                        {
+                            "name": "table_row_inserted_count_between_fail",
+                            "testDefinitionName": "tableRowInsertedCountToBeBetween",
+                            "parameterValues": [
+                                {"name": "min", "value": "10"},
+                                {"name": "max", "value": "50"},
+                                {"name": "columnName", "value": "create_date"},
+                                {"name": "rangeType", "value": "DAY"},
+                                {"name": "rangeInterval", "value": "1"},
+                            ],
+                        },
+                        {
+                            "name": "table_row_inserted_count_between_success",
+                            "testDefinitionName": "tableRowInsertedCountToBeBetween",
+                            "parameterValues": [
+                                {"name": "min", "value": "590"},
+                                {"name": "max", "value": "600"},
+                                {"name": "columnName", "value": "last_update"},
+                                {"name": "rangeType", "value": "YEAR"},
+                                {"name": "rangeInterval", "value": "12"},
+                            ],
+                        },
                     ],
                 }
             ),
@@ -109,14 +217,150 @@ def run_data_quality_workflow(
         metadata.delete(TestSuite, test_suite.id, recursive=True, hard_delete=True)
 
 
+def test_all_definition_exists(metadata, run_data_quality_workflow, db_service):
+    test_difinitions_glob = (
+        os.path.dirname(__file__)
+        + "/../../../.."
+        + "/openmetadata-service/src/main/resources/json/data/tests/**.json"
+    )
+    test_definitions: List[str] = []
+    for test_definition_file in glob.glob(test_difinitions_glob, recursive=False):
+        test_definitions.append(json.load(open(test_definition_file))["name"])
+    assert len(test_definitions) > 0
+    table: Table = metadata.get_by_name(
+        Table,
+        f"{db_service.fullyQualifiedName.root}.dvdrental.public.customer",
+        nullable=False,
+    )
+    tcs: List[TestCase] = metadata.list_entities(
+        TestCase,
+        fields=["*"],
+        params={
+            "entityLink": entity_link.get_entity_link(
+                Table, table.fullyQualifiedName.root
+            )
+        },
+    ).entities
+    tcs_dict = {tc.testDefinition.fullyQualifiedName: tc for tc in tcs}
+    excluded = {
+        # TODO implement these too
+        "columnValueLengthsToBeBetween",
+        "columnValueMaxToBeBetween",
+        "columnValueMinToBeBetween",
+        "columnValuesToBeUnique",
+        "tableDataToBeFresh",
+        "columnValuesToMatchRegex",
+        "columnValuesToNotMatchRegex",
+        "columnValueStdDevToBeBetween",
+        "columnValuesToBeNotNull",
+        "columnValueMedianToBeBetween",
+        "columnValuesSumToBeBetween",
+        "columnValuesToBeInSet",
+        "columnValuesMissingCount",
+        "columnValuesToBeNotInSet",
+        "columnValueMeanToBeBetween",
+        "columnValuesToBeBetween",
+        "tableDiff",
+    }
+    missing = set()
+    for test_definition in test_definitions:
+        if test_definition in tcs_dict:
+            assert (
+                test_definition not in excluded
+            ), f"Remove test from excluded list: {test_definition}"
+        else:
+            if test_definition in excluded:
+                continue
+            missing.add(test_definition.fullyQualifiedName.root)
+    assert not missing, f"Missing test cases: {missing}"
+
+
 @pytest.mark.parametrize(
     "test_case_name,expected_status",
     [
-        ("first_name_includes_tom_and_jerry_wo_enum", TestCaseStatus.Success),
-        ("first_name_includes_tom_and_jerry", TestCaseStatus.Success),
-        ("first_name_is_tom_or_jerry", TestCaseStatus.Failed),
-        ("id_no_bounds", TestCaseStatus.Success),
+        (
+            "first_name_includes_tom_and_jerry_wo_enum",
+            TestCaseResult(
+                timestamp=0,
+                testCaseStatus=TestCaseStatus.Success,
+                passedRows=2,
+                failedRows=597,
+            ),
+        ),
+        (
+            "first_name_includes_tom_and_jerry",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Success),
+        ),
+        (
+            "first_name_is_tom_or_jerry",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Failed),
+        ),
+        (
+            "id_no_bounds",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Success),
+        ),
+        (
+            "column_values_not_match_regex",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Success),
+        ),
+        (
+            "table_column_count_between",
+            TestCaseResult(
+                timestamp=0,
+                testCaseStatus=TestCaseStatus.Success,
+                testResultValue=[TestResultValue(name="columnCount", value="11")],
+            ),
+        ),
+        (
+            "table_column_count_equal",
+            TestCaseResult(
+                timestamp=0,
+                testCaseStatus=TestCaseStatus.Success,
+                testResultValue=[TestResultValue(name="columnCount", value="11")],
+            ),
+        ),
+        (
+            "table_column_name_exists",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Success),
+        ),
+        (
+            "table_column_names_match_set",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Success),
+        ),
+        (
+            "custom_sql_query_count",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Success),
+        ),
+        (
+            "custom_sql_query_rows",
+            TestCaseResult(
+                timestamp=0,
+                testCaseStatus=TestCaseStatus.Failed,
+                testResultValues=[{"name": "resultRowCount", "value": "599"}],
+            ),
+        ),
+        (
+            "table_row_count_between",
+            TestCaseResult(
+                timestamp=0,
+                testCaseStatus=TestCaseStatus.Success,
+                testResultValue=[TestResultValue(name="rowCount", value="599")],
+            ),
+        ),
+        (
+            "table_row_count_equal",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Success),
+        ),
+        (
+            "table_row_inserted_count_between_fail",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Failed),
+        ),
+        (
+            "table_row_inserted_count_between_success",
+            TestCaseResult(timestamp=0, testCaseStatus=TestCaseStatus.Success),
+        ),
     ],
+    ids=lambda *x: x[0],
 )
 def test_data_quality(
     run_data_quality_workflow, metadata: OpenMetadata, test_case_name, expected_status
@@ -128,7 +372,12 @@ def test_data_quality(
         (t for t in test_cases if t.name.root == test_case_name), None
     )
     assert test_case is not None
-    assert test_case.testCaseResult.testCaseStatus == expected_status
+    assert_equal_pydantic_objects(
+        expected_status.model_copy(
+            update={"timestamp": test_case.testCaseResult.timestamp}
+        ),
+        test_case.testCaseResult,
+    )
 
 
 @pytest.fixture()
diff --git a/ingestion/tests/unit/data_quality/validations/test_base_handler.py b/ingestion/tests/unit/data_quality/validations/test_base_handler.py
new file mode 100644
index 000000000000..fab476e1ddd3
--- /dev/null
+++ b/ingestion/tests/unit/data_quality/validations/test_base_handler.py
@@ -0,0 +1,34 @@
+from ast import literal_eval
+
+import pytest
+
+from metadata.data_quality.validations.base_test_handler import BaseTestValidator
+from metadata.generated.schema.tests.testCase import TestCaseParameterValue
+
+
+@pytest.mark.parametrize(
+    "param_values, name, type_, default, expected",
+    [
+        ([TestCaseParameterValue(name="str", value="test")], "str", str, None, "test"),
+        (
+            [TestCaseParameterValue(name="param", value="[1, 2, 3]")],
+            "param",
+            literal_eval,
+            None,
+            [1, 2, 3],
+        ),
+        ([TestCaseParameterValue(name="param", value="123")], "param", int, None, 123),
+        (
+            [TestCaseParameterValue(name="param", value=None)],
+            "param",
+            str,
+            "default",
+            "default",
+        ),
+    ],
+)
+def test_get_test_case_param_value(param_values, name, type_, default, expected):
+    result = BaseTestValidator.get_test_case_param_value(
+        param_values, name, type_, default
+    )
+    assert result == expected
diff --git a/ingestion/tests/unit/data_quality/validations/test_utils.py b/ingestion/tests/unit/data_quality/validations/test_utils.py
new file mode 100644
index 000000000000..4b4ce41a4248
--- /dev/null
+++ b/ingestion/tests/unit/data_quality/validations/test_utils.py
@@ -0,0 +1,22 @@
+import pytest
+
+from metadata.data_quality.validations.utils import get_bool_test_case_param
+from metadata.generated.schema.tests.testCase import TestCaseParameterValue
+
+
+@pytest.mark.parametrize(
+    "test_case_param_vals, name, expected",
+    [
+        ([TestCaseParameterValue(name="param1", value="true")], "param1", True),
+        ([TestCaseParameterValue(name="param1", value="false")], "param1", False),
+        ([TestCaseParameterValue(name="param1", value="True")], "param1", True),
+        ([TestCaseParameterValue(name="param1", value="False")], "param1", False),
+        ([TestCaseParameterValue(name="param1", value="TRUE")], "param1", True),
+        ([TestCaseParameterValue(name="param1", value="FALSE")], "param1", False),
+        ([TestCaseParameterValue(name="param1", value="invalid")], "param1", False),
+        ([], "param1", False),
+        ([TestCaseParameterValue(name="param2", value="true")], "param1", False),
+    ],
+)
+def test_get_bool_test_case_param(test_case_param_vals, name, expected):
+    assert get_bool_test_case_param(test_case_param_vals, name) == expected
diff --git a/ingestion/tests/unit/metadata/ingestion/source/database/snowflake/profiler/test_system_metrics.py b/ingestion/tests/unit/metadata/ingestion/source/database/snowflake/profiler/test_system_metrics.py
index c2fcbdc090f9..48809d434cfc 100644
--- a/ingestion/tests/unit/metadata/ingestion/source/database/snowflake/profiler/test_system_metrics.py
+++ b/ingestion/tests/unit/metadata/ingestion/source/database/snowflake/profiler/test_system_metrics.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from metadata.ingestion.source.database.snowflake.profiler.system_metrics import (
+from metadata.ingestion.source.database.snowflake.profiler.system import (
     PUBLIC_SCHEMA,
     SnowflakeTableResovler,
 )
diff --git a/ingestion/tests/unit/metadata/utils/test_time_utils.py b/ingestion/tests/unit/metadata/utils/test_time_utils.py
index 13ef5c9d9c0a..8ff2099a4359 100644
--- a/ingestion/tests/unit/metadata/utils/test_time_utils.py
+++ b/ingestion/tests/unit/metadata/utils/test_time_utils.py
@@ -1,8 +1,9 @@
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 
 import pytest
 
-from metadata.utils.time_utils import timedelta_to_string
+from metadata.generated.schema.type.basic import Timestamp
+from metadata.utils.time_utils import timedelta_to_string, timestamp_to_datetime
 
 
 @pytest.mark.parametrize(
@@ -23,3 +24,54 @@
 )
 def test_timedelta_to_string(parameter, expected):
     assert timedelta_to_string(parameter).startswith(expected)
+
+
+@pytest.mark.parametrize(
+    "timestamp, expected_datetime",
+    [
+        (
+            Timestamp(root=1638316800000),
+            datetime(2021, 12, 1, 0, 0, tzinfo=timezone.utc),
+        ),
+        (
+            Timestamp(root=1609459200000),
+            datetime(2021, 1, 1, 0, 0, tzinfo=timezone.utc),
+        ),
+        (Timestamp(root=0), datetime(1970, 1, 1, 0, 0, tzinfo=timezone.utc)),
+    ],
+)
+def test_timestamp_to_datetime(timestamp, expected_datetime):
+    assert timestamp_to_datetime(timestamp) == expected_datetime
+
+
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from metadata.utils.time_utils import datetime_to_timestamp
+
+
+@pytest.mark.parametrize(
+    "datetime_value, milliseconds, expected_timestamp",
+    [
+        # Naive datetime (assumed to be in UTC)
+        (datetime(2021, 12, 1, 0, 0, 0), False, 1638316800),
+        (datetime(2021, 12, 1, 0, 0, 0), True, 1638316800000),
+        # Timezone-aware datetime (UTC)
+        (datetime(2021, 12, 1, 0, 0, 0, tzinfo=timezone.utc), False, 1638316800),
+        (datetime(2021, 12, 1, 0, 0, 0, tzinfo=timezone.utc), True, 1638316800000),
+        # Timezone-aware datetime (non-UTC)
+        (
+            datetime(2021, 12, 1, 0, 0, 0, tzinfo=timezone(timedelta(hours=1))),
+            False,
+            1638313200,
+        ),
+        (
+            datetime(2021, 12, 1, 0, 0, 0, tzinfo=timezone(timedelta(hours=1))),
+            True,
+            1638313200000,
+        ),
+    ],
+)
+def test_datetime_to_timestamp(datetime_value, milliseconds, expected_timestamp):
+    assert datetime_to_timestamp(datetime_value, milliseconds) == expected_timestamp
diff --git a/ingestion/tests/unit/profiler/test_utils.py b/ingestion/tests/unit/profiler/test_utils.py
index 523ad24c6452..7743673cf250 100644
--- a/ingestion/tests/unit/profiler/test_utils.py
+++ b/ingestion/tests/unit/profiler/test_utils.py
@@ -22,7 +22,7 @@
 from sqlalchemy.sql.sqltypes import Integer, String
 
 from metadata.ingestion.source.database.snowflake.models import SnowflakeQueryLogEntry
-from metadata.ingestion.source.database.snowflake.profiler.system_metrics import (
+from metadata.ingestion.source.database.snowflake.profiler.system import (
     SnowflakeTableResovler,
     get_snowflake_system_queries,
 )
diff --git a/ingestion/tests/unit/profiler/test_workflow.py b/ingestion/tests/unit/profiler/test_workflow.py
index 8cc358c6f9d8..aeeb3df971d1 100644
--- a/ingestion/tests/unit/profiler/test_workflow.py
+++ b/ingestion/tests/unit/profiler/test_workflow.py
@@ -26,9 +26,6 @@
     Table,
     TableProfilerConfig,
 )
-from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
-    OpenMetadataConnection,
-)
 from metadata.generated.schema.entity.services.databaseService import (
     DatabaseService,
     DatabaseServiceType,
@@ -36,6 +33,7 @@
 from metadata.generated.schema.metadataIngestion.databaseServiceProfilerPipeline import (
     DatabaseServiceProfilerPipeline,
 )
+from metadata.generated.schema.metadataIngestion.workflow import WorkflowConfig
 from metadata.generated.schema.type.entityReference import EntityReference
 from metadata.profiler.api.models import ProfilerProcessorConfig
 from metadata.profiler.interface.sqlalchemy.profiler_interface import (
@@ -122,7 +120,7 @@ def test_init_workflow(mocked_method, mocked_orm):  # pylint: disable=unused-arg
     mocked_method.assert_called()
 
     assert isinstance(workflow.source.source_config, DatabaseServiceProfilerPipeline)
-    assert isinstance(workflow.metadata_config, OpenMetadataConnection)
+    assert isinstance(workflow.workflow_config, WorkflowConfig)
 
     profiler_processor_step = workflow.steps[0]
     assert isinstance(profiler_processor_step.profiler_config, ProfilerProcessorConfig)
diff --git a/ingestion/tests/unit/resources/datasets/manifest_v8.json b/ingestion/tests/unit/resources/datasets/manifest_v8.json
index e6a640e9c1e3..12d49be7fad7 100644
--- a/ingestion/tests/unit/resources/datasets/manifest_v8.json
+++ b/ingestion/tests/unit/resources/datasets/manifest_v8.json
@@ -77,7 +77,8 @@
             },
             "tags": [
                 "model_tag_one",
-                "model_tag_two"
+                "model_tag_two",
+                "22.8.5.1"
             ],
             "refs": [
                 [
diff --git a/ingestion/tests/unit/test_dbt.py b/ingestion/tests/unit/test_dbt.py
index 937428d01a9b..11b18438a1ad 100644
--- a/ingestion/tests/unit/test_dbt.py
+++ b/ingestion/tests/unit/test_dbt.py
@@ -135,6 +135,14 @@
                 state="Suggested",
                 href=None,
             ),
+            TagLabel(
+                tagFQN='dbtTags."22.8.5.1"',
+                description=None,
+                source="Classification",
+                labelType="Automated",
+                state="Suggested",
+                href=None,
+            ),
         ],
         columns=[
             Column(
@@ -426,6 +434,12 @@ def test_dbt_manifest_v8(self, get_tag_label, es_search_from_fqn, get_dbt_owner)
                 state=State.Suggested.value,
                 source=TagSource.Classification.value,
             ),
+            TagLabel(
+                tagFQN='dbtTags."22.8.5.1"',
+                labelType=LabelType.Automated.value,
+                state=State.Suggested.value,
+                source=TagSource.Classification.value,
+            ),
         ]
         self.execute_test(
             MOCK_SAMPLE_MANIFEST_V8,
diff --git a/ingestion/tests/unit/test_json_schema_parser.py b/ingestion/tests/unit/test_json_schema_parser.py
index fd1bb1d9ca89..7c5ce9acc401 100644
--- a/ingestion/tests/unit/test_json_schema_parser.py
+++ b/ingestion/tests/unit/test_json_schema_parser.py
@@ -91,8 +91,81 @@ class JsonSchemaParserTests(TestCase):
         }
     }"""
 
+    sample_array_json_schema = """
+    {
+        "$schema": "http://json-schema.org/draft-07/schema#",
+        "type": "object",
+        "properties": {
+            "firstName": {
+                "type": "string"
+            },
+            "lastName": {
+                "type": "string"
+            },
+            "age": {
+                "type": "integer"
+            },
+            "address": {
+                "type": "object",
+                "properties": {
+                    "streetAddress": {
+                        "type": "string"
+                    },
+                    "city": {
+                        "type": "string"
+                    },
+                    "state": {
+                        "type": "string"
+                    },
+                    "postalCode": {
+                        "type": "string"
+                    }
+                },
+                "required": [
+                    "streetAddress",
+                    "city",
+                    "state",
+                    "postalCode"
+                ]
+            },
+            "phoneNumbers": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "type": {
+                            "type": "string"
+                        },
+                        "number": {
+                            "type": "string"
+                        }
+                    },
+                    "required": [
+                        "type",
+                        "number"
+                    ]
+                }
+            },
+            "hobbies": {
+                "type": "array",
+                "items": {
+                    "type": "string"
+                }
+            }
+        },
+        "required": [
+            "firstName",
+            "lastName",
+            "age",
+            "address",
+            "phoneNumbers"
+        ]
+    }
+    """
+
     parsed_schema = parse_json_schema(sample_json_schema)
     parsed_postgres_schema = parse_json_schema(sample_postgres_json_schema, Column)
+    parsed_array_schema = parse_json_schema(sample_array_json_schema)
 
     def test_schema_name(self):
         self.assertEqual(self.parsed_schema[0].name.root, "Person")
@@ -143,3 +216,37 @@ def test_parse_postgres_json_fields(self):
         )
         self.assertEqual(len(self.parsed_postgres_schema[0].children), 3)
         self.assertEqual(len(self.parsed_postgres_schema[0].children[1].children), 4)
+
+    def test_parse_postgres_json_fields(self):
+        self.assertEqual(self.parsed_array_schema[0].name.root, "default")
+        self.assertEqual(len(self.parsed_array_schema[0].children), 6)
+
+        # Validate the complex array datatype
+        self.assertEqual(
+            self.parsed_array_schema[0].children[4].name.root, "phoneNumbers"
+        )
+        self.assertEqual(self.parsed_array_schema[0].children[4].dataType.name, "ARRAY")
+        self.assertEqual(
+            self.parsed_array_schema[0].children[4].dataTypeDisplay, "ARRAY<RECORD>"
+        )
+        self.assertEqual(len(self.parsed_array_schema[0].children[4].children), 2)
+        self.assertEqual(
+            self.parsed_array_schema[0].children[4].children[0].name.root, "type"
+        )
+        self.assertEqual(
+            self.parsed_array_schema[0].children[4].children[0].dataType.name, "STRING"
+        )
+        self.assertEqual(
+            self.parsed_array_schema[0].children[4].children[1].name.root, "number"
+        )
+        self.assertEqual(
+            self.parsed_array_schema[0].children[4].children[1].dataType.name, "STRING"
+        )
+
+        # Validate the primitive array datatype
+        self.assertEqual(self.parsed_array_schema[0].children[5].name.root, "hobbies")
+        self.assertEqual(self.parsed_array_schema[0].children[5].dataType.name, "ARRAY")
+        self.assertEqual(
+            self.parsed_array_schema[0].children[5].dataTypeDisplay, "ARRAY<STRING>"
+        )
+        self.assertIsNone(self.parsed_array_schema[0].children[5].children)
diff --git a/ingestion/tests/unit/test_workflow_parse.py b/ingestion/tests/unit/test_workflow_parse.py
index 14d9463ed70b..f14d4c200450 100644
--- a/ingestion/tests/unit/test_workflow_parse.py
+++ b/ingestion/tests/unit/test_workflow_parse.py
@@ -19,6 +19,9 @@
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
+from metadata.generated.schema.entity.services.connections.api.restConnection import (
+    RestConnection,
+)
 from metadata.generated.schema.entity.services.connections.dashboard.tableauConnection import (
     TableauConnection,
 )
@@ -115,6 +118,10 @@ def test_get_connection_class(self):
         connection = get_connection_class(source_type, get_service_type(source_type))
         self.assertEqual(connection, KafkaConnection)
 
+        source_type = "Rest"
+        connection = get_connection_class(source_type, get_service_type(source_type))
+        self.assertEqual(connection, RestConnection)
+
     def test_get_source_config_class(self):
         """
         Check that we can correctly build the connection module ingredients
diff --git a/ingestion/tests/unit/topology/api/test_rest.py b/ingestion/tests/unit/topology/api/test_rest.py
index 8e6d8e69f17f..995b1de3a3a3 100644
--- a/ingestion/tests/unit/topology/api/test_rest.py
+++ b/ingestion/tests/unit/topology/api/test_rest.py
@@ -21,9 +21,9 @@
     CreateAPICollectionRequest,
 )
 from metadata.generated.schema.entity.services.apiService import (
+    ApiConnection,
     ApiService,
-    ApiServiceConnection,
-    APIServiceType,
+    ApiServiceType,
 )
 from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
@@ -43,7 +43,7 @@
         "serviceName": "openapi_rest",
         "serviceConnection": {
             "config": {
-                "type": "REST",
+                "type": "Rest",
                 "openAPISchemaURL": "https://petstore3.swagger.io/api/v3/openapi.json",
             }
         },
@@ -91,8 +91,8 @@
     id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb",
     name="openapi_rest",
     fullyQualifiedName=FullyQualifiedEntityName("openapi_rest"),
-    connection=ApiServiceConnection(),
-    serviceType=APIServiceType.REST,
+    connection=ApiConnection(),
+    serviceType=ApiServiceType.Rest,
 )
 EXPECTED_COLLECTION_REQUEST = [
     Either(
diff --git a/ingestion/tests/unit/topology/pipeline/test_gluepipeline.py b/ingestion/tests/unit/topology/pipeline/test_gluepipeline.py
new file mode 100644
index 000000000000..428d6e900255
--- /dev/null
+++ b/ingestion/tests/unit/topology/pipeline/test_gluepipeline.py
@@ -0,0 +1,341 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Test dbt cloud using the topology
+"""
+import json
+from unittest import TestCase
+from unittest.mock import patch
+
+from metadata.generated.schema.api.data.createPipeline import CreatePipelineRequest
+from metadata.generated.schema.entity.data.pipeline import Pipeline, Task
+from metadata.generated.schema.entity.services.pipelineService import (
+    PipelineConnection,
+    PipelineService,
+    PipelineServiceType,
+)
+from metadata.generated.schema.metadataIngestion.workflow import (
+    OpenMetadataWorkflowConfig,
+)
+from metadata.generated.schema.type.basic import (
+    EntityName,
+    FullyQualifiedEntityName,
+    Markdown,
+    SourceUrl,
+)
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.source.pipeline.gluepipeline.metadata import GluepipelineSource
+
+mock_glue_config = {
+    "source": {
+        "type": "gluepipeline",
+        "serviceName": "local_gluepipeline",
+        "serviceConnection": {
+            "config": {
+                "type": "GluePipeline",
+                "awsConfig": {
+                    "awsAccessKeyId": "aws_access_key_id",
+                    "awsSecretAccessKey": "aws_secret_access_key",
+                    "awsRegion": "us-east-2",
+                    "endPointURL": "https://endpoint.com/",
+                },
+            },
+        },
+        "sourceConfig": {"config": {"type": "PipelineMetadata"}},
+    },
+    "sink": {"type": "metadata-rest", "config": {}},
+    "workflowConfig": {
+        "openMetadataServerConfig": {
+            "hostPort": "http://localhost:8585/api",
+            "authProvider": "openmetadata",
+            "securityConfig": {
+                "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
+            },
+        }
+    },
+}
+
+EXPECTED_JOB_DETAILS = json.loads(
+    """
+{
+    "Name": "redshift workflow",
+    "Description": "redshift workflow description",
+    "DefaultRunProperties": {},
+    "CreatedOn": "2024-09-20 15:46:36.668000",
+    "LastModifiedOn": "2024-09-20 15:46:36.668000",
+    "LastRun": {
+        "Name": "redshift workflow",
+        "WorkflowRunId": "wr_6db99d3ea932db0739f03ba5ae56e4b635b7878261f75af062e1223a7272c50e",
+        "WorkflowRunProperties": {},
+        "StartedOn": "2024-09-30 17:07:24.032000",
+        "CompletedOn": "2024-09-30 17:08:24.032000",
+        "Status": "COMPLETED",
+        "Statistics": {
+            "TotalActions": 1,
+            "TimeoutActions": 0,
+            "FailedActions": 1,
+            "StoppedActions": 0,
+            "SucceededActions": 0,
+            "RunningActions": 0,
+            "ErroredActions": 0,
+            "WaitingActions": 0
+        },
+        "Graph": {
+            "Nodes": [
+                {
+                    "Type": "TRIGGER",
+                    "Name": "redshift_event",
+                    "UniqueId": "wnode_98c85bc1e19d969e35e0687b2ec586822271463c72dd556f90cfe6421a2517ee",
+                    "TriggerDetails": {
+                        "Trigger": {
+                            "Name": "redshift_event",
+                            "WorkflowName": "redshift workflow",
+                            "Type": "ON_DEMAND",
+                            "State": "CREATED",
+                            "Actions": [
+                                {
+                                    "JobName": "Redshift DBT Job"
+                                }
+                            ]
+                        }
+                    }
+                },
+                {
+                    "Type": "JOB",
+                    "Name": "Redshift DBT Job",
+                    "UniqueId": "wnode_0cbf9f52c41002015ebc46fe70a9b0ea64ff7dba891cf141d6dcbf5580fe7123",
+                    "JobDetails": {
+                        "JobRuns": [
+                            {
+                                "Id": "jr_108804857dd29cb1857c92d3e8bf0b48f7685c246e56125b713eb6ea7ebfe4e2",
+                                "Attempt": 0,
+                                "TriggerName": "redshift_event",
+                                "JobName": "Redshift DBT Job",
+                                "JobMode": "VISUAL",
+                                "JobRunQueuingEnabled": false,
+                                "StartedOn": "2024-09-30 17:07:59.185000",
+                                "LastModifiedOn": "2024-09-30 17:08:03.003000",
+                                "CompletedOn": "2024-09-30 17:08:03.003000",
+                                "JobRunState": "FAILED",
+                                "ErrorMessage": "Error Message",
+                                "PredecessorRuns": [],
+                                "AllocatedCapacity": 10,
+                                "ExecutionTime": 0,
+                                "Timeout": 2880,
+                                "MaxCapacity": 10.0,
+                                "WorkerType": "G.1X",
+                                "NumberOfWorkers": 10,
+                                "LogGroupName": "/aws-glue/jobs",
+                                "GlueVersion": "4.0",
+                                "ExecutionClass": "STANDARD"
+                            }
+                        ]
+                    }
+                }
+            ],
+            "Edges": [
+                {
+                    "SourceId": "wnode_98c85bc1e19d969e35e0687b2ec586822271463c72dd556f90cfe6421a2517ee",
+                    "DestinationId": "wnode_0cbf9f52c41002015ebc46fe70a9b0ea64ff7dba891cf141d6dcbf5580fe7123"
+                }
+            ]
+        }
+    },
+    "Graph": {
+        "Nodes": [
+            {
+                "Type": "TRIGGER",
+                "Name": "redshift_event",
+                "UniqueId": "wnode_98c85bc1e19d969e35e0687b2ec586822271463c72dd556f90cfe6421a2517ee",
+                "TriggerDetails": {
+                    "Trigger": {
+                        "Name": "redshift_event",
+                        "WorkflowName": "redshift workflow",
+                        "Type": "ON_DEMAND",
+                        "State": "CREATED",
+                        "Actions": [
+                            {
+                                "JobName": "Redshift DBT Job"
+                            }
+                        ]
+                    }
+                }
+            },
+            {
+                "Type": "JOB",
+                "Name": "Redshift DBT Job",
+                "UniqueId": "wnode_0cbf9f52c41002015ebc46fe70a9b0ea64ff7dba891cf141d6dcbf5580fe7123",
+                "JobDetails": {}
+            }
+        ],
+        "Edges": [
+            {
+                "SourceId": "wnode_98c85bc1e19d969e35e0687b2ec586822271463c72dd556f90cfe6421a2517ee",
+                "DestinationId": "wnode_0cbf9f52c41002015ebc46fe70a9b0ea64ff7dba891cf141d6dcbf5580fe7123"
+            }
+        ]
+    }
+}
+"""
+)
+
+EXPECTED_CREATED_PIPELINES = CreatePipelineRequest(
+    name=EntityName(root="redshift workflow"),
+    displayName="redshift workflow",
+    description=None,
+    dataProducts=None,
+    sourceUrl=SourceUrl(
+        root="https://us-east-2.console.aws.amazon.com/glue/home?region=us-east-2#/v2/etl-configuration/workflows/view/redshift workflow"
+    ),
+    concurrency=None,
+    pipelineLocation=None,
+    startDate=None,
+    tasks=[
+        Task(
+            name="redshift_event",
+            displayName="redshift_event",
+            fullyQualifiedName=None,
+            description=None,
+            sourceUrl=None,
+            downstreamTasks=["Redshift DBT Job"],
+            taskType="TRIGGER",
+            taskSQL=None,
+            startDate=None,
+            endDate=None,
+            tags=None,
+            owners=None,
+        ),
+        Task(
+            name="Redshift DBT Job",
+            displayName="Redshift DBT Job",
+            fullyQualifiedName=None,
+            description=None,
+            sourceUrl=None,
+            downstreamTasks=[],
+            taskType="JOB",
+            taskSQL=None,
+            startDate=None,
+            endDate=None,
+            tags=None,
+            owners=None,
+        ),
+    ],
+    tags=None,
+    owners=None,
+    service=FullyQualifiedEntityName(root="gluepipeline_test"),
+    extension=None,
+    scheduleInterval=None,
+    domain=None,
+    lifeCycle=None,
+    sourceHash=None,
+)
+
+MOCK_PIPELINE_SERVICE = PipelineService(
+    id="85811038-099a-11ed-861d-0242ac120002",
+    name="gluepipeline_test",
+    fullyQualifiedName=FullyQualifiedEntityName("gluepipeline_test"),
+    connection=PipelineConnection(),
+    serviceType=PipelineServiceType.DBTCloud,
+)
+
+MOCK_PIPELINE = Pipeline(
+    id="2aaa012e-099a-11ed-861d-0242ac120002",
+    name=EntityName(root="redshift workflow"),
+    fullyQualifiedName="gluepipeline_test.redshift workflow",
+    displayName="OpenMetadata DBTCloud Workflow",
+    description=Markdown(root="Example Job Description"),
+    dataProducts=None,
+    sourceUrl=SourceUrl(
+        root="https://abc12.us1.dbt.com/deploy/70403103922125/projects/70403103926818/jobs/70403103936332"
+    ),
+    concurrency=None,
+    pipelineLocation=None,
+    startDate=None,
+    tasks=[
+        Task(
+            name="70403110257794",
+            displayName=None,
+            fullyQualifiedName=None,
+            description=None,
+            sourceUrl=SourceUrl(
+                root="https://abc12.us1.dbt.com/deploy/70403103922125/projects/70403103926818/runs/70403110257794/"
+            ),
+            downstreamTasks=None,
+            taskType=None,
+            taskSQL=None,
+            startDate="2024-05-27 10:42:20.621788+00:00",
+            endDate="2024-05-28 10:42:52.622408+00:00",
+            tags=None,
+            owners=None,
+        ),
+        Task(
+            name="70403111615088",
+            displayName=None,
+            fullyQualifiedName=None,
+            description=None,
+            sourceUrl=SourceUrl(
+                root="https://abc12.us1.dbt.com/deploy/70403103922125/projects/70403103926818/runs/70403111615088/"
+            ),
+            downstreamTasks=None,
+            taskType=None,
+            taskSQL=None,
+            startDate="None",
+            endDate="None",
+            tags=None,
+            owners=None,
+        ),
+    ],
+    tags=None,
+    owners=None,
+    service=EntityReference(
+        id="85811038-099a-11ed-861d-0242ac120002", type="pipelineService"
+    ),
+    extension=None,
+    scheduleInterval="6 */12 * * 0,1,2,3,4,5,6",
+    domain=None,
+    lifeCycle=None,
+    sourceHash=None,
+)
+
+EXPECTED_PIPELINE_NAME = "redshift workflow"
+
+
+class GluePipelineUnitTest(TestCase):
+    """
+    DBTCloud unit tests
+    """
+
+    @patch(
+        "metadata.ingestion.source.pipeline.pipeline_service.PipelineServiceSource.test_connection"
+    )
+    def __init__(self, methodName, test_connection) -> None:
+        super().__init__(methodName)
+        test_connection.return_value = False
+
+        config = OpenMetadataWorkflowConfig.model_validate(mock_glue_config)
+        self.gluepipeline = GluepipelineSource.create(
+            mock_glue_config["source"],
+            config.workflowConfig.openMetadataServerConfig,
+        )
+        self.gluepipeline.context.get().__dict__["pipeline"] = MOCK_PIPELINE.name.root
+        self.gluepipeline.context.get().__dict__[
+            "pipeline_service"
+        ] = MOCK_PIPELINE_SERVICE.name.root
+
+    def test_pipeline_name(self):
+        assert (
+            self.gluepipeline.get_pipeline_name(EXPECTED_JOB_DETAILS)
+            == EXPECTED_PIPELINE_NAME
+        )
+
+    def test_pipelines(self):
+        pipeline = list(self.gluepipeline.yield_pipeline(EXPECTED_JOB_DETAILS))[0].right
+        assert pipeline == EXPECTED_CREATED_PIPELINES
diff --git a/ingestion/tests/unit/workflow/test_application_workflow.py b/ingestion/tests/unit/workflow/test_application_workflow.py
new file mode 100644
index 000000000000..df725f4d493f
--- /dev/null
+++ b/ingestion/tests/unit/workflow/test_application_workflow.py
@@ -0,0 +1,51 @@
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+"""
+Validate the initialization of the App Workflow
+"""
+import yaml
+
+from metadata.workflow.application import ApplicationWorkflow, AppRunner
+
+
+class TestApp(AppRunner):
+    """Test App class"""
+
+    def close(self) -> None:
+        """I am a test"""
+
+    def run(self) -> None:
+        """I am a test"""
+
+
+def test_init_app() -> None:
+    """We can properly instantiate the app"""
+
+    config = f"""
+            sourcePythonClass: "{__name__}.TestApp"
+            appConfig:
+              type: Automator
+              resources:
+                type: [table]
+                queryFilter: "..."
+              actions:
+                - type: LineagePropagationAction
+                  overwriteMetadata: false
+            workflowConfig:
+              openMetadataServerConfig:
+                hostPort: "http://localhost:8585/api"
+                authProvider: "openmetadata"
+                securityConfig:
+                  jwtToken: "..."
+            """
+
+    workflow = ApplicationWorkflow.create(yaml.safe_load(config))
+    assert isinstance(workflow, ApplicationWorkflow)
diff --git a/openmetadata-docs/content/partials/v1.5/connectors/storage/connectors-list.md b/openmetadata-docs/content/partials/v1.5/connectors/storage/connectors-list.md
index 8b7ad617ad67..3a796ba54172 100644
--- a/openmetadata-docs/content/partials/v1.5/connectors/storage/connectors-list.md
+++ b/openmetadata-docs/content/partials/v1.5/connectors/storage/connectors-list.md
@@ -2,6 +2,6 @@
 
 {% connectorInfoCard name="S3 Storage" stage="PROD" href="/connectors/storage/s3" platform="OpenMetadata" / %}
 {% connectorInfoCard name="ADLS" stage="PROD" href="/connectors/storage/adls" platform="Collate" / %}
-{% connectorInfoCard name="GCS" stage="PROD" href="/connectors/storage/gcs" platform="Collate" / %}
+{% connectorInfoCard name="GCS" stage="PROD" href="/connectors/storage/gcs" platform="OpenMetadata" / %}
 
 {% /connectorsListContainer %}
\ No newline at end of file
diff --git a/openmetadata-docs/content/partials/v1.6/connectors/storage/connectors-list.md b/openmetadata-docs/content/partials/v1.6/connectors/storage/connectors-list.md
index 8b7ad617ad67..3a796ba54172 100644
--- a/openmetadata-docs/content/partials/v1.6/connectors/storage/connectors-list.md
+++ b/openmetadata-docs/content/partials/v1.6/connectors/storage/connectors-list.md
@@ -2,6 +2,6 @@
 
 {% connectorInfoCard name="S3 Storage" stage="PROD" href="/connectors/storage/s3" platform="OpenMetadata" / %}
 {% connectorInfoCard name="ADLS" stage="PROD" href="/connectors/storage/adls" platform="Collate" / %}
-{% connectorInfoCard name="GCS" stage="PROD" href="/connectors/storage/gcs" platform="Collate" / %}
+{% connectorInfoCard name="GCS" stage="PROD" href="/connectors/storage/gcs" platform="OpenMetadata" / %}
 
 {% /connectorsListContainer %}
\ No newline at end of file
diff --git a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md
index 09a6f8e63ed0..24bc4ebc30df 100644
--- a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md
+++ b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md
@@ -113,75 +113,14 @@ We believe this update will bring greater consistency and clarity to our version
 
 # Backward Incompatible Changes
 
-## 1.5.0
+## 1.6.0
 
-### Multi Owners
-OpenMetadata allows a single user or a team to be tagged as owners for any data assets. In Release 1.5.0, we allow users to tag multiple individual owners or a single team. This will allow organizations to add ownership to multiple individuals without necessarily needing to create a team around them like previously.
+### Ingestion Workflow Status
 
-This is a backward incompatible change, if you are using APIs, please make sure the owner field is now changed to “owners”
+We are updating how we compute the success percentage. Previously, we took into account for partial success the results
+of the Source (e.g., the tables we were able to properly retrieve from Snowflake, Redshift, etc.). This means that we had 
+an error threshold in there were if up to 90% of the tables were successfully ingested, we would still consider the
+workflow as successful. However, any errors when sending the information to OpenMetadata would be considered as a failure.
 
-### Import/Export Format
-To support the multi-owner format, we have now changed how we export and import the CSV file in glossary, services, database, schema, table, etc. The new format will be
-user:userName;team:TeamName
-
-If you are importing an older file, please make sure to make this change.
-
-### Pydantic V2
-The core of OpenMetadata are the JSON Schemas that define the metadata standard. These schemas are automatically translated into Java, Typescript, and Python code with Pydantic classes.
-
-In this release, we have [migrated](https://docs.pydantic.dev/latest/migration/) the codebase from Pydantic V1 to Pydantic V2.
-
-### Deployment Related Changes (OSS only)
-
-`./bootstrap/bootstrap_storage.sh` **removed**
-
-OpenMetadata community has built rolling upgrades to database schema and the data to make upgrades easier. This tool is now called as ./bootstrap/openmetadata-ops.sh and has been part of our releases since 1.3. The `bootstrap_storage.sh` doesn’t support new native schemas in OpenMetadata. Hence, we have deleted this tool from this release.
-
-While upgrading, please refer to our Upgrade Notes in the documentation. Always follow the best practices provided there.
-
-### Database Connection Pooling
-
-OpenMetadata uses Jdbi to handle database-related operations such as read/write/delete. In this release, we introduced additional configs to help with connection pooling, allowing the efficient use of a database with low resources.
-
-Please update the defaults if your cluster is running at a large scale to scale up the connections efficiently.
-
-For the new configuration, please refer to the [doc](https://docs.open-metadata.org/latest/deployment/database-connection-pooling) here
-
-### Data Insights
-
-The Data Insights application is meant to give you a quick glance at your data's state and allow you to take action based on the information you receive. To continue pursuing this objective, the application was completely refactored to allow customizability.
-
-Part of this refactor was making Data Insights an internal application, no longer relying on an external pipeline. This means triggering Data Insights from the Python SDK will no longer be possible.
-
-With this change you will need to run a backfill on the Data Insights for the last couple of days since the Data Assets data changed.
-
-### UI Changes
-
-#### New Explore Page
-
-Explore page displays hierarchically organized data assets by grouping them into `services > database > schema > tables/stored procedures`. This helps users organically find the data asset they are looking for based on a known database or schema they were using. This is a new feature and changes the way the Explore page was built in previous releases.
-
-#### Connector Schema Changes
-
-In the latest release, several updates and enhancements have been made to the JSON schema across various connectors. These changes aim to improve security, configurability, and expand integration capabilities. Here's a detailed breakdown of the updates:
-
-- **KafkaConnect**: Added `schemaRegistryTopicSuffixName` to enhance topic configuration flexibility for schema registries.
-- **GCS Datalake**: Introduced `bucketNames` field, allowing users to specify targeted storage buckets within the Google Cloud Storage environment.
-- **OpenLineage**: Added `saslConfig` to enhance security by enabling SASL (Simple Authentication and Security Layer) configuration.
-- **Salesforce**: Added sslConfig to strengthen the security layer for Salesforce connections by supporting SSL.
-- **DeltaLake**: Updated schema by moving metastoreConnection to a newly created `metastoreConfig.json` file. Additionally, introduced `configSource` to better define source configurations, with new support for `metastoreConfig.json` and `storageConfig.json`.
-- **Iceberg RestCatalog**: Removed clientId and `clientSecret` as mandatory fields, making the schema more flexible for different authentication methods.
-- **DBT Cloud Pipelines**: Added as a new connector to support cloud-native data transformation workflows using DBT.
-- **Looker**: Expanded support to include connections using GitLab integration, offering more flexible and secure version control.
-- **Tableau**: Enhanced support by adding capabilities for connecting with `TableauPublishedDatasource` and `TableauEmbeddedDatasource`, providing more granular control over data visualization and reporting.
-
-### Include DDL
-During the Database Metadata ingestion, we can optionally pick up the DDL for both tables and views. During the metadata ingestion, we use the view DDLs to generate the View Lineage.
-
-To reduce the processing time for out-of-the-box workflows, we are disabling the include DDL by default, whereas before, it was enabled, which potentially led to long-running workflows.
-
-### Secrets Manager
-Starting with the release 1.5.0, the JWT Token for the bots will be sent to the Secrets Manager if you configured one. It won't appear anymore in your dag_generated_configs in Airflow.
-
-### Python SDK
-The `metadata insight` command has been removed. Since Data Insights application was moved to be an internal system application instead of relying on external pipelines the SDK command to run the pipeline was removed.
+Now, we're changing this behavior to consider the success rate of all the steps involved in the workflow. The UI will
+then show more `Partial Success` statuses rather than `Failed`, properly reflecting the real state of the workflow.
diff --git a/openmetadata-docs/content/v1.4.x/connectors/pipeline/glue-pipeline/yaml.md b/openmetadata-docs/content/v1.4.x/connectors/pipeline/glue-pipeline/yaml.md
index e6a57bcfcf62..86b773919e10 100644
--- a/openmetadata-docs/content/v1.4.x/connectors/pipeline/glue-pipeline/yaml.md
+++ b/openmetadata-docs/content/v1.4.x/connectors/pipeline/glue-pipeline/yaml.md
@@ -112,11 +112,11 @@ This is a sample config for Glue:
 
 ```yaml {% isCodeBlock=true %}
 source:
-  type: glue
+  type: gluepipeline 
   serviceName: local_glue
   serviceConnection:
     config:
-      type: Glue
+      type: GluePipeline
       awsConfig:
 ```
 ```yaml {% srNumber=1 %}
diff --git a/openmetadata-docs/content/v1.4.x/deployment/bare-metal/index.md b/openmetadata-docs/content/v1.4.x/deployment/bare-metal/index.md
index f9bb71f11abe..05447310fd90 100644
--- a/openmetadata-docs/content/v1.4.x/deployment/bare-metal/index.md
+++ b/openmetadata-docs/content/v1.4.x/deployment/bare-metal/index.md
@@ -53,7 +53,7 @@ You can refer a sample script [here](https://github.com/open-metadata/OpenMetada
 
 ## Elasticsearch (version 8.X)
 
-OpenMetadata supports ElasticSearch version up to 8.10.2. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
+OpenMetadata supports ElasticSearch version up to 8.11.4. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
 [Installing ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
 
 Please follow the instructions here to [install ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/setup.html).
@@ -169,7 +169,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support 
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version between 12 or higher
 
 For Production Systems, we recommend Amazon RDS to be in Multiple Availability Zones. For Amazon OpenSearch (or ElasticSearch) Service, we recommend Multiple Availability Zones with minimum 3 Master Nodes.
diff --git a/openmetadata-docs/content/v1.4.x/deployment/docker/index.md b/openmetadata-docs/content/v1.4.x/deployment/docker/index.md
index 338a79dfb120..8b8f21bf7789 100644
--- a/openmetadata-docs/content/v1.4.x/deployment/docker/index.md
+++ b/openmetadata-docs/content/v1.4.x/deployment/docker/index.md
@@ -243,7 +243,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version 12 or higher
 
 Note:-
diff --git a/openmetadata-docs/content/v1.4.x/deployment/kubernetes/aks.md b/openmetadata-docs/content/v1.4.x/deployment/kubernetes/aks.md
index 7c606ecfd6c1..3e82e230511d 100644
--- a/openmetadata-docs/content/v1.4.x/deployment/kubernetes/aks.md
+++ b/openmetadata-docs/content/v1.4.x/deployment/kubernetes/aks.md
@@ -15,7 +15,7 @@ We support
 
 - Azure SQL (MySQL) engine version 8 or higher
 - Azure SQL (PostgreSQL) engine version 12 or higher
-- Elastic Cloud (ElasticSearch version 8.10.2)
+- Elastic Cloud (ElasticSearch version 8.11.4)
 
 Once you have the Azure SQL and Elastic Cloud on Azure configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 
diff --git a/openmetadata-docs/content/v1.4.x/deployment/kubernetes/on-prem.md b/openmetadata-docs/content/v1.4.x/deployment/kubernetes/on-prem.md
index ab84a05697f2..8751547ca565 100644
--- a/openmetadata-docs/content/v1.4.x/deployment/kubernetes/on-prem.md
+++ b/openmetadata-docs/content/v1.4.x/deployment/kubernetes/on-prem.md
@@ -22,7 +22,7 @@ We support
 
 - MySQL engine version 8 or higher
 - PostgreSQL engine version 12 or higher
-- ElasticSearch version 8.X (upto 8.10.2) or OpenSearch Version 2.X (upto 2.7)
+- ElasticSearch version 8.X (upto 8.11.4) or OpenSearch Version 2.X (upto 2.7)
 
 Once you have the External Database and Search Engine configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 
diff --git a/openmetadata-docs/content/v1.4.x/deployment/upgrade/versions/110-to-120.md b/openmetadata-docs/content/v1.4.x/deployment/upgrade/versions/110-to-120.md
index c4d742d006b5..2124ccc57a2a 100644
--- a/openmetadata-docs/content/v1.4.x/deployment/upgrade/versions/110-to-120.md
+++ b/openmetadata-docs/content/v1.4.x/deployment/upgrade/versions/110-to-120.md
@@ -160,7 +160,7 @@ For Kubernetes Deployment, `openmetadata.config.database.dbParams` is available
 ### Version Upgrades
 
 - The OpenMetadata Server is now based on **JDK 17**
-- OpenMetadata now **requires** **Elasticsearch** version **8.10.2** or **Opensearch** version **2.7**
+- OpenMetadata now **requires** **Elasticsearch** version **8.11.4** or **Opensearch** version **2.7**
 
 There is no direct migration to bump the indexes to the new supported versions. You might see errors like:
 
diff --git a/openmetadata-docs/content/v1.4.x/menu.md b/openmetadata-docs/content/v1.4.x/menu.md
index db5ad502fb4c..a91c580bbd82 100644
--- a/openmetadata-docs/content/v1.4.x/menu.md
+++ b/openmetadata-docs/content/v1.4.x/menu.md
@@ -5,8 +5,6 @@ site_menu:
 
   - category: Quickstart
     url: /quick-start
-    color: violet-70
-    icon: quickstart
   - category: Quickstart / Try OpenMetadata in Docker
     url: /quick-start/local-docker-deployment
   - category: Quickstart / Try OpenMetadata in Kubernetes
@@ -16,8 +14,6 @@ site_menu:
 
   - category: Deployment
     url: /deployment
-    color: violet-70
-    icon: deployment
   - category: Deployment / Bare Metal Deployment
     url: /deployment/bare-metal
   - category: Deployment / Bare Metal Deployment / Enable Security
@@ -208,8 +204,6 @@ site_menu:
 
   - category: Connectors
     url: /connectors
-    color: violet-70
-    icon: openmetadata
 
   - category: Connectors / Database
     url: /connectors/database
@@ -653,8 +647,6 @@ site_menu:
 
   - category: How-to Guides
     url: /how-to-guides
-    color: violet-70
-    icon: openmetadata
 
   - category: How-to Guides / Admin Guide
     url: /how-to-guides/admin-guide
@@ -875,8 +867,6 @@ site_menu:
 
   - category: Releases
     url: /releases
-    color: violet-70
-    icon: overview
   - category: Releases / Latest Release
     url: /releases/latest-release
   - category: Releases / Supported Releases
@@ -959,8 +949,6 @@ site_menu:
 
   - category: Main Concepts
     url: /main-concepts
-    color: violet-70
-    icon: main-concepts
   - category: Main Concepts / High Level Design
     url: /main-concepts/high-level-design
   - category: Main Concepts / Backend DB
@@ -1888,8 +1876,6 @@ site_menu:
 
   - category: Developers
     url: /developers
-    color: violet-70
-    icon: developers
   - category: Developers / Architecture
     url: /developers/architecture
   - category: Developers / Architecture / Understand Code Layout
@@ -1935,8 +1921,6 @@ site_menu:
 
   - category: SDK
     url: /sdk
-    color: violet-70
-    icon: sdk
   - category: SDK / Python SDK
     url: /sdk/python
   - category: SDK / Python SDK / Entities
diff --git a/openmetadata-docs/content/v1.5.x/Metapilot/how-to-use-metapilot.md b/openmetadata-docs/content/v1.5.x/Metapilot/how-to-use-metapilot.md
new file mode 100644
index 000000000000..3bf1b2a56350
--- /dev/null
+++ b/openmetadata-docs/content/v1.5.x/Metapilot/how-to-use-metapilot.md
@@ -0,0 +1,110 @@
+---
+title: How to Use MetaPilot
+slug: /metapilot/how-to-use-metapilot
+collate: true
+---
+
+# How to Use MetaPilot
+
+## 1. Setting Up MetaPilot
+- Navigate to **Settings > Applications** in the Collate platform.
+
+{% image
+src="/images/v1.5/metapilot/setting-up-metapilot-1.png"
+alt="setting up metapilot"
+caption="Navigate to Settings > Applications"
+/%}
+
+- Install MetaPilot by following the on-screen instructions.
+
+{% image
+src="/images/v1.5/metapilot/setting-up-metapilot-2.png"
+alt="Install MetaPilot"
+caption="Install MetaPilot"
+/%}
+
+- Select the databases for which you want MetaPilot to automatically generate descriptions. You can configure multiple databases and set a default database for the MetaPilot chatbot to work with.
+
+{% image
+src="/images/v1.5/metapilot/setting-up-metapilot-3.png"
+alt="automatically generate descriptions"
+caption="automatically generate descriptions"
+/%}
+
+- **Scheduling**: Schedule MetaPilot to run regularly, automatically generating metadata at predefined intervals (e.g., weekly).
+
+{% image
+src="/images/v1.5/metapilot/setting-up-metapilot-4.png"
+alt="Schedule MetaPilot"
+caption="Schedule MetaPilot"
+/%}
+
+## 2. Using the MetaPilot Chatbot
+- The MetaPilot chatbot icon appears on every page after installation.
+
+{% image
+src="/images/v1.5/metapilot/using-metapilot-chatbot-1.png"
+alt="chatbot icon"
+caption="chatbot icon"
+/%}
+
+- Interact with the chatbot by typing natural language questions. For example:
+  - “Show me sales data for Q1.”
+  - “What is the average revenue per customer?”
+
+- MetaPilot will generate the corresponding SQL query and provide a detailed explanation of the query logic.
+
+{% image
+src="/images/v1.5/metapilot/using-metapilot-chatbot-2.png"
+alt="natural language questions"
+caption="natural language questions"
+/%}
+
+- Users can refine queries by providing further instructions, and the chatbot will adjust the SQL query accordingly.
+
+## 3. Optimizing and Fixing SQL Queries
+
+- The **Metadata Usage** workflows will ingest the queries being run against the tables. You can see how long each query has been executed in the **Queries** tab.
+
+{% image
+src="/images/v1.5/metapilot/fixing-sql-queries-1.png"
+alt="metadata usage workflows"
+caption="metadata usage workflows"
+/%}
+
+- If a query runs inefficiently, ask the chatbot to optimize it by typing: “Optimize this query.”
+
+{% image
+src="/images/v1.5/metapilot/fixing-sql-queries-3.png"
+alt="Optimize this query"
+caption="Optimize this query"
+/%}
+
+- MetaPilot will return a more efficient version of the SQL query, which you can then copy and execute in your database.
+
+- If the query contains errors or isn’t functioning correctly, ask MetaPilot: “Can you fix this query?” 
+
+- MetaPilot will correct the query and provide a working version.
+
+{% image
+src="/images/v1.5/metapilot/fixing-sql-queries-4.png"
+alt="Can you fix this query"
+caption="Can you fix this query"
+/%}
+
+## 4. Reviewing Generated Metadata
+- Once MetaPilot generates descriptions for tables and columns, navigate to the **database view** to review the metadata.
+- You can accept or reject each suggestion individually or choose to accept all suggestions in bulk.
+
+{% image
+src="/images/v1.5/metapilot/reviewing-generated-metadata.png"
+alt="reviewing Generated Metadata"
+caption="reviewing Generated Metadata"
+/%}
+
+- MetaPilot allows you to document entire datasets in a matter of minutes, significantly reducing the manual effort required to maintain metadata.
+
+## Best Practices
+- **Regular Scheduling**: Schedule MetaPilot to run at regular intervals to ensure your metadata is always up-to-date, especially when dealing with frequently changing datasets.
+- **Leverage the Chatbot for Query Writing**: Encourage both technical and non-technical users to use the MetaPilot chatbot for SQL query generation. It simplifies complex query writing and ensures accurate results.
+- **Optimize Queries Regularly**: Monitor the performance of your queries and use MetaPilot to optimize them, especially when working with large datasets or queries that require complex joins and filters.
diff --git a/openmetadata-docs/content/v1.5.x/Metapilot/index.md b/openmetadata-docs/content/v1.5.x/Metapilot/index.md
new file mode 100644
index 000000000000..43f031cf4e0a
--- /dev/null
+++ b/openmetadata-docs/content/v1.5.x/Metapilot/index.md
@@ -0,0 +1,85 @@
+---
+title: Metapilot
+slug: /metapilot
+collate: true
+---
+
+# Metapilot Technical Documentation
+
+{%  youtube videoId="6glMYLzxNqk" start="0:00" end="04:20" width="560px" height="315px" /%}
+
+MetaPilot is an AI-powered tool within Collate that simplifies and enhances metadata management. By integrating generative AI, MetaPilot assists users in automating the documentation of data assets, writing and optimizing SQL queries, and interacting with data through natural language. This first-of-its-kind data copilot improves productivity by automating tedious tasks and providing intelligent insights into your data environment.
+
+## Key Features
+
+- **Automated Data Documentation**: Automatically generates metadata descriptions for tables and columns, saving time and effort for data owners and stewards.
+- **Natural Language SQL Query Generation**: Allows users to interact with MetaPilot through a chatbot to generate SQL queries by simply asking questions in plain English.
+- **SQL Query Optimization and Fixing**: Capable of optimizing and troubleshooting SQL queries to improve their performance and efficiency.
+
+## Why MetaPilot is Useful
+
+### Metadata Management Challenges
+
+Managing metadata across multiple data assets can be overwhelming due to the influx of new data and changing team dynamics. MetaPilot addresses these challenges by:
+
+- Automating metadata description generation.
+- Simplifying the creation and optimization of SQL queries.
+- Reducing manual effort and enhancing data quality.
+
+### Time-Saving Features
+
+Documenting thousands of tables manually is tedious and time-consuming. MetaPilot automates metadata generation, allowing data teams to focus on high-value tasks and ensuring that data assets are consistently documented and understood across the organization.
+
+## Use Cases
+
+### 1. Automatic Data Asset Documentation
+
+{% image
+src="/images/v1.5/metapilot/reviewing-generated-metadata.png"
+alt="Automatic Data Asset Documentation"
+caption="Auto Generate data Asset Documentation"
+/%}
+
+- **Problem**: Manually creating metadata descriptions for large datasets is labor-intensive and error-prone.
+- **Solution**: MetaPilot’s generative AI automates the process, analyzing database structures and suggesting accurate descriptions for tables and columns.
+- **How It Works**: After configuring MetaPilot, it scans the database schema and generates metadata descriptions automatically. Users can review these descriptions and approve or reject them in bulk or individually.
+- **Benefit**: Streamlines the documentation process, ensuring consistent and up-to-date metadata across all datasets.
+
+### 2. Natural Language SQL Query Generation
+
+{% image
+src="/images/v1.5/metapilot/using-metapilot-chatbot-2.png"
+alt="Natural Language SQL Query Generation"
+caption="Natural Language SQL Query Generation"
+/%}
+
+- **Problem**: Non-technical users often struggle with writing SQL queries to extract insights from databases.
+- **Solution**: MetaPilot’s chatbot allows users to ask questions in natural language, generating SQL queries and providing explanations to help extract the required data quickly and easily.
+- **How It Works**: Users can click on the MetaPilot chatbot widget and ask questions like "Show me sales data from last quarter." MetaPilot generates the corresponding SQL query and explains its logic.
+- **Benefit**: Democratizes data access by enabling users of all technical levels to interact with data without deep SQL knowledge.
+
+### 3. SQL Query Optimization and Troubleshooting
+
+{% image
+src="/images/v1.5/metapilot/fixing-sql-queries-3.png"
+alt="SQL Query Optimization and Troubleshooting"
+caption="SQL Query Optimization"
+/%}
+
+- **Problem**: SQL queries can become complex and inefficient, leading to performance issues and increased costs.
+- **Solution**: MetaPilot optimizes inefficient queries to improve performance, saving both time and resources.
+- **How It Works**: If a query runs too long, users can request MetaPilot to optimize it. MetaPilot provides a more efficient SQL query version that can be implemented immediately.
+- **Benefit**: Enhances query performance, reduces costs associated with inefficient queries, and speeds up data processing.
+
+### 4. Fixing SQL Queries
+
+{% image
+src="/images/v1.5/metapilot/fixing-sql-queries-4.png"
+alt="Fixing SQL Queries"
+caption="Fixing SQL Queries"
+/%}
+
+- **Problem**: Complex queries can cause issues even for SQL experts.
+- **Solution**: MetaPilot can fix problematic SQL queries, ensuring they run correctly and efficiently.
+- **How It Works**: Users can ask MetaPilot, "Can you fix this query for me?" It analyzes the query, detects issues, and returns a corrected version ready for use.
+- **Benefit**: Simplifies writing and maintaining queries, allowing data teams to focus on analysis rather than troubleshooting.
diff --git a/openmetadata-docs/content/v1.5.x/collate-menu.md b/openmetadata-docs/content/v1.5.x/collate-menu.md
index 21c35dfa2f7c..08d034076c29 100644
--- a/openmetadata-docs/content/v1.5.x/collate-menu.md
+++ b/openmetadata-docs/content/v1.5.x/collate-menu.md
@@ -5,8 +5,6 @@ site_menu:
 
   - category: Getting Started
     url: /getting-started
-    color: violet-70
-    icon: openmetadata
   
   - category: Getting Started / Day 1
     url: /getting-started/day-1
@@ -27,8 +25,6 @@ site_menu:
 
   - category: Connectors
     url: /connectors
-    color: violet-70
-    icon: openmetadata
 
   - category: Connectors / Database
     url: /connectors/database
@@ -498,8 +494,6 @@ site_menu:
 
   - category: How-to Guides
     url: /how-to-guides
-    color: violet-70
-    icon: openmetadata
 
   - category: How-to Guides / Admin Guide
     url: /how-to-guides/admin-guide
@@ -682,6 +676,10 @@ site_menu:
 
   - category: How-to Guides / Data Governance
     url: /how-to-guides/data-governance
+  - category: How-to Guides / Data Governance / Automation
+    url: /how-to-guides/data-governance/automation
+  - category: How-to Guides / Data Governance / Automation / How to Set Up Automations in Collate 
+    url: /how-to-guides/data-governance/automation/set-up-automation
   - category: How-to Guides / Data Governance / Glossary
     url: /how-to-guides/data-governance/glossary
   - category: How-to Guides / Data Governance / Glossary / What is a Glossary Term
@@ -725,8 +723,6 @@ site_menu:
 
   - category: Enable Security
     url: /security
-    color: violet-70
-    icon: deployment
 
   - category: Enable Security / Basic Authentication
     url: /security/basic-auth
@@ -759,8 +755,6 @@ site_menu:
    
   - category: Releases
     url: /releases
-    color: violet-70
-    icon: overview
   - category: Releases / Latest Release
     url: /releases/latest-release
   - category: Releases / Supported Releases
@@ -855,8 +849,6 @@ site_menu:
 
   - category: Main Concepts
     url: /main-concepts
-    color: violet-70
-    icon: main-concepts
   - category: Main Concepts / High Level Design
     url: /main-concepts/high-level-design
   - category: Main Concepts / Backend DB
@@ -1790,8 +1782,6 @@ site_menu:
 
   - category: SDK
     url: /sdk
-    color: violet-70
-    icon: sdk
   - category: SDK / Python SDK
     url: /sdk/python
   - category: SDK / Python SDK / Entities
@@ -1870,4 +1860,8 @@ site_menu:
     url: /sdk/java
   - category: SDK / Go
     url: /sdk/go
+  - category: Metapilot
+    url: /metapilot
+  - category: Metapilot / How to Use MetaPilot
+    url: /metapilot/how-to-use-metapilot
 ---
diff --git a/openmetadata-docs/content/v1.5.x/connectors/ingestion/lineage/spark-lineage.md b/openmetadata-docs/content/v1.5.x/connectors/ingestion/lineage/spark-lineage.md
index 3e48cfc5eaa0..43a7ebf8cc44 100644
--- a/openmetadata-docs/content/v1.5.x/connectors/ingestion/lineage/spark-lineage.md
+++ b/openmetadata-docs/content/v1.5.x/connectors/ingestion/lineage/spark-lineage.md
@@ -343,3 +343,40 @@ spark.openmetadata.transport.timeout 30
 ```
 
 After all these steps are completed you can start/restart your compute instance and you are ready to extract the lineage from spark to OpenMetadata.
+
+
+## Using Spark Agent with Glue
+
+Follow the below steps in order to use OpenMetadata Spark Agent with glue.
+
+### 1. Specify the OpenMetadata Spark Agent JAR URL
+
+1. Upload the OpenMetadata Spark Agent Jar to S3
+2. Navigate to the glue job,In the Job details tab, navigate to Advanced properties → Libraries → Dependent Jars path
+3. Add the S3 url of OpenMetadata Spark Agent Jar in the Dependent Jars path.
+
+{% image
+  src="/images/v1.5/connectors/spark/glue-job-jar.png"
+  alt="Glue Job Configure Jar"
+  caption="Glue Job Configure Jar"
+ /%}
+
+
+### 2. Add Spark configuration in Job Parameters
+
+In the same Job details tab, add a new property under Job parameters:
+
+1. Add the `--conf` property with following value, make sure to customize this configuration as described in the above documentation.
+
+```
+spark.extraListeners=org.openmetadata.spark.agent.OpenMetadataSparkListener --conf spark.openmetadata.transport.hostPort=https://your-org.host:port  --conf spark.openmetadata.transport.type=openmetadata --conf spark.openmetadata.transport.jwtToken=<jwt-token> --conf spark.openmetadata.transport.pipelineServiceName=glue_spark_pipeline_service --conf spark.openmetadata.transport.pipelineName=glue_pipeline_name --conf spark.openmetadata.transport.timeout=30
+```
+
+2. Add the `--user-jars-first` parameter and set its value to `true`
+
+{% image
+  src="/images/v1.5/connectors/spark/glue-job-params.png"
+  alt="Glue Job Configure Params"
+  caption="Glue Job Configure Params"
+ /%}
+
diff --git a/openmetadata-docs/content/v1.5.x/connectors/metadata/alation/index.md b/openmetadata-docs/content/v1.5.x/connectors/metadata/alation/index.md
index e732fe1d71ab..a7ff96c95ded 100644
--- a/openmetadata-docs/content/v1.5.x/connectors/metadata/alation/index.md
+++ b/openmetadata-docs/content/v1.5.x/connectors/metadata/alation/index.md
@@ -122,9 +122,9 @@ Choose either postgres or mysql connection depending on the db:
 
 **ingestDatasources**: Specifies if databases, schemas and tables should be included while ingesting. By default is set to `true`.
 
-**ingestDomains**: Specifies if hidden domains and subdomains should be included while ingesting. By default is set to `true`.
+**ingestDomains**: Specifies if domains and subdomains should be included while ingesting. By default is set to `true`.
 
-**ingestDashboards**: Specifies if hidden BI sources and dashboards should be included while ingesting. By default is set to `true`.
+**ingestDashboards**: Specifies if BI sources and dashboards should be included while ingesting. By default is set to `true`.
 
 **alationTagClassificationName**: Specify the classification name under which the tags from alation will be created in OpenMetadata. By default it is set to `alationTags`.
 
diff --git a/openmetadata-docs/content/v1.5.x/connectors/pipeline/glue-pipeline/yaml.md b/openmetadata-docs/content/v1.5.x/connectors/pipeline/glue-pipeline/yaml.md
index 5898468a0f98..4468dc424867 100644
--- a/openmetadata-docs/content/v1.5.x/connectors/pipeline/glue-pipeline/yaml.md
+++ b/openmetadata-docs/content/v1.5.x/connectors/pipeline/glue-pipeline/yaml.md
@@ -112,11 +112,11 @@ This is a sample config for Glue:
 
 ```yaml {% isCodeBlock=true %}
 source:
-  type: glue
+  type: gluepipeline 
   serviceName: local_glue
   serviceConnection:
     config:
-      type: Glue
+      type: GluePipeline
       awsConfig:
 ```
 ```yaml {% srNumber=1 %}
diff --git a/openmetadata-docs/content/v1.5.x/deployment/backup-restore-metadata.md b/openmetadata-docs/content/v1.5.x/deployment/backup-restore-metadata.md
index 5e704c7e56de..306423cf28f8 100644
--- a/openmetadata-docs/content/v1.5.x/deployment/backup-restore-metadata.md
+++ b/openmetadata-docs/content/v1.5.x/deployment/backup-restore-metadata.md
@@ -70,15 +70,16 @@ Ingest some data...
 
 ```shell
 BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql"
-DOCKER_COMPOSE_FILE="docker/development/docker-compose.yml"
+export COMPOSE_FILE="docker/development/docker-compose.yml"
 # backup
-docker compose -f $DOCKER_COMPOSE_FILE exec ingestion mysqldump --no-tablespaces -u openmetadata_user -popenmetadata_password -h mysql -P 3306 openmetadata_db > $BACKUP_FILE
+docker compose exec ingestion mysqldump --no-tablespaces -u openmetadata_user -popenmetadata_password -h mysql -P 3306 openmetadata_db > $BACKUP_FILE
 # create the restore database
-docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "create database restore;"
-docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "grant all privileges on restore.* to 'openmetadata_user'@'%';"
-docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "flush privileges;"
+docker compose exec mysql  mysql -u root -ppassword -e "create database restore;"
+docker compose exec mysql  mysql -u root -ppassword -e "grant all privileges on restore.* to 'openmetadata_user'@'%';"
+docker compose exec mysql  mysql -u root -ppassword -e "GRANT SUPER, SYSTEM_VARIABLES_ADMIN, SESSION_VARIABLES_ADMIN ON *.* TO 'openmetadata_user'@'%';"
+docker compose exec mysql  mysql -u root -ppassword -e "flush privileges;"
 # restore from the backup
-docker compose -f $DOCKER_COMPOSE_FILE exec -T ingestion mysql -u openmetadata_user -popenmetadata_password -h mysql -P 3306 restore < $BACKUP_FILE
+docker compose exec -T ingestion mysql -u openmetadata_user -popenmetadata_password -h mysql -P 3306 restore < $BACKUP_FILE
 ```
 
 ### 3. Restart the docker deployment with the restored database
@@ -102,14 +103,14 @@ Ingest some data...
 
 ```shell
 BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql"
-DOCKER_COMPOSE_FILE="docker/development/docker-compose-postgres.yml"
+export COMPOSE_FILE="docker/development/docker-compose-postgres.yml"
 # backup
-docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password ingestion pg_dump -U openmetadata_user -h postgresql -d openmetadata_db > $BACKUP_FILE
+docker compose exec -e PGPASSWORD=openmetadata_password ingestion pg_dump -U openmetadata_user -h postgresql -d openmetadata_db > $BACKUP_FILE
 # create the restore database
-docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "create database restore;"
-docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "ALTER DATABASE restore OWNER TO openmetadata_user;"
+docker compose exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "create database restore;"
+docker compose exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "ALTER DATABASE restore OWNER TO openmetadata_user;"
 # restore from the backup
-docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password -T ingestion psql -U openmetadata_user -h postgresql -d restore < $BACKUP_FILE
+docker compose exec -e PGPASSWORD=openmetadata_password -T ingestion psql -U openmetadata_user -h postgresql -d restore < $BACKUP_FILE
 ```
 
 ### 3. Restart the docker deployment with the restored database
diff --git a/openmetadata-docs/content/v1.5.x/deployment/bare-metal/index.md b/openmetadata-docs/content/v1.5.x/deployment/bare-metal/index.md
index c993747ef63c..180b1a29c9d5 100644
--- a/openmetadata-docs/content/v1.5.x/deployment/bare-metal/index.md
+++ b/openmetadata-docs/content/v1.5.x/deployment/bare-metal/index.md
@@ -58,7 +58,7 @@ If you are facing an error `ERROR: could not compute MD5 hash: disabled for FIPS
 
 ## Elasticsearch (version 8.X)
 
-OpenMetadata supports ElasticSearch version up to 8.10.2. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
+OpenMetadata supports ElasticSearch version up to 8.11.4. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
 [Installing ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
 
 Please follow the instructions here to [install ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/setup.html).
@@ -174,7 +174,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support 
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version between 12 or higher
 
 For Production Systems, we recommend Amazon RDS to be in Multiple Availability Zones. For Amazon OpenSearch (or ElasticSearch) Service, we recommend Multiple Availability Zones with minimum 3 Master Nodes.
diff --git a/openmetadata-docs/content/v1.5.x/deployment/docker/index.md b/openmetadata-docs/content/v1.5.x/deployment/docker/index.md
index b5a8227e3645..0e3689962547 100644
--- a/openmetadata-docs/content/v1.5.x/deployment/docker/index.md
+++ b/openmetadata-docs/content/v1.5.x/deployment/docker/index.md
@@ -244,7 +244,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version 12 or higher
 
 Note:-
diff --git a/openmetadata-docs/content/v1.5.x/deployment/kubernetes/aks.md b/openmetadata-docs/content/v1.5.x/deployment/kubernetes/aks.md
index 5c4abeb5b242..39d3402c610b 100644
--- a/openmetadata-docs/content/v1.5.x/deployment/kubernetes/aks.md
+++ b/openmetadata-docs/content/v1.5.x/deployment/kubernetes/aks.md
@@ -16,7 +16,7 @@ We support
 
 - Azure SQL (MySQL) engine version 8 or higher
 - Azure SQL (PostgreSQL) engine version 12 or higher
-- Elastic Cloud (ElasticSearch version 8.10.2)
+- Elastic Cloud (ElasticSearch version 8.11.4)
 
 Once you have the Azure SQL and Elastic Cloud on Azure configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 
diff --git a/openmetadata-docs/content/v1.5.x/deployment/kubernetes/on-prem.md b/openmetadata-docs/content/v1.5.x/deployment/kubernetes/on-prem.md
index e231b1440d6d..ed0217709f4d 100644
--- a/openmetadata-docs/content/v1.5.x/deployment/kubernetes/on-prem.md
+++ b/openmetadata-docs/content/v1.5.x/deployment/kubernetes/on-prem.md
@@ -23,7 +23,7 @@ We support
 
 - MySQL engine version 8 or higher
 - PostgreSQL engine version 12 or higher
-- ElasticSearch version 8.X (upto 8.10.2) or OpenSearch Version 2.X (upto 2.7)
+- ElasticSearch version 8.X (upto 8.11.4) or OpenSearch Version 2.X (upto 2.7)
 
 Once you have the External Database and Search Engine configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 
diff --git a/openmetadata-docs/content/v1.5.x/deployment/upgrade/versions/110-to-120.md b/openmetadata-docs/content/v1.5.x/deployment/upgrade/versions/110-to-120.md
index b661db7eb18e..c8a61dba23a4 100644
--- a/openmetadata-docs/content/v1.5.x/deployment/upgrade/versions/110-to-120.md
+++ b/openmetadata-docs/content/v1.5.x/deployment/upgrade/versions/110-to-120.md
@@ -161,7 +161,7 @@ For Kubernetes Deployment, `openmetadata.config.database.dbParams` is available
 ### Version Upgrades
 
 - The OpenMetadata Server is now based on **JDK 17**
-- OpenMetadata now **requires** **Elasticsearch** version **8.10.2** or **Opensearch** version **2.7**
+- OpenMetadata now **requires** **Elasticsearch** version **8.11.4** or **Opensearch** version **2.7**
 
 There is no direct migration to bump the indexes to the new supported versions. You might see errors like:
 
diff --git a/openmetadata-docs/content/v1.5.x/how-to-guides/data-governance/automation/index.md b/openmetadata-docs/content/v1.5.x/how-to-guides/data-governance/automation/index.md
new file mode 100644
index 000000000000..c48e1bee9516
--- /dev/null
+++ b/openmetadata-docs/content/v1.5.x/how-to-guides/data-governance/automation/index.md
@@ -0,0 +1,78 @@
+---
+title: Collate Automations Documentation
+slug: /how-to-guides/data-governance/automation
+collate: true
+---
+
+# Collate Automations
+
+{% youtube videoId="ug08aLUyTyE" start="0:00" end="14:52" width="560px" height="315px" /%}
+
+## Overview
+
+Collate's **Automation** feature is a powerful tool designed to simplify and streamline metadata management tasks. By automating repetitive actions such as assigning owners, domains, or tagging data, Collate helps maintain consistency in metadata across an organization's datasets. These automations reduce manual effort and ensure that metadata is always up-to-date, accurate, and governed according to predefined policies.
+
+## Why Automations are Useful
+
+Managing metadata manually can be challenging, particularly in dynamic environments where data constantly evolves. Collate's Automation feature addresses several key pain points:
+
+- **Maintaining Consistency**: Automation helps ensure that metadata such as ownership, tags, and descriptions are applied consistently across all data assets.
+- **Saving Time**: Automations allow data teams to focus on higher-value tasks by eliminating the need for manual updates and maintenance.
+- **Enforcing Governance Policies**: Automations help ensure that data follows organizational policies at all times by automatically applying governance rules (e.g., assigning data owners or domains).
+- **Data Quality and Accountability**: Data quality suffers without clear ownership. Automating ownership assignments helps ensure that data quality issues are addressed efficiently.
+
+## Key Use Cases for Collate Automations
+
+### 1. Bulk Ownership and Domain Assignment
+
+{% image
+src="/images/v1.5/how-to-guides/governance/bulk-ownership-and.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+- **Problem**: Many data assets lack proper ownership and domain assignment, leading to governance and accountability issues. Manually assigning owners can be error-prone and time-consuming.
+- **Solution**: Automations can bulk-assign ownership and domains to datasets, ensuring all data assets are correctly categorized and owned. This process can be applied to tables, schemas, or other assets within Collate.
+- **Benefit**: This use case ensures data assets have a designated owner and are organized under the appropriate domain, making data more discoverable and accountable.
+
+### 2. Bulk Tagging and Glossary Term Assignment
+
+{% image
+src="/images/v1.5/how-to-guides/governance/bulk-tagging-glossary.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+- **Problem**: Manually applying the same tags or glossary terms to multiple datasets can be inefficient and inconsistent.
+- **Solution**: Automations allow users to bulk-apply tags (e.g., PII) or glossary terms (e.g., Customer ID) to specific datasets, ensuring uniformity across the platform.
+- **Benefit**: This automation reduces the risk of missing important tags like PII-sensitive and ensures that key metadata elements are applied consistently across datasets.
+
+### 3. Metadata Propagation via Lineage
+
+{% image
+src="/images/v1.5/how-to-guides/governance/metadata-propogation.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+- **Problem**: When metadata such as tags, descriptions, or glossary terms are updated in one part of the data lineage, they may not be propagated across related datasets, leading to inconsistencies.
+- **Solution**: Use automations to propagate metadata across related datasets, ensuring that all relevant data inherits the correct metadata properties from the source dataset.
+- **Benefit**: Metadata consistency is ensured across the entire data lineage, reducing the need for manual updates and maintaining a single source of truth.
+
+### 4. Automatic PII Detection and Tagging
+
+{% image
+src="/images/v1.5/how-to-guides/governance/automatic-detection.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+- **Problem**: Manually identifying and tagging Personally Identifiable Information (PII) across large datasets is labor-intensive and prone to errors.
+- **Solution**: Automations can automatically detect PII data (e.g., emails, usernames) and apply relevant tags to ensure that sensitive data is flagged appropriately for compliance.
+- **Benefit**: Ensures compliance with data protection regulations by consistently tagging sensitive data, reducing the risk of non-compliance.
+
+## Best Practices
+
+- **Validate Assets Before Applying Actions**: Always use the **Explore** page to verify the assets that will be affected by the automation. This ensures that only the intended datasets are updated.
+- **Use Automation Logs**: Regularly check the **Recent Runs** logs to monitor automation activity and ensure that they are running as expected.
+- **Propagate Metadata Thoughtfully**: When propagating metadata via lineage, make sure that the source metadata is correct before applying it across multiple datasets.
diff --git a/openmetadata-docs/content/v1.5.x/how-to-guides/data-governance/automation/set-up-automation.md b/openmetadata-docs/content/v1.5.x/how-to-guides/data-governance/automation/set-up-automation.md
new file mode 100644
index 000000000000..0b57f0a55b05
--- /dev/null
+++ b/openmetadata-docs/content/v1.5.x/how-to-guides/data-governance/automation/set-up-automation.md
@@ -0,0 +1,67 @@
+---
+title: How to Set Up Automations in OpenMetadata
+slug: /how-to-guides/data-governance/automation/set-up-automation
+collate: true
+---
+
+# How to Set Up Automations in Collate
+
+### Step 1: Access the Automations Section
+In the OpenMetadata UI, navigate to **Govern>Automations**.  
+This will take you to the Automations page where you can view and manage your existing automations.
+
+{% image
+src="/images/v1.5/how-to-guides/governance/automation-1.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+### Step 2: Add a New Automation
+In the Automations page, click the **Add Automation** button located at the top right of the page.  
+A pop-up window will appear to begin the process of adding a new automation.
+
+{% image
+src="/images/v1.5/how-to-guides/governance/automation-2.png"
+alt="Add Automation"
+caption="Add Automation"
+/%}
+
+### Step 3: Fill in Automation Details
+In the pop-up window, provide the necessary information to set up the automation:
+- **Automation Name**: Give a meaningful name to the automation for easy identification.
+- **Description**: Add a brief description explaining what this automation will do (e.g., "Daily metadata ingestion for database XYZ").
+- **Logic/Conditions**: Define any conditions or specific criteria needed for this automation to work (e.g., specific tables or columns to be included).  
+  Ensure that the logic is set up as per your specific requirements to make the automation useful for your workflows.
+
+{% image
+src="/images/v1.5/how-to-guides/governance/automation-4.png"
+alt="Automation details"
+caption="Automation details"
+/%}
+
+{% image
+src="/images/v1.5/how-to-guides/governance/automation-5.png"
+alt="Automation logics"
+caption="Automation logics"
+/%}
+
+### Step 4: Configure Automation Interval
+Once you've filled in the required details, click **Next**.  
+On the next page, you’ll be prompted to select the interval for the automation. This defines how frequently the automation should run (e.g., daily, weekly, or custom intervals).  
+Review your settings and click **Automate** once you are satisfied with the configuration.
+
+{% image
+src="/images/v1.5/how-to-guides/governance/automation-6.png"
+alt="Automation Interval"
+caption="Automation Interval"
+/%}
+
+### Step 5: Manage Your Automation
+After completing the setup, your automation will appear in the Automations list.  
+To manage the automation, click on the three dots next to the automation entry. From here, you can **edit**, **re-deploy**, **delete**, etc.
+
+{% image
+src="/images/v1.5/how-to-guides/governance/automation-7.png"
+alt="Manage Your Automation"
+caption="Manage Your Automation"
+/%}
diff --git a/openmetadata-docs/content/v1.5.x/how-to-guides/data-quality-observability/quality/adding-test-suits.md b/openmetadata-docs/content/v1.5.x/how-to-guides/data-quality-observability/quality/adding-test-suites.md
similarity index 96%
rename from openmetadata-docs/content/v1.5.x/how-to-guides/data-quality-observability/quality/adding-test-suits.md
rename to openmetadata-docs/content/v1.5.x/how-to-guides/data-quality-observability/quality/adding-test-suites.md
index bcaa39b7b443..78d7a89f9c0d 100644
--- a/openmetadata-docs/content/v1.5.x/how-to-guides/data-quality-observability/quality/adding-test-suits.md
+++ b/openmetadata-docs/content/v1.5.x/how-to-guides/data-quality-observability/quality/adding-test-suites.md
@@ -1,6 +1,6 @@
 ---
-title: Adding test suits through the UI
-slug: /how-to-guides/data-quality-observability/quality/adding-test-suits
+title: Adding test suites through the UI
+slug: /how-to-guides/data-quality-observability/quality/adding-test-suites
 ---
 
 # Adding Test Suites Through the UI
diff --git a/openmetadata-docs/content/v1.5.x/how-to-guides/index.md b/openmetadata-docs/content/v1.5.x/how-to-guides/index.md
index 84af90a4eb6a..fcc33a8cc675 100644
--- a/openmetadata-docs/content/v1.5.x/how-to-guides/index.md
+++ b/openmetadata-docs/content/v1.5.x/how-to-guides/index.md
@@ -5,19 +5,6 @@ slug: /how-to-guides
 
 # How-to Guides
 
-## Getting Started
-
-Set up and explore OpenMetadata's core features, from basic configuration to advanced functionalities, for a seamless onboarding experience.
-
-{% tilesContainer %}
-{% tile
-    title="Getting Started"
-    description="Unlock metadata insights for informed business decisions."
-    link="/how-to-guides/getting-started"
-    icon="discovery"
-/%}
-{% /tilesContainer %}
-
 The How-to Guides will give you a walk through on accomplishing the basic to the most advanced things in OpenMetadata. These step-by-step guides will help get an overview of the features and also help explore the various functionalities.  
 
 ## Features in OpenMetadata
diff --git a/openmetadata-docs/content/v1.5.x/menu.md b/openmetadata-docs/content/v1.5.x/menu.md
index 14eac2bf02f8..d42b506dbba7 100644
--- a/openmetadata-docs/content/v1.5.x/menu.md
+++ b/openmetadata-docs/content/v1.5.x/menu.md
@@ -5,19 +5,21 @@ site_menu:
 
   - category: Quickstart
     url: /quick-start
-    color: violet-70
-    icon: quickstart
   - category: Quickstart / Try OpenMetadata in Docker
     url: /quick-start/local-docker-deployment
   - category: Quickstart / Try OpenMetadata in Kubernetes
     url: /quick-start/local-kubernetes-deployment
   - category: Quickstart / Try the OpenMetadata Sandbox
     url: /quick-start/sandbox
+  - category: Quickstart / Getting Started  
+    url: /quick-start/getting-started
+  - category: Quickstart / Day 1  
+    url: /quick-start/getting-started/day-1
+  - category: Quickstart / Day 1 / Database Service Setup
+    url: /quick-start/getting-started/day-1/database-service-setup
 
   - category: Deployment
     url: /deployment
-    color: violet-70
-    icon: deployment
   - category: Deployment / Bare Metal Deployment
     url: /deployment/bare-metal
   - category: Deployment / Bare Metal Deployment / Enable Security
@@ -215,8 +217,6 @@ site_menu:
 
   - category: Connectors
     url: /connectors
-    color: violet-70
-    icon: openmetadata
 
   - category: Connectors / Database
     url: /connectors/database
@@ -386,12 +386,6 @@ site_menu:
     url: /connectors/database/sqlite
   - category: Connectors / Database / SQLite / Run Externally
     url: /connectors/database/sqlite/yaml
-  - category: Connectors / Database / Synapse
-    url: /connectors/database/synapse
-  - category: Connectors / Database / Synapse / Run Externally
-    url: /connectors/database/synapse/yaml
-  - category: Connectors / Database / Synapse / Troubleshooting
-    url: /connectors/database/synapse/troubleshooting
   - category: Connectors / Database / S3 Datalake
     url: /connectors/database/s3-datalake
   - category: Connectors / Database / S3 Datalake / Run Externally
@@ -577,10 +571,6 @@ site_menu:
     url: /connectors/storage/gcs
   - category: Connectors / Storage / GCS / Run Externally
     url: /connectors/storage/gcs/yaml
-  - category: Connectors / Storage / ADLS
-    url: /connectors/storage/adls
-  - category: Connectors / Storage / ADLS / Run Externally
-    url: /connectors/storage/adls/yaml
 
   - category: Connectors / Search
     url: /connectors/search
@@ -599,10 +589,6 @@ site_menu:
     url: /connectors/metadata/atlas
   - category: Connectors / Metadata / Atlas / Run Externally
     url: /connectors/metadata/atlas/yaml
-  - category: Connectors / Metadata / Alation
-    url: /connectors/metadata/alation
-  - category: Connectors / Metadata / Alation / Run Externally
-    url: /connectors/metadata/alation/yaml
   - category: Connectors / Metadata / Alation Sink
     url: /connectors/metadata/alationsink
   - category: Connectors / Metadata / Alation Sink / Run Externally
@@ -694,23 +680,7 @@ site_menu:
 
   - category: How-to Guides
     url: /how-to-guides
-    color: violet-70
-    icon: openmetadata
 
-  - category: How-to Guides / Data Quality Observability / Visualize  
-    url: /how-to-guides/data-quality-observability/visualize
-  - category: How-to Guides / Data Quality Observability / Test Cases From YAML Config  
-    url: /how-to-guides/data-quality-observability/quality/test-cases-from-yaml-config
-  - category: How-to Guides / Data Quality Observability / Adding Test Suits  
-    url: /how-to-guides/data-quality-observability/quality/adding-test-suits
-  - category: How-to Guides / Data Quality Observability / Adding Test Cases  
-    url: /how-to-guides/data-quality-observability/quality/adding-test-cases
-  - category: How-to Guides / Getting Started  
-    url: /how-to-guides/getting-started
-  - category: How-to Guides / Day 1  
-    url: /how-to-guides/getting-started/day-1
-  - category: How-to Guides / Day 1 / Database Service Setup
-    url: /how-to-guides/getting-started/day-1/database-service-setup
   - category: How-to Guides / Admin Guide
     url: /how-to-guides/admin-guide
   - category: How-to Guides / Admin Guide / How to Ingest Metadata
@@ -822,6 +792,14 @@ site_menu:
     url: /how-to-guides/data-quality-observability/quality/test
   - category: How-to Guides / Data Quality and Observability / Data Quality / Configure Data Quality
     url: /how-to-guides/data-quality-observability/quality/configure
+  - category: How-to Guides / Data Quality Observability / Data Quality / Adding Test Cases  
+    url: /how-to-guides/data-quality-observability/quality/adding-test-cases
+  - category: How-to Guides / Data Quality Observability / Data Quality / Adding Test Suites  
+    url: /how-to-guides/data-quality-observability/quality/adding-test-suites
+  - category: How-to Guides / Data Quality Observability / Data Quality / Test Cases From YAML Config  
+    url: /how-to-guides/data-quality-observability/quality/test-cases-from-yaml-config
+  - category: How-to Guides / Data Quality Observability / Data Quality / How to Visualize Test Results
+    url: /how-to-guides/data-quality-observability/quality/visualize
   - category: How-to Guides / Data Quality and Observability / Data Quality / Tests - YAML Config
     url: /how-to-guides/data-quality-observability/quality/tests-yaml
   - category: How-to Guides / Data Quality and Observability / Data Quality / Custom Tests
@@ -931,8 +909,6 @@ site_menu:
 
   - category: Releases
     url: /releases
-    color: violet-70
-    icon: overview
   - category: Releases / Latest Release
     url: /releases/latest-release
   - category: Releases / Supported Releases
@@ -1027,8 +1003,6 @@ site_menu:
 
   - category: Main Concepts
     url: /main-concepts
-    color: violet-70
-    icon: main-concepts
   - category: Main Concepts / High Level Design
     url: /main-concepts/high-level-design
   - category: Main Concepts / Backend DB
@@ -1962,8 +1936,6 @@ site_menu:
 
   - category: Developers
     url: /developers
-    color: violet-70
-    icon: developers
   - category: Developers / Architecture
     url: /developers/architecture
   - category: Developers / Architecture / Understand Code Layout
@@ -2011,8 +1983,6 @@ site_menu:
 
   - category: SDK
     url: /sdk
-    color: violet-70
-    icon: sdk
   - category: SDK / Python SDK
     url: /sdk/python
   - category: SDK / Python SDK / Entities
diff --git a/openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/day-1/database-service-setup.md b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/database-service-setup.md
similarity index 93%
rename from openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/day-1/database-service-setup.md
rename to openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/database-service-setup.md
index a67d0dc90c28..c2c3340fb30c 100644
--- a/openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/day-1/database-service-setup.md
+++ b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/database-service-setup.md
@@ -1,6 +1,6 @@
 ---
 title: Database service setup
-slug: /how-to-guides/getting-started/day-1/database-service-setup
+slug: /quick-start/getting-started/day-1/database-service-setup
 ---
 
 ## Setting Up a Database Service for Metadata Extraction
@@ -41,4 +41,4 @@ Click Test Connection to verify the setup. This will check if OpenMetadata can r
 {% image
   src="/images/v1.5/getting-started/test-connection.png"
   alt="Verifying the Test Connection"
-  caption="Verifying the Test Connection" /%}
+  caption="Verifying the Test Connection" /%}
\ No newline at end of file
diff --git a/openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/day-1/index.md b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/index.md
similarity index 98%
rename from openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/day-1/index.md
rename to openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/index.md
index 6019df091257..5a4af7e59086 100644
--- a/openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/day-1/index.md
+++ b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/day-1/index.md
@@ -1,6 +1,6 @@
 ---
 title: Day 1
-slug: /how-to-guides/getting-started/day-1
+slug: /quick-start/getting-started/day-1
 ---
 
 # Getting Started: Day 1
@@ -39,7 +39,7 @@ There's two options on how to set up a data connector:
 {% tile
   title="Run the connector in OpenMetadata"
   description="Guide to start ingesting metadata seamlessly from your data sources."
-  link="/how-to-guides/getting-started/day-1/database-service-setup"
+  link="/quick-start/getting-started/day-1/database-service-setup"
   icon="discovery"
 /%}
 {% /tilesContainer %}
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/index.md b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/index.md
similarity index 97%
rename from openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/index.md
rename to openmetadata-docs/content/v1.5.x/quick-start/getting-started/index.md
index d1044aa836fd..778794a85fd3 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/index.md
+++ b/openmetadata-docs/content/v1.5.x/quick-start/getting-started/index.md
@@ -1,6 +1,6 @@
 ---
 title: Getting Started with OpenMetadata for Data cataloging
-slug: /how-to-guides/getting-started
+slug: /quick-start/getting-started
 ---
 
 # Getting Started
@@ -65,7 +65,7 @@ In this section, you will find a series of guides to help you get started with O
 {% tile
   title="Day 1: Connect your Data Sources and invite users"
   description="Discover the right data assets to make timely business decisions."
-  link="/how-to-guides/getting-started/day-1"
+  link="/quick-start/getting-started/day-1"
   icon="discovery"
 /%}
 {% /tilesContainer %}
diff --git a/openmetadata-docs/content/v1.5.x/quick-start/index.md b/openmetadata-docs/content/v1.5.x/quick-start/index.md
index 9bd90f0eff7a..95b2873e39cc 100644
--- a/openmetadata-docs/content/v1.5.x/quick-start/index.md
+++ b/openmetadata-docs/content/v1.5.x/quick-start/index.md
@@ -44,3 +44,14 @@ Get OpenMetadata up and running with kubernetes in under 5 minutes!
 {% /inlineCallout %}
 
 {% /inlineCalloutContainer %}
+
+Set up and explore OpenMetadata's core features, from basic configuration to advanced functionalities, for a seamless onboarding experience.
+
+{% tilesContainer %}
+{% tile
+    title="Getting Started"
+    description="Unlock metadata insights for informed business decisions."
+    link="/how-to-guides/getting-started"
+    icon="discovery"
+/%}
+{% /tilesContainer %}
\ No newline at end of file
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/Metapilot/how-to-use-metapilot.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/Metapilot/how-to-use-metapilot.md
new file mode 100644
index 000000000000..e78793cca165
--- /dev/null
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/Metapilot/how-to-use-metapilot.md
@@ -0,0 +1,110 @@
+---
+title: How to Use MetaPilot
+slug: /metapilot/how-to-use-metapilot
+collate: true
+---
+
+# How to Use MetaPilot
+
+## 1. Setting Up MetaPilot
+- Navigate to **Settings > Applications** in the Collate platform.
+
+{% image
+src="/images/v1.6/metapilot/setting-up-metapilot-1.png"
+alt="setting up metapilot"
+caption="Navigate to Settings > Applications"
+/%}
+
+- Install MetaPilot by following the on-screen instructions.
+
+{% image
+src="/images/v1.6/metapilot/setting-up-metapilot-2.png"
+alt="Install MetaPilot"
+caption="Install MetaPilot"
+/%}
+
+- Select the databases for which you want MetaPilot to automatically generate descriptions. You can configure multiple databases and set a default database for the MetaPilot chatbot to work with.
+
+{% image
+src="/images/v1.6/metapilot/setting-up-metapilot-3.png"
+alt="automatically generate descriptions"
+caption="automatically generate descriptions"
+/%}
+
+- **Scheduling**: Schedule MetaPilot to run regularly, automatically generating metadata at predefined intervals (e.g., weekly).
+
+{% image
+src="/images/v1.6/metapilot/setting-up-metapilot-4.png"
+alt="Schedule MetaPilot"
+caption="Schedule MetaPilot"
+/%}
+
+## 2. Using the MetaPilot Chatbot
+- The MetaPilot chatbot icon appears on every page after installation.
+
+{% image
+src="/images/v1.6/metapilot/using-metapilot-chatbot-1.png"
+alt="chatbot icon"
+caption="chatbot icon"
+/%}
+
+- Interact with the chatbot by typing natural language questions. For example:
+  - “Show me sales data for Q1.”
+  - “What is the average revenue per customer?”
+
+- MetaPilot will generate the corresponding SQL query and provide a detailed explanation of the query logic.
+
+{% image
+src="/images/v1.6/metapilot/using-metapilot-chatbot-2.png"
+alt="natural language questions"
+caption="natural language questions"
+/%}
+
+- Users can refine queries by providing further instructions, and the chatbot will adjust the SQL query accordingly.
+
+## 3. Optimizing and Fixing SQL Queries
+
+- The **Metadata Usage** workflows will ingest the queries being run against the tables. You can see how long each query has been executed in the **Queries** tab.
+
+{% image
+src="/images/v1.6/metapilot/fixing-sql-queries-1.png"
+alt="metadata usage workflows"
+caption="metadata usage workflows"
+/%}
+
+- If a query runs inefficiently, ask the chatbot to optimize it by typing: “Optimize this query.”
+
+{% image
+src="/images/v1.6/metapilot/fixing-sql-queries-3.png"
+alt="Optimize this query"
+caption="Optimize this query"
+/%}
+
+- MetaPilot will return a more efficient version of the SQL query, which you can then copy and execute in your database.
+
+- If the query contains errors or isn’t functioning correctly, ask MetaPilot: “Can you fix this query?” 
+
+- MetaPilot will correct the query and provide a working version.
+
+{% image
+src="/images/v1.6/metapilot/fixing-sql-queries-4.png"
+alt="Can you fix this query"
+caption="Can you fix this query"
+/%}
+
+## 4. Reviewing Generated Metadata
+- Once MetaPilot generates descriptions for tables and columns, navigate to the **database view** to review the metadata.
+- You can accept or reject each suggestion individually or choose to accept all suggestions in bulk.
+
+{% image
+src="/images/v1.6/metapilot/reviewing-generated-metadata.png"
+alt="reviewing Generated Metadata"
+caption="reviewing Generated Metadata"
+/%}
+
+- MetaPilot allows you to document entire datasets in a matter of minutes, significantly reducing the manual effort required to maintain metadata.
+
+## Best Practices
+- **Regular Scheduling**: Schedule MetaPilot to run at regular intervals to ensure your metadata is always up-to-date, especially when dealing with frequently changing datasets.
+- **Leverage the Chatbot for Query Writing**: Encourage both technical and non-technical users to use the MetaPilot chatbot for SQL query generation. It simplifies complex query writing and ensures accurate results.
+- **Optimize Queries Regularly**: Monitor the performance of your queries and use MetaPilot to optimize them, especially when working with large datasets or queries that require complex joins and filters.
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/Metapilot/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/Metapilot/index.md
new file mode 100644
index 000000000000..310dc0dbfb09
--- /dev/null
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/Metapilot/index.md
@@ -0,0 +1,85 @@
+---
+title: Metapilot
+slug: /metapilot
+collate: true
+---
+
+# Metapilot Technical Documentation
+
+{%  youtube videoId="6glMYLzxNqk" start="0:00" end="04:20" width="560px" height="315px" /%}
+
+MetaPilot is an AI-powered tool within Collate that simplifies and enhances metadata management. By integrating generative AI, MetaPilot assists users in automating the documentation of data assets, writing and optimizing SQL queries, and interacting with data through natural language. This first-of-its-kind data copilot improves productivity by automating tedious tasks and providing intelligent insights into your data environment.
+
+## Key Features
+
+- **Automated Data Documentation**: Automatically generates metadata descriptions for tables and columns, saving time and effort for data owners and stewards.
+- **Natural Language SQL Query Generation**: Allows users to interact with MetaPilot through a chatbot to generate SQL queries by simply asking questions in plain English.
+- **SQL Query Optimization and Fixing**: Capable of optimizing and troubleshooting SQL queries to improve their performance and efficiency.
+
+## Why MetaPilot is Useful
+
+### Metadata Management Challenges
+
+Managing metadata across multiple data assets can be overwhelming due to the influx of new data and changing team dynamics. MetaPilot addresses these challenges by:
+
+- Automating metadata description generation.
+- Simplifying the creation and optimization of SQL queries.
+- Reducing manual effort and enhancing data quality.
+
+### Time-Saving Features
+
+Documenting thousands of tables manually is tedious and time-consuming. MetaPilot automates metadata generation, allowing data teams to focus on high-value tasks and ensuring that data assets are consistently documented and understood across the organization.
+
+## Use Cases
+
+### 1. Automatic Data Asset Documentation
+
+{% image
+src="/images/v1.6/metapilot/reviewing-generated-metadata.png"
+alt="Automatic Data Asset Documentation"
+caption="Auto Generate data Asset Documentation"
+/%}
+
+- **Problem**: Manually creating metadata descriptions for large datasets is labor-intensive and error-prone.
+- **Solution**: MetaPilot’s generative AI automates the process, analyzing database structures and suggesting accurate descriptions for tables and columns.
+- **How It Works**: After configuring MetaPilot, it scans the database schema and generates metadata descriptions automatically. Users can review these descriptions and approve or reject them in bulk or individually.
+- **Benefit**: Streamlines the documentation process, ensuring consistent and up-to-date metadata across all datasets.
+
+### 2. Natural Language SQL Query Generation
+
+{% image
+src="/images/v1.6/metapilot/using-metapilot-chatbot-2.png"
+alt="Natural Language SQL Query Generation"
+caption="Natural Language SQL Query Generation"
+/%}
+
+- **Problem**: Non-technical users often struggle with writing SQL queries to extract insights from databases.
+- **Solution**: MetaPilot’s chatbot allows users to ask questions in natural language, generating SQL queries and providing explanations to help extract the required data quickly and easily.
+- **How It Works**: Users can click on the MetaPilot chatbot widget and ask questions like "Show me sales data from last quarter." MetaPilot generates the corresponding SQL query and explains its logic.
+- **Benefit**: Democratizes data access by enabling users of all technical levels to interact with data without deep SQL knowledge.
+
+### 3. SQL Query Optimization and Troubleshooting
+
+{% image
+src="/images/v1.6/metapilot/fixing-sql-queries-3.png"
+alt="SQL Query Optimization and Troubleshooting"
+caption="SQL Query Optimization"
+/%}
+
+- **Problem**: SQL queries can become complex and inefficient, leading to performance issues and increased costs.
+- **Solution**: MetaPilot optimizes inefficient queries to improve performance, saving both time and resources.
+- **How It Works**: If a query runs too long, users can request MetaPilot to optimize it. MetaPilot provides a more efficient SQL query version that can be implemented immediately.
+- **Benefit**: Enhances query performance, reduces costs associated with inefficient queries, and speeds up data processing.
+
+### 4. Fixing SQL Queries
+
+{% image
+src="/images/v1.6/metapilot/fixing-sql-queries-4.png"
+alt="Fixing SQL Queries"
+caption="Fixing SQL Queries"
+/%}
+
+- **Problem**: Complex queries can cause issues even for SQL experts.
+- **Solution**: MetaPilot can fix problematic SQL queries, ensuring they run correctly and efficiently.
+- **How It Works**: Users can ask MetaPilot, "Can you fix this query for me?" It analyzes the query, detects issues, and returns a corrected version ready for use.
+- **Benefit**: Simplifies writing and maintaining queries, allowing data teams to focus on analysis rather than troubleshooting.
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/collate-menu.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/collate-menu.md
index 8457fdf5be7d..86a34d7e6921 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/collate-menu.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/collate-menu.md
@@ -5,8 +5,6 @@ site_menu:
 
   - category: Enable Security
     url: /deployment
-    color: violet-70
-    icon: deployment
   - category: Enable Security / Basic Authentication
     url: /deployment/security/basic-auth
   - category: Enable Security / Ldap Authentication
@@ -38,8 +36,6 @@ site_menu:
 
   - category: Connectors
     url: /connectors
-    color: violet-70
-    icon: openmetadata
 
   - category: Connectors / Database
     url: /connectors/database
@@ -518,8 +514,6 @@ site_menu:
 
   - category: How-to Guides
     url: /how-to-guides
-    color: violet-70
-    icon: openmetadata
 
   - category: How-to Guides / Admin Guide
     url: /how-to-guides/admin-guide
@@ -702,6 +696,10 @@ site_menu:
 
   - category: How-to Guides / Data Governance
     url: /how-to-guides/data-governance
+  - category: How-to Guides / Data Governance / Automation
+    url: /how-to-guides/data-governance/automation
+  - category: How-to Guides / Data Governance / Automation / How to Set Up Automations in Collate 
+    url: /how-to-guides/data-governance/automation/set-up-automation
   - category: How-to Guides / Data Governance / Glossary
     url: /how-to-guides/data-governance/glossary
   - category: How-to Guides / Data Governance / Glossary / What is a Glossary Term
@@ -745,8 +743,6 @@ site_menu:
 
   - category: Getting Started
     url: /getting-started
-    color: violet-70
-    icon: openmetadata
 
   - category: Getting Started / Day 1
     url: /getting-started/day-1
@@ -765,8 +761,6 @@ site_menu:
 
   - category: Releases
     url: /releases
-    color: violet-70
-    icon: overview
   - category: Releases / Latest Release
     url: /releases/latest-release
   - category: Releases / Supported Releases
@@ -861,8 +855,6 @@ site_menu:
 
   - category: Main Concepts
     url: /main-concepts
-    color: violet-70
-    icon: main-concepts
   - category: Main Concepts / High Level Design
     url: /main-concepts/high-level-design
   - category: Main Concepts / Backend DB
@@ -1796,8 +1788,6 @@ site_menu:
 
   - category: SDK
     url: /sdk
-    color: violet-70
-    icon: sdk
   - category: SDK / Python SDK
     url: /sdk/python
   - category: SDK / Python SDK / Entities
@@ -1876,4 +1866,10 @@ site_menu:
     url: /sdk/java
   - category: SDK / Go
     url: /sdk/go
+
+  - category: Metapilot
+    url: /metapilot
+  
+   category: Metapilot / How to Use MetaPilot
+    url: /metapilot/how-to-use-metapilot
 ---
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/lineage/spark-lineage.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/lineage/spark-lineage.md
index 3e48cfc5eaa0..3bc39c4340e9 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/lineage/spark-lineage.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/ingestion/lineage/spark-lineage.md
@@ -343,3 +343,39 @@ spark.openmetadata.transport.timeout 30
 ```
 
 After all these steps are completed you can start/restart your compute instance and you are ready to extract the lineage from spark to OpenMetadata.
+
+
+## Using Spark Agent with Glue
+
+Follow the below steps in order to use OpenMetadata Spark Agent with glue.
+
+### 1. Specify the OpenMetadata Spark Agent JAR URL
+
+1. Upload the OpenMetadata Spark Agent Jar to S3
+2. Navigate to the glue job,In the Job details tab, navigate to Advanced properties → Libraries → Dependent Jars path
+3. Add the S3 url of OpenMetadata Spark Agent Jar in the Dependent Jars path.
+
+{% image
+  src="/images/v1.6/connectors/spark/glue-job-jar.png"
+  alt="Glue Job Configure Jar"
+  caption="Glue Job Configure Jar"
+ /%}
+
+
+### 2. Add Spark configuration in Job Parameters
+
+In the same Job details tab, add a new property under Job parameters:
+
+1. Add the `--conf` property with following value, make sure to customize this configuration as described in the above documentation.
+
+```
+spark.extraListeners=org.openmetadata.spark.agent.OpenMetadataSparkListener --conf spark.openmetadata.transport.hostPort=https://your-org.host:port  --conf spark.openmetadata.transport.type=openmetadata --conf spark.openmetadata.transport.jwtToken=<jwt-token> --conf spark.openmetadata.transport.pipelineServiceName=glue_spark_pipeline_service --conf spark.openmetadata.transport.pipelineName=glue_pipeline_name --conf spark.openmetadata.transport.timeout=30
+```
+
+2. Add the `--user-jars-first` parameter and set its value to `true`
+
+{% image
+  src="/images/v1.6/connectors/spark/glue-job-params.png"
+  alt="Glue Job Configure Params"
+  caption="Glue Job Configure Params"
+ /%}
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/metadata/alation/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/metadata/alation/index.md
index e732fe1d71ab..a7ff96c95ded 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/metadata/alation/index.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/metadata/alation/index.md
@@ -122,9 +122,9 @@ Choose either postgres or mysql connection depending on the db:
 
 **ingestDatasources**: Specifies if databases, schemas and tables should be included while ingesting. By default is set to `true`.
 
-**ingestDomains**: Specifies if hidden domains and subdomains should be included while ingesting. By default is set to `true`.
+**ingestDomains**: Specifies if domains and subdomains should be included while ingesting. By default is set to `true`.
 
-**ingestDashboards**: Specifies if hidden BI sources and dashboards should be included while ingesting. By default is set to `true`.
+**ingestDashboards**: Specifies if BI sources and dashboards should be included while ingesting. By default is set to `true`.
 
 **alationTagClassificationName**: Specify the classification name under which the tags from alation will be created in OpenMetadata. By default it is set to `alationTags`.
 
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/glue-pipeline/yaml.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/glue-pipeline/yaml.md
index 5898468a0f98..4468dc424867 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/glue-pipeline/yaml.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/connectors/pipeline/glue-pipeline/yaml.md
@@ -112,11 +112,11 @@ This is a sample config for Glue:
 
 ```yaml {% isCodeBlock=true %}
 source:
-  type: glue
+  type: gluepipeline 
   serviceName: local_glue
   serviceConnection:
     config:
-      type: Glue
+      type: GluePipeline
       awsConfig:
 ```
 ```yaml {% srNumber=1 %}
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/backup-restore-metadata.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/backup-restore-metadata.md
index 5e704c7e56de..306423cf28f8 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/backup-restore-metadata.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/backup-restore-metadata.md
@@ -70,15 +70,16 @@ Ingest some data...
 
 ```shell
 BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql"
-DOCKER_COMPOSE_FILE="docker/development/docker-compose.yml"
+export COMPOSE_FILE="docker/development/docker-compose.yml"
 # backup
-docker compose -f $DOCKER_COMPOSE_FILE exec ingestion mysqldump --no-tablespaces -u openmetadata_user -popenmetadata_password -h mysql -P 3306 openmetadata_db > $BACKUP_FILE
+docker compose exec ingestion mysqldump --no-tablespaces -u openmetadata_user -popenmetadata_password -h mysql -P 3306 openmetadata_db > $BACKUP_FILE
 # create the restore database
-docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "create database restore;"
-docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "grant all privileges on restore.* to 'openmetadata_user'@'%';"
-docker compose -f $DOCKER_COMPOSE_FILE exec mysql  mysql -u root -ppassword -e "flush privileges;"
+docker compose exec mysql  mysql -u root -ppassword -e "create database restore;"
+docker compose exec mysql  mysql -u root -ppassword -e "grant all privileges on restore.* to 'openmetadata_user'@'%';"
+docker compose exec mysql  mysql -u root -ppassword -e "GRANT SUPER, SYSTEM_VARIABLES_ADMIN, SESSION_VARIABLES_ADMIN ON *.* TO 'openmetadata_user'@'%';"
+docker compose exec mysql  mysql -u root -ppassword -e "flush privileges;"
 # restore from the backup
-docker compose -f $DOCKER_COMPOSE_FILE exec -T ingestion mysql -u openmetadata_user -popenmetadata_password -h mysql -P 3306 restore < $BACKUP_FILE
+docker compose exec -T ingestion mysql -u openmetadata_user -popenmetadata_password -h mysql -P 3306 restore < $BACKUP_FILE
 ```
 
 ### 3. Restart the docker deployment with the restored database
@@ -102,14 +103,14 @@ Ingest some data...
 
 ```shell
 BACKUP_FILE="backup_$(date +%Y%m%d%H%M).sql"
-DOCKER_COMPOSE_FILE="docker/development/docker-compose-postgres.yml"
+export COMPOSE_FILE="docker/development/docker-compose-postgres.yml"
 # backup
-docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password ingestion pg_dump -U openmetadata_user -h postgresql -d openmetadata_db > $BACKUP_FILE
+docker compose exec -e PGPASSWORD=openmetadata_password ingestion pg_dump -U openmetadata_user -h postgresql -d openmetadata_db > $BACKUP_FILE
 # create the restore database
-docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "create database restore;"
-docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "ALTER DATABASE restore OWNER TO openmetadata_user;"
+docker compose exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "create database restore;"
+docker compose exec -e PGPASSWORD=openmetadata_password postgresql psql -U postgres -c "ALTER DATABASE restore OWNER TO openmetadata_user;"
 # restore from the backup
-docker compose -f $DOCKER_COMPOSE_FILE exec -e PGPASSWORD=openmetadata_password -T ingestion psql -U openmetadata_user -h postgresql -d restore < $BACKUP_FILE
+docker compose exec -e PGPASSWORD=openmetadata_password -T ingestion psql -U openmetadata_user -h postgresql -d restore < $BACKUP_FILE
 ```
 
 ### 3. Restart the docker deployment with the restored database
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/bare-metal/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/bare-metal/index.md
index 3df742a45630..0299492340f6 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/bare-metal/index.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/bare-metal/index.md
@@ -54,7 +54,7 @@ You can refer a sample script [here](https://github.com/open-metadata/OpenMetada
 
 ## Elasticsearch (version 8.X)
 
-OpenMetadata supports ElasticSearch version up to 8.10.2. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
+OpenMetadata supports ElasticSearch version up to 8.11.4. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
 [Installing ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
 
 Please follow the instructions here to [install ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/setup.html).
@@ -170,7 +170,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support 
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version between 12 or higher
 
 For Production Systems, we recommend Amazon RDS to be in Multiple Availability Zones. For Amazon OpenSearch (or ElasticSearch) Service, we recommend Multiple Availability Zones with minimum 3 Master Nodes.
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/docker/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/docker/index.md
index b5a8227e3645..0e3689962547 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/docker/index.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/docker/index.md
@@ -244,7 +244,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version 12 or higher
 
 Note:-
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/aks.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/aks.md
index 5c4abeb5b242..39d3402c610b 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/aks.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/aks.md
@@ -16,7 +16,7 @@ We support
 
 - Azure SQL (MySQL) engine version 8 or higher
 - Azure SQL (PostgreSQL) engine version 12 or higher
-- Elastic Cloud (ElasticSearch version 8.10.2)
+- Elastic Cloud (ElasticSearch version 8.11.4)
 
 Once you have the Azure SQL and Elastic Cloud on Azure configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/on-prem.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/on-prem.md
index e231b1440d6d..ed0217709f4d 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/on-prem.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/on-prem.md
@@ -23,7 +23,7 @@ We support
 
 - MySQL engine version 8 or higher
 - PostgreSQL engine version 12 or higher
-- ElasticSearch version 8.X (upto 8.10.2) or OpenSearch Version 2.X (upto 2.7)
+- ElasticSearch version 8.X (upto 8.11.4) or OpenSearch Version 2.X (upto 2.7)
 
 Once you have the External Database and Search Engine configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/upgrade/versions/110-to-120.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/upgrade/versions/110-to-120.md
index b661db7eb18e..c8a61dba23a4 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/upgrade/versions/110-to-120.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/upgrade/versions/110-to-120.md
@@ -161,7 +161,7 @@ For Kubernetes Deployment, `openmetadata.config.database.dbParams` is available
 ### Version Upgrades
 
 - The OpenMetadata Server is now based on **JDK 17**
-- OpenMetadata now **requires** **Elasticsearch** version **8.10.2** or **Opensearch** version **2.7**
+- OpenMetadata now **requires** **Elasticsearch** version **8.11.4** or **Opensearch** version **2.7**
 
 There is no direct migration to bump the indexes to the new supported versions. You might see errors like:
 
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-governance/automation/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-governance/automation/index.md
new file mode 100644
index 000000000000..5a1cb1cd7ce7
--- /dev/null
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-governance/automation/index.md
@@ -0,0 +1,78 @@
+---
+title: Collate Automations Documentation
+slug: /how-to-guides/data-governance/automation
+collate: true
+---
+
+# Collate Automations
+
+{% youtube videoId="ug08aLUyTyE" start="0:00" end="14:52" width="560px" height="315px" /%}
+
+## Overview
+
+Collate's **Automation** feature is a powerful tool designed to simplify and streamline metadata management tasks. By automating repetitive actions such as assigning owners, domains, or tagging data, Collate helps maintain consistency in metadata across an organization's datasets. These automations reduce manual effort and ensure that metadata is always up-to-date, accurate, and governed according to predefined policies.
+
+## Why Automations are Useful
+
+Managing metadata manually can be challenging, particularly in dynamic environments where data constantly evolves. Collate's Automation feature addresses several key pain points:
+
+- **Maintaining Consistency**: Automation helps ensure that metadata such as ownership, tags, and descriptions are applied consistently across all data assets.
+- **Saving Time**: Automations allow data teams to focus on higher-value tasks by eliminating the need for manual updates and maintenance.
+- **Enforcing Governance Policies**: Automations help ensure that data follows organizational policies at all times by automatically applying governance rules (e.g., assigning data owners or domains).
+- **Data Quality and Accountability**: Data quality suffers without clear ownership. Automating ownership assignments helps ensure that data quality issues are addressed efficiently.
+
+## Key Use Cases for Collate Automations
+
+### 1. Bulk Ownership and Domain Assignment
+
+{% image
+src="/images/v1.6/how-to-guides/governance/bulk-ownership-and.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+- **Problem**: Many data assets lack proper ownership and domain assignment, leading to governance and accountability issues. Manually assigning owners can be error-prone and time-consuming.
+- **Solution**: Automations can bulk-assign ownership and domains to datasets, ensuring all data assets are correctly categorized and owned. This process can be applied to tables, schemas, or other assets within Collate.
+- **Benefit**: This use case ensures data assets have a designated owner and are organized under the appropriate domain, making data more discoverable and accountable.
+
+### 2. Bulk Tagging and Glossary Term Assignment
+
+{% image
+src="/images/v1.6/how-to-guides/governance/bulk-tagging-glossary.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+- **Problem**: Manually applying the same tags or glossary terms to multiple datasets can be inefficient and inconsistent.
+- **Solution**: Automations allow users to bulk-apply tags (e.g., PII) or glossary terms (e.g., Customer ID) to specific datasets, ensuring uniformity across the platform.
+- **Benefit**: This automation reduces the risk of missing important tags like PII-sensitive and ensures that key metadata elements are applied consistently across datasets.
+
+### 3. Metadata Propagation via Lineage
+
+{% image
+src="/images/v1.6/how-to-guides/governance/metadata-propogation.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+- **Problem**: When metadata such as tags, descriptions, or glossary terms are updated in one part of the data lineage, they may not be propagated across related datasets, leading to inconsistencies.
+- **Solution**: Use automations to propagate metadata across related datasets, ensuring that all relevant data inherits the correct metadata properties from the source dataset.
+- **Benefit**: Metadata consistency is ensured across the entire data lineage, reducing the need for manual updates and maintaining a single source of truth.
+
+### 4. Automatic PII Detection and Tagging
+
+{% image
+src="/images/v1.6/how-to-guides/governance/automatic-detection.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+- **Problem**: Manually identifying and tagging Personally Identifiable Information (PII) across large datasets is labor-intensive and prone to errors.
+- **Solution**: Automations can automatically detect PII data (e.g., emails, usernames) and apply relevant tags to ensure that sensitive data is flagged appropriately for compliance.
+- **Benefit**: Ensures compliance with data protection regulations by consistently tagging sensitive data, reducing the risk of non-compliance.
+
+## Best Practices
+
+- **Validate Assets Before Applying Actions**: Always use the **Explore** page to verify the assets that will be affected by the automation. This ensures that only the intended datasets are updated.
+- **Use Automation Logs**: Regularly check the **Recent Runs** logs to monitor automation activity and ensure that they are running as expected.
+- **Propagate Metadata Thoughtfully**: When propagating metadata via lineage, make sure that the source metadata is correct before applying it across multiple datasets.
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-governance/automation/set-up-automation.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-governance/automation/set-up-automation.md
new file mode 100644
index 000000000000..a4afc9e61f46
--- /dev/null
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-governance/automation/set-up-automation.md
@@ -0,0 +1,67 @@
+---
+title: How to Set Up Automations in OpenMetadata
+slug: /how-to-guides/data-governance/automation/set-up-automation
+collate: true
+---
+
+# How to Set Up Automations in Collate
+
+### Step 1: Access the Automations Section
+In the OpenMetadata UI, navigate to **Govern>Automations**.  
+This will take you to the Automations page where you can view and manage your existing automations.
+
+{% image
+src="/images/v1.6/how-to-guides/governance/automation-1.png"
+alt="Getting started with Automation"
+caption="Getting started with Automation"
+/%}
+
+### Step 2: Add a New Automation
+In the Automations page, click the **Add Automation** button located at the top right of the page.  
+A pop-up window will appear to begin the process of adding a new automation.
+
+{% image
+src="/images/v1.6/how-to-guides/governance/automation-2.png"
+alt="Add Automation"
+caption="Add Automation"
+/%}
+
+### Step 3: Fill in Automation Details
+In the pop-up window, provide the necessary information to set up the automation:
+- **Automation Name**: Give a meaningful name to the automation for easy identification.
+- **Description**: Add a brief description explaining what this automation will do (e.g., "Daily metadata ingestion for database XYZ").
+- **Logic/Conditions**: Define any conditions or specific criteria needed for this automation to work (e.g., specific tables or columns to be included).  
+  Ensure that the logic is set up as per your specific requirements to make the automation useful for your workflows.
+
+{% image
+src="/images/v1.6/how-to-guides/governance/automation-4.png"
+alt="Automation details"
+caption="Automation details"
+/%}
+
+{% image
+src="/images/v1.6/how-to-guides/governance/automation-5.png"
+alt="Automation logics"
+caption="Automation logics"
+/%}
+
+### Step 4: Configure Automation Interval
+Once you've filled in the required details, click **Next**.  
+On the next page, you’ll be prompted to select the interval for the automation. This defines how frequently the automation should run (e.g., daily, weekly, or custom intervals).  
+Review your settings and click **Automate** once you are satisfied with the configuration.
+
+{% image
+src="/images/v1.6/how-to-guides/governance/automation-6.png"
+alt="Automation Interval"
+caption="Automation Interval"
+/%}
+
+### Step 5: Manage Your Automation
+After completing the setup, your automation will appear in the Automations list.  
+To manage the automation, click on the three dots next to the automation entry. From here, you can **edit**, **re-deploy**, **delete**, etc.
+
+{% image
+src="/images/v1.6/how-to-guides/governance/automation-7.png"
+alt="Manage Your Automation"
+caption="Manage Your Automation"
+/%}
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/adding-test-suits.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/adding-test-suites.md
similarity index 96%
rename from openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/adding-test-suits.md
rename to openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/adding-test-suites.md
index 11c01feb655b..e699578d3e6a 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/adding-test-suits.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/adding-test-suites.md
@@ -1,6 +1,6 @@
 ---
-title: Adding test suits through the UI
-slug: /how-to-guides/data-quality-observability/quality/adding-test-suits
+title: Adding test suites through the UI
+slug: /how-to-guides/data-quality-observability/quality/adding-test-suites
 ---
 
 # Adding Test Suites Through the UI
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/tests-yaml.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/tests-yaml.md
index a6fc743fa37a..b79f7b38743e 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/tests-yaml.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/tests-yaml.md
@@ -487,6 +487,7 @@ Consistency
 * `table2`: The table against which the comparison will be done. Must be the fully qualified name as defined in OpenMetadata
 * `threshold`: The threshold of different rows above which the test should fail -- default to 0
 * `where`: Any `where` clause to pass 
+* `caseSensitiveColumns`: Whether the column comparison should be case sensitive or not. Default to `false`.
 
 **Behavior**
 
@@ -513,6 +514,8 @@ parameterValues:
     value: 10
   - name: where
     value: id != 999
+  - name: caseSensitiveColumns
+    value: false
 ```
 
 **JSON Config**
@@ -543,6 +546,10 @@ parameterValues:
     {
       "name": "where",
       "value": "id != 999"
+    },
+    {
+      "name": "caseSensitiveColumns",
+      "value": false
     }
   ]
 }
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/visualize.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/visualize.md
index 6237df4647da..8e0582a736ef 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/visualize.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/data-quality-observability/quality/visualize.md
@@ -1,6 +1,6 @@
 ---
 title: How to Visualize Test Results
-slug: /how-to-guides/data-quality-observability/visualize
+slug: /how-to-guides/data-quality-observability/quality/visualize
 ---
 
 # How to Visualize Test Results
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/index.md
index 84af90a4eb6a..fcc33a8cc675 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/index.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/index.md
@@ -5,19 +5,6 @@ slug: /how-to-guides
 
 # How-to Guides
 
-## Getting Started
-
-Set up and explore OpenMetadata's core features, from basic configuration to advanced functionalities, for a seamless onboarding experience.
-
-{% tilesContainer %}
-{% tile
-    title="Getting Started"
-    description="Unlock metadata insights for informed business decisions."
-    link="/how-to-guides/getting-started"
-    icon="discovery"
-/%}
-{% /tilesContainer %}
-
 The How-to Guides will give you a walk through on accomplishing the basic to the most advanced things in OpenMetadata. These step-by-step guides will help get an overview of the features and also help explore the various functionalities.  
 
 ## Features in OpenMetadata
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/menu.md
index 4073dae3b862..00d1abf6ced4 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/menu.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/menu.md
@@ -5,19 +5,21 @@ site_menu:
 
   - category: Quickstart
     url: /quick-start
-    color: violet-70
-    icon: quickstart
   - category: Quickstart / Try OpenMetadata in Docker
     url: /quick-start/local-docker-deployment
   - category: Quickstart / Try OpenMetadata in Kubernetes
     url: /quick-start/local-kubernetes-deployment
   - category: Quickstart / Try the OpenMetadata Sandbox
     url: /quick-start/sandbox
+  - category: Quickstart / Getting Started  
+    url: /quick-start/getting-started
+  - category: Quickstart / Day 1  
+    url: /quick-start/getting-started/day-1
+  - category: Quickstart / Day 1 / Database Service Setup
+    url: /quick-start/getting-started/day-1/database-service-setup
 
   - category: Deployment
     url: /deployment
-    color: violet-70
-    icon: deployment
   - category: Deployment / Bare Metal Deployment
     url: /deployment/bare-metal
   - category: Deployment / Bare Metal Deployment / Enable Security
@@ -215,8 +217,6 @@ site_menu:
 
   - category: Connectors
     url: /connectors
-    color: violet-70
-    icon: openmetadata
 
   - category: Connectors / API
     url: /connectors/api
@@ -386,12 +386,6 @@ site_menu:
     url: /connectors/database/sqlite
   - category: Connectors / Database / SQLite / Run Externally
     url: /connectors/database/sqlite/yaml
-  - category: Connectors / Database / Synapse
-    url: /connectors/database/synapse
-  - category: Connectors / Database / Synapse / Run Externally
-    url: /connectors/database/synapse/yaml
-  - category: Connectors / Database / Synapse / Troubleshooting
-    url: /connectors/database/synapse/troubleshooting
   - category: Connectors / Database / S3 Datalake
     url: /connectors/database/s3-datalake
   - category: Connectors / Database / S3 Datalake / Run Externally
@@ -527,11 +521,6 @@ site_menu:
     url: /connectors/pipeline/kafkaconnect
   - category: Connectors / Pipeline / KafkaConnect / Run Externally
     url: /connectors/pipeline/kafkaconnect/yaml
-  - category: Connectors / Pipeline / Matillion
-    url: /connectors/pipeline/matillion
-    isCollateOnly: true
-  - category: Connectors / Pipeline / Matillion / Run Externally
-    url: /connectors/pipeline/matillion/yaml
   - category: Connectors / Pipeline / Databricks Pipeline
     url: /connectors/pipeline/databricks-pipeline
   - category: Connectors / Pipeline / Databricks Pipeline / Run Externally
@@ -575,10 +564,6 @@ site_menu:
     url: /connectors/ml-model/sagemaker
   - category: Connectors / ML Model / Sagemaker / Run Externally
     url: /connectors/ml-model/sagemaker/yaml
-  - category: Connectors / ML Model / VertexAI
-    url: /connectors/ml-model/vertexai
-  - category: Connectors / ML Model / VertexAI / Run Externally
-    url: /connectors/ml-model/vertexai/yaml
 
   - category: Connectors / Storage
     url: /connectors/storage
@@ -590,10 +575,6 @@ site_menu:
     url: /connectors/storage/gcs
   - category: Connectors / Storage / GCS / Run Externally
     url: /connectors/storage/gcs/yaml
-  - category: Connectors / Storage / ADLS
-    url: /connectors/storage/adls
-  - category: Connectors / Storage / ADLS / Run Externally
-    url: /connectors/storage/adls/yaml
 
   - category: Connectors / Search
     url: /connectors/search
@@ -612,10 +593,6 @@ site_menu:
     url: /connectors/metadata/atlas
   - category: Connectors / Metadata / Atlas / Run Externally
     url: /connectors/metadata/atlas/yaml
-  - category: Connectors / Metadata / Alation
-    url: /connectors/metadata/alation
-  - category: Connectors / Metadata / Alation / Run Externally
-    url: /connectors/metadata/alation/yaml
   - category: Connectors / Metadata / Alation Sink
     url: /connectors/metadata/alationsink
   - category: Connectors / Metadata / Alation Sink / Run Externally
@@ -707,23 +684,7 @@ site_menu:
 
   - category: How-to Guides
     url: /how-to-guides
-    color: violet-70
-    icon: openmetadata
 
- - category: How-to Guides / Data Quality Observability / Visualize  
-    url: /how-to-guides/data-quality-observability/visualize
-  - category: How-to Guides / Data Quality Observability / Test Cases From YAML Config  
-    url: /how-to-guides/data-quality-observability/quality/test-cases-from-yaml-config
-  - category: How-to Guides / Data Quality Observability / Adding Test Suits  
-    url: /how-to-guides/data-quality-observability/quality/adding-test-suits
-  - category: How-to Guides / Data Quality Observability / Adding Test Cases  
-    url: /how-to-guides/data-quality-observability/quality/adding-test-cases
-  - category: How-to Guides / Getting Started  
-    url: /how-to-guides/getting-started
-  - category: How-to Guides / Day 1  
-    url: /how-to-guides/getting-started/day-1
-  - category: How-to Guides / Day 1 / Database Service Setup
-    url: /how-to-guides/getting-started/day-1/database-service-setup
   - category: How-to Guides / Admin Guide
     url: /how-to-guides/admin-guide
   - category: How-to Guides / Admin Guide / How to Ingest Metadata
@@ -835,6 +796,14 @@ site_menu:
     url: /how-to-guides/data-quality-observability/quality/test
   - category: How-to Guides / Data Quality and Observability / Data Quality / Configure Data Quality
     url: /how-to-guides/data-quality-observability/quality/configure
+  - category: How-to Guides / Data Quality Observability / Data Quality / Adding Test Cases  
+    url: /how-to-guides/data-quality-observability/quality/adding-test-cases
+  - category: How-to Guides / Data Quality Observability / Data Quality / Adding Test Suites  
+    url: /how-to-guides/data-quality-observability/quality/adding-test-suites
+  - category: How-to Guides / Data Quality Observability / Data Quality / Test Cases From YAML Config  
+    url: /how-to-guides/data-quality-observability/quality/test-cases-from-yaml-config
+  - category: How-to Guides / Data Quality Observability / Data Quality / How to Visualize Test Results
+    url: /how-to-guides/data-quality-observability/quality/visualize
   - category: How-to Guides / Data Quality and Observability / Data Quality / Tests - YAML Config
     url: /how-to-guides/data-quality-observability/quality/tests-yaml
   - category: How-to Guides / Data Quality and Observability / Data Quality / Custom Tests
@@ -944,8 +913,6 @@ site_menu:
 
   - category: Releases
     url: /releases
-    color: violet-70
-    icon: overview
   - category: Releases / Latest Release
     url: /releases/latest-release
   - category: Releases / Supported Releases
@@ -1042,8 +1009,6 @@ site_menu:
 
   - category: Main Concepts
     url: /main-concepts
-    color: violet-70
-    icon: main-concepts
   - category: Main Concepts / High Level Design
     url: /main-concepts/high-level-design
   - category: Main Concepts / Backend DB
@@ -1979,8 +1944,6 @@ site_menu:
 
   - category: Developers
     url: /developers
-    color: violet-70
-    icon: developers
   - category: Developers / Architecture
     url: /developers/architecture
   - category: Developers / Architecture / Understand Code Layout
@@ -2028,8 +1991,6 @@ site_menu:
 
   - category: SDK
     url: /sdk
-    color: violet-70
-    icon: sdk
   - category: SDK / Python SDK
     url: /sdk/python
   - category: SDK / Python SDK / Entities
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/day-1/database-service-setup.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/getting-started/day-1/database-service-setup.md
similarity index 96%
rename from openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/day-1/database-service-setup.md
rename to openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/getting-started/day-1/database-service-setup.md
index c8310a1dfff4..90f4c691bec8 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/day-1/database-service-setup.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/getting-started/day-1/database-service-setup.md
@@ -1,6 +1,6 @@
 ---
 title: Database service setup
-slug: /how-to-guides/getting-started/day-1/database-service-setup
+slug: /quick-start/getting-started/day-1/database-service-setup
 ---
 
 ## Setting Up a Database Service for Metadata Extraction
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/day-1/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/getting-started/day-1/index.md
similarity index 98%
rename from openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/day-1/index.md
rename to openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/getting-started/day-1/index.md
index fd0d47d403d7..6bc58494e729 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/how-to-guides/getting-started/day-1/index.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/getting-started/day-1/index.md
@@ -1,6 +1,6 @@
 ---
 title: Day 1
-slug: /how-to-guides/getting-started/day-1
+slug: /quick-start/getting-started/day-1
 ---
 
 # Getting Started: Day 1
@@ -39,7 +39,7 @@ There's two options on how to set up a data connector:
 {% tile
   title="Run the connector in OpenMetadata"
   description="Guide to start ingesting metadata seamlessly from your data sources."
-  link="/how-to-guides/getting-started/day-1/database-service-setup"
+  link="/quick-start/getting-started/day-1/database-service-setup"
   icon="discovery"
 /%}
 {% /tilesContainer %}
diff --git a/openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/getting-started/index.md
similarity index 97%
rename from openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/index.md
rename to openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/getting-started/index.md
index d1044aa836fd..778794a85fd3 100644
--- a/openmetadata-docs/content/v1.5.x/how-to-guides/getting-started/index.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/getting-started/index.md
@@ -1,6 +1,6 @@
 ---
 title: Getting Started with OpenMetadata for Data cataloging
-slug: /how-to-guides/getting-started
+slug: /quick-start/getting-started
 ---
 
 # Getting Started
@@ -65,7 +65,7 @@ In this section, you will find a series of guides to help you get started with O
 {% tile
   title="Day 1: Connect your Data Sources and invite users"
   description="Discover the right data assets to make timely business decisions."
-  link="/how-to-guides/getting-started/day-1"
+  link="/quick-start/getting-started/day-1"
   icon="discovery"
 /%}
 {% /tilesContainer %}
diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/index.md
index 9bd90f0eff7a..95b2873e39cc 100644
--- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/index.md
+++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/quick-start/index.md
@@ -44,3 +44,14 @@ Get OpenMetadata up and running with kubernetes in under 5 minutes!
 {% /inlineCallout %}
 
 {% /inlineCalloutContainer %}
+
+Set up and explore OpenMetadata's core features, from basic configuration to advanced functionalities, for a seamless onboarding experience.
+
+{% tilesContainer %}
+{% tile
+    title="Getting Started"
+    description="Unlock metadata insights for informed business decisions."
+    link="/how-to-guides/getting-started"
+    icon="discovery"
+/%}
+{% /tilesContainer %}
\ No newline at end of file
diff --git a/openmetadata-docs/images/v1.5/connectors/spark/glue-job-jar.png b/openmetadata-docs/images/v1.5/connectors/spark/glue-job-jar.png
new file mode 100644
index 000000000000..5ce7b558770c
Binary files /dev/null and b/openmetadata-docs/images/v1.5/connectors/spark/glue-job-jar.png differ
diff --git a/openmetadata-docs/images/v1.5/connectors/spark/glue-job-params.png b/openmetadata-docs/images/v1.5/connectors/spark/glue-job-params.png
new file mode 100644
index 000000000000..3cef8e39272a
Binary files /dev/null and b/openmetadata-docs/images/v1.5/connectors/spark/glue-job-params.png differ
diff --git a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/add-test-case.png b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/add-test-case.png
index 3ef4e0827141..0ab13a503208 100644
Binary files a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/add-test-case.png and b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/add-test-case.png differ
diff --git a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/add-test-defintion.png b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/add-test-defintion.png
index 2294788cef78..4d8049764df4 100644
Binary files a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/add-test-defintion.png and b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/add-test-defintion.png differ
diff --git a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/profiler-tab-view.png b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/profiler-tab-view.png
index 8f0a4c149069..29381cbac6aa 100644
Binary files a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/profiler-tab-view.png and b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/profiler-tab-view.png differ
diff --git a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/table-results-entity.png b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/table-results-entity.png
index 265fa6708d19..126e7ceb9685 100644
Binary files a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/table-results-entity.png and b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/table-results-entity.png differ
diff --git a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-case-page.png b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-case-page.png
index b1d31b87b8fa..85748bf2fbf3 100644
Binary files a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-case-page.png and b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-case-page.png differ
diff --git a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-suite-page.png b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-suite-page.png
index b002453a0f0d..7a252d3ba13d 100644
Binary files a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-suite-page.png and b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-suite-page.png differ
diff --git a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-suite-results.png b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-suite-results.png
index c3eff2340546..cbff22aa35b2 100644
Binary files a/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-suite-results.png and b/openmetadata-docs/images/v1.5/features/ingestion/workflows/data-quality/test-suite-results.png differ
diff --git a/openmetadata-docs/images/v1.5/features/ingestion/workflows/profiler/profiler-summary-column.png b/openmetadata-docs/images/v1.5/features/ingestion/workflows/profiler/profiler-summary-column.png
index e962225b0169..5f3d3aa87fdd 100644
Binary files a/openmetadata-docs/images/v1.5/features/ingestion/workflows/profiler/profiler-summary-column.png and b/openmetadata-docs/images/v1.5/features/ingestion/workflows/profiler/profiler-summary-column.png differ
diff --git a/openmetadata-docs/images/v1.5/features/ingestion/workflows/profiler/profiler-summary-table.png b/openmetadata-docs/images/v1.5/features/ingestion/workflows/profiler/profiler-summary-table.png
index 84a6595d99c5..9745f0d7d4d0 100644
Binary files a/openmetadata-docs/images/v1.5/features/ingestion/workflows/profiler/profiler-summary-table.png and b/openmetadata-docs/images/v1.5/features/ingestion/workflows/profiler/profiler-summary-table.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/automatic-detection.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/automatic-detection.png
new file mode 100644
index 000000000000..c5b8077414f8
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/automatic-detection.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-1.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-1.png
new file mode 100644
index 000000000000..f6cefd8c8d78
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-1.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-2.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-2.png
new file mode 100644
index 000000000000..cef6b5428fbd
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-2.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-3.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-3.png
new file mode 100644
index 000000000000..b9b195c7ca09
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-3.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-4.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-4.png
new file mode 100644
index 000000000000..4c49ede5a0b7
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-4.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-5.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-5.png
new file mode 100644
index 000000000000..7f18a8a25134
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-5.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-6.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-6.png
new file mode 100644
index 000000000000..155a4f05fb74
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-6.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-7.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-7.png
new file mode 100644
index 000000000000..23b11e4b2b7a
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/automation-7.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/bulk-ownership-and.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/bulk-ownership-and.png
new file mode 100644
index 000000000000..b15cef41a7c4
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/bulk-ownership-and.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/bulk-tagging-glossary.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/bulk-tagging-glossary.png
new file mode 100644
index 000000000000..8497a169eb76
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/bulk-tagging-glossary.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/governance/metadata-propogation.png b/openmetadata-docs/images/v1.5/how-to-guides/governance/metadata-propogation.png
new file mode 100644
index 000000000000..92562cb3cfdd
Binary files /dev/null and b/openmetadata-docs/images/v1.5/how-to-guides/governance/metadata-propogation.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/quality/dq1.png b/openmetadata-docs/images/v1.5/how-to-guides/quality/dq1.png
index 230b7d4f5734..cfb44ce83470 100644
Binary files a/openmetadata-docs/images/v1.5/how-to-guides/quality/dq1.png and b/openmetadata-docs/images/v1.5/how-to-guides/quality/dq1.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/quality/quality1.png b/openmetadata-docs/images/v1.5/how-to-guides/quality/quality1.png
index a6516128cc3d..e8941dabdd91 100644
Binary files a/openmetadata-docs/images/v1.5/how-to-guides/quality/quality1.png and b/openmetadata-docs/images/v1.5/how-to-guides/quality/quality1.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/quality/test2.png b/openmetadata-docs/images/v1.5/how-to-guides/quality/test2.png
index d8999b1b24c9..a065fa655c4d 100644
Binary files a/openmetadata-docs/images/v1.5/how-to-guides/quality/test2.png and b/openmetadata-docs/images/v1.5/how-to-guides/quality/test2.png differ
diff --git a/openmetadata-docs/images/v1.5/how-to-guides/quality/test4.png b/openmetadata-docs/images/v1.5/how-to-guides/quality/test4.png
index efd405875f00..ab86982c3f5f 100644
Binary files a/openmetadata-docs/images/v1.5/how-to-guides/quality/test4.png and b/openmetadata-docs/images/v1.5/how-to-guides/quality/test4.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/fixing-sql-queries-1.png b/openmetadata-docs/images/v1.5/metapilot/fixing-sql-queries-1.png
new file mode 100644
index 000000000000..911a405173c4
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/fixing-sql-queries-1.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/fixing-sql-queries-3.png b/openmetadata-docs/images/v1.5/metapilot/fixing-sql-queries-3.png
new file mode 100644
index 000000000000..4dd4e3e6c20f
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/fixing-sql-queries-3.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/fixing-sql-queries-4.png b/openmetadata-docs/images/v1.5/metapilot/fixing-sql-queries-4.png
new file mode 100644
index 000000000000..38ec4e25fbb9
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/fixing-sql-queries-4.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/reviewing-generated-metadata.png b/openmetadata-docs/images/v1.5/metapilot/reviewing-generated-metadata.png
new file mode 100644
index 000000000000..fcf8da1451e3
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/reviewing-generated-metadata.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-1.png b/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-1.png
new file mode 100644
index 000000000000..08d80960e158
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-1.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-2.png b/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-2.png
new file mode 100644
index 000000000000..2ca38a1626f9
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-2.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-3.png b/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-3.png
new file mode 100644
index 000000000000..44c07bd7ce68
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-3.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-4.png b/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-4.png
new file mode 100644
index 000000000000..1f87fa2f48bb
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/setting-up-metapilot-4.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/using-metapilot-chatbot-1.png b/openmetadata-docs/images/v1.5/metapilot/using-metapilot-chatbot-1.png
new file mode 100644
index 000000000000..94d9a064af10
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/using-metapilot-chatbot-1.png differ
diff --git a/openmetadata-docs/images/v1.5/metapilot/using-metapilot-chatbot-2.png b/openmetadata-docs/images/v1.5/metapilot/using-metapilot-chatbot-2.png
new file mode 100644
index 000000000000..b2114ad13347
Binary files /dev/null and b/openmetadata-docs/images/v1.5/metapilot/using-metapilot-chatbot-2.png differ
diff --git a/openmetadata-docs/images/v1.6/connectors/spark/glue-job-jar.png b/openmetadata-docs/images/v1.6/connectors/spark/glue-job-jar.png
new file mode 100644
index 000000000000..5ce7b558770c
Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/spark/glue-job-jar.png differ
diff --git a/openmetadata-docs/images/v1.6/connectors/spark/glue-job-params.png b/openmetadata-docs/images/v1.6/connectors/spark/glue-job-params.png
new file mode 100644
index 000000000000..3cef8e39272a
Binary files /dev/null and b/openmetadata-docs/images/v1.6/connectors/spark/glue-job-params.png differ
diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/add-test-case.png b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/add-test-case.png
index 3ef4e0827141..0ab13a503208 100644
Binary files a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/add-test-case.png and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/add-test-case.png differ
diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/add-test-defintion.png b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/add-test-defintion.png
index 2294788cef78..4d8049764df4 100644
Binary files a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/add-test-defintion.png and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/add-test-defintion.png differ
diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/profiler-tab-view.png b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/profiler-tab-view.png
index 8f0a4c149069..29381cbac6aa 100644
Binary files a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/profiler-tab-view.png and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/profiler-tab-view.png differ
diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/table-results-entity.png b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/table-results-entity.png
index 265fa6708d19..126e7ceb9685 100644
Binary files a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/table-results-entity.png and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/table-results-entity.png differ
diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-case-page.png b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-case-page.png
index b1d31b87b8fa..85748bf2fbf3 100644
Binary files a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-case-page.png and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-case-page.png differ
diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-suite-page.png b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-suite-page.png
index b002453a0f0d..7a252d3ba13d 100644
Binary files a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-suite-page.png and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-suite-page.png differ
diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-suite-results.png b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-suite-results.png
index c3eff2340546..cbff22aa35b2 100644
Binary files a/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-suite-results.png and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/data-quality/test-suite-results.png differ
diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/profiler/profiler-summary-column.png b/openmetadata-docs/images/v1.6/features/ingestion/workflows/profiler/profiler-summary-column.png
index e962225b0169..5f3d3aa87fdd 100644
Binary files a/openmetadata-docs/images/v1.6/features/ingestion/workflows/profiler/profiler-summary-column.png and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/profiler/profiler-summary-column.png differ
diff --git a/openmetadata-docs/images/v1.6/features/ingestion/workflows/profiler/profiler-summary-table.png b/openmetadata-docs/images/v1.6/features/ingestion/workflows/profiler/profiler-summary-table.png
index 84a6595d99c5..9745f0d7d4d0 100644
Binary files a/openmetadata-docs/images/v1.6/features/ingestion/workflows/profiler/profiler-summary-table.png and b/openmetadata-docs/images/v1.6/features/ingestion/workflows/profiler/profiler-summary-table.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/automatic-detection.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/automatic-detection.png
new file mode 100644
index 000000000000..c5b8077414f8
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/automatic-detection.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-1.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-1.png
new file mode 100644
index 000000000000..f6cefd8c8d78
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-1.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-2.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-2.png
new file mode 100644
index 000000000000..cef6b5428fbd
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-2.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-3.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-3.png
new file mode 100644
index 000000000000..b9b195c7ca09
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-3.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-4.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-4.png
new file mode 100644
index 000000000000..4c49ede5a0b7
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-4.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-5.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-5.png
new file mode 100644
index 000000000000..7f18a8a25134
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-5.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-6.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-6.png
new file mode 100644
index 000000000000..155a4f05fb74
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-6.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-7.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-7.png
new file mode 100644
index 000000000000..23b11e4b2b7a
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/automation-7.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/bulk-ownership-and.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/bulk-ownership-and.png
new file mode 100644
index 000000000000..b15cef41a7c4
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/bulk-ownership-and.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/bulk-tagging-glossary.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/bulk-tagging-glossary.png
new file mode 100644
index 000000000000..8497a169eb76
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/bulk-tagging-glossary.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/governance/metadata-propogation.png b/openmetadata-docs/images/v1.6/how-to-guides/governance/metadata-propogation.png
new file mode 100644
index 000000000000..92562cb3cfdd
Binary files /dev/null and b/openmetadata-docs/images/v1.6/how-to-guides/governance/metadata-propogation.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/quality/dq1.png b/openmetadata-docs/images/v1.6/how-to-guides/quality/dq1.png
index 230b7d4f5734..cfb44ce83470 100644
Binary files a/openmetadata-docs/images/v1.6/how-to-guides/quality/dq1.png and b/openmetadata-docs/images/v1.6/how-to-guides/quality/dq1.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/quality/quality1.png b/openmetadata-docs/images/v1.6/how-to-guides/quality/quality1.png
index a6516128cc3d..e8941dabdd91 100644
Binary files a/openmetadata-docs/images/v1.6/how-to-guides/quality/quality1.png and b/openmetadata-docs/images/v1.6/how-to-guides/quality/quality1.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/quality/test2.png b/openmetadata-docs/images/v1.6/how-to-guides/quality/test2.png
index d8999b1b24c9..a065fa655c4d 100644
Binary files a/openmetadata-docs/images/v1.6/how-to-guides/quality/test2.png and b/openmetadata-docs/images/v1.6/how-to-guides/quality/test2.png differ
diff --git a/openmetadata-docs/images/v1.6/how-to-guides/quality/test4.png b/openmetadata-docs/images/v1.6/how-to-guides/quality/test4.png
index efd405875f00..ab86982c3f5f 100644
Binary files a/openmetadata-docs/images/v1.6/how-to-guides/quality/test4.png and b/openmetadata-docs/images/v1.6/how-to-guides/quality/test4.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/fixing-sql-queries-1.png b/openmetadata-docs/images/v1.6/metapilot/fixing-sql-queries-1.png
new file mode 100644
index 000000000000..911a405173c4
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/fixing-sql-queries-1.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/fixing-sql-queries-3.png b/openmetadata-docs/images/v1.6/metapilot/fixing-sql-queries-3.png
new file mode 100644
index 000000000000..4dd4e3e6c20f
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/fixing-sql-queries-3.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/fixing-sql-queries-4.png b/openmetadata-docs/images/v1.6/metapilot/fixing-sql-queries-4.png
new file mode 100644
index 000000000000..38ec4e25fbb9
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/fixing-sql-queries-4.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/reviewing-generated-metadata.png b/openmetadata-docs/images/v1.6/metapilot/reviewing-generated-metadata.png
new file mode 100644
index 000000000000..fcf8da1451e3
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/reviewing-generated-metadata.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-1.png b/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-1.png
new file mode 100644
index 000000000000..08d80960e158
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-1.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-2.png b/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-2.png
new file mode 100644
index 000000000000..2ca38a1626f9
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-2.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-3.png b/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-3.png
new file mode 100644
index 000000000000..44c07bd7ce68
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-3.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-4.png b/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-4.png
new file mode 100644
index 000000000000..1f87fa2f48bb
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/setting-up-metapilot-4.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/using-metapilot-chatbot-1.png b/openmetadata-docs/images/v1.6/metapilot/using-metapilot-chatbot-1.png
new file mode 100644
index 000000000000..94d9a064af10
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/using-metapilot-chatbot-1.png differ
diff --git a/openmetadata-docs/images/v1.6/metapilot/using-metapilot-chatbot-2.png b/openmetadata-docs/images/v1.6/metapilot/using-metapilot-chatbot-2.png
new file mode 100644
index 000000000000..b2114ad13347
Binary files /dev/null and b/openmetadata-docs/images/v1.6/metapilot/using-metapilot-chatbot-2.png differ
diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml
index 3a907a8ae6ea..7aea1ae9a4d8 100644
--- a/openmetadata-service/pom.xml
+++ b/openmetadata-service/pom.xml
@@ -294,6 +294,10 @@
       <groupId>io.dropwizard.modules</groupId>
       <artifactId>dropwizard-web</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.github.erosb</groupId>
+      <artifactId>everit-json-schema</artifactId>
+    </dependency>
 
     <!-- Dependencies for cloudwatch monitoring -->
     <dependency>
@@ -710,7 +714,7 @@
                 <systemPropertyVariables>
                   <jdbcContainerClassName>org.testcontainers.containers.PostgreSQLContainer</jdbcContainerClassName>
                   <jdbcContainerImage>postgres:15</jdbcContainerImage>
-                  <elasticSearchContainerClassName>docker.elastic.co/elasticsearch/elasticsearch:8.10.2</elasticSearchContainerClassName>
+                  <elasticSearchContainerClassName>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</elasticSearchContainerClassName>
                   <openSearchContainerClassName>opensearchproject/opensearch:2.7.0</openSearchContainerClassName>
                   <runESTestCases>false</runESTestCases>
                 </systemPropertyVariables>
@@ -750,7 +754,7 @@
                 <systemPropertyVariables>
                   <jdbcContainerClassName>org.testcontainers.containers.MySQLContainer</jdbcContainerClassName>
                   <jdbcContainerImage>mysql:8.3.0</jdbcContainerImage>
-                  <elasticSearchContainerClassName>docker.elastic.co/elasticsearch/elasticsearch:8.10.2</elasticSearchContainerClassName>
+                  <elasticSearchContainerClassName>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</elasticSearchContainerClassName>
                   <openSearchContainerClassName>opensearchproject/opensearch:2.7.0</openSearchContainerClassName>
                   <runESTestCases>false</runESTestCases>
                 </systemPropertyVariables>
diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java
index 1d6ed073a1a9..d09e4b7ccc72 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java
@@ -112,7 +112,7 @@ public static List<String> fieldToInternalArray(String field) {
   }
 
   /**
-   * Parses a field containing key-value pairs separated by semicolons, correctly handling quotes.
+   * Parses a field containing key-value pairs separated by FIELD_SEPARATOR, correctly handling quotes.
    * Each key-value pair may also be enclosed in quotes, especially if it contains delimiter like (SEPARATOR , FIELD_SEPARATOR).
    * Input Example:
    * "key1:value1;key2:value2;\"key3:value;with;semicolon\""
@@ -124,7 +124,8 @@ public static List<String> fieldToExtensionStrings(String field) throws IOExcept
       return List.of();
     }
 
-    // Replace semicolons within quoted strings with a placeholder
+    // Case when semicolon is part of the fieldValue - Replace semicolons within quoted strings with
+    // a placeholder
     String preprocessedField =
         Pattern.compile("\"([^\"]*)\"") // Matches content inside double quotes
             .matcher(field)
@@ -146,9 +147,7 @@ public static List<String> fieldToExtensionStrings(String field) throws IOExcept
           .flatMap(CSVRecord::stream)
           .map(
               value ->
-                  value
-                      .replace("__SEMICOLON__", ";")
-                      .replace("\\n", "\n")) // Restore original semicolons and newlines
+                  value.replace("__SEMICOLON__", ";")) // Restore original semicolons and newlines
           .map(
               value ->
                   value.startsWith("\"") && value.endsWith("\"") // Remove outer quotes if present
@@ -158,6 +157,48 @@ public static List<String> fieldToExtensionStrings(String field) throws IOExcept
     }
   }
 
+  /**
+   * Parses a field containing column values separated by SEPARATOR, correctly handling quotes.
+   * Each value  enclosed in quotes, especially if it contains delimiter like SEPARATOR.
+   * Input Example:
+   * "value1,value2,\"value,with,comma\""
+   * Output: [value1, value2, value,with,comma]
+   *
+   */
+  public static List<String> fieldToColumns(String field) throws IOException {
+    if (field == null || field.isBlank()) {
+      return Collections.emptyList();
+    }
+
+    // Case when comma is part of the columnValue - Replace commas within quoted strings with a
+    // placeholder
+    String preprocessedField =
+        Pattern.compile("\"([^\"]*)\"")
+            .matcher(field)
+            .replaceAll(mr -> "\"" + mr.group(1).replace(",", "__COMMA__") + "\"");
+
+    preprocessedField = preprocessedField.replace("\n", "\\n").replace("\"", "\\\"");
+
+    CSVFormat format = CSVFormat.DEFAULT.withDelimiter(',').withQuote('"').withEscape('\\');
+
+    List<String> columns;
+    try (CSVParser parser = CSVParser.parse(new StringReader(preprocessedField), format)) {
+      columns =
+          parser.getRecords().stream()
+              .flatMap(CSVRecord::stream)
+              .map(value -> value.replace("__COMMA__", ","))
+              .map(
+                  value ->
+                      value.startsWith("\"")
+                              && value.endsWith("\"") // Remove outer quotes if present
+                          ? value.substring(1, value.length() - 1)
+                          : value)
+              .collect(Collectors.toList());
+    }
+
+    return columns;
+  }
+
   public static String quote(String field) {
     return String.format("\"%s\"", field);
   }
@@ -270,6 +311,13 @@ private static String quoteCsvField(String str) {
     return str;
   }
 
+  private static String quoteCsvFieldForSeparator(String str) {
+    if (str.contains(SEPARATOR)) {
+      return quote(str);
+    }
+    return str;
+  }
+
   public static List<String> addExtension(List<String> csvRecord, Object extension) {
     if (extension == null) {
       csvRecord.add(null);
@@ -310,6 +358,8 @@ private static String formatMapValue(Map<String, Object> valueMap) {
       return formatEntityReference(valueMap);
     } else if (isTimeInterval(valueMap)) {
       return formatTimeInterval(valueMap);
+    } else if (isTableType(valueMap)) {
+      return formatTableRows(valueMap);
     }
 
     return valueMap.toString();
@@ -320,11 +370,7 @@ private static String formatListValue(List<?> list) {
       return "";
     }
 
-    if (list.get(0) instanceof Map && isEnumWithDescriptions((Map<String, Object>) list.get(0))) {
-      return list.stream()
-          .map(item -> ((Map<String, Object>) item).get("key").toString())
-          .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR));
-    } else if (list.get(0) instanceof Map) {
+    if (list.get(0) instanceof Map) {
       return list.stream()
           .map(item -> formatMapValue((Map<String, Object>) item))
           .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR));
@@ -343,8 +389,8 @@ private static boolean isTimeInterval(Map<String, Object> valueMap) {
     return valueMap.containsKey("start") && valueMap.containsKey("end");
   }
 
-  private static boolean isEnumWithDescriptions(Map<String, Object> valueMap) {
-    return valueMap.containsKey("key") && valueMap.containsKey("description");
+  private static boolean isTableType(Map<String, Object> valueMap) {
+    return valueMap.containsKey("rows") && valueMap.containsKey("columns");
   }
 
   private static String formatEntityReference(Map<String, Object> valueMap) {
@@ -354,4 +400,19 @@ private static String formatEntityReference(Map<String, Object> valueMap) {
   private static String formatTimeInterval(Map<String, Object> valueMap) {
     return valueMap.get("start") + ENTITY_TYPE_SEPARATOR + valueMap.get("end");
   }
+
+  private static String formatTableRows(Map<String, Object> valueMap) {
+    List<String> columns = (List<String>) valueMap.get("columns");
+    List<Map<String, Object>> rows = (List<Map<String, Object>>) valueMap.get("rows");
+
+    return rows.stream()
+        .map(
+            row ->
+                columns.stream()
+                    .map(
+                        column ->
+                            quoteCsvFieldForSeparator(row.getOrDefault(column, "").toString()))
+                    .collect(Collectors.joining(SEPARATOR)))
+        .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR));
+  }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
index c169465526e9..0b2e0ccb5e93 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java
@@ -18,6 +18,7 @@
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
 import static org.openmetadata.csv.CsvUtil.ENTITY_TYPE_SEPARATOR;
 import static org.openmetadata.csv.CsvUtil.FIELD_SEPARATOR;
+import static org.openmetadata.csv.CsvUtil.fieldToColumns;
 import static org.openmetadata.csv.CsvUtil.fieldToEntities;
 import static org.openmetadata.csv.CsvUtil.fieldToExtensionStrings;
 import static org.openmetadata.csv.CsvUtil.fieldToInternalArray;
@@ -40,14 +41,14 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
 import javax.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.csv.CSVFormat;
@@ -68,7 +69,7 @@
 import org.openmetadata.schema.type.csv.CsvFile;
 import org.openmetadata.schema.type.csv.CsvHeader;
 import org.openmetadata.schema.type.csv.CsvImportResult;
-import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig;
+import org.openmetadata.schema.type.customProperties.TableConfig;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.TypeRegistry;
 import org.openmetadata.service.jdbi3.EntityRepository;
@@ -382,7 +383,7 @@ private void validateExtension(
               parseEntityReferences(printer, csvRecord, fieldNumber, fieldValue.toString(), isList);
         }
         case "date-cp", "dateTime-cp", "time-cp" -> fieldValue =
-            getFormattedDateTimeField(
+            parseFormattedDateTimeField(
                 printer,
                 csvRecord,
                 fieldNumber,
@@ -395,29 +396,13 @@ private void validateExtension(
           fieldValue = enumKeys.isEmpty() ? null : enumKeys;
         }
         case "timeInterval" -> fieldValue =
-            handleTimeInterval(printer, csvRecord, fieldNumber, fieldName, fieldValue);
-        case "number", "integer", "timestamp" -> {
-          try {
-            fieldValue = Long.parseLong(fieldValue.toString());
-          } catch (NumberFormatException e) {
-            importFailure(
-                printer,
-                invalidCustomPropertyValue(
-                    fieldNumber, fieldName, customPropertyType, fieldValue.toString()),
-                csvRecord);
-            fieldValue = null;
-          }
-        }
-        case "enumWithDescriptions" -> {
-          fieldValue =
-              parseEnumWithDescriptions(
-                  printer,
-                  csvRecord,
-                  fieldNumber,
-                  fieldName,
-                  fieldValue.toString(),
-                  propertyConfig);
-        }
+            parseTimeInterval(printer, csvRecord, fieldNumber, fieldName, fieldValue);
+        case "number", "integer", "timestamp" -> fieldValue =
+            parseLongField(
+                printer, csvRecord, fieldNumber, fieldName, customPropertyType, fieldValue);
+        case "table-cp" -> fieldValue =
+            parseTableType(printer, csvRecord, fieldNumber, fieldName, fieldValue, propertyConfig);
+
         default -> {}
       }
       // Validate the field against the JSON schema
@@ -461,71 +446,7 @@ private Object parseEntityReferences(
     return isList ? entityReferences : entityReferences.isEmpty() ? null : entityReferences.get(0);
   }
 
-  private Object parseEnumWithDescriptions(
-      CSVPrinter printer,
-      CSVRecord csvRecord,
-      int fieldNumber,
-      String fieldName,
-      String fieldValue,
-      String propertyConfig)
-      throws IOException {
-    List<String> enumKeys = listOrEmpty(fieldToInternalArray(fieldValue));
-    List<Object> enumObjects = new ArrayList<>();
-
-    JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig);
-    if (propertyConfigNode == null) {
-      importFailure(
-          printer,
-          invalidCustomPropertyFieldFormat(
-              fieldNumber,
-              fieldName,
-              "enumWithDescriptions",
-              "Invalid propertyConfig of enumWithDescriptions: " + fieldValue),
-          csvRecord);
-      return null;
-    }
-
-    Map<String, JsonNode> keyToObjectMap =
-        StreamSupport.stream(propertyConfigNode.get("values").spliterator(), false)
-            .collect(Collectors.toMap(node -> node.get("key").asText(), node -> node));
-    EnumWithDescriptionsConfig config =
-        JsonUtils.treeToValue(propertyConfigNode, EnumWithDescriptionsConfig.class);
-    if (!config.getMultiSelect() && enumKeys.size() > 1) {
-      importFailure(
-          printer,
-          invalidCustomPropertyFieldFormat(
-              fieldNumber,
-              fieldName,
-              "enumWithDescriptions",
-              "only one key is allowed for non-multiSelect enumWithDescriptions"),
-          csvRecord);
-      return null;
-    }
-
-    for (String key : enumKeys) {
-      try {
-        JsonNode valueObject = keyToObjectMap.get(key);
-        if (valueObject == null) {
-          importFailure(
-              printer,
-              invalidCustomPropertyValue(
-                  fieldNumber,
-                  fieldName,
-                  "enumWithDescriptions",
-                  key + " not found in propertyConfig of " + fieldName),
-              csvRecord);
-          return null;
-        }
-        enumObjects.add(valueObject);
-      } catch (Exception e) {
-        importFailure(printer, e.getMessage(), csvRecord);
-      }
-    }
-
-    return enumObjects;
-  }
-
-  protected String getFormattedDateTimeField(
+  protected String parseFormattedDateTimeField(
       CSVPrinter printer,
       CSVRecord csvRecord,
       int fieldNumber,
@@ -561,7 +482,7 @@ protected String getFormattedDateTimeField(
     }
   }
 
-  private Map<String, Long> handleTimeInterval(
+  private Map<String, Long> parseTimeInterval(
       CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, String fieldName, Object fieldValue)
       throws IOException {
     List<String> timestampValues = fieldToEntities(fieldValue.toString());
@@ -588,6 +509,70 @@ private Map<String, Long> handleTimeInterval(
     return timestampMap;
   }
 
+  private Object parseLongField(
+      CSVPrinter printer,
+      CSVRecord csvRecord,
+      int fieldNumber,
+      String fieldName,
+      String customPropertyType,
+      Object fieldValue)
+      throws IOException {
+    try {
+      return Long.parseLong(fieldValue.toString());
+    } catch (NumberFormatException e) {
+      importFailure(
+          printer,
+          invalidCustomPropertyValue(
+              fieldNumber, fieldName, customPropertyType, fieldValue.toString()),
+          csvRecord);
+      return null;
+    }
+  }
+
+  private Object parseTableType(
+      CSVPrinter printer,
+      CSVRecord csvRecord,
+      int fieldNumber,
+      String fieldName,
+      Object fieldValue,
+      String propertyConfig)
+      throws IOException {
+    List<String> tableValues = listOrEmpty(fieldToInternalArray(fieldValue.toString()));
+    List<Map<String, String>> rows = new ArrayList<>();
+    TableConfig tableConfig =
+        JsonUtils.treeToValue(JsonUtils.readTree(propertyConfig), TableConfig.class);
+
+    for (String row : tableValues) {
+      List<String> columns = listOrEmpty(fieldToColumns(row));
+      Map<String, String> rowMap = new LinkedHashMap<>();
+      Iterator<String> columnIterator = tableConfig.getColumns().iterator();
+      Iterator<String> valueIterator = columns.iterator();
+
+      if (columns.size() > tableConfig.getColumns().size()) {
+        importFailure(
+            printer,
+            invalidCustomPropertyValue(
+                fieldNumber,
+                fieldName,
+                "table",
+                "Column count should be less than or equal to " + tableConfig.getColumns().size()),
+            csvRecord);
+        return null;
+      }
+
+      while (columnIterator.hasNext() && valueIterator.hasNext()) {
+        rowMap.put(columnIterator.next(), valueIterator.next());
+      }
+
+      rows.add(rowMap);
+    }
+
+    Map<String, Object> tableJson = new LinkedHashMap<>();
+    tableJson.put("rows", rows);
+    tableJson.put("columns", tableConfig.getColumns());
+    return tableJson;
+  }
+
   private void validateAndUpdateExtension(
       CSVPrinter printer,
       CSVRecord csvRecord,
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java
index 1d42626762d4..c62a0fb85e56 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ApplicationHandler.java
@@ -130,6 +130,7 @@ public Object runAppInit(App app, CollectionDAO daoCollection, SearchRepository
   public void runMethodFromApplication(
       App app, CollectionDAO daoCollection, SearchRepository searchRepository, String methodName) {
     // Native Application
+    setAppRuntimeProperties(app);
     try {
       Object resource = runAppInit(app, daoCollection, searchRepository);
       // Call method on demand
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java
index ba680d1a347f..5ff8272962f2 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/ConfigurationReader.java
@@ -4,18 +4,20 @@
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
 import io.dropwizard.configuration.ConfigurationException;
 import io.dropwizard.configuration.EnvironmentVariableSubstitutor;
-import io.dropwizard.configuration.FileConfigurationSourceProvider;
+import io.dropwizard.configuration.ResourceConfigurationSourceProvider;
 import io.dropwizard.configuration.SubstitutingSourceProvider;
 import io.dropwizard.configuration.YamlConfigurationFactory;
-import java.io.File;
 import java.io.IOException;
 import java.net.URL;
 import java.util.Map;
 import org.apache.commons.text.StringSubstitutor;
 import org.openmetadata.schema.api.configuration.apps.AppPrivateConfig;
 import org.openmetadata.service.util.JsonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class ConfigurationReader {
+  private static final Logger log = LoggerFactory.getLogger(ConfigurationReader.class);
   private final StringSubstitutor substitutor;
   private final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
   private final YamlConfigurationFactory<Object> factory =
@@ -39,17 +41,18 @@ public AppPrivateConfig readConfigFromResource(String appName)
     if (resource == null) {
       throw new IOException("Configuration file not found: " + configFilePath);
     }
-    File configFile = new File(resource.getFile());
-    return JsonUtils.convertValue(readConfigFile(configFile), AppPrivateConfig.class);
+    log.debug("Loaded config file from resource: {}", configFilePath);
+    return JsonUtils.convertValue(readConfigResource(configFilePath), AppPrivateConfig.class);
   }
 
-  public Map<String, Object> readConfigFile(File configFile)
+  public Map<String, Object> readConfigResource(String resourcePath)
       throws IOException, ConfigurationException {
     try {
       return (Map<String, Object>)
           factory.build(
-              new SubstitutingSourceProvider(new FileConfigurationSourceProvider(), substitutor),
-              configFile.getAbsolutePath());
+              new SubstitutingSourceProvider(
+                  new ResourceConfigurationSourceProvider(), substitutor),
+              resourcePath);
     } catch (ClassCastException e) {
       throw new RuntimeException("Configuration file is not a valid YAML file", e);
     }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java
index 998b1fae172a..748413f379e5 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/insights/DataInsightsReportApp.java
@@ -218,7 +218,7 @@ private DataInsightTotalAssetTemplate createTotalAssetTemplate(
     dateWithCount.forEach((key, value) -> dateMap.put(key, value.intValue()));
     processDateMapToNormalize(dateMap);
 
-    int changeInTotalAssets = (int) (currentCount - previousCount);
+    int changeInTotalAssets = (int) Math.abs(currentCount - previousCount);
 
     if (previousCount == 0D) {
       // it should be undefined
@@ -276,7 +276,7 @@ private DataInsightDescriptionAndOwnerTemplate createDescriptionTemplate(
       currentPercentCompleted = (currentCompletedDescription / currentTotalAssetCount) * 100;
     }
 
-    int changeCount = (int) (currentCompletedDescription - previousCompletedDescription);
+    int changeCount = (int) Math.abs(currentCompletedDescription - previousCompletedDescription);
 
     return getTemplate(
         DataInsightDescriptionAndOwnerTemplate.MetricType.DESCRIPTION,
@@ -327,7 +327,7 @@ private DataInsightDescriptionAndOwnerTemplate createOwnershipTemplate(
       currentPercentCompleted = (currentHasOwner / currentTotalAssetCount) * 100;
     }
 
-    int changeCount = (int) (currentHasOwner - previousHasOwner);
+    int changeCount = (int) Math.abs(currentHasOwner - previousHasOwner);
 
     return getTemplate(
         DataInsightDescriptionAndOwnerTemplate.MetricType.OWNER,
@@ -376,7 +376,7 @@ private DataInsightDescriptionAndOwnerTemplate createTierTemplate(
       currentPercentCompleted = (currentHasTier / currentTotalAssetCount) * 100;
     }
 
-    int changeCount = (int) (currentHasTier - previousHasTier);
+    int changeCount = (int) Math.abs(currentHasTier - previousHasTier);
 
     // TODO: Understand if we actually use this tierData for anything.
     Map<String, Double> tierData = new HashMap<>();
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java
index adb45d380433..244ea2b33ec5 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java
@@ -1,6 +1,8 @@
 package org.openmetadata.service.apps.bundles.searchIndex;
 
 import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER;
+import static org.openmetadata.service.Entity.TEST_CASE_RESOLUTION_STATUS;
+import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
 import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS;
 import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB;
 import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY;
@@ -90,6 +92,7 @@ public class SearchIndexApp extends AbstractNativeApplication {
           "storedProcedure",
           "storageService",
           "testCaseResolutionStatus",
+          "testCaseResult",
           "apiService",
           "apiEndpoint",
           "apiCollection",
@@ -101,7 +104,8 @@ public class SearchIndexApp extends AbstractNativeApplication {
           ReportData.ReportDataType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA.value(),
           ReportData.ReportDataType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA.value(),
           ReportData.ReportDataType.AGGREGATED_COST_ANALYSIS_REPORT_DATA.value(),
-          "testCaseResolutionStatus");
+          TEST_CASE_RESOLUTION_STATUS,
+          TEST_CASE_RESULT);
   private final List<Source> paginatedSources = new ArrayList<>();
   private Processor entityProcessor;
   private Processor entityTimeSeriesProcessor;
@@ -174,6 +178,8 @@ private void cleanUpStaleJobsFromRuns() {
   }
 
   private void initializeJob() {
+    List<Source> paginatedEntityTimeSeriesSources = new ArrayList<>();
+
     // Remove any Stale Jobs
     cleanUpStaleJobsFromRuns();
 
@@ -205,9 +211,11 @@ private void initializeJob() {
                 if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) {
                   source.setCursor(jobData.getAfterCursor());
                 }
-                paginatedSources.add(source);
+                paginatedEntityTimeSeriesSources.add(source);
               }
             });
+    // Add Time Series Sources at the End of the List to Process them last
+    paginatedSources.addAll(paginatedEntityTimeSeriesSources);
     if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
       this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords);
       this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords);
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluator.java b/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluator.java
index 13836d950d8e..8db0110d1577 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluator.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluator.java
@@ -15,6 +15,7 @@
 import static org.openmetadata.service.Entity.USER;
 
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.regex.Matcher;
@@ -28,6 +29,7 @@
 import org.openmetadata.schema.entity.teams.Team;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.tests.TestCase;
+import org.openmetadata.schema.tests.TestSuite;
 import org.openmetadata.schema.tests.type.TestCaseResult;
 import org.openmetadata.schema.tests.type.TestCaseStatus;
 import org.openmetadata.schema.type.ChangeEvent;
@@ -97,27 +99,24 @@ public boolean matchAnyOwnerName(List<String> ownerNameList) {
     if (nullOrEmpty(ownerReferences)) {
       entity =
           Entity.getEntity(
-              changeEvent.getEntityType(), entity.getId(), "owner", Include.NON_DELETED);
+              changeEvent.getEntityType(), entity.getId(), "owners", Include.NON_DELETED);
       ownerReferences = entity.getOwners();
     }
     if (!nullOrEmpty(ownerReferences)) {
-      for (EntityReference owner : ownerReferences) {
-        if (USER.equals(owner.getType())) {
-          User user = Entity.getEntity(Entity.USER, owner.getId(), "", Include.NON_DELETED);
-          for (String name : ownerNameList) {
-            if (user.getName().equals(name)) {
-              return true;
-            }
-          }
-        } else if (TEAM.equals(owner.getType())) {
-          Team team = Entity.getEntity(Entity.TEAM, owner.getId(), "", Include.NON_DELETED);
-          for (String name : ownerNameList) {
-            if (team.getName().equals(name)) {
-              return true;
-            }
-          }
-        }
-      }
+      return matchOwners(ownerReferences, ownerNameList);
+    }
+
+    if (changeEvent.getEntityType().equals(TEST_CASE)) {
+      // If we did not match on the owner name and are dealing with a test case,
+      // check if the match happens on the test suite owner name
+      TestCase testCase =
+          Entity.getEntity(
+              changeEvent.getEntityType(),
+              entity.getId(),
+              "testSuites,owners",
+              Include.NON_DELETED);
+      Optional<List<TestSuite>> testSuites = Optional.ofNullable(testCase.getTestSuites());
+      return testSuites.filter(suites -> testSuiteOwnerMatcher(suites, ownerNameList)).isPresent();
     }
     return false;
   }
@@ -147,6 +146,15 @@ public boolean matchAnyEntityFqn(List<String> entityNames) {
         return true;
       }
     }
+
+    if (changeEvent.getEntityType().equals(TEST_CASE)) {
+      // If we did not match on the entity FQN and are dealing with a test case,
+      // check if the match happens on the test suite FQN
+      TestCase testCase = ((TestCase) entity);
+      Optional<List<TestSuite>> testSuites = Optional.ofNullable(testCase.getTestSuites());
+      return testSuites.filter(suites -> testSuiteMatcher(suites, entityNames)).isPresent();
+    }
+
     return false;
   }
 
@@ -455,6 +463,14 @@ public boolean matchAnyDomain(List<String> fieldChangeUpdate) {
         }
       }
     }
+
+    if (changeEvent.getEntityType().equals(TEST_CASE)) {
+      // If we did not match on the domain and are dealing with a test case,
+      // check if the match happens on the test suite domain
+      TestCase testCase = ((TestCase) entity);
+      Optional<List<TestSuite>> testSuites = Optional.ofNullable(testCase.getTestSuites());
+      return testSuites.filter(suites -> testSuiteMatcher(suites, fieldChangeUpdate)).isPresent();
+    }
     return false;
   }
 
@@ -543,4 +559,50 @@ public static Thread getThread(ChangeEvent event) {
               JsonUtils.pojoToJson(event.getEntity())));
     }
   }
+
+  private boolean testSuiteMatcher(List<TestSuite> testSuites, List<String> entityNames) {
+    for (TestSuite testSuite : testSuites) {
+      for (String name : entityNames) {
+        Pattern pattern = Pattern.compile(name);
+        Matcher matcherTestSuiteFQN = pattern.matcher(testSuite.getFullyQualifiedName());
+        if (matcherTestSuiteFQN.find()) return true;
+        if (testSuite.getDomain() != null) {
+          Matcher matcherDomainFQN = pattern.matcher(testSuite.getDomain().getFullyQualifiedName());
+          if (matcherDomainFQN.find()) return true;
+        }
+      }
+    }
+    return false;
+  }
+
+  private boolean testSuiteOwnerMatcher(List<TestSuite> testSuites, List<String> ownerNameList) {
+    boolean match;
+    for (TestSuite testSuite : testSuites) {
+      List<EntityReference> owners = testSuite.getOwners();
+      match = matchOwners(owners, ownerNameList);
+      if (match) return true;
+    }
+    return false;
+  }
+
+  private boolean matchOwners(List<EntityReference> ownerReferences, List<String> ownerNameList) {
+    for (EntityReference owner : ownerReferences) {
+      if (USER.equals(owner.getType())) {
+        User user = Entity.getEntity(Entity.USER, owner.getId(), "", Include.NON_DELETED);
+        for (String name : ownerNameList) {
+          if (user.getName().equals(name)) {
+            return true;
+          }
+        }
+      } else if (TEAM.equals(owner.getType())) {
+        Team team = Entity.getEntity(Entity.TEAM, owner.getId(), "", Include.NON_DELETED);
+        for (String name : ownerNameList) {
+          if (team.getName().equals(name)) {
+            return true;
+          }
+        }
+      }
+    }
+    return false;
+  }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java
index d56fbf97b8eb..b52680c2a1a7 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/CatalogExceptionMessage.java
@@ -253,6 +253,10 @@ public static String jsonValidationError(String fieldName, String validationMess
     return String.format("Custom field %s has invalid JSON %s", fieldName, validationMessages);
   }
 
+  public static String customPropertyConfigError(String fieldName, String validationMessages) {
+    return String.format("Custom Property %s has invalid value %s", fieldName, validationMessages);
+  }
+
   public static String invalidParent(Team parent, String child, TeamType childType) {
     return String.format(
         "Team %s of type %s can't be of parent of team %s of type %s",
@@ -314,6 +318,11 @@ public static String mutuallyExclusiveLabels(TagLabel tag1, TagLabel tag2) {
         tag1.getTagFQN(), tag2.getTagFQN());
   }
 
+  public static String disabledTag(TagLabel tag) {
+    return String.format(
+        "Tag label %s is disabled and can't be assigned to a data asset.", tag.getTagFQN());
+  }
+
   public static String csvNotSupported(String entityType) {
     return String.format(
         "Upload/download CSV for bulk operations is not supported for entity [%s]", entityType);
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java
index 7bf3732ea2ff..805ad8fa9047 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/formatter/util/FormatterUtil.java
@@ -16,6 +16,8 @@
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
 import static org.openmetadata.schema.type.EventType.ENTITY_CREATED;
 import static org.openmetadata.service.Entity.FIELD_EXTENSION;
+import static org.openmetadata.service.Entity.TEST_CASE;
+import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
 import static org.openmetadata.service.Entity.THREAD;
 import static org.openmetadata.service.formatter.factory.ParserFactory.getFieldParserObject;
 import static org.openmetadata.service.formatter.field.DefaultFieldFormatter.getFieldNameChange;
@@ -31,11 +33,16 @@
 import javax.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.schema.EntityInterface;
+import org.openmetadata.schema.EntityTimeSeriesInterface;
 import org.openmetadata.schema.entity.feed.Thread;
+import org.openmetadata.schema.tests.TestCase;
+import org.openmetadata.schema.tests.type.TestCaseResult;
 import org.openmetadata.schema.type.ChangeDescription;
 import org.openmetadata.schema.type.ChangeEvent;
 import org.openmetadata.schema.type.EventType;
 import org.openmetadata.schema.type.FieldChange;
+import org.openmetadata.schema.type.Include;
+import org.openmetadata.service.Entity;
 import org.openmetadata.service.formatter.decorators.MessageDecorator;
 import org.openmetadata.service.formatter.factory.ParserFactory;
 import org.openmetadata.service.formatter.field.DefaultFieldFormatter;
@@ -237,6 +244,11 @@ private static ChangeEvent extractChangeEvent(
       return createChangeEventForThread(updateBy, eventType, thread);
     }
 
+    // if the response entity is an EntityTimeseriesInterface, then create a ChangeEvent from it
+    if (responseContext.getEntity() instanceof EntityTimeSeriesInterface entityTimeSeries) {
+      return createChangeEventForEntity(updateBy, eventType, entityTimeSeries);
+    }
+
     LOG.debug("Unknown event type in Change Event :  {}", eventType.value());
     return null;
   }
@@ -253,6 +265,12 @@ private static ChangeEvent createChangeEventForEntity(
         .withEntityFullyQualifiedName(entityInterface.getEntityReference().getFullyQualifiedName());
   }
 
+  private static ChangeEvent createChangeEventForEntity(
+      String updateBy, EventType eventType, EntityTimeSeriesInterface entityTimeSeries) {
+    return getChangeEventForEntityTimeSeries(
+        updateBy, eventType, entityTimeSeries.getEntityReference().getType(), entityTimeSeries);
+  }
+
   private static ChangeEvent createChangeEventForThread(
       String updateBy, EventType eventType, Thread threadEntity) {
     return getChangeEventForThread(updateBy, eventType, THREAD, threadEntity)
@@ -285,6 +303,39 @@ private static ChangeEvent getChangeEvent(
         .withCurrentVersion(entityInterface.getVersion());
   }
 
+  private static ChangeEvent getChangeEventForEntityTimeSeries(
+      String updateBy,
+      EventType eventType,
+      String entityType,
+      EntityTimeSeriesInterface entityTimeSeries) {
+    if (entityTimeSeries instanceof TestCaseResult) {
+      eventType =
+          EventType
+              .ENTITY_UPDATED; // workaround as adding a test case result is sent as a POST request
+      TestCaseResult testCaseResult =
+          JsonUtils.readOrConvertValue(entityTimeSeries, TestCaseResult.class);
+      TestCase testCase =
+          Entity.getEntityByName(
+              TEST_CASE,
+              testCaseResult.getTestCaseFQN(),
+              TEST_CASE_RESULT + ",testSuites",
+              Include.ALL);
+      ChangeEvent changeEvent =
+          getChangeEvent(updateBy, eventType, testCase.getEntityReference().getType(), testCase);
+      return changeEvent
+          .withChangeDescription(
+              new ChangeDescription()
+                  .withFieldsUpdated(
+                      List.of(
+                          new FieldChange()
+                              .withName(TEST_CASE_RESULT)
+                              .withNewValue(testCase.getTestCaseResult()))))
+          .withEntity(testCase)
+          .withEntityFullyQualifiedName(testCase.getFullyQualifiedName());
+    }
+    return null;
+  }
+
   private static ChangeEvent getChangeEventForThread(
       String updateBy, EventType eventType, String entityType, Thread thread) {
     return new ChangeEvent()
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APICollectionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APICollectionRepository.java
index 538c5b8c1141..8c6551c0988a 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APICollectionRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APICollectionRepository.java
@@ -18,7 +18,7 @@
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.schema.EntityInterface;
 import org.openmetadata.schema.entity.data.APICollection;
-import org.openmetadata.schema.entity.services.APIService;
+import org.openmetadata.schema.entity.services.ApiService;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.Relationship;
@@ -108,7 +108,7 @@ public EntityRepository<APICollection>.EntityUpdater getUpdater(
   }
 
   private void populateService(APICollection apiCollection) {
-    APIService service = Entity.getEntity(apiCollection.getService(), "", Include.NON_DELETED);
+    ApiService service = Entity.getEntity(apiCollection.getService(), "", Include.NON_DELETED);
     apiCollection.setService(service.getEntityReference());
     apiCollection.setServiceType(service.getServiceType());
   }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APIServiceRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APIServiceRepository.java
index aab2ec3560b3..5d6f9f619937 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APIServiceRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/APIServiceRepository.java
@@ -1,19 +1,18 @@
 package org.openmetadata.service.jdbi3;
 
-import org.openmetadata.schema.entity.services.APIService;
+import org.openmetadata.schema.entity.services.ApiService;
 import org.openmetadata.schema.entity.services.ServiceType;
-import org.openmetadata.schema.type.APIServiceConnection;
+import org.openmetadata.schema.type.ApiConnection;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.resources.services.apiservices.APIServiceResource;
 
-public class APIServiceRepository
-    extends ServiceEntityRepository<APIService, APIServiceConnection> {
+public class APIServiceRepository extends ServiceEntityRepository<ApiService, ApiConnection> {
   public APIServiceRepository() {
     super(
         APIServiceResource.COLLECTION_PATH,
         Entity.API_SERVICE,
         Entity.getCollectionDAO().apiServiceDAO(),
-        APIServiceConnection.class,
+        ApiConnection.class,
         "",
         ServiceType.API);
     supportsSearch = true;
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java
index 2eea460d72ea..601d2a9d0f69 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ClassificationRepository.java
@@ -13,18 +13,18 @@
 
 package org.openmetadata.service.jdbi3;
 
+import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
 import static org.openmetadata.service.Entity.CLASSIFICATION;
 import static org.openmetadata.service.Entity.TAG;
 import static org.openmetadata.service.search.SearchClient.TAG_SEARCH_INDEX;
 
-import java.sql.ResultSet;
-import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.UUID;
+
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.jdbi.v3.core.mapper.RowMapper;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.schema.entity.classification.Classification;
 import org.openmetadata.schema.entity.classification.Tag;
@@ -32,25 +32,27 @@
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.ProviderType;
 import org.openmetadata.schema.type.Relationship;
-import org.openmetadata.schema.type.TagLabel;
 import org.openmetadata.schema.type.TagLabel.TagSource;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.CatalogExceptionMessage;
 import org.openmetadata.service.jdbi3.CollectionDAO.EntityRelationshipRecord;
 import org.openmetadata.service.resources.tags.ClassificationResource;
+import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.EntityUtil.Fields;
 import org.openmetadata.service.util.JsonUtils;
 
 @Slf4j
 public class ClassificationRepository extends EntityRepository<Classification> {
+  static final String ROLES_FIELD = "roles";
+
   public ClassificationRepository() {
     super(
         ClassificationResource.TAG_COLLECTION_PATH,
         Entity.CLASSIFICATION,
         Classification.class,
         Entity.getCollectionDAO().classificationDAO(),
-        "",
-        "");
+        ROLES_FIELD,
+        ROLES_FIELD);
     quoteFqn = true;
     supportsSearch = true;
     renameAllowed = true;
@@ -68,6 +70,8 @@ public void setFields(Classification classification, Fields fields) {
         fields.contains("termCount") ? getTermCount(classification) : null);
     classification.withUsageCount(
         fields.contains("usageCount") ? getUsageCount(classification) : null);
+    classification.setRoles(
+        fields.contains(ROLES_FIELD) ? getRoles(classification) : classification.getRoles());
   }
 
   @Override
@@ -80,19 +84,23 @@ public void clearFields(Classification classification, Fields fields) {
 
   @Override
   public void prepare(Classification entity, boolean update) {
-    /* Nothing to do */
+    validateRoles(entity.getRoles());
   }
 
   @Override
   public void storeEntity(Classification classification, boolean update) {
+    List<EntityReference> roles = classification.getRoles();
+    classification.withRoles(null);
     store(classification, update);
+    classification.withRoles(roles);
   }
 
   @Override
   public void storeRelationships(Classification entity) {
-    // No relationships to store beyond what is stored in the super class
+    assignRoles(entity, entity.getRoles());
   }
 
+
   private int getTermCount(Classification classification) {
     ListFilter filter =
         new ListFilter(Include.NON_DELETED)
@@ -106,17 +114,18 @@ private Integer getUsageCount(Classification classification) {
         .getTagCount(TagSource.CLASSIFICATION.ordinal(), classification.getFullyQualifiedName());
   }
 
-  public static class TagLabelMapper implements RowMapper<TagLabel> {
-    @Override
-    public TagLabel map(ResultSet r, org.jdbi.v3.core.statement.StatementContext ctx)
-        throws SQLException {
-      return new TagLabel()
-          .withLabelType(TagLabel.LabelType.values()[r.getInt("labelType")])
-          .withState(TagLabel.State.values()[r.getInt("state")])
-          .withTagFQN(r.getString("tagFQN"));
+  private void assignRoles(Classification classification, List<EntityReference> roles) {
+    roles = listOrEmpty(roles);
+    for (EntityReference role : roles) {
+      addRelationship(
+          classification.getId(), role.getId(), CLASSIFICATION, Entity.ROLE, Relationship.HAS);
     }
   }
 
+  private List<EntityReference> getRoles(Classification classification) {
+    return findTo(classification.getId(), CLASSIFICATION, Relationship.HAS, Entity.ROLE);
+  }
+
   @Override
   public void entityRelationshipReindex(Classification original, Classification updated) {
     super.entityRelationshipReindex(original, updated);
@@ -160,6 +169,7 @@ public ClassificationUpdater(
     public void entitySpecificUpdate() {
       // Mutually exclusive cannot be updated
       updated.setMutuallyExclusive(original.getMutuallyExclusive());
+      updateRoles(original, updated);
       recordChange("disabled", original.getDisabled(), updated.getDisabled());
       updateName(original, updated);
     }
@@ -205,5 +215,21 @@ private void invalidateTags(UUID tagId) {
         invalidateTags(tagRecord.getId());
       }
     }
+
+    private void updateRoles(Classification original, Classification updated) {
+      deleteFrom(original.getId(), CLASSIFICATION, Relationship.HAS, Entity.ROLE);
+      assignRoles(updated, updated.getRoles());
+
+      List<EntityReference> origRoles = listOrEmpty(original.getRoles());
+      List<EntityReference> updatedRoles = listOrEmpty(updated.getRoles());
+
+      origRoles.sort(EntityUtil.compareEntityReference);
+      updatedRoles.sort(EntityUtil.compareEntityReference);
+
+      List<EntityReference> added = new ArrayList<>();
+      List<EntityReference> deleted = new ArrayList<>();
+      recordListChange(
+          ROLES_FIELD, origRoles, updatedRoles, added, deleted, EntityUtil.entityReferenceMatch);
+    }
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
index 2e9336fb3a34..a5a471df3667 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
@@ -98,7 +98,7 @@
 import org.openmetadata.schema.entity.domains.Domain;
 import org.openmetadata.schema.entity.events.EventSubscription;
 import org.openmetadata.schema.entity.policies.Policy;
-import org.openmetadata.schema.entity.services.APIService;
+import org.openmetadata.schema.entity.services.ApiService;
 import org.openmetadata.schema.entity.services.DashboardService;
 import org.openmetadata.schema.entity.services.DatabaseService;
 import org.openmetadata.schema.entity.services.MessagingService;
@@ -161,6 +161,9 @@ public interface CollectionDAO {
   @CreateSqlObject
   AppExtensionTimeSeries appExtensionTimeSeriesDao();
 
+  @CreateSqlObject
+  AppsDataStore appStoreDAO();
+
   @CreateSqlObject
   EntityExtensionTimeSeriesDAO entityExtensionTimeSeriesDao();
 
@@ -288,7 +291,7 @@ public interface CollectionDAO {
   SearchServiceDAO searchServiceDAO();
 
   @CreateSqlObject
-  APIServiceDAO apiServiceDAO();
+  ApiServiceDAO apiServiceDAO();
 
   @CreateSqlObject
   ContainerDAO containerDAO();
@@ -636,15 +639,15 @@ default String getNameHashColumn() {
     }
   }
 
-  interface APIServiceDAO extends EntityDAO<APIService> {
+  interface ApiServiceDAO extends EntityDAO<ApiService> {
     @Override
     default String getTableName() {
       return "api_service_entity";
     }
 
     @Override
-    default Class<APIService> getEntityClass() {
-      return APIService.class;
+    default Class<ApiService> getEntityClass() {
+      return ApiService.class;
     }
 
     @Override
@@ -4243,6 +4246,52 @@ default String getTimeSeriesTableName() {
     }
   }
 
+  interface AppsDataStore {
+    @ConnectionAwareSqlUpdate(
+        value =
+            "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json)",
+        connectionType = MYSQL)
+    @ConnectionAwareSqlUpdate(
+        value =
+            "INSERT INTO apps_data_store(identifier, type, json) VALUES (:identifier, :type, :json :: jsonb)",
+        connectionType = POSTGRES)
+    void insert(
+        @Bind("identifier") String identifier,
+        @Bind("type") String type,
+        @Bind("json") String json);
+
+    @ConnectionAwareSqlUpdate(
+        value =
+            "UPDATE apps_data_store set json = :json where identifier = :identifier AND type=:type",
+        connectionType = MYSQL)
+    @ConnectionAwareSqlUpdate(
+        value =
+            "UPDATE apps_data_store set json = (:json :: jsonb) where identifier = :identifier AND type=:type",
+        connectionType = POSTGRES)
+    void update(
+        @Bind("identifier") String identifier,
+        @Bind("type") String type,
+        @Bind("json") String json);
+
+    @SqlUpdate("DELETE FROM apps_data_store WHERE identifier = :identifier AND type = :type")
+    void delete(@Bind("identifier") String identifier, @Bind("type") String type);
+
+    @SqlQuery(
+        "SELECT count(*) FROM apps_data_store where identifier = :identifier AND type = :type")
+    int listAppDataCount(@Bind("identifier") String identifier, @Bind("type") String type);
+
+    @SqlQuery(
+        "SELECT json FROM apps_data_store where identifier in (<identifier>) AND type = :type")
+    List<String> listAppsDataWithIds(
+        @BindList("identifier") List<String> identifier, @Bind("type") String type);
+
+    @SqlQuery("SELECT json FROM apps_data_store where type = :type")
+    List<String> listAppsDataWithType(@Bind("type") String type);
+
+    @SqlQuery("SELECT json FROM apps_data_store where identifier = :identifier AND type = :type")
+    String findAppData(@Bind("identifier") String identifier, @Bind("type") String type);
+  }
+
   interface AppExtensionTimeSeries {
     @ConnectionAwareSqlUpdate(
         value =
@@ -4533,6 +4582,30 @@ void insert(
         @Bind("json") String json,
         @Bind("incidentStateId") String incidentStateId);
 
+    @SqlQuery(
+        """
+              SELECT dqdts1.json FROM
+              data_quality_data_time_series dqdts1
+              INNER JOIN (
+                  SELECT tc.fqnHash
+                  FROM entity_relationship er
+                  INNER JOIN test_case tc ON er.toId = tc.id
+                  where fromEntity = 'testSuite' AND toEntity = 'testCase' and fromId = :testSuiteId
+              ) ts ON dqdts1.entityFQNHash = ts.fqnHash
+              LEFT JOIN data_quality_data_time_series dqdts2 ON
+                  (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
+              WHERE dqdts2.entityFQNHash IS NULL""")
+    List<String> listLastTestCaseResultsForTestSuite(@BindMap Map<String, String> params);
+
+    @SqlQuery(
+        """
+            SELECT dqdts1.json FROM
+            data_quality_data_time_series dqdts1
+            LEFT JOIN data_quality_data_time_series dqdts2 ON
+                (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
+            WHERE dqdts2.entityFQNHash IS NULL AND dqdts1.entityFQNHash = :testCaseFQN""")
+    String listLastTestCaseResult(@BindFQN("testCaseFQN") String testCaseFQN);
+
     default void insert(
         String testCaseFQN,
         String extension,
@@ -4548,6 +4621,10 @@ default void insert(
           json,
           incidentStateId != null ? incidentStateId.toString() : null);
     }
+
+    default List<String> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
+      return listLastTestCaseResultsForTestSuite(Map.of("testSuiteId", testSuiteId.toString()));
+    }
   }
 
   class EntitiesCountRowMapper implements RowMapper<EntitiesCount> {
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
index 0a42523c2f98..e31adbb6ab65 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java
@@ -51,6 +51,7 @@
 import static org.openmetadata.service.exception.CatalogExceptionMessage.csvNotSupported;
 import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
 import static org.openmetadata.service.resources.tags.TagLabelUtil.addDerivedTags;
+import static org.openmetadata.service.resources.tags.TagLabelUtil.checkDisabledTags;
 import static org.openmetadata.service.resources.tags.TagLabelUtil.checkMutuallyExclusive;
 import static org.openmetadata.service.util.EntityUtil.compareTagLabel;
 import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch;
@@ -67,7 +68,6 @@
 import static org.openmetadata.service.util.EntityUtil.tagLabelMatch;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.cache.CacheBuilder;
@@ -101,8 +101,8 @@
 import java.util.function.BiPredicate;
 import java.util.function.Function;
 import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
 import javax.json.JsonPatch;
+import javax.validation.ConstraintViolationException;
 import javax.validation.constraints.NotNull;
 import javax.ws.rs.core.Response.Status;
 import javax.ws.rs.core.UriInfo;
@@ -145,7 +145,7 @@
 import org.openmetadata.schema.type.api.BulkOperationResult;
 import org.openmetadata.schema.type.api.BulkResponse;
 import org.openmetadata.schema.type.csv.CsvImportResult;
-import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig;
+import org.openmetadata.schema.type.customProperties.TableConfig;
 import org.openmetadata.schema.utils.EntityInterfaceUtil;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.OpenMetadataApplicationConfig;
@@ -1430,19 +1430,8 @@ private void validateExtension(T entity) {
       String customPropertyType = TypeRegistry.getCustomPropertyType(entityType, fieldName);
       String propertyConfig = TypeRegistry.getCustomPropertyConfig(entityType, fieldName);
 
-      try {
-        validateAndUpdateExtensionBasedOnPropertyType(
-            entity,
-            (ObjectNode) jsonNode,
-            fieldName,
-            fieldValue,
-            customPropertyType,
-            propertyConfig);
-      } catch (DateTimeParseException e) {
-        throw new IllegalArgumentException(
-            CatalogExceptionMessage.dateTimeValidationError(fieldName, propertyConfig));
-      }
-
+      validateAndUpdateExtensionBasedOnPropertyType(
+          entity, (ObjectNode) jsonNode, fieldName, fieldValue, customPropertyType, propertyConfig);
       Set<ValidationMessage> validationMessages = jsonSchema.validate(entry.getValue());
       if (!validationMessages.isEmpty()) {
         throw new IllegalArgumentException(
@@ -1466,8 +1455,7 @@ private void validateAndUpdateExtensionBasedOnPropertyType(
                 fieldValue.textValue(), customPropertyType, propertyConfig, fieldName);
         jsonNode.put(fieldName, formattedValue);
       }
-      case "enumWithDescriptions" -> handleEnumWithDescriptions(
-          fieldName, fieldValue, propertyConfig, jsonNode, entity);
+      case "table-cp" -> validateTableType(fieldValue, propertyConfig, fieldName);
       default -> {}
     }
   }
@@ -1505,37 +1493,48 @@ private String getFormattedDateTimeField(
     }
   }
 
-  private void handleEnumWithDescriptions(
-      String fieldName, JsonNode fieldValue, String propertyConfig, ObjectNode jsonNode, T entity) {
-    JsonNode propertyConfigNode = JsonUtils.readTree(propertyConfig);
-    EnumWithDescriptionsConfig config =
-        JsonUtils.treeToValue(propertyConfigNode, EnumWithDescriptionsConfig.class);
+  private void validateTableType(JsonNode fieldValue, String propertyConfig, String fieldName) {
+    TableConfig tableConfig =
+        JsonUtils.convertValue(JsonUtils.readTree(propertyConfig), TableConfig.class);
+    org.openmetadata.schema.type.customProperties.Table tableValue =
+        JsonUtils.convertValue(
+            JsonUtils.readTree(String.valueOf(fieldValue)),
+            org.openmetadata.schema.type.customProperties.Table.class);
+    Set<String> configColumns = tableConfig.getColumns();
 
-    if (!config.getMultiSelect() && fieldValue.size() > 1) {
-      throw new IllegalArgumentException(
-          "Only one key is allowed for non-multiSelect enumWithDescriptions");
-    }
-    // Replace each enumWithDescriptions key in the fieldValue with the corresponding object from
-    // the propertyConfig
-    Map<String, JsonNode> keyToObjectMap =
-        StreamSupport.stream(propertyConfigNode.get("values").spliterator(), false)
-            .collect(Collectors.toMap(node -> node.get("key").asText(), node -> node));
+    try {
+      JsonUtils.validateJsonSchema(
+          tableValue, org.openmetadata.schema.type.customProperties.Table.class);
 
-    if (fieldValue.isArray()) {
-      ArrayNode newArray = JsonUtils.getObjectNode().arrayNode();
-      fieldValue.forEach(
-          valueNode -> {
-            String key = valueNode.isTextual() ? valueNode.asText() : valueNode.get("key").asText();
-            JsonNode valueObject = keyToObjectMap.get(key);
+      Set<String> fieldColumns = new HashSet<>();
+      fieldValue.get("columns").forEach(column -> fieldColumns.add(column.asText()));
 
-            if (valueObject == null) {
-              throw new IllegalArgumentException("Key not found in propertyConfig: " + key);
-            }
-            newArray.add(valueNode.isTextual() ? valueObject : valueNode);
-          });
+      Set<String> undefinedColumns = new HashSet<>(fieldColumns);
+      undefinedColumns.removeAll(configColumns);
+      if (!undefinedColumns.isEmpty()) {
+        throw new IllegalArgumentException(
+            "Expected columns: "
+                + configColumns
+                + ", but found undefined columns: "
+                + undefinedColumns);
+      }
+
+      Set<String> rowFieldNames = new HashSet<>();
+      fieldValue.get("rows").forEach(row -> row.fieldNames().forEachRemaining(rowFieldNames::add));
 
-      jsonNode.replace(fieldName, newArray);
-      entity.setExtension(JsonUtils.treeToValue(jsonNode, Object.class));
+      undefinedColumns = new HashSet<>(rowFieldNames);
+      undefinedColumns.removeAll(configColumns);
+      if (!undefinedColumns.isEmpty()) {
+        throw new IllegalArgumentException("Rows contain undefined columns: " + undefinedColumns);
+      }
+    } catch (ConstraintViolationException e) {
+      String validationErrors =
+          e.getConstraintViolations().stream()
+              .map(violation -> violation.getPropertyPath() + " " + violation.getMessage())
+              .collect(Collectors.joining(", "));
+
+      throw new IllegalArgumentException(
+          CatalogExceptionMessage.jsonValidationError(fieldName, validationErrors));
     }
   }
 
@@ -1602,7 +1601,7 @@ protected void applyColumnTags(List<Column> columns) {
   protected void applyTags(T entity) {
     if (supportsTags) {
       // Add entity level tags by adding tag to the entity relationship
-      applyTags(entity.getTags(), entity.getFullyQualifiedName());
+      applyTags(entity.getTags(), entity.getFullyQualifiedName(), entity.getUpdatedBy());
     }
   }
 
@@ -1610,10 +1609,7 @@ protected void applyTags(T entity) {
   @Transaction
   public final void applyTags(List<TagLabel> tagLabels, String targetFQN) {
     for (TagLabel tagLabel : listOrEmpty(tagLabels)) {
-      // Apply tagLabel to targetFQN that identifies an entity or field
-      boolean isTagDerived = tagLabel.getLabelType().equals(TagLabel.LabelType.DERIVED);
-      // Derived Tags should not create Relationships, and needs to be built on the during Read
-      if (!isTagDerived) {
+      if (!tagLabel.getLabelType().equals(TagLabel.LabelType.DERIVED)) {
         daoCollection
             .tagUsageDAO()
             .applyTag(
@@ -2325,6 +2321,8 @@ protected void validateTags(T entity) {
     validateTags(entity.getTags());
     entity.setTags(addDerivedTags(entity.getTags()));
     checkMutuallyExclusive(entity.getTags());
+    checkDisabledTags(entity.getTags());
+    checkTagsPermissions(entity.getTags(), entity.getUpdatedBy());
   }
 
   protected void validateTags(List<TagLabel> labels) {
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java
index 539b182f73dc..30859058a91b 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java
@@ -29,7 +29,9 @@
 import org.openmetadata.schema.type.Relationship;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.EntityNotFoundException;
+import org.openmetadata.service.search.SearchAggregation;
 import org.openmetadata.service.search.SearchClient;
+import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.SearchListFilter;
 import org.openmetadata.service.search.SearchRepository;
 import org.openmetadata.service.search.SearchSortFilter;
@@ -408,12 +410,12 @@ public ResultList<T> listLatestFromSearch(
     setExcludeSearchFields(searchListFilter);
     String aggregationPath = "$.sterms#byTerms.buckets";
     String aggregationStr =
-        "{\"aggregations\":{\"byTerms\":{\"terms\": {\"field\":\"%s\",\"size\":100},\"aggs\":{\"latest\":"
-            + "{\"top_hits\":{\"size\":1,\"sort_field\":\"timestamp\",\"sort_order\":\"desc\"}}}}}}";
+        "bucketName=byTerms:aggType=terms:field=%s&size=100,"
+            + "bucketName=latest:aggType=top_hits:size=1&sort_field=timestamp&sort_order=desc";
     aggregationStr = String.format(aggregationStr, groupBy);
-    JsonObject aggregation = JsonUtils.readJson(aggregationStr).asJsonObject();
+    SearchAggregation searchAggregation = SearchIndexUtils.buildAggregationTree(aggregationStr);
     JsonObject jsonObjResults =
-        searchRepository.aggregate(q, entityType, aggregation, searchListFilter);
+        searchRepository.aggregate(q, entityType, searchAggregation, searchListFilter);
 
     Optional<List> jsonObjects =
         JsonUtils.readJsonAtPath(jsonObjResults.toString(), aggregationPath, List.class);
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java
index ae328a2f60d9..d85d1aa1534a 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java
@@ -13,7 +13,9 @@
 
 package org.openmetadata.service.jdbi3;
 
+import static org.openmetadata.schema.type.Include.ALL;
 import static org.openmetadata.schema.type.Include.NON_DELETED;
+import static org.openmetadata.service.Entity.CLASSIFICATION;
 import static org.openmetadata.service.Entity.TAG;
 import static org.openmetadata.service.util.EntityUtil.entityReferenceMatch;
 import static org.openmetadata.service.util.EntityUtil.getId;
@@ -24,6 +26,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
+import org.openmetadata.schema.entity.classification.Classification;
 import org.openmetadata.schema.entity.classification.Tag;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.ProviderType;
@@ -62,6 +65,15 @@ public void prepare(Tag entity, boolean update) {
     entity.setClassification(classification);
   }
 
+  @Override
+  public void setInheritedFields(Tag tag, Fields fields) {
+    Classification parent =  Entity.getEntity(CLASSIFICATION, tag.getClassification().getId(), "roles", ALL);
+    if (parent.getDisabled() != null && parent.getDisabled()) {
+      tag.setDisabled(true);
+    }
+     tag.setInheritedRoles(parent.getRoles());
+  }
+
   @Override
   public void storeEntity(Tag tag, boolean update) {
     EntityReference classification = tag.getClassification();
@@ -69,7 +81,7 @@ public void storeEntity(Tag tag, boolean update) {
 
     // Parent and Classification are not stored as part of JSON. Build it on the fly based on
     // relationships
-    tag.withClassification(null).withParent(null);
+    tag.withClassification(null).withParent(null).withInheritedRoles(null);
     store(tag, update);
     tag.withClassification(classification).withParent(parent);
   }
@@ -78,12 +90,14 @@ public void storeEntity(Tag tag, boolean update) {
   public void restorePatchAttributes(Tag original, Tag updated) {
     super.restorePatchAttributes(original, updated);
     updated.setChildren(original.getChildren());
+    updated.setInheritedRoles(original.getInheritedRoles());
   }
 
   @Override
   public void storeRelationships(Tag entity) {
     addClassificationRelationship(entity);
     addParentRelationship(entity);
+    entity.setInheritedRoles(getInheritedRoles(entity));
   }
 
   @Override
@@ -128,11 +142,18 @@ public void setFields(Tag tag, Fields fields) {
     if (fields.contains("usageCount")) {
       tag.withUsageCount(getUsageCount(tag));
     }
+    tag.setInheritedRoles(fields.contains("roles") ? getInheritedRoles(tag) : null);
   }
 
   @Override
   public void clearFields(Tag tag, Fields fields) {
     tag.withUsageCount(fields.contains("usageCount") ? tag.getUsageCount() : null);
+    tag.setInheritedRoles(fields.contains("roles") ? tag.getInheritedRoles() : null);
+  }
+
+  private List<EntityReference> getInheritedRoles(Tag tag) {
+    Classification classification = Entity.getEntity(CLASSIFICATION, tag.getClassification().getId(), "roles", ALL);
+    return classification.getRoles();
   }
 
   private Integer getUsageCount(Tag tag) {
@@ -172,7 +193,7 @@ public TagUpdater(Tag original, Tag updated, Operation operation) {
     public void entitySpecificUpdate() {
       recordChange(
           "mutuallyExclusive", original.getMutuallyExclusive(), updated.getMutuallyExclusive());
-      recordChange("disabled,", original.getDisabled(), updated.getDisabled());
+      recordChange("disabled", original.getDisabled(), updated.getDisabled());
       updateName(original, updated);
       updateParent(original, updated);
     }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
index bef5cd30e748..40d15e09a1a5 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
@@ -12,6 +12,7 @@
 import static org.openmetadata.service.Entity.TEST_DEFINITION;
 import static org.openmetadata.service.Entity.TEST_SUITE;
 import static org.openmetadata.service.Entity.getEntityByName;
+import static org.openmetadata.service.Entity.getEntityTimeSeriesRepository;
 import static org.openmetadata.service.Entity.populateEntityFieldTags;
 import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
 import static org.openmetadata.service.security.mask.PIIMasker.maskSampleData;
@@ -206,7 +207,8 @@ private List<TestSuite> getTestSuites(TestCase test) {
     return records.stream()
         .map(
             testSuiteId ->
-                Entity.<TestSuite>getEntity(TEST_SUITE, testSuiteId.getId(), "", Include.ALL, false)
+                Entity.<TestSuite>getEntity(
+                        TEST_SUITE, testSuiteId.getId(), "owners,domain", Include.ALL, false)
                     .withInherited(true)
                     .withChangeDescription(null))
         .toList();
@@ -378,6 +380,7 @@ private ResultSummary getResultSummary(
 
   @SneakyThrows
   private TestCaseResult getTestCaseResult(TestCase testCase) {
+    TestCaseResult testCaseResult;
     if (testCase.getTestCaseResult() != null) {
       // we'll return the saved state if it exists otherwise we'll fetch it from the database
       // Should be the case if listing from the search repo. as the test case result
@@ -386,10 +389,21 @@ private TestCaseResult getTestCaseResult(TestCase testCase) {
     }
     SearchListFilter searchListFilter = new SearchListFilter();
     searchListFilter.addQueryParam("testCaseFQN", testCase.getFullyQualifiedName());
-    EntityTimeSeriesRepository<?> timeSeriesRepository =
-        Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT);
-    return (TestCaseResult)
-        timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
+    TestCaseResultRepository timeSeriesRepository =
+        (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
+    try {
+      testCaseResult =
+          timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
+    } catch (Exception e) {
+      // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
+      // to database
+      LOG.debug(
+          "Error fetching test case result from search. Fetching from test case results from database",
+          e);
+      testCaseResult =
+          timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName());
+    }
+    return testCaseResult;
   }
 
   public ResultList<TestCaseResult> getTestCaseResults(String fqn, Long startTs, Long endTs) {
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java
index d9c937fd8073..7af2339fb0f7 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java
@@ -96,6 +96,21 @@ public Response addTestCaseResult(
     return Response.created(uriInfo.getRequestUri()).entity(testCaseResult).build();
   }
 
+  public ResultList<TestCaseResult> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
+    List<String> json =
+        ((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
+            .listLastTestCaseResultsForTestSuite(testSuiteId);
+    List<TestCaseResult> testCaseResults = JsonUtils.readObjects(json, TestCaseResult.class);
+    return new ResultList<>(testCaseResults, null, null, testCaseResults.size());
+  }
+
+  public TestCaseResult listLastTestCaseResult(String testCaseFQN) {
+    String json =
+        ((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
+            .listLastTestCaseResult(testCaseFQN);
+    return JsonUtils.readValue(json, TestCaseResult.class);
+  }
+
   @Override
   protected void postCreate(TestCaseResult entity) {
     super.postCreate(entity);
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java
index 848c171fa6ef..a582ea559982 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java
@@ -8,6 +8,7 @@
 import static org.openmetadata.service.Entity.TEST_CASE;
 import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
 import static org.openmetadata.service.Entity.TEST_SUITE;
+import static org.openmetadata.service.Entity.getEntity;
 import static org.openmetadata.service.Entity.getEntityTimeSeriesRepository;
 import static org.openmetadata.service.util.FullyQualifiedName.quoteName;
 
@@ -24,6 +25,7 @@
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
+import org.openmetadata.schema.EntityInterface;
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.tests.DataQualityReport;
 import org.openmetadata.schema.tests.ResultSummary;
@@ -37,9 +39,12 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.resources.dqtests.TestSuiteResource;
 import org.openmetadata.service.resources.feeds.MessageParser;
+import org.openmetadata.service.search.SearchAggregation;
 import org.openmetadata.service.search.SearchClient;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.SearchListFilter;
+import org.openmetadata.service.search.indexes.SearchIndex;
+import org.openmetadata.service.search.models.IndexMapping;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.FullyQualifiedName;
 import org.openmetadata.service.util.JsonUtils;
@@ -51,38 +56,6 @@ public class TestSuiteRepository extends EntityRepository<TestSuite> {
   private static final String UPDATE_FIELDS = "tests";
   private static final String PATCH_FIELDS = "tests";
 
-  private static final String EXECUTION_SUMMARY_AGGS =
-      """
-        {
-          "aggregations": {
-              "status_counts": {
-                "terms": {
-                  "field": "testCaseResult.testCaseStatus"
-                }
-              }
-            }
-        }
-        """;
-
-  private static final String ENTITY_EXECUTION_SUMMARY_AGGS =
-      """
-  {
-    "aggregations": {
-      "entityLinks": {
-        "terms": {
-          "field": "entityLink.nonNormalized"
-        },
-        "aggs": {
-          "status_counts": {
-            "terms": {
-              "field": "testCaseResult.testCaseStatus"
-            }
-          }
-        }
-      }
-    }
-  }""";
-
   private static final String ENTITY_EXECUTION_SUMMARY_FILTER =
       """
   {
@@ -147,7 +120,8 @@ public void setFields(TestSuite entity, EntityUtil.Fields fields) {
   public void setInheritedFields(TestSuite testSuite, EntityUtil.Fields fields) {
     if (Boolean.TRUE.equals(testSuite.getExecutable())) {
       Table table =
-          Entity.getEntity(TABLE, testSuite.getExecutableEntityReference().getId(), "owners", ALL);
+          Entity.getEntity(
+              TABLE, testSuite.getExecutableEntityReference().getId(), "owners,domain", ALL);
       inheritOwners(testSuite, fields, table);
       inheritDomain(testSuite, fields, table);
     }
@@ -239,24 +213,29 @@ private TestSummary getEntityTestCasesExecutionSummary(JsonObject aggregation) {
 
   public DataQualityReport getDataQualityReport(String q, String aggQuery, String index)
       throws IOException {
-    Map<String, Object> aggregationString = SearchIndexUtils.buildAggregationString(aggQuery);
-    return searchRepository.genericAggregation(q, index, aggregationString);
+    SearchAggregation searchAggregation = SearchIndexUtils.buildAggregationTree(aggQuery);
+    return searchRepository.genericAggregation(q, index, searchAggregation);
   }
 
   public TestSummary getTestSummary(UUID testSuiteId) {
-    JsonObject aggregationJson = JsonUtils.readJson(EXECUTION_SUMMARY_AGGS).asJsonObject();
     try {
       TestSummary testSummary;
       if (testSuiteId == null) {
+        String aggregationStr =
+            "bucketName=status_counts:aggType=terms:field=testCaseResult.testCaseStatus";
+        SearchAggregation searchAggregation = SearchIndexUtils.buildAggregationTree(aggregationStr);
         JsonObject testCaseResultSummary =
-            searchRepository.aggregate(null, TEST_CASE, aggregationJson, new SearchListFilter());
+            searchRepository.aggregate(null, TEST_CASE, searchAggregation, new SearchListFilter());
         testSummary = getTestCasesExecutionSummary(testCaseResultSummary);
       } else {
+        String aggregationStr =
+            "bucketName=entityLinks:aggType=terms:field=entityLink.nonNormalized,"
+                + "bucketName=status_counts:aggType=terms:field=testCaseResult.testCaseStatus";
+        SearchAggregation searchAggregation = SearchIndexUtils.buildAggregationTree(aggregationStr);
         String query = ENTITY_EXECUTION_SUMMARY_FILTER.formatted(testSuiteId);
         // don't want to get it from the cache as test results summary may be stale
-        aggregationJson = JsonUtils.readJson(ENTITY_EXECUTION_SUMMARY_AGGS).asJsonObject();
         JsonObject testCaseResultSummary =
-            searchRepository.aggregate(query, TEST_CASE, aggregationJson, new SearchListFilter());
+            searchRepository.aggregate(query, TEST_CASE, searchAggregation, new SearchListFilter());
         testSummary = getEntityTestCasesExecutionSummary(testCaseResultSummary);
       }
       return testSummary;
@@ -266,17 +245,52 @@ public TestSummary getTestSummary(UUID testSuiteId) {
     return null;
   }
 
+  @Override
+  protected void postCreate(TestSuite entity) {
+    super.postCreate(entity);
+    if (Boolean.TRUE.equals(entity.getExecutable())
+        && entity.getExecutableEntityReference() != null) {
+      // Update table index with test suite field
+      EntityInterface entityInterface =
+          getEntity(entity.getExecutableEntityReference(), "testSuite", ALL);
+      IndexMapping indexMapping =
+          searchRepository.getIndexMapping(entity.getExecutableEntityReference().getType());
+      SearchClient searchClient = searchRepository.getSearchClient();
+      SearchIndex index =
+          searchRepository
+              .getSearchIndexFactory()
+              .buildIndex(entity.getExecutableEntityReference().getType(), entityInterface);
+      Map<String, Object> doc = index.buildSearchIndexDoc();
+      searchClient.updateEntity(
+          indexMapping.getIndexName(searchRepository.getClusterAlias()),
+          entity.getExecutableEntityReference().getId().toString(),
+          doc,
+          "ctx._source.testSuite = params.testSuite;");
+    }
+  }
+
   @SneakyThrows
   private List<ResultSummary> getResultSummary(UUID testSuiteId) {
     List<ResultSummary> resultSummaries = new ArrayList<>();
+    ResultList<TestCaseResult> latestTestCaseResultResults;
     String groupBy = "testCaseFQN.keyword";
     SearchListFilter searchListFilter = new SearchListFilter();
     searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString());
-    EntityTimeSeriesRepository<TestCaseResult> entityTimeSeriesRepository =
+    TestCaseResultRepository entityTimeSeriesRepository =
         (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
-    ResultList<TestCaseResult> latestTestCaseResultResults =
-        entityTimeSeriesRepository.listLatestFromSearch(
-            EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
+    try {
+      latestTestCaseResultResults =
+          entityTimeSeriesRepository.listLatestFromSearch(
+              EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
+    } catch (Exception e) {
+      // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
+      // to database
+      LOG.debug(
+          "Error fetching test case result from search. Fetching from test case results from database",
+          e);
+      latestTestCaseResultResults =
+          entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
+    }
 
     latestTestCaseResultResults
         .getData()
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java
index c78250586adc..c835589fad57 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java
@@ -22,6 +22,7 @@
 import static org.openmetadata.service.util.EntityUtil.customFieldMatch;
 import static org.openmetadata.service.util.EntityUtil.getCustomField;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -29,6 +30,7 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
+import javax.validation.ConstraintViolationException;
 import javax.ws.rs.core.UriInfo;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.Triple;
@@ -40,11 +42,11 @@
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.Relationship;
-import org.openmetadata.schema.type.customproperties.EnumConfig;
-import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig;
-import org.openmetadata.schema.type.customproperties.Value;
+import org.openmetadata.schema.type.customProperties.EnumConfig;
+import org.openmetadata.schema.type.customProperties.TableConfig;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.TypeRegistry;
+import org.openmetadata.service.exception.CatalogExceptionMessage;
 import org.openmetadata.service.resources.types.TypeResource;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.EntityUtil.Fields;
@@ -173,8 +175,7 @@ private List<CustomProperty> getCustomProperties(Type type) {
   private void validateProperty(CustomProperty customProperty) {
     switch (customProperty.getPropertyType().getName()) {
       case "enum" -> validateEnumConfig(customProperty.getCustomPropertyConfig());
-      case "enumWithDescriptions" -> validateEnumWithDescriptionsConfig(
-          customProperty.getCustomPropertyConfig());
+      case "table-cp" -> validateTableTypeConfig(customProperty.getCustomPropertyConfig());
       case "date-cp" -> validateDateFormat(
           customProperty.getCustomPropertyConfig(), getDateTokens(), "Invalid date format");
       case "dateTime-cp" -> validateDateFormat(
@@ -233,25 +234,31 @@ private void validateEnumConfig(CustomPropertyConfig config) {
     }
   }
 
-  private void validateEnumWithDescriptionsConfig(CustomPropertyConfig config) {
-    if (config != null) {
-      EnumWithDescriptionsConfig enumWithDescriptionsConfig =
-          JsonUtils.convertValue(config.getConfig(), EnumWithDescriptionsConfig.class);
-      if (enumWithDescriptionsConfig == null
-          || (enumWithDescriptionsConfig.getValues() != null
-              && enumWithDescriptionsConfig.getValues().isEmpty())) {
-        throw new IllegalArgumentException(
-            "EnumWithDescriptions Custom Property Type must have customPropertyConfig populated with values.");
-      }
-      JsonUtils.validateJsonSchema(config.getConfig(), EnumWithDescriptionsConfig.class);
-      if (enumWithDescriptionsConfig.getValues().stream().map(Value::getKey).distinct().count()
-          != enumWithDescriptionsConfig.getValues().size()) {
-        throw new IllegalArgumentException(
-            "EnumWithDescriptions Custom Property key cannot have duplicates.");
-      }
-    } else {
+  private void validateTableTypeConfig(CustomPropertyConfig config) {
+    if (config == null) {
+      throw new IllegalArgumentException("Table Custom Property Type must have config populated.");
+    }
+
+    JsonNode configNode = JsonUtils.valueToTree(config.getConfig());
+    TableConfig tableConfig = JsonUtils.convertValue(config.getConfig(), TableConfig.class);
+
+    List<String> columns = new ArrayList<>();
+    configNode.path("columns").forEach(node -> columns.add(node.asText()));
+    Set<String> uniqueColumns = new HashSet<>(columns);
+    if (uniqueColumns.size() != columns.size()) {
+      throw new IllegalArgumentException("Column names must be unique.");
+    }
+
+    try {
+      JsonUtils.validateJsonSchema(config.getConfig(), TableConfig.class);
+    } catch (ConstraintViolationException e) {
+      String validationErrors =
+          e.getConstraintViolations().stream()
+              .map(violation -> violation.getPropertyPath() + " " + violation.getMessage())
+              .collect(Collectors.joining(", "));
+
       throw new IllegalArgumentException(
-          "EnumWithDescriptions Custom Property Type must have customPropertyConfig.");
+          CatalogExceptionMessage.customPropertyConfigError("table", validationErrors));
     }
   }
 
@@ -413,27 +420,6 @@ private void validatePropertyConfigUpdate(
           throw new IllegalArgumentException(
               "Existing Enum Custom Property values cannot be removed.");
         }
-      } else if (origProperty.getPropertyType().getName().equals("enumWithDescriptions")) {
-        EnumWithDescriptionsConfig origConfig =
-            JsonUtils.convertValue(
-                origProperty.getCustomPropertyConfig().getConfig(),
-                EnumWithDescriptionsConfig.class);
-        EnumWithDescriptionsConfig updatedConfig =
-            JsonUtils.convertValue(
-                updatedProperty.getCustomPropertyConfig().getConfig(),
-                EnumWithDescriptionsConfig.class);
-        HashSet<String> updatedValues =
-            updatedConfig.getValues().stream()
-                .map(Value::getKey)
-                .collect(Collectors.toCollection(HashSet::new));
-        if (updatedValues.size() != updatedConfig.getValues().size()) {
-          throw new IllegalArgumentException(
-              "EnumWithDescriptions Custom Property values cannot have duplicates.");
-        } else if (!updatedValues.containsAll(
-            origConfig.getValues().stream().map(Value::getKey).collect(Collectors.toSet()))) {
-          throw new IllegalArgumentException(
-              "Existing EnumWithDescriptions Custom Property values cannot be removed.");
-        }
       }
     }
   }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcess.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcess.java
index 2400d6799766..1eb4ee0b5adc 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcess.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcess.java
@@ -3,6 +3,7 @@
 import java.util.List;
 import java.util.Map;
 import org.jdbi.v3.core.Handle;
+import org.jdbi.v3.core.Jdbi;
 import org.openmetadata.service.migration.QueryStatus;
 import org.openmetadata.service.migration.context.MigrationOps;
 
@@ -15,7 +16,7 @@ interface MigrationProcessCallback {
   // Ex: if the server is 1.0.0 the migration version for that server is 1.0.0
   // This version is used to sort all the upgrade migrations and apply them in order
 
-  void initialize(Handle handle);
+  void initialize(Handle handle, Jdbi jdbi);
 
   List<MigrationOps> getMigrationOps();
 
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcessImpl.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcessImpl.java
index 75897e005539..d3317e477005 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcessImpl.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationProcessImpl.java
@@ -9,6 +9,7 @@
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import org.jdbi.v3.core.Handle;
+import org.jdbi.v3.core.Jdbi;
 import org.openmetadata.schema.api.security.AuthenticationConfiguration;
 import org.openmetadata.sdk.PipelineServiceClientInterface;
 import org.openmetadata.service.clients.pipeline.PipelineServiceClientFactory;
@@ -23,6 +24,7 @@
 public class MigrationProcessImpl implements MigrationProcess {
   protected MigrationDAO migrationDAO;
   protected CollectionDAO collectionDAO;
+  protected Jdbi jdbi;
   protected Handle handle;
   protected PipelineServiceClientInterface pipelineServiceClient;
   protected AuthenticationConfiguration authenticationConfiguration;
@@ -35,8 +37,9 @@ public MigrationProcessImpl(MigrationFile migrationFile) {
   }
 
   @Override
-  public void initialize(Handle handle) {
+  public void initialize(Handle handle, Jdbi jdbi) {
     this.handle = handle;
+    this.jdbi = jdbi;
     this.collectionDAO = handle.attach(CollectionDAO.class);
     this.migrationDAO = handle.attach(MigrationDAO.class);
     this.pipelineServiceClient =
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java
index d40083b32da4..7f6cb2be0d88 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/api/MigrationWorkflow.java
@@ -159,11 +159,22 @@ private List<MigrationProcess> filterAndGetMigrationsToRun(
     try {
       for (MigrationFile file : applyMigrations) {
         file.parseSQLFiles();
-        String clazzName = file.getMigrationProcessClassName();
-        MigrationProcess process =
-            (MigrationProcess)
-                Class.forName(clazzName).getConstructor(MigrationFile.class).newInstance(file);
-        processes.add(process);
+        String extClazzName = null;
+        if (file.version.contains("collate")) {
+          extClazzName = file.getMigrationProcessExtClassName();
+        }
+        if (extClazzName != null) {
+          MigrationProcess collateProcess =
+              (MigrationProcess)
+                  Class.forName(extClazzName).getConstructor(MigrationFile.class).newInstance(file);
+          processes.add(collateProcess);
+        } else {
+          String clazzName = file.getMigrationProcessClassName();
+          MigrationProcess openMetadataProcess =
+              (MigrationProcess)
+                  Class.forName(clazzName).getConstructor(MigrationFile.class).newInstance(file);
+          processes.add(openMetadataProcess);
+        }
       }
     } catch (Exception e) {
       LOG.error("Failed to list and add migrations to run due to ", e);
@@ -254,7 +265,7 @@ public void runMigrationWorkflows() {
           row.add(process.getVersion());
           try {
             // Initialize
-            runStepAndAddStatus(row, () -> process.initialize(transactionHandler));
+            runStepAndAddStatus(row, () -> process.initialize(transactionHandler, jdbi));
 
             // Schema Changes
             runSchemaChanges(row, process);
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v157/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v157/Migration.java
new file mode 100644
index 000000000000..612cef3d47f5
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v157/Migration.java
@@ -0,0 +1,20 @@
+package org.openmetadata.service.migration.mysql.v157;
+
+import static org.openmetadata.service.migration.utils.v157.MigrationUtil.migrateTableDiffParams;
+
+import lombok.SneakyThrows;
+import org.openmetadata.service.migration.api.MigrationProcessImpl;
+import org.openmetadata.service.migration.utils.MigrationFile;
+
+public class Migration extends MigrationProcessImpl {
+
+  public Migration(MigrationFile migrationFile) {
+    super(migrationFile);
+  }
+
+  @Override
+  @SneakyThrows
+  public void runDataMigration() {
+    migrateTableDiffParams(handle, collectionDAO, authenticationConfiguration, false);
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java
index 9be8ca5e22fc..bf3d4769338a 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java
@@ -1,6 +1,7 @@
 package org.openmetadata.service.migration.mysql.v160;
 
 import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addAppExtensionName;
+import static org.openmetadata.service.migration.utils.v160.MigrationUtil.migrateServiceTypesAndConnections;
 
 import lombok.SneakyThrows;
 import org.openmetadata.service.migration.api.MigrationProcessImpl;
@@ -16,5 +17,6 @@ public Migration(MigrationFile migrationFile) {
   @SneakyThrows
   public void runDataMigration() {
     addAppExtensionName(handle, collectionDAO, authenticationConfiguration, false);
+    migrateServiceTypesAndConnections(handle, false);
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v157/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v157/Migration.java
new file mode 100644
index 000000000000..c263f81b3461
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v157/Migration.java
@@ -0,0 +1,20 @@
+package org.openmetadata.service.migration.postgres.v157;
+
+import static org.openmetadata.service.migration.utils.v157.MigrationUtil.migrateTableDiffParams;
+
+import lombok.SneakyThrows;
+import org.openmetadata.service.migration.api.MigrationProcessImpl;
+import org.openmetadata.service.migration.utils.MigrationFile;
+
+public class Migration extends MigrationProcessImpl {
+
+  public Migration(MigrationFile migrationFile) {
+    super(migrationFile);
+  }
+
+  @Override
+  @SneakyThrows
+  public void runDataMigration() {
+    migrateTableDiffParams(handle, collectionDAO, authenticationConfiguration, false);
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java
index d3f0651feced..ca1f9547f54c 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java
@@ -1,6 +1,7 @@
 package org.openmetadata.service.migration.postgres.v160;
 
 import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addAppExtensionName;
+import static org.openmetadata.service.migration.utils.v160.MigrationUtil.migrateServiceTypesAndConnections;
 
 import lombok.SneakyThrows;
 import org.openmetadata.service.migration.api.MigrationProcessImpl;
@@ -16,5 +17,6 @@ public Migration(MigrationFile migrationFile) {
   @SneakyThrows
   public void runDataMigration() {
     addAppExtensionName(handle, collectionDAO, authenticationConfiguration, true);
+    migrateServiceTypesAndConnections(handle, true);
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/MigrationFile.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/MigrationFile.java
index 66d834e94e61..b93fbdb2d4eb 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/MigrationFile.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/MigrationFile.java
@@ -113,6 +113,18 @@ public String getMigrationProcessClassName() {
     return clazzName;
   }
 
+  public String getMigrationProcessExtClassName() {
+    String clazzName =
+        String.format(
+            "io.collate.service.migration.%s.%s.Migration", dbPackageName, getVersionPackageName());
+    try {
+      Class.forName(clazzName);
+    } catch (ClassNotFoundException e) {
+      return null;
+    }
+    return clazzName;
+  }
+
   public String getMigrationsFilePath() {
     if (connectionType == ConnectionType.MYSQL) {
       return Paths.get(dir.getAbsolutePath(), "mysql").toString();
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v157/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v157/MigrationUtil.java
new file mode 100644
index 000000000000..1beaacce4231
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v157/MigrationUtil.java
@@ -0,0 +1,63 @@
+package org.openmetadata.service.migration.utils.v157;
+
+import static org.openmetadata.service.Entity.TEST_CASE;
+import static org.openmetadata.service.Entity.TEST_DEFINITION;
+
+import java.util.List;
+import java.util.Objects;
+import lombok.extern.slf4j.Slf4j;
+import org.jdbi.v3.core.Handle;
+import org.openmetadata.schema.api.security.AuthenticationConfiguration;
+import org.openmetadata.schema.tests.TestCase;
+import org.openmetadata.schema.tests.TestCaseParameterValue;
+import org.openmetadata.schema.tests.TestDefinition;
+import org.openmetadata.schema.type.Relationship;
+import org.openmetadata.service.jdbi3.CollectionDAO;
+import org.openmetadata.service.util.JsonUtils;
+
+@Slf4j
+public class MigrationUtil {
+  private static final String TABLE_DIFF = "tableDiff";
+
+  public static void migrateTableDiffParams(
+      Handle handle,
+      CollectionDAO daoCollection,
+      AuthenticationConfiguration config,
+      boolean postgres) {
+    int pageSize = 1000;
+    int offset = 0;
+    while (true) {
+      List<String> jsons = daoCollection.testCaseDAO().listAfterWithOffset(pageSize, offset);
+      if (jsons.isEmpty()) {
+        break;
+      }
+      offset += pageSize;
+      for (String json : jsons) {
+        TestCase testCase = JsonUtils.readValue(json, TestCase.class);
+        TestDefinition td = getTestDefinition(daoCollection, testCase);
+        if (Objects.nonNull(td) && Objects.equals(td.getName(), TABLE_DIFF)) {
+          LOG.debug("Adding caseSensitiveColumns=true table diff test case: {}", testCase.getId());
+          testCase
+              .getParameterValues()
+              .add(new TestCaseParameterValue().withName("caseSensitiveColumns").withValue("true"));
+          daoCollection.testCaseDAO().update(testCase);
+        }
+      }
+    }
+  }
+
+  public static TestDefinition getTestDefinition(CollectionDAO dao, TestCase testCase) {
+    List<CollectionDAO.EntityRelationshipRecord> records =
+        dao.relationshipDAO()
+            .findFrom(
+                testCase.getId(), TEST_CASE, Relationship.CONTAINS.ordinal(), TEST_DEFINITION);
+    if (records.size() > 1) {
+      throw new RuntimeException(
+          "Multiple test definitions found for test case: " + testCase.getId());
+    }
+    if (records.isEmpty()) {
+      return null;
+    }
+    return dao.testDefinitionDAO().findEntityById(records.get(0).getId());
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java
index 1feac38ff753..8c1608f51950 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java
@@ -24,7 +24,7 @@ public class MigrationUtil {
   private static final String UPDATE_MYSQL_APP_EXTENSION =
       "UPDATE apps_extension_time_series SET json = JSON_SET(json, '$.appName', :appName) WHERE appId = :appId AND extension = 'status'";
   private static final String UPDATE_PG_APP_EXTENSION =
-      "UPDATE apps_extension_time_series SET json = jsonb_set(json_data, '{appName}', :appName::jsonb) WHERE appId = :appId AND extension = 'status'";
+      "UPDATE apps_extension_time_series SET json = jsonb_set(json, '{appName}', to_jsonb(:appName)) WHERE appId = :appId AND extension = 'status'";
 
   // We'll list the entries in app_extension_time_series, clean those whose appId
   // is not installed, and for those that appId matches from installed Apps, we'll
@@ -80,4 +80,94 @@ private static void updateAppExtension(Handle handle, App app, boolean postgres)
     }
     update.bind("appId", app.getId().toString()).bind("appName", app.getName()).execute();
   }
+
+  public static void migrateServiceTypesAndConnections(Handle handle, boolean postgresql) {
+    LOG.info("Starting service type and connection type migrations");
+    try {
+      migrateServiceTypeInApiEndPointServiceType(handle, postgresql);
+      migrateServiceTypeInApiServiceEntity(handle, postgresql);
+      migrateConnectionTypeInApiServiceEntity(handle, postgresql);
+      migrateServiceTypeInApiCollectionEntity(handle, postgresql);
+      LOG.info("Successfully completed service type and connection type migrations");
+    } catch (Exception e) {
+      LOG.error("Error occurred during migration", e);
+    }
+  }
+
+  private static void migrateServiceTypeInApiEndPointServiceType(
+      Handle handle, boolean postgresql) {
+    LOG.info("Starting migrateServiceTypeInApiEndPointServiceType");
+    String query;
+
+    if (postgresql) {
+      query =
+          "UPDATE api_endpoint_entity SET json = jsonb_set(json, '{serviceType}', '\"Rest\"', false) WHERE jsonb_extract_path_text(json, 'serviceType') = 'REST'";
+    } else {
+      query =
+          "UPDATE api_endpoint_entity SET json = JSON_SET(json, '$.serviceType', 'Rest') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'REST'";
+    }
+
+    try {
+      handle.execute(query);
+    } catch (Exception e) {
+      LOG.error("Error updating", e);
+    }
+  }
+
+  private static void migrateServiceTypeInApiServiceEntity(Handle handle, boolean postgresql) {
+    LOG.info("Starting migrateServiceTypeInApiServiceEntity");
+
+    String query;
+    if (postgresql) {
+      query =
+          "UPDATE api_service_entity SET json = jsonb_set(json, '{serviceType}', '\"Rest\"', false) WHERE jsonb_extract_path_text(json, 'serviceType') = 'REST'";
+    } else {
+      query =
+          "UPDATE api_service_entity SET json = JSON_SET(json, '$.serviceType', 'Rest') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'REST'";
+    }
+
+    try {
+      handle.execute(query);
+    } catch (Exception e) {
+      LOG.error("Error updating", e);
+    }
+  }
+
+  private static void migrateServiceTypeInApiCollectionEntity(Handle handle, boolean postgresql) {
+    LOG.info("Starting runApiCollectionEntityServiceTypeDataMigrations");
+
+    String query;
+    if (postgresql) {
+      query =
+          "UPDATE api_collection_entity SET json = jsonb_set(json, '{serviceType}', '\"Rest\"', false) WHERE jsonb_extract_path_text(json, 'serviceType') = 'REST'";
+    } else {
+      query =
+          "UPDATE api_collection_entity SET json = JSON_SET(json, '$.serviceType', 'Rest') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.serviceType')) = 'REST'";
+    }
+
+    try {
+      handle.execute(query);
+    } catch (Exception e) {
+      LOG.error("Error updating", e);
+    }
+  }
+
+  private static void migrateConnectionTypeInApiServiceEntity(Handle handle, boolean postgresql) {
+    LOG.info("Starting runApiServiceEntityConnectionTypeMigrate");
+
+    String query;
+    if (postgresql) {
+      query =
+          "UPDATE api_service_entity SET json = jsonb_set(json, '{connection,config,type}', '\"Rest\"', false) WHERE jsonb_extract_path_text(json, 'connection', 'config', 'type') = 'REST'";
+    } else {
+      query =
+          "UPDATE api_service_entity SET json = JSON_SET(json, '$.connection.config.type', 'Rest') WHERE JSON_UNQUOTE(JSON_EXTRACT(json, '$.connection.config.type')) = 'REST'";
+    }
+
+    try {
+      handle.execute(query);
+    } catch (Exception e) {
+      LOG.error("Error updating", e);
+    }
+  }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java
index 01049f76a36b..11540d291828 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java
@@ -64,7 +64,7 @@
     description = "Apps marketplace holds to application available for Open-metadata")
 @Produces(MediaType.APPLICATION_JSON)
 @Consumes(MediaType.APPLICATION_JSON)
-@Collection(name = "apps/marketplace", order = 8)
+@Collection(name = "apps/marketplace", order = 7)
 @Slf4j
 public class AppMarketPlaceResource
     extends EntityResource<AppMarketPlaceDefinition, AppMarketPlaceRepository> {
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java
index a870b17667eb..a52a36bda939 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java
@@ -94,7 +94,7 @@
     description = "Apps are internal/external apps used to something on top of Open-metadata.")
 @Produces(MediaType.APPLICATION_JSON)
 @Consumes(MediaType.APPLICATION_JSON)
-@Collection(name = "apps")
+@Collection(name = "apps", order = 8)
 @Slf4j
 public class AppResource extends EntityResource<App, AppRepository> {
   public static final String COLLECTION_PATH = "v1/apps/";
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java
index 60b0e55aa3b2..6941d0e86dc0 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java
@@ -122,6 +122,7 @@ protected List<MetadataOperation> getEntitySpecificOperations() {
         MetadataOperation.VIEW_DATA_PROFILE,
         MetadataOperation.VIEW_SAMPLE_DATA,
         MetadataOperation.VIEW_USAGE,
+        MetadataOperation.VIEW_PROFILER_GLOBAL_CONFIGURATION,
         MetadataOperation.EDIT_TESTS,
         MetadataOperation.EDIT_QUERIES,
         MetadataOperation.EDIT_DATA_PROFILE,
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java
index 82a60062514e..4e96e18eba93 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java
@@ -92,7 +92,7 @@ public class TestCaseResource extends EntityResource<TestCase, TestCaseRepositor
 
   static final String FIELDS = "owners,testSuite,testDefinition,testSuites,incidentId,domain,tags";
   static final String SEARCH_FIELDS_EXCLUDE =
-      "testPlatforms,table,database,databaseSchema,service,testSuite,dataQualityDimension,testCaseType";
+      "testPlatforms,table,database,databaseSchema,service,testSuite,dataQualityDimension,testCaseType,originEntityFQN";
 
   @Override
   public TestCase addHref(UriInfo uriInfo, TestCase test) {
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java
index 7b8d5979c899..f1eba18f6b88 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java
@@ -230,6 +230,9 @@ public ResultList<TestSuite> listFromSearch(
           @QueryParam("includeEmptyTestSuites")
           @DefaultValue("true")
           Boolean includeEmptyTestSuites,
+      @Parameter(description = "Filter a test suite by domain.", schema = @Schema(type = "string"))
+          @QueryParam("domain")
+          String domain,
       @Parameter(
               description = "Filter a test suite by fully qualified name.",
               schema = @Schema(type = "string"))
@@ -283,6 +286,7 @@ public ResultList<TestSuite> listFromSearch(
     searchListFilter.addQueryParam("includeEmptyTestSuites", includeEmptyTestSuites);
     searchListFilter.addQueryParam("fullyQualifiedName", fullyQualifiedName);
     searchListFilter.addQueryParam("excludeFields", SEARCH_FIELDS_EXCLUDE);
+    searchListFilter.addQueryParam("domain", domain);
     if (!nullOrEmpty(owner)) {
       EntityInterface entity;
       try {
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java
index 60335068a943..1a23c9b1c9a9 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java
@@ -47,11 +47,11 @@
 import javax.ws.rs.core.UriInfo;
 import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.schema.api.data.RestoreEntity;
-import org.openmetadata.schema.api.services.CreateAPIService;
-import org.openmetadata.schema.entity.services.APIService;
+import org.openmetadata.schema.api.services.CreateApiService;
+import org.openmetadata.schema.entity.services.ApiService;
 import org.openmetadata.schema.entity.services.ServiceType;
 import org.openmetadata.schema.entity.services.connections.TestConnectionResult;
-import org.openmetadata.schema.type.APIServiceConnection;
+import org.openmetadata.schema.type.ApiConnection;
 import org.openmetadata.schema.type.EntityHistory;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.MetadataOperation;
@@ -75,12 +75,12 @@
 @Consumes(MediaType.APPLICATION_JSON)
 @Collection(name = "apiServices")
 public class APIServiceResource
-    extends ServiceEntityResource<APIService, APIServiceRepository, APIServiceConnection> {
+    extends ServiceEntityResource<ApiService, APIServiceRepository, ApiConnection> {
   public static final String COLLECTION_PATH = "v1/services/apiServices/";
   static final String FIELDS = "pipelines,owners,tags,domain";
 
   @Override
-  public APIService addHref(UriInfo uriInfo, APIService service) {
+  public ApiService addHref(UriInfo uriInfo, ApiService service) {
     super.addHref(uriInfo, service);
     Entity.withHref(uriInfo, service.getPipelines());
     return service;
@@ -96,7 +96,7 @@ protected List<MetadataOperation> getEntitySpecificOperations() {
     return null;
   }
 
-  public static class APIServiceList extends ResultList<APIService> {
+  public static class APIServiceList extends ResultList<ApiService> {
     /* Required for serde */
   }
 
@@ -118,7 +118,7 @@ public static class APIServiceList extends ResultList<APIService> {
                                 org.openmetadata.service.resources.services.apiservices
                                     .APIServiceResource.APIServiceList.class)))
       })
-  public ResultList<APIService> list(
+  public ResultList<ApiService> list(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
       @Parameter(
@@ -165,12 +165,12 @@ public ResultList<APIService> list(
             content =
                 @Content(
                     mediaType = "application/json",
-                    schema = @Schema(implementation = APIService.class))),
+                    schema = @Schema(implementation = ApiService.class))),
         @ApiResponse(
             responseCode = "404",
             description = "API service for instance {id} is not found")
       })
-  public APIService get(
+  public ApiService get(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
       @PathParam("id") UUID id,
@@ -185,7 +185,7 @@ public APIService get(
           @QueryParam("include")
           @DefaultValue("non-deleted")
           Include include) {
-    APIService apiService = getInternal(uriInfo, securityContext, id, fieldsParam, include);
+    ApiService apiService = getInternal(uriInfo, securityContext, id, fieldsParam, include);
     return decryptOrNullify(securityContext, apiService);
   }
 
@@ -202,12 +202,12 @@ public APIService get(
             content =
                 @Content(
                     mediaType = "application/json",
-                    schema = @Schema(implementation = APIService.class))),
+                    schema = @Schema(implementation = ApiService.class))),
         @ApiResponse(
             responseCode = "404",
             description = "API service for instance {id} is not found")
       })
-  public APIService getByName(
+  public ApiService getByName(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
       @PathParam("name") String name,
@@ -222,7 +222,7 @@ public APIService getByName(
           @QueryParam("include")
           @DefaultValue("non-deleted")
           Include include) {
-    APIService apiService =
+    ApiService apiService =
         getByNameInternal(
             uriInfo, securityContext, EntityInterfaceUtil.quoteName(name), fieldsParam, include);
     return decryptOrNullify(securityContext, apiService);
@@ -241,9 +241,9 @@ public APIService getByName(
             content =
                 @Content(
                     mediaType = "application/json",
-                    schema = @Schema(implementation = APIService.class)))
+                    schema = @Schema(implementation = ApiService.class)))
       })
-  public APIService addTestConnectionResult(
+  public ApiService addTestConnectionResult(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
       @Parameter(description = "Id of the service", schema = @Schema(type = "UUID"))
@@ -252,7 +252,7 @@ public APIService addTestConnectionResult(
       @Valid TestConnectionResult testConnectionResult) {
     OperationContext operationContext = new OperationContext(entityType, MetadataOperation.CREATE);
     authorizer.authorize(securityContext, operationContext, getResourceContextById(id));
-    APIService service = repository.addTestConnectionResult(id, testConnectionResult);
+    ApiService service = repository.addTestConnectionResult(id, testConnectionResult);
     return decryptOrNullify(securityContext, service);
   }
 
@@ -283,7 +283,7 @@ public EntityHistory listVersions(
             .map(
                 json -> {
                   try {
-                    APIService apiService = JsonUtils.readValue((String) json, APIService.class);
+                    ApiService apiService = JsonUtils.readValue((String) json, ApiService.class);
                     return JsonUtils.pojoToJson(decryptOrNullify(securityContext, apiService));
                   } catch (Exception e) {
                     return json;
@@ -307,12 +307,12 @@ public EntityHistory listVersions(
             content =
                 @Content(
                     mediaType = "application/json",
-                    schema = @Schema(implementation = APIService.class))),
+                    schema = @Schema(implementation = ApiService.class))),
         @ApiResponse(
             responseCode = "404",
             description = "API service for instance {id} and version {version} is not found")
       })
-  public APIService getVersion(
+  public ApiService getVersion(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
       @Parameter(description = "API service Id", schema = @Schema(type = "string")) @PathParam("id")
@@ -322,13 +322,13 @@ public APIService getVersion(
               schema = @Schema(type = "string", example = "0.1 or 1.1"))
           @PathParam("version")
           String version) {
-    APIService apiService = super.getVersionInternal(securityContext, id, version);
+    ApiService apiService = super.getVersionInternal(securityContext, id, version);
     return decryptOrNullify(securityContext, apiService);
   }
 
   @POST
   @Operation(
-      operationId = "createAPIService",
+      operationId = "createApiService",
       summary = "Create API service",
       description = "Create a new API service.",
       responses = {
@@ -338,16 +338,16 @@ public APIService getVersion(
             content =
                 @Content(
                     mediaType = "application/json",
-                    schema = @Schema(implementation = APIService.class))),
+                    schema = @Schema(implementation = ApiService.class))),
         @ApiResponse(responseCode = "400", description = "Bad request")
       })
   public Response create(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
-      @Valid CreateAPIService create) {
-    APIService service = getService(create, securityContext.getUserPrincipal().getName());
+      @Valid CreateApiService create) {
+    ApiService service = getService(create, securityContext.getUserPrincipal().getName());
     Response response = create(uriInfo, securityContext, service);
-    decryptOrNullify(securityContext, (APIService) response.getEntity());
+    decryptOrNullify(securityContext, (ApiService) response.getEntity());
     return response;
   }
 
@@ -363,16 +363,16 @@ public Response create(
             content =
                 @Content(
                     mediaType = "application/json",
-                    schema = @Schema(implementation = APIService.class))),
+                    schema = @Schema(implementation = ApiService.class))),
         @ApiResponse(responseCode = "400", description = "Bad request")
       })
   public Response createOrUpdate(
       @Context UriInfo uriInfo,
       @Context SecurityContext securityContext,
-      @Valid CreateAPIService update) {
-    APIService service = getService(update, securityContext.getUserPrincipal().getName());
+      @Valid CreateApiService update) {
+    ApiService service = getService(update, securityContext.getUserPrincipal().getName());
     Response response = createOrUpdate(uriInfo, securityContext, unmask(service));
-    decryptOrNullify(securityContext, (APIService) response.getEntity());
+    decryptOrNullify(securityContext, (ApiService) response.getEntity());
     return response;
   }
 
@@ -504,7 +504,7 @@ public Response delete(
             content =
                 @Content(
                     mediaType = "application/json",
-                    schema = @Schema(implementation = APIService.class)))
+                    schema = @Schema(implementation = ApiService.class)))
       })
   public Response restoreAPIService(
       @Context UriInfo uriInfo,
@@ -513,20 +513,20 @@ public Response restoreAPIService(
     return restoreEntity(uriInfo, securityContext, restore.getId());
   }
 
-  private APIService getService(CreateAPIService create, String user) {
+  private ApiService getService(CreateApiService create, String user) {
     return repository
-        .copy(new APIService(), create, user)
+        .copy(new ApiService(), create, user)
         .withServiceType(create.getServiceType())
         .withConnection(create.getConnection());
   }
 
   @Override
-  protected APIService nullifyConnection(APIService service) {
+  protected ApiService nullifyConnection(ApiService service) {
     return service.withConnection(null);
   }
 
   @Override
-  protected String extractServiceType(APIService service) {
+  protected String extractServiceType(ApiService service) {
     return service.getServiceType().value();
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java
index 743632c80256..c463fac3fb6e 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java
@@ -34,6 +34,7 @@
 import org.openmetadata.schema.settings.SettingsType;
 import org.openmetadata.schema.system.ValidationResponse;
 import org.openmetadata.schema.type.Include;
+import org.openmetadata.schema.type.MetadataOperation;
 import org.openmetadata.schema.util.EntitiesCount;
 import org.openmetadata.schema.util.ServicesCount;
 import org.openmetadata.sdk.PipelineServiceClientInterface;
@@ -46,6 +47,8 @@
 import org.openmetadata.service.resources.Collection;
 import org.openmetadata.service.security.Authorizer;
 import org.openmetadata.service.security.JwtFilter;
+import org.openmetadata.service.security.policyevaluator.OperationContext;
+import org.openmetadata.service.security.policyevaluator.ResourceContext;
 import org.openmetadata.service.util.ResultList;
 import org.openmetadata.service.util.email.EmailUtil;
 
@@ -145,8 +148,18 @@ public Settings getSettingByName(
                     schema = @Schema(implementation = Settings.class)))
       })
   public Settings getProfilerConfigurationSetting(
-      @Context UriInfo uriInfo, @Context SecurityContext securityContext) {
-    authorizer.authorizeAdminOrBot(securityContext);
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @Parameter(
+              description = "Entity type for which to get the global profiler configuration",
+              schema = @Schema(type = "string"))
+          @QueryParam("entityType")
+          @DefaultValue("table")
+          String entityType) {
+    ResourceContext resourceContext = new ResourceContext(entityType);
+    OperationContext operationContext =
+        new OperationContext(entityType, MetadataOperation.VIEW_PROFILER_GLOBAL_CONFIGURATION);
+    authorizer.authorize(securityContext, operationContext, resourceContext);
     return systemRepository.getConfigWithKey(SettingsType.PROFILER_CONFIGURATION.value());
   }
 
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java
index 18a60fa1a986..695ebd427edf 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java
@@ -78,7 +78,7 @@
 public class ClassificationResource
     extends EntityResource<Classification, ClassificationRepository> {
   public static final String TAG_COLLECTION_PATH = "/v1/classifications/";
-  static final String FIELDS = "usageCount,termCount";
+  static final String FIELDS = "usageCount,termCount,roles";
 
   static class ClassificationList extends ResultList<Classification> {
     /* Required for serde */
@@ -458,6 +458,7 @@ public static Classification getClassification(
     return repository
         .copy(new Classification(), create, updatedBy)
         .withFullyQualifiedName(create.getName())
+        .withRoles(create.getRoles())
         .withProvider(create.getProvider())
         .withMutuallyExclusive(create.getMutuallyExclusive());
   }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java
index d544aa5c3aaa..cf2cb628cbfc 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagLabelUtil.java
@@ -31,10 +31,13 @@
 import org.openmetadata.schema.entity.classification.Tag;
 import org.openmetadata.schema.entity.data.Glossary;
 import org.openmetadata.schema.entity.data.GlossaryTerm;
+import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.TagLabel;
 import org.openmetadata.schema.type.TagLabel.TagSource;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.CatalogExceptionMessage;
+import org.openmetadata.service.security.AuthorizationException;
+import org.openmetadata.service.security.policyevaluator.SubjectContext;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.FullyQualifiedName;
 
@@ -144,6 +147,31 @@ public static void checkMutuallyExclusive(List<TagLabel> tagLabels) {
     }
   }
 
+  public static void checkDisabledTags(List<TagLabel> tagLabels) {
+    for (TagLabel tagLabel : listOrEmpty(tagLabels)) {
+      if (tagLabel.getSource().equals(TagSource.CLASSIFICATION)) {
+        Tag tag = Entity.getCollectionDAO().tagDAO().findEntityByName(tagLabel.getTagFQN());
+        if (tag.getDisabled()) {
+          throw new IllegalArgumentException(CatalogExceptionMessage.disabledTag(tagLabel));
+        }
+      }
+    }
+  }
+
+  public static void checkTagsPermissions(List<TagLabel> tagLabels, String user) {
+    if (tagLabels == null || tagLabels.isEmpty()) {
+      return;
+    }
+    List<TagLabel> classificationTags = tagLabels.stream()
+        .filter(tagLabel -> tagLabel.getSource() != TagSource.GLOSSARY)
+        .toList();
+    if (classificationTags.isEmpty()) {
+      return; // No classification tags to check
+    }
+    SubjectContext subjectContext = SubjectContext.getSubjectContext(user);
+
+  }
+
   public static void checkMutuallyExclusiveForParentAndSubField(
       String assetFqn,
       String assetFqnHash,
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java
index 3dce57ac92d4..bd88a043e3b3 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java
@@ -89,7 +89,7 @@
     order = 5) // initialize after Classification, and before Glossary and GlossaryTerm
 public class TagResource extends EntityResource<Tag, TagRepository> {
   public static final String TAG_COLLECTION_PATH = "/v1/tags/";
-  static final String FIELDS = "children,usageCount";
+  static final String FIELDS = "children,usageCount,roles";
 
   static class TagList extends ResultList<Tag> {
     /* Required for serde */
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java
index becbd478d591..983357d9b5c4 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java
@@ -26,6 +26,8 @@
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import javax.json.JsonPatch;
 import javax.validation.Valid;
@@ -68,6 +70,7 @@
 import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.util.RestUtil.PutResponse;
 import org.openmetadata.service.util.ResultList;
+import org.openmetadata.service.util.SchemaFieldExtractor;
 
 @Path("/v1/metadata/types")
 @Tag(
@@ -466,6 +469,57 @@ public Response addOrUpdateProperty(
     return response.toResponse();
   }
 
+  @GET
+  @Path("/fields/{entityType}")
+  @Produces(MediaType.APPLICATION_JSON)
+  public Response getEntityTypeFields(
+      @Context UriInfo uriInfo,
+      @Context SecurityContext securityContext,
+      @PathParam("entityType") String entityType,
+      @QueryParam("include") @DefaultValue("non-deleted") Include include) {
+
+    try {
+      Fields fieldsParam = new Fields(Set.of("customProperties"));
+      Type typeEntity = repository.getByName(uriInfo, entityType, fieldsParam, include, false);
+      SchemaFieldExtractor extractor = new SchemaFieldExtractor();
+      List<SchemaFieldExtractor.FieldDefinition> fieldsList =
+          extractor.extractFields(typeEntity, entityType);
+      return Response.ok(fieldsList).type(MediaType.APPLICATION_JSON).build();
+
+    } catch (Exception e) {
+      LOG.error("Error processing schema for entity type: " + entityType, e);
+      return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
+          .entity(
+              "Error processing schema for entity type: "
+                  + entityType
+                  + ". Exception: "
+                  + e.getMessage())
+          .build();
+    }
+  }
+
+  @GET
+  @Path("/customProperties")
+  @Produces(MediaType.APPLICATION_JSON)
+  public Response getAllCustomPropertiesByEntityType(
+      @Context UriInfo uriInfo, @Context SecurityContext securityContext) {
+    try {
+      SchemaFieldExtractor extractor = new SchemaFieldExtractor();
+      Map<String, List<SchemaFieldExtractor.FieldDefinition>> customPropertiesMap =
+          extractor.extractAllCustomProperties(uriInfo, repository);
+      return Response.ok(customPropertiesMap).build();
+    } catch (Exception e) {
+      LOG.error("Error fetching custom properties: {}", e.getMessage(), e);
+      return Response.status(Response.Status.INTERNAL_SERVER_ERROR)
+          .entity(
+              "Error processing schema for entity type: "
+                  + entityType
+                  + ". Exception: "
+                  + e.getMessage())
+          .build();
+    }
+  }
+
   private Type getType(CreateType create, String user) {
     return repository
         .copy(new Type(), create, user)
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
index b9bbdc7d10c9..5815f984d7cf 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/EntityBuilderConstant.java
@@ -26,7 +26,7 @@ public class EntityBuilderConstant {
   public static final String FIELD_DISPLAY_NAME_NGRAM = "displayName.ngram";
   public static final String PRE_TAG = "<span class=\"text-highlighter\">";
   public static final String POST_TAG = "</span>";
-  public static final Integer MAX_AGGREGATE_SIZE = 50;
+  public static final Integer MAX_AGGREGATE_SIZE = 10000;
   public static final Integer MAX_RESULT_HITS = 10000;
   public static final String QUERY = "query";
   public static final String QUERY_NGRAM = "query.ngram";
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchAggregation.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchAggregation.java
new file mode 100644
index 000000000000..496dbaa17ee0
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchAggregation.java
@@ -0,0 +1,67 @@
+package org.openmetadata.service.search;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import lombok.Getter;
+import org.openmetadata.schema.tests.type.DataQualityReportMetadata;
+
+@Getter
+public class SearchAggregation {
+  public SearchAggregationNode aggregationTree;
+  List<List<Map<String, String>>> rawAggregationMetadata;
+
+  public SearchAggregation(
+      SearchAggregationNode aggregationTree,
+      List<List<Map<String, String>>> rawAggregationMetadata) {
+    this.aggregationTree = aggregationTree;
+    this.rawAggregationMetadata = rawAggregationMetadata;
+  }
+
+  /*
+   * Get the metadata for the aggregation results. We'll use the metadata to build the report and
+   * to traverse the aggregation tree. 3 types of metadata are returned:
+   *   1. dimensions: the list of dimensions
+   *   2. metrics: the list of metrics
+   *   3. keys: the list of keys to traverse the aggregation tree
+   *
+   * @param aggregationMapList the list of aggregations
+   * @return the metadata
+   */
+  public DataQualityReportMetadata getAggregationMetadata() {
+    DataQualityReportMetadata metadata = new DataQualityReportMetadata();
+    List<String> dimensions = new ArrayList<>();
+    List<String> metrics = new ArrayList<>();
+    List<String> keys = new ArrayList<>();
+
+    for (List<Map<String, String>> aggregationsMap : rawAggregationMetadata) {
+      // remove bucket_selector from metadata as it is a filter and neither a dimension nor a metric
+      aggregationsMap.removeIf(
+          aggregationMap -> aggregationMap.get("aggType").contains("bucket_selector"));
+      for (int j = 0; j < aggregationsMap.size(); j++) {
+        Map<String, String> aggregationMap = aggregationsMap.get(j);
+        String aggType = aggregationMap.get("aggType");
+        String field = aggregationMap.get("field");
+
+        boolean isLeaf = j == aggregationsMap.size() - 1;
+        if (isLeaf) {
+          // leaf aggregation
+          if (!aggType.contains("term")) {
+            metrics.add(field);
+          } else {
+            dimensions.add(field);
+            metrics.add("document_count");
+          }
+        } else {
+          dimensions.add(field);
+        }
+        String formattedAggType = aggType.contains("term") ? "s%s".formatted(aggType) : aggType;
+        keys.add("%s#%s".formatted(formattedAggType, aggregationMap.get("bucketName")));
+      }
+    }
+
+    metadata.withKeys(keys).withDimensions(dimensions).withMetrics(metrics);
+
+    return metadata;
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchAggregationNode.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchAggregationNode.java
new file mode 100644
index 000000000000..fc6a4a841fe0
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchAggregationNode.java
@@ -0,0 +1,30 @@
+package org.openmetadata.service.search;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+
+@Setter
+@Getter
+public class SearchAggregationNode {
+  String type;
+  String name;
+  Map<String, String> value;
+  List<SearchAggregationNode> children = new ArrayList<>();
+
+  public SearchAggregationNode(String type, String name, Map<String, String> value) {
+    this.type = type;
+    this.name = name;
+    this.value = value;
+  }
+
+  public SearchAggregationNode() {
+    // Default constructor
+  }
+
+  public void addChild(SearchAggregationNode child) {
+    children.add(child);
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
index 0160dcbca156..f0fe1eba24ab 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java
@@ -163,11 +163,12 @@ Map<String, Object> searchLineageInternal(
 
   Response aggregate(String index, String fieldName, String value, String query) throws IOException;
 
-  JsonObject aggregate(String query, String index, JsonObject aggregationJson, String filters)
+  JsonObject aggregate(
+      String query, String index, SearchAggregation searchAggregation, String filters)
       throws IOException;
 
   DataQualityReport genericAggregation(
-      String query, String index, Map<String, Object> aggregationMetadata) throws IOException;
+      String query, String index, SearchAggregation aggregationMetadata) throws IOException;
 
   Response suggest(SearchRequest request) throws IOException;
 
@@ -186,6 +187,8 @@ DataQualityReport genericAggregation(
 
   void deleteEntityByFields(List<String> indexName, List<Pair<String, String>> fieldAndValue);
 
+  void deleteEntityByFQNPrefix(String indexName, String fqnPrefix);
+
   void softDeleteOrRestoreEntity(String indexName, String docId, String scriptTxt);
 
   void softDeleteOrRestoreChildren(
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java
index 6cc25604d5f0..36cea84741c8 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexFactory.java
@@ -23,15 +23,8 @@
 import org.openmetadata.schema.entity.data.Topic;
 import org.openmetadata.schema.entity.domains.DataProduct;
 import org.openmetadata.schema.entity.domains.Domain;
-import org.openmetadata.schema.entity.services.APIService;
-import org.openmetadata.schema.entity.services.DashboardService;
-import org.openmetadata.schema.entity.services.DatabaseService;
-import org.openmetadata.schema.entity.services.MessagingService;
-import org.openmetadata.schema.entity.services.MetadataService;
-import org.openmetadata.schema.entity.services.MlModelService;
-import org.openmetadata.schema.entity.services.PipelineService;
-import org.openmetadata.schema.entity.services.SearchService;
-import org.openmetadata.schema.entity.services.StorageService;
+import org.openmetadata.schema.entity.services.*;
+import org.openmetadata.schema.entity.services.ApiService;
 import org.openmetadata.schema.entity.services.ingestionPipelines.IngestionPipeline;
 import org.openmetadata.schema.entity.teams.Team;
 import org.openmetadata.schema.entity.teams.User;
@@ -118,7 +111,7 @@ public SearchIndex buildIndex(String entityType, Object entity) {
       case Entity.MESSAGING_SERVICE -> new MessagingServiceIndex((MessagingService) entity);
       case Entity.MLMODEL_SERVICE -> new MlModelServiceIndex((MlModelService) entity);
       case Entity.SEARCH_SERVICE -> new SearchServiceIndex((SearchService) entity);
-      case Entity.API_SERVICE -> new APIServiceIndex((APIService) entity);
+      case Entity.API_SERVICE -> new APIServiceIndex((ApiService) entity);
       case Entity.SEARCH_INDEX -> new SearchEntityIndex(
           (org.openmetadata.schema.entity.data.SearchIndex) entity);
       case Entity.PIPELINE_SERVICE -> new PipelineServiceIndex((PipelineService) entity);
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java
index edede39ad542..ba210001f734 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchIndexUtils.java
@@ -4,7 +4,6 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -20,6 +19,7 @@
 import org.openmetadata.schema.tests.type.DataQualityReportMetadata;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.TagLabel;
+import org.openmetadata.service.util.Utilities;
 
 public final class SearchIndexUtils {
 
@@ -93,9 +93,15 @@ private static void handleLeafMetricsAggregation(
       List<Datum> reportData,
       Map<String, String> nodeData,
       String metric) {
-    Optional<String> val = Optional.ofNullable(aggregationResults.getString("value_as_string"));
-    val.ifPresentOrElse(s -> nodeData.put(metric, s), () -> nodeData.put(metric, null));
+    String valueStr = null;
+    if (aggregationResults.containsKey("value")) {
+      JsonNumber value = aggregationResults.getJsonNumber("value");
+      if (value != null) valueStr = value.toString();
+    } else {
+      valueStr = aggregationResults.getString("value_as_string", null);
+    }
 
+    nodeData.put(metric, valueStr);
     Datum datum = new Datum();
     for (Map.Entry<String, String> entry : nodeData.entrySet()) {
       datum.withAdditionalProperty(entry.getKey(), entry.getValue());
@@ -104,51 +110,6 @@ private static void handleLeafMetricsAggregation(
     reportData.add(datum);
   }
 
-  /*
-   * Get the metadata for the aggregation results. We'll use the metadata to build the report and
-   * to traverse the aggregation tree. 3 types of metadata are returned:
-   *   1. dimensions: the list of dimensions
-   *   2. metrics: the list of metrics
-   *   3. keys: the list of keys to traverse the aggregation tree
-   *
-   * @param aggregationMapList the list of aggregations
-   * @return the metadata
-   */
-  private static DataQualityReportMetadata getAggregationMetadata(
-      List<List<Map<String, String>>> aggregationMapList) {
-    DataQualityReportMetadata metadata = new DataQualityReportMetadata();
-    List<String> dimensions = new ArrayList<>();
-    List<String> metrics = new ArrayList<>();
-    List<String> keys = new ArrayList<>();
-
-    for (List<Map<String, String>> aggregationsMap : aggregationMapList) {
-      for (int j = 0; j < aggregationsMap.size(); j++) {
-        Map<String, String> aggregationMap = aggregationsMap.get(j);
-        String aggType = aggregationMap.get("aggType");
-        String field = aggregationMap.get("field");
-
-        boolean isLeaf = j == aggregationsMap.size() - 1;
-        if (isLeaf) {
-          // leaf aggregation
-          if (!aggType.contains("term")) {
-            metrics.add(field);
-          } else {
-            dimensions.add(field);
-            metrics.add("document_count");
-          }
-        } else {
-          dimensions.add(field);
-        }
-        String formattedAggType = aggType.contains("term") ? "s%s".formatted(aggType) : aggType;
-        keys.add("%s#%s".formatted(formattedAggType, aggregationMap.get("bucketName")));
-      }
-    }
-
-    metadata.withKeys(keys).withDimensions(dimensions).withMetrics(metrics);
-
-    return metadata;
-  }
-
   /*
    * Traverse the aggregation results and build the report data. Note that the method supports
    * n levels of nested aggregations, but does not support sibling aggregations.
@@ -175,9 +136,8 @@ private static void traverseAggregationResults(
       return;
     }
 
-    String currentKey =
-        keys.get(0); // The current key represent the node in the aggregation tree (i.e. the current
-    // bucket)
+    // The current key represent the node in the aggregation tree (i.e. the current bucket)
+    String currentKey = keys.get(0);
     Optional<JsonObject> aggregation =
         Optional.ofNullable(SearchClient.getAggregationObject(aggregationResults, currentKey));
 
@@ -186,6 +146,18 @@ private static void traverseAggregationResults(
           Optional<JsonArray> buckets =
               Optional.ofNullable(SearchClient.getAggregationBuckets(agg));
           if (buckets.isEmpty()) {
+            if ((keys.size() > 1) && (agg.containsKey(keys.get(1)))) {
+              // If the current node in the aggregation tree does not have further buckets
+              // but contains the next level of aggregation, it means we are in the nested
+              // aggregation
+              traverseAggregationResults(
+                  agg,
+                  reportData,
+                  nodeData,
+                  keys.subList(1, keys.size()),
+                  metric,
+                  dimensions.subList(1, dimensions.size()));
+            }
             // If the current node in the aggregation tree does not have further bucket
             // it means we are in the leaf of the metric aggregation. We'll add the metric
             handleLeafMetricsAggregation(agg, reportData, nodeData, metric);
@@ -226,8 +198,7 @@ private static void traverseAggregationResults(
   }
 
   public static DataQualityReport parseAggregationResults(
-      Optional<JsonObject> aggregationResults, List<List<Map<String, String>>> aggregationMapList) {
-    DataQualityReportMetadata metadata = getAggregationMetadata(aggregationMapList);
+      Optional<JsonObject> aggregationResults, DataQualityReportMetadata metadata) {
     List<Datum> reportData = new ArrayList<>();
 
     aggregationResults.ifPresent(
@@ -244,78 +215,71 @@ public static DataQualityReport parseAggregationResults(
     return report.withMetadata(metadata).withData(reportData);
   }
 
-  /*
-   * Build the aggregation string for the given aggregation
-   *
-   * @param aggregation the aggregation to build the string for.
-   *   The aggregation string is in the form
-   * `bucketName:aggType:key=value,bucketName:aggType:key=value;bucketName:aggType:key=value`
-   * where `,` represents a nested aggregation and `;` represents a sibling aggregation
-   * NOTE: As of 07/25/2024 sibling aggregation parsing and processing has not been added
-   * @return the aggregation string
-   */
-  public static Map<String, Object> buildAggregationString(String aggregation) {
-    Map<String, Object> metadata = new HashMap<>();
-
-    StringBuilder aggregationString = new StringBuilder();
-    String[] siblings = aggregation.split(";");
-    List<List<Map<String, String>>> aggregationsMapList = new ArrayList<>();
+  public static List<TagLabel> parseTags(List<TagLabel> tags) {
+    if (tags == null) {
+      return Collections.emptyList();
+    }
+    return tags;
+  }
 
+  public static SearchAggregation buildAggregationTree(String aggregationString) {
+    List<List<Map<String, String>>> aggregationsMetadata = new ArrayList<>();
+    SearchAggregationNode root = new SearchAggregationNode("root", "root", null);
+    String[] siblings = aggregationString.split(";");
     for (String sibling : siblings) {
-      List<Map<String, String>> aggregationsMap = new ArrayList<>();
-      String[] nested = sibling.split(",");
-      for (int i = 0; i < nested.length; i++) {
-        Map<String, String> aggregationMap = new HashMap<>();
-        String[] parts = nested[i].split(":");
-
-        Iterator<String> partsIterator = Arrays.stream(parts).iterator();
-
-        while (partsIterator.hasNext()) {
-          String part = partsIterator.next();
-          if (!partsIterator.hasNext()) {
-            // last element = key=value pairs of the aggregation
-            String[] subParts = part.split("&");
-            Arrays.stream(subParts)
-                .forEach(
-                    subPart -> {
-                      String[] kvPairs = subPart.split("=");
-                      aggregationString
-                          .append("\"")
-                          .append(kvPairs[0])
-                          .append("\":\"")
-                          .append(kvPairs[1])
-                          .append("\"");
-                      aggregationMap.put(kvPairs[0], kvPairs[1]);
-                      // add comma if not the last element
-                      if (Arrays.asList(subParts).indexOf(subPart) < subParts.length - 1)
-                        aggregationString.append(",");
-                    });
-            aggregationString.append("}");
-          } else {
-            String[] kvPairs = part.split("=");
-            aggregationString.append("\"").append(kvPairs[1]).append("\":{");
-            aggregationMap.put(kvPairs[0], kvPairs[1]);
+      List<Map<String, String>> siblingAggregationsMetadata = new ArrayList<>();
+      SearchAggregationNode currentNode = root;
+      String[] nestedAggregations = sibling.split(Utilities.doubleQuoteRegexEscape(","), -1);
+      for (String aggregation : nestedAggregations) {
+        SearchAggregationNode bucketNode = new SearchAggregationNode();
+        String[] nestedAggregationSiblings = aggregation.split("::");
+        for (int j = 0; j < nestedAggregationSiblings.length; j++) {
+          Map<String, String> nestedAggregationMetadata = new HashMap<>();
+
+          String nestedAggregationSibling = nestedAggregationSiblings[j];
+          String[] parts = nestedAggregationSibling.split(":(?!:)");
+          for (String part : parts) {
+            if (part.contains("&")) {
+              String[] subParts = part.split("&");
+              Map<String, String> params = new HashMap<>();
+              for (String subPart : subParts) {
+                String[] kvPairs = subPart.split(Utilities.doubleQuoteRegexEscape("="), -1);
+                params.put(kvPairs[0], Utilities.cleanUpDoubleQuotes(kvPairs[1]));
+                nestedAggregationMetadata.put(
+                    kvPairs[0], Utilities.cleanUpDoubleQuotes(kvPairs[1]));
+              }
+              bucketNode.setValue(params);
+            } else {
+              String[] kvPairs = part.split(Utilities.doubleQuoteRegexEscape("="), -1);
+              switch (kvPairs[0]) {
+                case "aggType":
+                  bucketNode.setType(Utilities.cleanUpDoubleQuotes(kvPairs[1]));
+                  nestedAggregationMetadata.put(
+                      kvPairs[0], Utilities.cleanUpDoubleQuotes(kvPairs[1]));
+                  break;
+                case "bucketName":
+                  bucketNode.setName(Utilities.cleanUpDoubleQuotes(kvPairs[1]));
+                  nestedAggregationMetadata.put(
+                      kvPairs[0], Utilities.cleanUpDoubleQuotes(kvPairs[1]));
+                  break;
+                default:
+                  nestedAggregationMetadata.put(
+                      kvPairs[0], Utilities.cleanUpDoubleQuotes(kvPairs[1]));
+                  bucketNode.setValue(
+                      Map.of(kvPairs[0], Utilities.cleanUpDoubleQuotes(kvPairs[1])));
+              }
+            }
           }
+          currentNode.addChild(bucketNode);
+          if (j < nestedAggregationSiblings.length - 1) {
+            bucketNode = new SearchAggregationNode();
+          }
+          siblingAggregationsMetadata.add(nestedAggregationMetadata);
         }
-
-        if (i < nested.length - 1) {
-          aggregationString.append(",\"aggs\":{");
-        }
-        aggregationsMap.add(aggregationMap);
+        currentNode = bucketNode;
       }
-      // nested aggregations will add the "aggs" key if nested.length > 1, hence *2
-      aggregationString.append("}".repeat(((nested.length - 1) * 2) + 1));
-      aggregationsMapList.add(aggregationsMap);
-    }
-    metadata.put("aggregationStr", aggregationString.toString());
-    metadata.put("aggregationMapList", aggregationsMapList);
-    return metadata;
-  }
-
-  public static List<TagLabel> parseTags(List<TagLabel> tags) {
-    if (tags == null) {
-      return Collections.emptyList();
+      aggregationsMetadata.add(siblingAggregationsMetadata);
     }
-    return tags;
+    return new SearchAggregation(root, aggregationsMetadata);
   }
 }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
index ea29b3e17d48..283f829d4274 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java
@@ -544,7 +544,7 @@ public void deleteByScript(String entityType, String scriptTxt, Map<String, Obje
       searchClient.deleteByScript(indexMapping.getIndexName(clusterAlias), scriptTxt, params);
     } catch (Exception ie) {
       LOG.error(
-          "Issue in Creating new search document for entityType [{}]. Reason[{}], Cause[{}], Stack [{}]",
+          "Issue in deleting  search document for entityType [{}]. Reason[{}], Cause[{}], Stack [{}]",
           entityType,
           ie.getMessage(),
           ie.getCause(),
@@ -572,6 +572,25 @@ public void deleteEntity(EntityInterface entity) {
     }
   }
 
+  public void deleteEntityByFQNPrefix(EntityInterface entity) {
+    if (entity != null) {
+      String entityType = entity.getEntityReference().getType();
+      String fqn = entity.getFullyQualifiedName();
+      IndexMapping indexMapping = entityIndexMap.get(entityType);
+      try {
+        searchClient.deleteEntityByFQNPrefix(indexMapping.getIndexName(clusterAlias), fqn);
+      } catch (Exception ie) {
+        LOG.error(
+            "Issue in Deleting the search document for entityFQN [{}] and entityType [{}]. Reason[{}], Cause[{}], Stack [{}]",
+            fqn,
+            entityType,
+            ie.getMessage(),
+            ie.getCause(),
+            ExceptionUtils.getStackTrace(ie));
+      }
+    }
+  }
+
   public void deleteTimeSeriesEntityById(EntityTimeSeriesInterface entity) {
     if (entity != null) {
       String entityId = entity.getId().toString();
@@ -634,6 +653,16 @@ public void deleteOrUpdateChildren(EntityInterface entity, IndexMapping indexMap
           new ImmutablePair<>(
               REMOVE_TAGS_CHILDREN_SCRIPT,
               Collections.singletonMap("fqn", entity.getFullyQualifiedName())));
+      case Entity.DASHBOARD -> {
+        String scriptTxt =
+            String.format(
+                "if (ctx._source.dashboards.size() == 1) { ctx._source.put('deleted', '%s') }",
+                true);
+        searchClient.softDeleteOrRestoreChildren(
+            indexMapping.getChildAliases(clusterAlias),
+            scriptTxt,
+            List.of(new ImmutablePair<>("dashboards.id", docId)));
+      }
       case Entity.TEST_SUITE -> {
         TestSuite testSuite = (TestSuite) entity;
         if (Boolean.TRUE.equals(testSuite.getExecutable())) {
@@ -682,6 +711,16 @@ public void softDeleteOrRestoredChildren(
           indexMapping.getChildAliases(clusterAlias),
           scriptTxt,
           List.of(new ImmutablePair<>("service.id", docId)));
+      case Entity.DASHBOARD -> {
+        scriptTxt =
+            String.format(
+                "if (ctx._source.dashboards.size() == 1) { ctx._source.put('deleted', '%s') }",
+                delete);
+        searchClient.softDeleteOrRestoreChildren(
+            indexMapping.getChildAliases(clusterAlias),
+            scriptTxt,
+            List.of(new ImmutablePair<>("dashboards.id", docId)));
+      }
       default -> searchClient.softDeleteOrRestoreChildren(
           indexMapping.getChildAliases(clusterAlias),
           scriptTxt,
@@ -826,14 +865,14 @@ public Response aggregate(String index, String fieldName, String value, String q
   }
 
   public JsonObject aggregate(
-      String query, String entityType, JsonObject aggregationJson, SearchListFilter filter)
+      String query, String entityType, SearchAggregation searchAggregation, SearchListFilter filter)
       throws IOException {
     return searchClient.aggregate(
-        query, entityType, aggregationJson, filter.getCondition(entityType));
+        query, entityType, searchAggregation, filter.getCondition(entityType));
   }
 
   public DataQualityReport genericAggregation(
-      String query, String index, Map<String, Object> aggregationMetadata) throws IOException {
+      String query, String index, SearchAggregation aggregationMetadata) throws IOException {
     return searchClient.genericAggregation(query, index, aggregationMetadata);
   }
 
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
index af76083dedd7..a37efbf4607f 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java
@@ -70,6 +70,7 @@
 import es.org.elasticsearch.index.query.MatchQueryBuilder;
 import es.org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import es.org.elasticsearch.index.query.Operator;
+import es.org.elasticsearch.index.query.PrefixQueryBuilder;
 import es.org.elasticsearch.index.query.QueryBuilder;
 import es.org.elasticsearch.index.query.QueryBuilders;
 import es.org.elasticsearch.index.query.QueryStringQueryBuilder;
@@ -95,10 +96,8 @@
 import es.org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
 import es.org.elasticsearch.search.aggregations.bucket.terms.Terms;
 import es.org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
-import es.org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
 import es.org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import es.org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
-import es.org.elasticsearch.search.aggregations.metrics.TopHitsAggregationBuilder;
 import es.org.elasticsearch.search.builder.SearchSourceBuilder;
 import es.org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import es.org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
@@ -159,11 +158,14 @@
 import org.openmetadata.service.jdbi3.DataInsightChartRepository;
 import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository;
 import org.openmetadata.service.jdbi3.TestCaseResultRepository;
+import org.openmetadata.service.search.SearchAggregation;
 import org.openmetadata.service.search.SearchClient;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.SearchRequest;
 import org.openmetadata.service.search.SearchSortFilter;
 import org.openmetadata.service.search.UpdateSearchEventsConstant;
+import org.openmetadata.service.search.elasticsearch.aggregations.ElasticAggregations;
+import org.openmetadata.service.search.elasticsearch.aggregations.ElasticAggregationsBuilder;
 import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchAggregatedUnusedAssetsCountAggregator;
 import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchAggregatedUnusedAssetsSizeAggregator;
 import org.openmetadata.service.search.elasticsearch.dataInsightAggregators.ElasticSearchAggregatedUsedvsUnusedAssetsCountAggregator;
@@ -232,9 +234,7 @@ public class ElasticSearchClient implements SearchClient {
           "chart_suggest",
           "field_suggest");
   private static final List<String> SOURCE_FIELDS_TO_EXCLUDE =
-      Stream.concat(
-              FIELDS_TO_REMOVE.stream(),
-              Stream.of("schemaDefinition", "testSuite", "customMetrics"))
+      Stream.concat(FIELDS_TO_REMOVE.stream(), Stream.of("schemaDefinition", "customMetrics"))
           .toList();
 
   static {
@@ -1115,93 +1115,12 @@ public Response aggregate(String index, String fieldName, String value, String q
     return Response.status(OK).entity(response).build();
   }
 
-  /*
-  Build dynamic aggregation from elasticsearch JSON like aggregation query.
-  See TestSuiteResourceTest for example usage (ln. 506) for tested aggregation query.
-
-  @param aggregations - JsonObject containing the aggregation query
-   */
-  public static List<AggregationBuilder> buildAggregation(JsonObject aggregations) {
-    List<AggregationBuilder> aggregationBuilders = new ArrayList<>();
-    for (String key : aggregations.keySet()) {
-      JsonObject aggregation = aggregations.getJsonObject(key);
-      Set<String> keySet = aggregation.keySet();
-      for (String aggregationType : keySet) {
-        switch (aggregationType) {
-          case "terms":
-            JsonObject termAggregation = aggregation.getJsonObject(aggregationType);
-            TermsAggregationBuilder termsAggregationBuilder =
-                AggregationBuilders.terms(key).field(termAggregation.getString("field"));
-            aggregationBuilders.add(termsAggregationBuilder);
-            break;
-          case "avg":
-            JsonObject avgAggregation = aggregation.getJsonObject(aggregationType);
-            AvgAggregationBuilder avgAggregationBuilder =
-                AggregationBuilders.avg(key).field(avgAggregation.getString("field"));
-            aggregationBuilders.add(avgAggregationBuilder);
-            break;
-          case "date_histogram":
-            JsonObject dateHistogramAggregation = aggregation.getJsonObject(aggregationType);
-            String calendarInterval = dateHistogramAggregation.getString("calendar_interval");
-            DateHistogramAggregationBuilder dateHistogramAggregationBuilder =
-                AggregationBuilders.dateHistogram(key)
-                    .field(dateHistogramAggregation.getString("field"))
-                    .calendarInterval(new DateHistogramInterval(calendarInterval));
-            aggregationBuilders.add(dateHistogramAggregationBuilder);
-            break;
-          case "top_hits":
-            JsonObject topHitsAggregation = aggregation.getJsonObject(aggregationType);
-            TopHitsAggregationBuilder topHitsAggregationBuilder =
-                AggregationBuilders.topHits(key)
-                    .size(topHitsAggregation.getInt("size"))
-                    .sort(
-                        topHitsAggregation.getString("sort_field"),
-                        SortOrder.fromString(topHitsAggregation.getString("sort_order")));
-            aggregationBuilders.add(topHitsAggregationBuilder);
-            break;
-          case "nested":
-            JsonObject nestedAggregation = aggregation.getJsonObject("nested");
-            AggregationBuilder nestedAggregationBuilder =
-                AggregationBuilders.nested(
-                    nestedAggregation.getString("path"), nestedAggregation.getString("path"));
-            JsonObject nestedAggregations = aggregation.getJsonObject("aggs");
-
-            List<AggregationBuilder> nestedAggregationBuilders =
-                buildAggregation(nestedAggregations);
-            for (AggregationBuilder nestedAggregationBuilder1 : nestedAggregationBuilders) {
-              nestedAggregationBuilder.subAggregation(nestedAggregationBuilder1);
-            }
-            aggregationBuilders.add(nestedAggregationBuilder);
-            break;
-          case "aggs":
-            // Sub aggregation logic
-            if (!keySet.contains("nested")) {
-              JsonObject subAggregation = aggregation.getJsonObject("aggs");
-              if (!nullOrEmpty(aggregationBuilders)) {
-                AggregationBuilder aggregationBuilder =
-                    aggregationBuilders.get(aggregationBuilders.size() - 1);
-                List<AggregationBuilder> subAggregationBuilders = buildAggregation(subAggregation);
-                for (AggregationBuilder subAggregationBuilder : subAggregationBuilders) {
-                  aggregationBuilder.subAggregation(subAggregationBuilder);
-                }
-              }
-              break;
-            }
-            break;
-          default:
-            break;
-        }
-      }
-    }
-    return aggregationBuilders;
-  }
-
   @Override
   public DataQualityReport genericAggregation(
-      String query, String index, Map<String, Object> aggregationMetadata) throws IOException {
-    String aggregationStr = (String) aggregationMetadata.get("aggregationStr");
-    JsonObject aggregationObj = JsonUtils.readJson("{%s}".formatted(aggregationStr)).asJsonObject();
-    List<AggregationBuilder> aggregationBuilder = buildAggregation(aggregationObj);
+      String query, String index, SearchAggregation aggregationMetadata) throws IOException {
+    List<ElasticAggregations> aggregationBuilder =
+        ElasticAggregationsBuilder.buildAggregation(
+            aggregationMetadata.getAggregationTree(), null, new ArrayList<>());
 
     // Create search request
     es.org.elasticsearch.action.search.SearchRequest searchRequest =
@@ -1223,8 +1142,12 @@ public DataQualityReport genericAggregation(
     }
     searchSourceBuilder.size(0).timeout(new TimeValue(30, TimeUnit.SECONDS));
 
-    for (AggregationBuilder aggregation : aggregationBuilder) {
-      searchSourceBuilder.aggregation(aggregation);
+    for (ElasticAggregations aggregation : aggregationBuilder) {
+      if (!aggregation.isPipelineAggregation()) {
+        searchSourceBuilder.aggregation(aggregation.getElasticAggregationBuilder());
+      } else {
+        searchSourceBuilder.aggregation(aggregation.getElasticPipelineAggregationBuilder());
+      }
     }
 
     searchRequest.source(searchSourceBuilder);
@@ -1233,19 +1156,20 @@ public DataQualityReport genericAggregation(
     Optional<JsonObject> aggregationResults =
         Optional.ofNullable(jsonResponse.getJsonObject("aggregations"));
     return SearchIndexUtils.parseAggregationResults(
-        aggregationResults,
-        (List<List<Map<String, String>>>) aggregationMetadata.get("aggregationMapList"));
+        aggregationResults, aggregationMetadata.getAggregationMetadata());
   }
 
   @Override
-  public JsonObject aggregate(String query, String index, JsonObject aggregationJson, String filter)
+  public JsonObject aggregate(
+      String query, String index, SearchAggregation searchAggregation, String filter)
       throws IOException {
-    JsonObject aggregations = aggregationJson.getJsonObject("aggregations");
-    if (aggregations == null) {
+    if (searchAggregation == null) {
       return null;
     }
 
-    List<AggregationBuilder> aggregationBuilder = buildAggregation(aggregations);
+    List<ElasticAggregations> aggregationBuilder =
+        ElasticAggregationsBuilder.buildAggregation(
+            searchAggregation.getAggregationTree(), null, new ArrayList<>());
     es.org.elasticsearch.action.search.SearchRequest searchRequest =
         new es.org.elasticsearch.action.search.SearchRequest(
             Entity.getSearchRepository().getIndexOrAliasName(index));
@@ -1265,8 +1189,12 @@ public JsonObject aggregate(String query, String index, JsonObject aggregationJs
 
     searchSourceBuilder.size(0).timeout(new TimeValue(30, TimeUnit.SECONDS));
 
-    for (AggregationBuilder aggregation : aggregationBuilder) {
-      searchSourceBuilder.aggregation(aggregation);
+    for (ElasticAggregations aggregation : aggregationBuilder) {
+      if (!aggregation.isPipelineAggregation()) {
+        searchSourceBuilder.aggregation(aggregation.getElasticAggregationBuilder());
+      } else {
+        searchSourceBuilder.aggregation(aggregation.getElasticPipelineAggregationBuilder());
+      }
     }
 
     searchRequest.source(searchSourceBuilder);
@@ -1433,6 +1361,23 @@ private static SearchSourceBuilder buildSearchAcrossIndexesBuilder(
         buildSearchQueryBuilder(query, SearchIndex.getAllFields());
     FunctionScoreQueryBuilder queryBuilder = boostScore(queryStringBuilder);
     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
+    searchSourceBuilder.aggregation(
+        AggregationBuilders.terms("database.name.keyword")
+            .field("database.name.keyword")
+            .size(MAX_AGGREGATE_SIZE));
+    searchSourceBuilder.aggregation(
+        AggregationBuilders.terms("databaseSchema.name.keyword")
+            .field("databaseSchema.name.keyword")
+            .size(MAX_AGGREGATE_SIZE));
+    return addAggregation(searchSourceBuilder);
+  }
+
+  private static SearchSourceBuilder buildDataAssetsSearchBuilder(
+      String query, int from, int size) {
+    QueryStringQueryBuilder queryStringBuilder =
+        buildSearchQueryBuilder(query, SearchIndex.getAllFields());
+    FunctionScoreQueryBuilder queryBuilder = boostScore(queryStringBuilder);
+    SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
     searchSourceBuilder.aggregation(
         AggregationBuilders.terms("database.name.keyword")
             .field("database.name.keyword")
@@ -1777,6 +1722,16 @@ public void deleteEntityByFields(
     }
   }
 
+  @Override
+  public void deleteEntityByFQNPrefix(String indexName, String fqnPrefix) {
+    if (isClientAvailable) {
+      DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexName);
+      deleteByQueryRequest.setQuery(
+          new PrefixQueryBuilder("fullyQualifiedName.keyword", fqnPrefix.toLowerCase()));
+      deleteEntityFromElasticSearchByQuery(deleteByQueryRequest);
+    }
+  }
+
   @Override
   public void softDeleteOrRestoreEntity(String indexName, String docId, String scriptTxt) {
     if (isClientAvailable) {
@@ -2422,7 +2377,8 @@ private static SearchSourceBuilder getSearchSourceBuilder(
           "storage_service_index",
           "search_service_index",
           "metadata_service_index" -> buildServiceSearchBuilder(q, from, size);
-      case "all", "dataAsset" -> buildSearchAcrossIndexesBuilder(q, from, size);
+      case "dataAsset" -> buildDataAssetsSearchBuilder(q, from, size);
+      case "all" -> buildSearchAcrossIndexesBuilder(q, from, size);
       default -> buildAggregateSearchBuilder(q, from, size);
     };
   }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticAggregations.java
new file mode 100644
index 000000000000..d12987f4e572
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticAggregations.java
@@ -0,0 +1,29 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import es.org.elasticsearch.search.aggregations.AggregationBuilder;
+import es.org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+public interface ElasticAggregations {
+  void createAggregation(SearchAggregationNode node);
+
+  default Boolean isPipelineAggregation() {
+    return false;
+  }
+
+  default void setAggregation(AggregationBuilder aggregation) {}
+
+  default void setSubAggregation(AggregationBuilder subAggregation) {}
+
+  default void setAggregation(PipelineAggregationBuilder aggregation) {}
+
+  default void setSubAggregation(PipelineAggregationBuilder subAggregation) {}
+
+  default AggregationBuilder getElasticAggregationBuilder() {
+    return null;
+  }
+
+  default PipelineAggregationBuilder getElasticPipelineAggregationBuilder() {
+    return null;
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticAggregationsBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticAggregationsBuilder.java
new file mode 100644
index 000000000000..12807454e84b
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticAggregationsBuilder.java
@@ -0,0 +1,47 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import java.util.List;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+public class ElasticAggregationsBuilder {
+  public static List<ElasticAggregations> buildAggregation(
+      SearchAggregationNode node,
+      ElasticAggregations aggregation,
+      List<ElasticAggregations> aggregations) {
+    String type = node.getType();
+    if (type == "root") {
+      for (SearchAggregationNode child : node.getChildren()) {
+        buildAggregation(child, null, aggregations);
+      }
+      return aggregations;
+    }
+    ElasticAggregations elasticAggregations = getAggregation(type);
+    elasticAggregations.createAggregation(node);
+    if (aggregation != null) {
+      if (elasticAggregations.isPipelineAggregation()) {
+        aggregation.setSubAggregation(elasticAggregations.getElasticPipelineAggregationBuilder());
+      } else {
+        aggregation.setSubAggregation(elasticAggregations.getElasticAggregationBuilder());
+      }
+    }
+    if (aggregation == null) aggregations.add(elasticAggregations);
+    for (SearchAggregationNode child : node.getChildren()) {
+      buildAggregation(child, elasticAggregations, aggregations);
+    }
+    return aggregations;
+  }
+
+  public static ElasticAggregations getAggregation(String aggregationType) {
+    return switch (aggregationType) {
+      case "bucket_selector" -> new ElasticBucketSelectorAggregations();
+      case "date_histogram" -> new ElasticDateHistogramAggregations();
+      case "terms" -> new ElasticTermsAggregations();
+      case "avg" -> new ElasticAvgAggregations();
+      case "min" -> new ElasticMinAggregations();
+      case "cardinality" -> new ElasticCardinalityAggregations();
+      case "nested" -> new ElasticNestedAggregations();
+      case "top_hits" -> new ElasticTopHitsAggregations();
+      default -> throw new IllegalArgumentException("Invalid aggregation type: " + aggregationType);
+    };
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticAvgAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticAvgAggregations.java
new file mode 100644
index 000000000000..8ebd55d409a6
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticAvgAggregations.java
@@ -0,0 +1,38 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import es.org.elasticsearch.search.aggregations.AggregationBuilder;
+import es.org.elasticsearch.search.aggregations.AggregationBuilders;
+import es.org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+@Setter
+@Getter
+public class ElasticAvgAggregations implements ElasticAggregations {
+  static final String aggregationType = "avg";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    AggregationBuilder aggregationBuilders =
+        AggregationBuilders.avg(node.getName()).field(params.get("field"));
+    setElasticAggregationBuilder(aggregationBuilders);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticBucketSelectorAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticBucketSelectorAggregations.java
new file mode 100644
index 000000000000..cc5a7bdc5cb0
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticBucketSelectorAggregations.java
@@ -0,0 +1,58 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
+
+import es.org.elasticsearch.script.Script;
+import es.org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import es.org.elasticsearch.search.aggregations.PipelineAggregatorBuilders;
+import es.org.elasticsearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+@Setter
+@Getter
+public class ElasticBucketSelectorAggregations implements ElasticAggregations {
+  private final String aggregationType = "bucket_selector";
+  PipelineAggregationBuilder elasticPipelineAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    String[] pathValues = Optional.ofNullable(params.get("pathValues")).orElse("").split(",");
+    String[] pathKeys = Optional.ofNullable(params.get("pathKeys")).orElse("").split(",");
+    String scriptStr = params.get("script");
+
+    if (!validateParams(pathKeys, pathValues, scriptStr)) {
+      throw new IllegalArgumentException(
+          "Invalid parameters. pathKeys & pathValues should be non-empty arrays of equal length"
+              + " and script should be non-empty");
+    }
+
+    Map<String, String> bucketsPaths = getBucketsPaths(pathKeys, pathValues);
+    Script script = new Script(scriptStr);
+    BucketSelectorPipelineAggregationBuilder bucketSelectorPipelineAggregationBuilder =
+        PipelineAggregatorBuilders.bucketSelector(node.getName(), bucketsPaths, script);
+    setElasticPipelineAggregationBuilder(bucketSelectorPipelineAggregationBuilder);
+  }
+
+  @Override
+  public Boolean isPipelineAggregation() {
+    return true;
+  }
+
+  private Map<String, String> getBucketsPaths(String[] pathKeys, String[] pathValues) {
+    return IntStream.range(0, pathKeys.length)
+        .mapToObj(i -> Map.entry(pathKeys[i], pathValues[i]))
+        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+  }
+
+  private Boolean validateParams(String[] pathKeys, String[] pathValues, String scriptStr) {
+    if ((pathKeys.length != pathValues.length) || (pathKeys.length == 0)) return false;
+    return !nullOrEmpty(scriptStr);
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticCardinalityAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticCardinalityAggregations.java
new file mode 100644
index 000000000000..0dfb4b7ec43d
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticCardinalityAggregations.java
@@ -0,0 +1,38 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import es.org.elasticsearch.search.aggregations.AggregationBuilder;
+import es.org.elasticsearch.search.aggregations.AggregationBuilders;
+import es.org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+@Setter
+@Getter
+public class ElasticCardinalityAggregations implements ElasticAggregations {
+  static final String aggregationType = "cardinality";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    AggregationBuilder aggregationBuilder =
+        AggregationBuilders.cardinality(node.getName()).field(params.get("field"));
+    setElasticAggregationBuilder(aggregationBuilder);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticDateHistogramAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticDateHistogramAggregations.java
new file mode 100644
index 000000000000..3817bc3377b6
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticDateHistogramAggregations.java
@@ -0,0 +1,43 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import es.org.elasticsearch.search.aggregations.AggregationBuilder;
+import es.org.elasticsearch.search.aggregations.AggregationBuilders;
+import es.org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+@Setter
+@Getter
+public class ElasticDateHistogramAggregations implements ElasticAggregations {
+  static final String aggregationType = "date_histogram";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    String calendarInterval = params.get("calendar_interval");
+    String field = params.get("field");
+    AggregationBuilder aggregationBuilder =
+        AggregationBuilders.dateHistogram(node.getName())
+            .field(field)
+            .calendarInterval(new DateHistogramInterval(calendarInterval));
+    setElasticAggregationBuilder(aggregationBuilder);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticMinAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticMinAggregations.java
new file mode 100644
index 000000000000..72035f8338fc
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticMinAggregations.java
@@ -0,0 +1,38 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import es.org.elasticsearch.search.aggregations.AggregationBuilder;
+import es.org.elasticsearch.search.aggregations.AggregationBuilders;
+import es.org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+@Setter
+@Getter
+public class ElasticMinAggregations implements ElasticAggregations {
+  static final String aggregationType = "min";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    AggregationBuilder aggregationBuilders =
+        AggregationBuilders.min(node.getName()).field(params.get("field"));
+    setElasticAggregationBuilder(aggregationBuilders);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticNestedAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticNestedAggregations.java
new file mode 100644
index 000000000000..d15fb29c9a4d
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticNestedAggregations.java
@@ -0,0 +1,31 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import es.org.elasticsearch.search.aggregations.AggregationBuilder;
+import es.org.elasticsearch.search.aggregations.AggregationBuilders;
+import es.org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+@Setter
+@Getter
+public class ElasticNestedAggregations implements ElasticAggregations {
+  public static final String aggregationType = "nested";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    NestedAggregationBuilder aggregationBuilders =
+        AggregationBuilders.nested(params.get("path"), params.get("path"));
+    setElasticAggregationBuilder(aggregationBuilders);
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTermsAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTermsAggregations.java
new file mode 100644
index 000000000000..c1f3bf450382
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTermsAggregations.java
@@ -0,0 +1,54 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
+
+import es.org.elasticsearch.search.aggregations.AggregationBuilder;
+import es.org.elasticsearch.search.aggregations.AggregationBuilders;
+import es.org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import es.org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
+import es.org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+@Setter
+@Getter
+public class ElasticTermsAggregations implements ElasticAggregations {
+  static final String aggregationType = "terms";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    String[] includes = null;
+    int size = -1;
+    Map<String, String> params = node.getValue();
+    String includesStr = params.get("include");
+    if (!nullOrEmpty(includesStr)) includes = includesStr.split(",");
+    String sizeStr = params.get("size");
+    if (!nullOrEmpty(sizeStr)) size = Integer.parseInt(params.get("size"));
+    TermsAggregationBuilder termsAggregationBuilder =
+        AggregationBuilders.terms(node.getName()).field(params.get("field"));
+
+    if (size > 0) termsAggregationBuilder.size(size);
+    if (!nullOrEmpty(includes)) {
+      IncludeExclude includeExclude = new IncludeExclude(includes, null);
+      termsAggregationBuilder.includeExclude(includeExclude);
+    }
+    setElasticAggregationBuilder(termsAggregationBuilder);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTopHitsAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTopHitsAggregations.java
new file mode 100644
index 000000000000..56e438efbf37
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/aggregations/ElasticTopHitsAggregations.java
@@ -0,0 +1,41 @@
+package org.openmetadata.service.search.elasticsearch.aggregations;
+
+import es.org.elasticsearch.search.aggregations.AggregationBuilder;
+import es.org.elasticsearch.search.aggregations.AggregationBuilders;
+import es.org.elasticsearch.search.aggregations.PipelineAggregationBuilder;
+import es.org.elasticsearch.search.sort.SortOrder;
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+@Setter
+@Getter
+public class ElasticTopHitsAggregations implements ElasticAggregations {
+  static final String aggregationType = "top_hits";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    AggregationBuilder aggregationBuilder =
+        AggregationBuilders.topHits(node.getName())
+            .size(Integer.parseInt(params.get("size")))
+            .sort(params.get("sort_field"), SortOrder.fromString(params.get("sort_order")));
+    setElasticAggregationBuilder(aggregationBuilder);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIServiceIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIServiceIndex.java
index 10bb01461bd9..5795f2721938 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIServiceIndex.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/APIServiceIndex.java
@@ -3,11 +3,11 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
-import org.openmetadata.schema.entity.services.APIService;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.search.models.SearchSuggest;
 
-public record APIServiceIndex(APIService apiService) implements SearchIndex {
+public record APIServiceIndex(org.openmetadata.schema.entity.services.ApiService apiService)
+    implements SearchIndex {
 
   @Override
   public List<SearchSuggest> getSuggest() {
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java
index d17805266cb0..92e82f5f4269 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java
@@ -11,6 +11,7 @@
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.service.Entity;
+import org.openmetadata.service.resources.feeds.MessageParser;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.models.SearchSuggest;
 
@@ -55,6 +56,8 @@ public Map<String, Object> buildSearchIndexDocInternal(Map<String, Object> doc)
     doc.put("dataQualityDimension", testDefinition.getDataQualityDimension());
     doc.put("followers", SearchIndexUtils.parseFollowers(testCase.getFollowers()));
     doc.put("testCaseType", testDefinition.getEntityType());
+    doc.put(
+        "originEntityFQN", MessageParser.EntityLink.parse(testCase.getEntityLink()).getEntityFQN());
     setParentRelationships(doc, testCase);
     return doc;
   }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
index f4af56337fbf..27760a9bef94 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java
@@ -77,6 +77,7 @@
 import org.openmetadata.service.jdbi3.DataInsightChartRepository;
 import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository;
 import org.openmetadata.service.jdbi3.TestCaseResultRepository;
+import org.openmetadata.service.search.SearchAggregation;
 import org.openmetadata.service.search.SearchClient;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.SearchRequest;
@@ -102,6 +103,8 @@
 import org.openmetadata.service.search.indexes.TopicIndex;
 import org.openmetadata.service.search.indexes.UserIndex;
 import org.openmetadata.service.search.models.IndexMapping;
+import org.openmetadata.service.search.opensearch.aggregations.OpenAggregations;
+import org.openmetadata.service.search.opensearch.aggregations.OpenAggregationsBuilder;
 import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchAggregatedUnusedAssetsCountAggregator;
 import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchAggregatedUnusedAssetsSizeAggregator;
 import org.openmetadata.service.search.opensearch.dataInsightAggregator.OpenSearchAggregatedUsedvsUnusedAssetsCountAggregator;
@@ -161,6 +164,7 @@
 import os.org.opensearch.index.query.MatchQueryBuilder;
 import os.org.opensearch.index.query.MultiMatchQueryBuilder;
 import os.org.opensearch.index.query.Operator;
+import os.org.opensearch.index.query.PrefixQueryBuilder;
 import os.org.opensearch.index.query.QueryBuilder;
 import os.org.opensearch.index.query.QueryBuilders;
 import os.org.opensearch.index.query.QueryStringQueryBuilder;
@@ -188,7 +192,6 @@
 import os.org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
 import os.org.opensearch.search.aggregations.metrics.MaxAggregationBuilder;
 import os.org.opensearch.search.aggregations.metrics.SumAggregationBuilder;
-import os.org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder;
 import os.org.opensearch.search.builder.SearchSourceBuilder;
 import os.org.opensearch.search.fetch.subphase.FetchSourceContext;
 import os.org.opensearch.search.fetch.subphase.highlight.HighlightBuilder;
@@ -226,9 +229,7 @@ public class OpenSearchClient implements SearchClient {
           "chart_suggest",
           "field_suggest");
   private static final List<String> SOURCE_FIELDS_TO_EXCLUDE =
-      Stream.concat(
-              FIELDS_TO_REMOVE.stream(),
-              Stream.of("schemaDefinition", "testSuite", "customMetrics"))
+      Stream.concat(FIELDS_TO_REMOVE.stream(), Stream.of("schemaDefinition", "customMetrics"))
           .toList();
 
   static {
@@ -1143,87 +1144,12 @@ public Response aggregate(String index, String fieldName, String value, String q
     return Response.status(OK).entity(response).build();
   }
 
-  /*
-  Build dynamic aggregation from elasticsearch JSON like aggregation query.
-  See TestSuiteResourceTest for example usage (ln. 506) for tested aggregation query.
-
-  @param aggregations - JsonObject containing the aggregation query
-  */
-  public static List<AggregationBuilder> buildAggregation(JsonObject aggregations) {
-    List<AggregationBuilder> aggregationBuilders = new ArrayList<>();
-    for (String key : aggregations.keySet()) {
-      JsonObject aggregation = aggregations.getJsonObject(key);
-      Set<String> keySet = aggregation.keySet();
-      for (String aggregationType : keySet) {
-        switch (aggregationType) {
-          case "terms":
-            JsonObject termAggregation = aggregation.getJsonObject(aggregationType);
-            TermsAggregationBuilder termsAggregationBuilder =
-                AggregationBuilders.terms(key).field(termAggregation.getString("field"));
-            aggregationBuilders.add(termsAggregationBuilder);
-            break;
-          case "nested":
-            JsonObject nestedAggregation = aggregation.getJsonObject("nested");
-            AggregationBuilder nestedAggregationBuilder =
-                AggregationBuilders.nested(
-                    nestedAggregation.getString("path"), nestedAggregation.getString("path"));
-            JsonObject nestedAggregations = aggregation.getJsonObject("aggs");
-
-            List<AggregationBuilder> nestedAggregationBuilders =
-                buildAggregation(nestedAggregations);
-            for (AggregationBuilder nestedAggregationBuilder1 : nestedAggregationBuilders) {
-              nestedAggregationBuilder.subAggregation(nestedAggregationBuilder1);
-            }
-            aggregationBuilders.add(nestedAggregationBuilder);
-            break;
-          case "date_histogram":
-            JsonObject dateHistogramAggregation = aggregation.getJsonObject(aggregationType);
-            String calendarInterval = dateHistogramAggregation.getString("calendar_interval");
-            DateHistogramAggregationBuilder dateHistogramAggregationBuilder =
-                AggregationBuilders.dateHistogram(key)
-                    .field(dateHistogramAggregation.getString("field"))
-                    .calendarInterval(new DateHistogramInterval(calendarInterval));
-            aggregationBuilders.add(dateHistogramAggregationBuilder);
-            break;
-          case "top_hits":
-            JsonObject topHitsAggregation = aggregation.getJsonObject(aggregationType);
-            TopHitsAggregationBuilder topHitsAggregationBuilder =
-                AggregationBuilders.topHits(key)
-                    .size(topHitsAggregation.getInt("size"))
-                    .sort(
-                        topHitsAggregation.getString("sort_field"),
-                        SortOrder.fromString(topHitsAggregation.getString("sort_order")));
-            aggregationBuilders.add(topHitsAggregationBuilder);
-            break;
-          case "aggs":
-            // Sub aggregation logic
-            if (!keySet.contains("nested")) {
-              JsonObject subAggregation = aggregation.getJsonObject("aggs");
-              if (!nullOrEmpty(aggregationBuilders)) {
-                AggregationBuilder aggregationBuilder =
-                    aggregationBuilders.get(aggregationBuilders.size() - 1);
-                List<AggregationBuilder> subAggregationBuilders = buildAggregation(subAggregation);
-                for (AggregationBuilder subAggregationBuilder : subAggregationBuilders) {
-                  aggregationBuilder.subAggregation(subAggregationBuilder);
-                }
-              }
-              break;
-            }
-            break;
-          default:
-            break;
-        }
-      }
-    }
-    return aggregationBuilders;
-  }
-
   @Override
   public DataQualityReport genericAggregation(
-      String query, String index, Map<String, Object> aggregationMetadata) throws IOException {
-    String aggregationStr = (String) aggregationMetadata.get("aggregationStr");
-    JsonObject aggregationObj = JsonUtils.readJson("{%s}".formatted(aggregationStr)).asJsonObject();
-    List<AggregationBuilder> aggregationBuilder = buildAggregation(aggregationObj);
+      String query, String index, SearchAggregation aggregationMetadata) throws IOException {
+    List<OpenAggregations> aggregationBuilder =
+        OpenAggregationsBuilder.buildAggregation(
+            aggregationMetadata.getAggregationTree(), null, new ArrayList<>());
 
     // Create search request
     os.org.opensearch.action.search.SearchRequest searchRequest =
@@ -1243,8 +1169,12 @@ public DataQualityReport genericAggregation(
     }
     searchSourceBuilder.size(0).timeout(new TimeValue(30, TimeUnit.SECONDS));
 
-    for (AggregationBuilder aggregation : aggregationBuilder) {
-      searchSourceBuilder.aggregation(aggregation);
+    for (OpenAggregations aggregation : aggregationBuilder) {
+      if (!aggregation.isPipelineAggregation()) {
+        searchSourceBuilder.aggregation(aggregation.getElasticAggregationBuilder());
+      } else {
+        searchSourceBuilder.aggregation(aggregation.getElasticPipelineAggregationBuilder());
+      }
     }
 
     searchRequest.source(searchSourceBuilder);
@@ -1253,19 +1183,20 @@ public DataQualityReport genericAggregation(
     Optional<JsonObject> aggregationResults =
         Optional.ofNullable(jsonResponse.getJsonObject("aggregations"));
     return SearchIndexUtils.parseAggregationResults(
-        aggregationResults,
-        (List<List<Map<String, String>>>) aggregationMetadata.get("aggregationMapList"));
+        aggregationResults, aggregationMetadata.getAggregationMetadata());
   }
 
   @Override
-  public JsonObject aggregate(String query, String index, JsonObject aggregationJson, String filter)
+  public JsonObject aggregate(
+      String query, String index, SearchAggregation searchAggregation, String filter)
       throws IOException {
-    JsonObject aggregations = aggregationJson.getJsonObject("aggregations");
-    if (aggregations == null) {
+    if (searchAggregation == null) {
       return null;
     }
 
-    List<AggregationBuilder> aggregationBuilder = buildAggregation(aggregations);
+    List<OpenAggregations> aggregationBuilder =
+        OpenAggregationsBuilder.buildAggregation(
+            searchAggregation.getAggregationTree(), null, new ArrayList<>());
     os.org.opensearch.action.search.SearchRequest searchRequest =
         new os.org.opensearch.action.search.SearchRequest(
             Entity.getSearchRepository().getIndexOrAliasName(index));
@@ -1283,8 +1214,12 @@ public JsonObject aggregate(String query, String index, JsonObject aggregationJs
 
     searchSourceBuilder.size(0).timeout(new TimeValue(30, TimeUnit.SECONDS));
 
-    for (AggregationBuilder aggregation : aggregationBuilder) {
-      searchSourceBuilder.aggregation(aggregation);
+    for (OpenAggregations aggregation : aggregationBuilder) {
+      if (!aggregation.isPipelineAggregation()) {
+        searchSourceBuilder.aggregation(aggregation.getElasticAggregationBuilder());
+      } else {
+        searchSourceBuilder.aggregation(aggregation.getElasticPipelineAggregationBuilder());
+      }
     }
 
     searchRequest.source(searchSourceBuilder);
@@ -1297,7 +1232,6 @@ public JsonObject aggregate(String query, String index, JsonObject aggregationJs
   @SneakyThrows
   public void updateSearch(UpdateRequest updateRequest) {
     if (updateRequest != null) {
-      updateRequest.docAsUpsert(true);
       updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
       LOG.debug(SENDING_REQUEST_TO_ELASTIC_SEARCH, updateRequest);
       client.update(updateRequest, RequestOptions.DEFAULT);
@@ -1402,6 +1336,24 @@ private static SearchSourceBuilder buildSearchAcrossIndexesBuilder(
     FunctionScoreQueryBuilder queryBuilder = boostScore(queryStringBuilder);
     queryBuilder.boostMode(CombineFunction.SUM);
     SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
+    searchSourceBuilder.aggregation(
+        AggregationBuilders.terms("database.name.keyword")
+            .field("database.name.keyword")
+            .size(MAX_AGGREGATE_SIZE));
+    searchSourceBuilder.aggregation(
+        AggregationBuilders.terms("databaseSchema.name.keyword")
+            .field("databaseSchema.name.keyword")
+            .size(MAX_AGGREGATE_SIZE));
+    return addAggregation(searchSourceBuilder);
+  }
+
+  private static SearchSourceBuilder buildDataAssetsSearchBuilder(
+      String query, int from, int size) {
+    QueryStringQueryBuilder queryStringBuilder =
+        buildSearchQueryBuilder(query, SearchIndex.getAllFields());
+    FunctionScoreQueryBuilder queryBuilder = boostScore(queryStringBuilder);
+    queryBuilder.boostMode(CombineFunction.SUM);
+    SearchSourceBuilder searchSourceBuilder = searchBuilder(queryBuilder, null, from, size);
     searchSourceBuilder.aggregation(
         AggregationBuilders.terms("database.name.keyword")
             .field("database.name.keyword")
@@ -1695,6 +1647,7 @@ public void createEntity(String indexName, String docId, String doc) {
     if (isClientAvailable) {
       UpdateRequest updateRequest = new UpdateRequest(indexName, docId);
       updateRequest.doc(doc, XContentType.JSON);
+      updateRequest.docAsUpsert(true);
       updateSearch(updateRequest);
     }
   }
@@ -1704,6 +1657,7 @@ public void createTimeSeriesEntity(String indexName, String docId, String doc) {
     if (isClientAvailable) {
       UpdateRequest updateRequest = new UpdateRequest(indexName, docId);
       updateRequest.doc(doc, XContentType.JSON);
+      updateRequest.docAsUpsert(true);
       updateSearch(updateRequest);
     }
   }
@@ -1727,6 +1681,16 @@ public void deleteEntity(String indexName, String docId) {
     }
   }
 
+  @Override
+  public void deleteEntityByFQNPrefix(String indexName, String fqnPrefix) {
+    if (isClientAvailable) {
+      DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(indexName);
+      deleteByQueryRequest.setQuery(
+          new PrefixQueryBuilder("fullyQualifiedName.keyword", fqnPrefix.toLowerCase()));
+      deleteEntityFromOpenSearchByQuery(deleteByQueryRequest);
+    }
+  }
+
   @Override
   public void deleteEntityByFields(
       List<String> indexName, List<Pair<String, String>> fieldAndValue) {
@@ -2376,7 +2340,8 @@ private static SearchSourceBuilder getSearchSourceBuilder(
           "storage_service_index",
           "search_service_index",
           "metadata_service_index" -> buildServiceSearchBuilder(q, from, size);
-      case "all", "dataAsset" -> buildSearchAcrossIndexesBuilder(q, from, size);
+      case "dataAsset" -> buildDataAssetsSearchBuilder(q, from, size);
+      case "all" -> buildSearchAcrossIndexesBuilder(q, from, size);
       default -> buildAggregateSearchBuilder(q, from, size);
     };
   }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenAggregations.java
new file mode 100644
index 000000000000..fe5a0bba57ae
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenAggregations.java
@@ -0,0 +1,29 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import org.openmetadata.service.search.SearchAggregationNode;
+import os.org.opensearch.search.aggregations.AggregationBuilder;
+import os.org.opensearch.search.aggregations.PipelineAggregationBuilder;
+
+public interface OpenAggregations {
+  void createAggregation(SearchAggregationNode node);
+
+  default Boolean isPipelineAggregation() {
+    return false;
+  }
+
+  default void setAggregation(AggregationBuilder aggregation) {}
+
+  default void setSubAggregation(AggregationBuilder subAggregation) {}
+
+  default void setAggregation(PipelineAggregationBuilder aggregation) {}
+
+  default void setSubAggregation(PipelineAggregationBuilder subAggregation) {}
+
+  default AggregationBuilder getElasticAggregationBuilder() {
+    return null;
+  }
+
+  default PipelineAggregationBuilder getElasticPipelineAggregationBuilder() {
+    return null;
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenAggregationsBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenAggregationsBuilder.java
new file mode 100644
index 000000000000..2d6ec69d30da
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenAggregationsBuilder.java
@@ -0,0 +1,47 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import java.util.List;
+import org.openmetadata.service.search.SearchAggregationNode;
+
+public class OpenAggregationsBuilder {
+  public static List<OpenAggregations> buildAggregation(
+      SearchAggregationNode node,
+      OpenAggregations aggregation,
+      List<OpenAggregations> aggregations) {
+    String type = node.getType();
+    if (type == "root") {
+      for (SearchAggregationNode child : node.getChildren()) {
+        buildAggregation(child, null, aggregations);
+      }
+      return aggregations;
+    }
+    OpenAggregations OpenAggregations = getAggregation(type);
+    OpenAggregations.createAggregation(node);
+    if (aggregation != null) {
+      if (OpenAggregations.isPipelineAggregation()) {
+        aggregation.setSubAggregation(OpenAggregations.getElasticPipelineAggregationBuilder());
+      } else {
+        aggregation.setSubAggregation(OpenAggregations.getElasticAggregationBuilder());
+      }
+    }
+    if (aggregation == null) aggregations.add(OpenAggregations);
+    for (SearchAggregationNode child : node.getChildren()) {
+      buildAggregation(child, OpenAggregations, aggregations);
+    }
+    return aggregations;
+  }
+
+  public static OpenAggregations getAggregation(String aggregationType) {
+    return switch (aggregationType) {
+      case "bucket_selector" -> new OpenBucketSelectorAggregations();
+      case "date_histogram" -> new OpenDateHistogramAggregations();
+      case "terms" -> new OpenTermsAggregations();
+      case "avg" -> new OpenAvgAggregations();
+      case "min" -> new OpenMinAggregations();
+      case "cardinality" -> new OpenCardinalityAggregations();
+      case "nested" -> new OpenNestedAggregations();
+      case "top_hits" -> new OpenTopHitsAggregations();
+      default -> throw new IllegalArgumentException("Invalid aggregation type: " + aggregationType);
+    };
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenAvgAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenAvgAggregations.java
new file mode 100644
index 000000000000..1b3b833223a5
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenAvgAggregations.java
@@ -0,0 +1,38 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+import os.org.opensearch.search.aggregations.AggregationBuilder;
+import os.org.opensearch.search.aggregations.AggregationBuilders;
+import os.org.opensearch.search.aggregations.PipelineAggregationBuilder;
+
+@Setter
+@Getter
+public class OpenAvgAggregations implements OpenAggregations {
+  static final String aggregationType = "avg";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    AggregationBuilder aggregationBuilders =
+        AggregationBuilders.avg(node.getName()).field(params.get("field"));
+    setElasticAggregationBuilder(aggregationBuilders);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenBucketSelectorAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenBucketSelectorAggregations.java
new file mode 100644
index 000000000000..af5405f87489
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenBucketSelectorAggregations.java
@@ -0,0 +1,58 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+import os.org.opensearch.script.Script;
+import os.org.opensearch.search.aggregations.PipelineAggregationBuilder;
+import os.org.opensearch.search.aggregations.PipelineAggregatorBuilders;
+import os.org.opensearch.search.aggregations.pipeline.BucketSelectorPipelineAggregationBuilder;
+
+@Setter
+@Getter
+public class OpenBucketSelectorAggregations implements OpenAggregations {
+  private final String aggregationType = "bucket_selector";
+  PipelineAggregationBuilder elasticPipelineAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    String[] pathValues = Optional.ofNullable(params.get("pathValues")).orElse("").split(",");
+    String[] pathKeys = Optional.ofNullable(params.get("pathKeys")).orElse("").split(",");
+    String scriptStr = params.get("script");
+
+    if (!validateParams(pathKeys, pathValues, scriptStr)) {
+      throw new IllegalArgumentException(
+          "Invalid parameters. pathKeys & pathValues should be non-empty arrays of equal length"
+              + " and script should be non-empty");
+    }
+
+    Map<String, String> bucketsPaths = getBucketsPaths(pathKeys, pathValues);
+    Script script = new Script(scriptStr);
+    BucketSelectorPipelineAggregationBuilder bucketSelectorPipelineAggregationBuilder =
+        PipelineAggregatorBuilders.bucketSelector(node.getName(), bucketsPaths, script);
+    setElasticPipelineAggregationBuilder(bucketSelectorPipelineAggregationBuilder);
+  }
+
+  @Override
+  public Boolean isPipelineAggregation() {
+    return true;
+  }
+
+  private Map<String, String> getBucketsPaths(String[] pathKeys, String[] pathValues) {
+    return IntStream.range(0, pathKeys.length)
+        .mapToObj(i -> Map.entry(pathKeys[i], pathValues[i]))
+        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+  }
+
+  private Boolean validateParams(String[] pathKeys, String[] pathValues, String scriptStr) {
+    if ((pathKeys.length != pathValues.length) || (pathKeys.length == 0)) return false;
+    return !nullOrEmpty(scriptStr);
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenCardinalityAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenCardinalityAggregations.java
new file mode 100644
index 000000000000..db37e13e3c81
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenCardinalityAggregations.java
@@ -0,0 +1,38 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+import os.org.opensearch.search.aggregations.AggregationBuilder;
+import os.org.opensearch.search.aggregations.AggregationBuilders;
+import os.org.opensearch.search.aggregations.PipelineAggregationBuilder;
+
+@Setter
+@Getter
+public class OpenCardinalityAggregations implements OpenAggregations {
+  static final String aggregationType = "cardinality";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    AggregationBuilder aggregationBuilder =
+        AggregationBuilders.cardinality(node.getName()).field(params.get("field"));
+    setElasticAggregationBuilder(aggregationBuilder);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenDateHistogramAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenDateHistogramAggregations.java
new file mode 100644
index 000000000000..38c3b375b1b9
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenDateHistogramAggregations.java
@@ -0,0 +1,42 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+import os.org.opensearch.search.aggregations.AggregationBuilder;
+import os.org.opensearch.search.aggregations.AggregationBuilders;
+import os.org.opensearch.search.aggregations.PipelineAggregationBuilder;
+import os.org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+
+@Setter
+@Getter
+public class OpenDateHistogramAggregations implements OpenAggregations {
+  static final String aggregationType = "date_histogram";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    String calendarInterval = params.get("calendar_interval");
+    AggregationBuilder aggregationBuilder =
+        AggregationBuilders.dateHistogram(node.getName())
+            .field(params.get("field"))
+            .calendarInterval(new DateHistogramInterval(calendarInterval));
+    setElasticAggregationBuilder(aggregationBuilder);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenMinAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenMinAggregations.java
new file mode 100644
index 000000000000..3f0747eb271b
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenMinAggregations.java
@@ -0,0 +1,38 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+import os.org.opensearch.search.aggregations.AggregationBuilder;
+import os.org.opensearch.search.aggregations.AggregationBuilders;
+import os.org.opensearch.search.aggregations.PipelineAggregationBuilder;
+
+@Setter
+@Getter
+public class OpenMinAggregations implements OpenAggregations {
+  static final String aggregationType = "min";
+  AggregationBuilder OpenAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    AggregationBuilder aggregationBuilders =
+        AggregationBuilders.min(node.getName()).field(params.get("field"));
+    setOpenAggregationBuilder(aggregationBuilders);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (OpenAggregationBuilder != null) {
+      OpenAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (OpenAggregationBuilder != null) {
+      OpenAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenNestedAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenNestedAggregations.java
new file mode 100644
index 000000000000..27f5e8e0e642
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenNestedAggregations.java
@@ -0,0 +1,31 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+import os.org.opensearch.search.aggregations.AggregationBuilder;
+import os.org.opensearch.search.aggregations.AggregationBuilders;
+import os.org.opensearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
+
+@Setter
+@Getter
+public class OpenNestedAggregations implements OpenAggregations {
+  public static final String aggregationType = "nested";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    NestedAggregationBuilder aggregationBuilders =
+        AggregationBuilders.nested(params.get("path"), params.get("path"));
+    setElasticAggregationBuilder(aggregationBuilders);
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTermsAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTermsAggregations.java
new file mode 100644
index 000000000000..269c3b9212e2
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTermsAggregations.java
@@ -0,0 +1,54 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
+
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+import os.org.opensearch.search.aggregations.AggregationBuilder;
+import os.org.opensearch.search.aggregations.AggregationBuilders;
+import os.org.opensearch.search.aggregations.PipelineAggregationBuilder;
+import os.org.opensearch.search.aggregations.bucket.terms.IncludeExclude;
+import os.org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+
+@Setter
+@Getter
+public class OpenTermsAggregations implements OpenAggregations {
+  static final String aggregationType = "terms";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    String[] includes = null;
+    int size = -1;
+    Map<String, String> params = node.getValue();
+    String includesStr = params.get("include");
+    String sizeStr = params.get("size");
+    if (!nullOrEmpty(includesStr)) includes = includesStr.split(",");
+    if (!nullOrEmpty(sizeStr)) size = Integer.parseInt(params.get("size"));
+    TermsAggregationBuilder termsAggregationBuilder =
+        AggregationBuilders.terms(node.getName()).field(params.get("field"));
+
+    if (size > 0) termsAggregationBuilder.size(size);
+    if (!nullOrEmpty(includes)) {
+      IncludeExclude includeExclude = new IncludeExclude(includes, null);
+      termsAggregationBuilder.includeExclude(includeExclude);
+    }
+    setElasticAggregationBuilder(termsAggregationBuilder);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTopHitsAggregations.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTopHitsAggregations.java
new file mode 100644
index 000000000000..30d44e74431e
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/aggregations/OpenTopHitsAggregations.java
@@ -0,0 +1,41 @@
+package org.openmetadata.service.search.opensearch.aggregations;
+
+import java.util.Map;
+import lombok.Getter;
+import lombok.Setter;
+import org.openmetadata.service.search.SearchAggregationNode;
+import os.org.opensearch.search.aggregations.AggregationBuilder;
+import os.org.opensearch.search.aggregations.AggregationBuilders;
+import os.org.opensearch.search.aggregations.PipelineAggregationBuilder;
+import os.org.opensearch.search.sort.SortOrder;
+
+@Setter
+@Getter
+public class OpenTopHitsAggregations implements OpenAggregations {
+  static final String aggregationType = "top_hits";
+  AggregationBuilder elasticAggregationBuilder;
+
+  @Override
+  public void createAggregation(SearchAggregationNode node) {
+    Map<String, String> params = node.getValue();
+    AggregationBuilder aggregationBuilder =
+        AggregationBuilders.topHits(node.getName())
+            .size(Integer.parseInt(params.get("size")))
+            .sort(params.get("sort_field"), SortOrder.fromString(params.get("sort_order")));
+    setElasticAggregationBuilder(aggregationBuilder);
+  }
+
+  @Override
+  public void setSubAggregation(PipelineAggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+
+  @Override
+  public void setSubAggregation(AggregationBuilder aggregation) {
+    if (elasticAggregationBuilder != null) {
+      elasticAggregationBuilder.subAggregation(aggregation);
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/security/RBACConditionEvaluator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/security/RBACConditionEvaluator.java
index 6bcb33bc8748..9d713d7a2247 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/search/security/RBACConditionEvaluator.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/security/RBACConditionEvaluator.java
@@ -1,7 +1,6 @@
 package org.openmetadata.service.search.security;
 
 import java.util.*;
-import java.util.stream.Collectors;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.MetadataOperation;
@@ -26,7 +25,7 @@ public class RBACConditionEvaluator {
   private final ExpressionParser spelParser = new SpelExpressionParser();
   private final StandardEvaluationContext spelContext;
   private static final Set<MetadataOperation> SEARCH_RELEVANT_OPS =
-      Set.of(MetadataOperation.VIEW_BASIC, MetadataOperation.VIEW_ALL);
+      Set.of(MetadataOperation.VIEW_BASIC, MetadataOperation.VIEW_ALL, MetadataOperation.ALL);
 
   public RBACConditionEvaluator(QueryBuilderFactory queryBuilderFactory) {
     this.queryBuilderFactory = queryBuilderFactory;
@@ -36,38 +35,89 @@ public RBACConditionEvaluator(QueryBuilderFactory queryBuilderFactory) {
   public OMQueryBuilder evaluateConditions(SubjectContext subjectContext) {
     User user = subjectContext.user();
     spelContext.setVariable("user", user);
-    ConditionCollector collector = new ConditionCollector(queryBuilderFactory);
+
+    ConditionCollector finalCollector = new ConditionCollector(queryBuilderFactory);
 
     for (Iterator<SubjectContext.PolicyContext> it =
             subjectContext.getPolicies(List.of(user.getEntityReference()));
         it.hasNext(); ) {
       SubjectContext.PolicyContext context = it.next();
+
+      ConditionCollector policyCollector = new ConditionCollector(queryBuilderFactory);
+      List<OMQueryBuilder> allowRuleQueries = new ArrayList<>();
+      List<OMQueryBuilder> denyRuleQueries = new ArrayList<>();
+
       for (CompiledRule rule : context.getRules()) {
-        if (rule.getCondition() != null
-            && rule.getOperations().stream().anyMatch(SEARCH_RELEVANT_OPS::contains)) {
-          ConditionCollector ruleCollector = new ConditionCollector(queryBuilderFactory);
-          if (!rule.getResources().isEmpty() && !rule.getResources().contains("All")) {
-            OMQueryBuilder indexFilter = getIndexFilter(rule.getResources());
-            ruleCollector.addMust(indexFilter);
-          }
-          SpelExpression parsedExpression =
-              (SpelExpression) spelParser.parseExpression(rule.getCondition());
-          preprocessExpression(parsedExpression.getAST(), ruleCollector);
-          OMQueryBuilder ruleQuery = ruleCollector.buildFinalQuery();
-          if (rule.getEffect().toString().equalsIgnoreCase("DENY")) {
-            collector.addMustNot(ruleQuery);
-          } else {
-            collector.addMust(ruleQuery);
-          }
+        boolean isDenyRule = rule.getEffect().toString().equalsIgnoreCase("DENY");
+        Set<MetadataOperation> ruleOperations = new HashSet<>(rule.getOperations());
+        ruleOperations.retainAll(SEARCH_RELEVANT_OPS);
+        if (ruleOperations.isEmpty()) {
+          // Skip this rule as it does not affect search results
+          continue;
         }
+
+        OMQueryBuilder ruleQuery = buildRuleQuery(rule, user);
+        if (ruleQuery == null || ruleQuery.isEmpty()) {
+          continue;
+        }
+        if (isDenyRule) {
+          denyRuleQueries.add(ruleQuery);
+        } else {
+          allowRuleQueries.add(ruleQuery);
+        }
+      }
+
+      if (!denyRuleQueries.isEmpty()) {
+        if (denyRuleQueries.size() == 1) {
+          policyCollector.addMustNot(denyRuleQueries.get(0));
+        } else {
+          OMQueryBuilder denyQuery = queryBuilderFactory.boolQuery().should(denyRuleQueries);
+          policyCollector.addMustNot(denyQuery);
+        }
+      }
+
+      if (!allowRuleQueries.isEmpty()) {
+        if (allowRuleQueries.size() == 1) {
+          policyCollector.addMust(allowRuleQueries.get(0));
+        } else {
+          OMQueryBuilder allowQuery = queryBuilderFactory.boolQuery().should(allowRuleQueries);
+          policyCollector.addMust(allowQuery);
+        }
+      } else {
+        policyCollector.addMust(queryBuilderFactory.matchAllQuery());
+      }
+
+      OMQueryBuilder policyFinalQuery = policyCollector.buildFinalQuery();
+      if (policyFinalQuery != null && !policyFinalQuery.isEmpty()) {
+        finalCollector.addMust(policyFinalQuery);
       }
     }
 
-    return collector.buildFinalQuery();
+    return finalCollector.buildFinalQuery();
+  }
+
+  private OMQueryBuilder buildRuleQuery(CompiledRule rule, User user) {
+    ConditionCollector ruleCollector = new ConditionCollector(queryBuilderFactory);
+    spelContext.setVariable("user", user);
+
+    // Apply index filtering if resources are specified and not "All"
+    if (!rule.getResources().isEmpty() && !rule.getResources().contains("All")) {
+      OMQueryBuilder indexFilter = getIndexFilter(rule.getResources());
+      ruleCollector.addMust(indexFilter);
+    }
+
+    if (rule.getCondition() != null && !rule.getCondition().trim().isEmpty()) {
+      SpelExpression parsedExpression =
+          (SpelExpression) spelParser.parseExpression(rule.getCondition());
+      preprocessExpression(parsedExpression.getAST(), ruleCollector);
+    } else {
+      ruleCollector.addMust(queryBuilderFactory.matchAllQuery());
+    }
+
+    return ruleCollector.buildFinalQuery();
   }
 
   private void preprocessExpression(SpelNode node, ConditionCollector collector) {
-    // Delay this check until after processing necessary expressions
     if (collector.isMatchNothing()) {
       return;
     }
@@ -82,35 +132,33 @@ private void preprocessExpression(SpelNode node, ConditionCollector collector) {
     } else if (node instanceof OpOr) {
       List<OMQueryBuilder> orQueries = new ArrayList<>();
       boolean allMatchNothing = true;
-      boolean hasTrueCondition = false; // Track if any condition evaluated to true
+      boolean hasTrueCondition = false;
 
       for (int i = 0; i < node.getChildCount(); i++) {
         ConditionCollector childCollector = new ConditionCollector(queryBuilderFactory);
         preprocessExpression(node.getChild(i), childCollector);
 
         if (childCollector.isMatchNothing()) {
-          continue; // If this child evaluates to match nothing, skip it
+          continue;
         }
 
         if (childCollector.isMatchAllQuery()) {
-          hasTrueCondition = true; // If any condition evaluates to true, mark it
-          break; // Short-circuit: if any condition in OR evaluates to true, the whole OR is true
+          hasTrueCondition = true;
+          break;
         }
 
         OMQueryBuilder childQuery = childCollector.buildFinalQuery();
         if (childQuery != null) {
-          allMatchNothing =
-              false; // If at least one child query is valid, it’s not all match nothing
+          allMatchNothing = false;
           orQueries.add(childQuery);
         }
       }
 
       if (hasTrueCondition) {
-        collector.addMust(queryBuilderFactory.matchAllQuery()); // OR is true, add match_all
+        collector.addMust(queryBuilderFactory.matchAllQuery());
       } else if (allMatchNothing) {
-        collector.setMatchNothing(true); // OR is false
+        collector.setMatchNothing(true);
       } else {
-        // Add the valid OR queries to the collector
         for (OMQueryBuilder orQuery : orQueries) {
           collector.addShould(orQuery);
         }
@@ -126,7 +174,7 @@ private void preprocessExpression(SpelNode node, ConditionCollector collector) {
       } else {
         OMQueryBuilder subQuery = subCollector.buildFinalQuery();
         if (subQuery != null && !subQuery.isEmpty()) {
-          collector.addMustNot(subQuery); // Add must_not without extra nesting
+          collector.addMustNot(subQuery);
         }
       }
     } else if (node instanceof MethodReference) {
@@ -165,7 +213,7 @@ private List<String> extractMethodArguments(MethodReference methodRef) {
     List<String> args = new ArrayList<>();
     for (int i = 0; i < methodRef.getChildCount(); i++) {
       SpelNode childNode = methodRef.getChild(i);
-      String value = childNode.toStringAST().replace("'", ""); // Remove single quotes
+      String value = childNode.toStringAST().replace("'", "");
       args.add(value);
     }
     return args;
@@ -181,27 +229,27 @@ public void matchAnyTag(List<String> tags, ConditionCollector collector) {
   public void matchAllTags(List<String> tags, ConditionCollector collector) {
     for (String tag : tags) {
       OMQueryBuilder tagQuery = queryBuilderFactory.termQuery("tags.tagFQN", tag);
-      collector.addMust(tagQuery); // Add directly to the collector's must clause
+      collector.addMust(tagQuery);
     }
   }
 
   public void isOwner(User user, ConditionCollector collector) {
     List<OMQueryBuilder> ownerQueries = new ArrayList<>();
-    ownerQueries.add(queryBuilderFactory.termQuery("owner.id", user.getId().toString()));
+    ownerQueries.add(queryBuilderFactory.termQuery("owners.id", user.getId().toString()));
 
     if (user.getTeams() != null) {
       for (EntityReference team : user.getTeams()) {
-        ownerQueries.add(queryBuilderFactory.termQuery("owner.id", team.getId().toString()));
+        ownerQueries.add(queryBuilderFactory.termQuery("owners.id", team.getId().toString()));
       }
     }
 
     for (OMQueryBuilder ownerQuery : ownerQueries) {
-      collector.addShould(ownerQuery); // Add directly to the collector's should clause
+      collector.addShould(ownerQuery);
     }
   }
 
   public void noOwner(ConditionCollector collector) {
-    OMQueryBuilder existsQuery = queryBuilderFactory.existsQuery("owner.id");
+    OMQueryBuilder existsQuery = queryBuilderFactory.existsQuery("owners.id");
     collector.addMustNot(existsQuery);
   }
 
@@ -253,7 +301,7 @@ private OMQueryBuilder getIndexFilter(List<String> resources) {
     List<String> indices =
         resources.stream()
             .map(resource -> Entity.getSearchRepository().getIndexOrAliasName(resource))
-            .collect(Collectors.toList());
+            .toList();
 
     return queryBuilderFactory.termsQuery("_index", indices);
   }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationException.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationException.java
index 64b757d26997..746b03d28482 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationException.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/AuthenticationException.java
@@ -43,8 +43,18 @@ public static AuthenticationException getTokenNotPresentException() {
     return new AuthenticationException(msg);
   }
 
-  public static AuthenticationException getInvalidTokenException() {
-    String msg = "Not Authorized! Invalid Token";
+  public static AuthenticationException getInvalidTokenException(String reason) {
+    String msg = String.format("Not Authorized! %s", reason);
+    return new AuthenticationException(msg);
+  }
+
+  public static AuthenticationException getInvalidTokenException(String reason, Exception e) {
+    String msg = String.format("Not Authorized! %s due to %s", reason, e);
+    return new AuthenticationException(msg);
+  }
+
+  public static AuthenticationException getExpiredTokenException() {
+    String msg = "Expired token!";
     return new AuthenticationException(msg);
   }
 
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java
index 844546438895..0e57365b069e 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java
@@ -212,14 +212,14 @@ public Map<String, Claim> validateJwtAndGetClaims(String token) {
     try {
       jwt = JWT.decode(token);
     } catch (JWTDecodeException e) {
-      throw new AuthenticationException("Invalid token", e);
+      throw AuthenticationException.getInvalidTokenException("Unable to decode the token.");
     }
 
     // Check if expired
     // If expiresAt is set to null, treat it as never expiring token
     if (jwt.getExpiresAt() != null
         && jwt.getExpiresAt().before(Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime())) {
-      throw new AuthenticationException("Expired token!");
+      throw AuthenticationException.getExpiredTokenException();
     }
 
     // Validate JWT with public key
@@ -228,7 +228,8 @@ public Map<String, Claim> validateJwtAndGetClaims(String token) {
     try {
       algorithm.verify(jwt);
     } catch (RuntimeException runtimeException) {
-      throw new AuthenticationException("Invalid token", runtimeException);
+      throw AuthenticationException.getInvalidTokenException(
+          "Token verification failed. Public key mismatch.", runtimeException);
     }
 
     Map<String, Claim> claims = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
@@ -266,7 +267,8 @@ private void validateBotToken(String tokenFromHeader, String userName) {
     if (tokenFromHeader.equals(BotTokenCache.getToken(userName))) {
       return;
     }
-    throw AuthenticationException.getInvalidTokenException();
+    throw AuthenticationException.getInvalidTokenException(
+        "The given token does not match the current bot's token!");
   }
 
   private void validatePersonalAccessToken(
@@ -282,7 +284,7 @@ private void validatePersonalAccessToken(
       if (userTokens != null && userTokens.contains(tokenFromHeader)) {
         return;
       }
-      throw AuthenticationException.getInvalidTokenException();
+      throw AuthenticationException.getInvalidTokenException("Invalid personal access token!");
     }
   }
 
@@ -292,7 +294,7 @@ private void validateTokenIsNotUsedAfterLogout(String authToken) {
       LogoutRequest previouslyLoggedOutEvent =
           JwtTokenCacheManager.getInstance().getLogoutEventForToken(authToken);
       if (previouslyLoggedOutEvent != null) {
-        throw new AuthenticationException("Expired token!");
+        throw AuthenticationException.getExpiredTokenException();
       }
     }
   }
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java
new file mode 100644
index 000000000000..a53c640e1fac
--- /dev/null
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/SchemaFieldExtractor.java
@@ -0,0 +1,607 @@
+package org.openmetadata.service.util;
+
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.ws.rs.core.UriInfo;
+import lombok.extern.slf4j.Slf4j;
+import org.everit.json.schema.*;
+import org.everit.json.schema.loader.SchemaClient;
+import org.everit.json.schema.loader.SchemaLoader;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+import org.openmetadata.schema.entity.Type;
+import org.openmetadata.schema.entity.type.CustomProperty;
+import org.openmetadata.schema.type.Include;
+import org.openmetadata.sdk.exception.SchemaProcessingException;
+import org.openmetadata.service.jdbi3.TypeRepository;
+
+@Slf4j
+public class SchemaFieldExtractor {
+
+  public SchemaFieldExtractor() {}
+
+  public List<FieldDefinition> extractFields(Type typeEntity, String entityType)
+      throws SchemaProcessingException {
+    String schemaPath = determineSchemaPath(entityType);
+    String schemaUri = "classpath:///" + schemaPath;
+    SchemaClient schemaClient = new CustomSchemaClient(schemaUri);
+    Map<String, String> fieldTypesMap = new LinkedHashMap<>();
+    Deque<Schema> processingStack = new ArrayDeque<>();
+    Set<String> processedFields = new HashSet<>();
+    Schema mainSchema = loadMainSchema(schemaPath, entityType, schemaUri, schemaClient);
+    extractFieldsFromSchema(mainSchema, "", fieldTypesMap, processingStack, processedFields);
+    addCustomProperties(
+        typeEntity, schemaUri, schemaClient, fieldTypesMap, processingStack, processedFields);
+    return convertMapToFieldList(fieldTypesMap);
+  }
+
+  public Map<String, List<FieldDefinition>> extractAllCustomProperties(
+      UriInfo uriInfo, TypeRepository repository) {
+    Map<String, List<FieldDefinition>> entityTypeToFields = new HashMap<>();
+    List<String> entityTypes = getAllEntityTypes();
+
+    for (String entityType : entityTypes) {
+      String schemaPath = determineSchemaPath(entityType);
+      String schemaUri = "classpath:///" + schemaPath;
+      SchemaClient schemaClient = new CustomSchemaClient(schemaUri);
+      EntityUtil.Fields fieldsParam = new EntityUtil.Fields(Set.of("customProperties"));
+      Type typeEntity = repository.getByName(uriInfo, entityType, fieldsParam, Include.ALL, false);
+      Map<String, String> fieldTypesMap = new LinkedHashMap<>();
+      Set<String> processedFields = new HashSet<>();
+      Deque<Schema> processingStack = new ArrayDeque<>();
+      addCustomProperties(
+          typeEntity, schemaUri, schemaClient, fieldTypesMap, processingStack, processedFields);
+      entityTypeToFields.put(entityType, convertMapToFieldList(fieldTypesMap));
+    }
+
+    return entityTypeToFields;
+  }
+
+  public static List<String> getAllEntityTypes() {
+    List<String> entityTypes = new ArrayList<>();
+    try {
+      String schemaDirectory = "json/schema/entity/";
+      Enumeration<URL> resources =
+          SchemaFieldExtractor.class.getClassLoader().getResources(schemaDirectory);
+      while (resources.hasMoreElements()) {
+        URL resourceUrl = resources.nextElement();
+        Path schemaDirPath = Paths.get(resourceUrl.toURI());
+
+        Files.walk(schemaDirPath)
+            .filter(Files::isRegularFile)
+            .filter(path -> path.toString().endsWith(".json"))
+            .forEach(
+                path -> {
+                  try (InputStream is = Files.newInputStream(path)) {
+                    JSONObject jsonSchema = new JSONObject(new JSONTokener(is));
+                    // Check if the schema is an entity type
+                    if (isEntityType(jsonSchema)) {
+                      String fileName = path.getFileName().toString();
+                      String entityType =
+                          fileName.substring(0, fileName.length() - 5); // Remove ".json"
+                      entityTypes.add(entityType);
+                      LOG.debug("Found entity type: {}", entityType);
+                    }
+                  } catch (Exception e) {
+                    LOG.error("Error reading schema file {}: {}", path, e.getMessage());
+                  }
+                });
+      }
+    } catch (Exception e) {
+      LOG.error("Error scanning schema directory: {}", e.getMessage());
+    }
+    return entityTypes;
+  }
+
+  private static boolean isEntityType(JSONObject jsonSchema) {
+    return "@om-entity-type".equals(jsonSchema.optString("$comment"));
+  }
+
+  private Schema loadMainSchema(
+      String schemaPath, String entityType, String schemaUri, SchemaClient schemaClient)
+      throws SchemaProcessingException {
+    InputStream schemaInputStream = getClass().getClassLoader().getResourceAsStream(schemaPath);
+    if (schemaInputStream == null) {
+      LOG.error("Schema file not found at path: {}", schemaPath);
+      throw new SchemaProcessingException(
+          "Schema file not found for entity type: " + entityType,
+          SchemaProcessingException.ErrorType.RESOURCE_NOT_FOUND);
+    }
+
+    JSONObject rawSchema = new JSONObject(new JSONTokener(schemaInputStream));
+    SchemaLoader schemaLoader =
+        SchemaLoader.builder()
+            .schemaJson(rawSchema)
+            .resolutionScope(schemaUri)
+            .schemaClient(schemaClient)
+            .build();
+
+    try {
+      Schema schema = schemaLoader.load().build();
+      LOG.debug("Schema '{}' loaded successfully.", schemaPath);
+      return schema;
+    } catch (Exception e) {
+      LOG.error("Error loading schema '{}': {}", schemaPath, e.getMessage());
+      throw new SchemaProcessingException(
+          "Error loading schema '" + schemaPath + "': " + e.getMessage(),
+          SchemaProcessingException.ErrorType.OTHER);
+    }
+  }
+
+  private void extractFieldsFromSchema(
+      Schema schema,
+      String parentPath,
+      Map<String, String> fieldTypesMap,
+      Deque<Schema> processingStack,
+      Set<String> processedFields) {
+    if (processingStack.contains(schema)) {
+      LOG.debug(
+          "Detected cyclic reference at path '{}'. Skipping further processing of this schema.",
+          parentPath);
+      return;
+    }
+
+    processingStack.push(schema);
+    try {
+      if (schema instanceof ObjectSchema objectSchema) {
+        for (Map.Entry<String, Schema> propertyEntry :
+            objectSchema.getPropertySchemas().entrySet()) {
+          String fieldName = propertyEntry.getKey();
+          Schema fieldSchema = propertyEntry.getValue();
+          String fullFieldName = parentPath.isEmpty() ? fieldName : parentPath + "." + fieldName;
+
+          if (processedFields.contains(fullFieldName)) {
+            LOG.debug(
+                "Field '{}' has already been processed. Skipping to prevent duplication.",
+                fullFieldName);
+            continue;
+          }
+
+          LOG.debug("Processing field '{}'", fullFieldName);
+
+          if (fieldSchema instanceof ReferenceSchema referenceSchema) {
+            handleReferenceSchema(
+                referenceSchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
+          } else if (fieldSchema instanceof ArraySchema arraySchema) {
+            handleArraySchema(
+                arraySchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
+          } else {
+            String fieldType = mapSchemaTypeToSimpleType(fieldSchema);
+            fieldTypesMap.putIfAbsent(fullFieldName, fieldType);
+            processedFields.add(fullFieldName);
+            LOG.debug("Added field '{}', Type: '{}'", fullFieldName, fieldType);
+            // Recursively process nested objects or arrays
+            if (fieldSchema instanceof ObjectSchema || fieldSchema instanceof ArraySchema) {
+              extractFieldsFromSchema(
+                  fieldSchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
+            }
+          }
+        }
+      } else if (schema instanceof ArraySchema arraySchema) {
+        handleArraySchema(arraySchema, parentPath, fieldTypesMap, processingStack, processedFields);
+      } else {
+        String fieldType = mapSchemaTypeToSimpleType(schema);
+        fieldTypesMap.putIfAbsent(parentPath, fieldType);
+        LOG.debug("Added field '{}', Type: '{}'", parentPath, fieldType);
+      }
+    } finally {
+      processingStack.pop();
+    }
+  }
+
+  private void handleReferenceSchema(
+      ReferenceSchema referenceSchema,
+      String fullFieldName,
+      Map<String, String> fieldTypesMap,
+      Deque<Schema> processingStack,
+      Set<String> processedFields) {
+
+    String refUri = referenceSchema.getReferenceValue();
+    String referenceType = determineReferenceType(refUri);
+
+    if (referenceType != null) {
+      fieldTypesMap.putIfAbsent(fullFieldName, referenceType);
+      processedFields.add(fullFieldName);
+      LOG.debug("Added field '{}', Type: '{}'", fullFieldName, referenceType);
+      if (referenceType.startsWith("array<") && referenceType.endsWith(">")) {
+        Schema itemSchema =
+            referenceSchema.getReferredSchema() instanceof ArraySchema
+                ? ((ArraySchema) referenceSchema.getReferredSchema()).getAllItemSchema()
+                : referenceSchema.getReferredSchema();
+        extractFieldsFromSchema(
+            itemSchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
+      } else if (!isPrimitiveType(referenceType)) {
+        Schema referredSchema = referenceSchema.getReferredSchema();
+        extractFieldsFromSchema(
+            referredSchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
+      }
+    } else {
+      fieldTypesMap.putIfAbsent(fullFieldName, "object");
+      processedFields.add(fullFieldName);
+      LOG.debug("Added field '{}', Type: 'object'", fullFieldName);
+      extractFieldsFromSchema(
+          referenceSchema.getReferredSchema(),
+          fullFieldName,
+          fieldTypesMap,
+          processingStack,
+          processedFields);
+    }
+  }
+
+  private void handleArraySchema(
+      ArraySchema arraySchema,
+      String fullFieldName,
+      Map<String, String> fieldTypesMap,
+      Deque<Schema> processingStack,
+      Set<String> processedFields) {
+
+    Schema itemsSchema = arraySchema.getAllItemSchema();
+
+    if (itemsSchema instanceof ReferenceSchema itemsReferenceSchema) {
+      String itemsRefUri = itemsReferenceSchema.getReferenceValue();
+      String itemsReferenceType = determineReferenceType(itemsRefUri);
+
+      if (itemsReferenceType != null) {
+        String arrayFieldType = "array<" + itemsReferenceType + ">";
+        fieldTypesMap.putIfAbsent(fullFieldName, arrayFieldType);
+        processedFields.add(fullFieldName);
+        LOG.debug("Added field '{}', Type: '{}'", fullFieldName, arrayFieldType);
+        Schema referredItemsSchema = itemsReferenceSchema.getReferredSchema();
+        extractFieldsFromSchema(
+            referredItemsSchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
+        return;
+      }
+    }
+    String arrayType = mapSchemaTypeToSimpleType(itemsSchema);
+    fieldTypesMap.putIfAbsent(fullFieldName, "array<" + arrayType + ">");
+    processedFields.add(fullFieldName);
+    LOG.debug("Added field '{}', Type: 'array<{}>'", fullFieldName, arrayType);
+
+    if (itemsSchema instanceof ObjectSchema || itemsSchema instanceof ArraySchema) {
+      extractFieldsFromSchema(
+          itemsSchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
+    }
+  }
+
+  private void addCustomProperties(
+      Type typeEntity,
+      String schemaUri,
+      SchemaClient schemaClient,
+      Map<String, String> fieldTypesMap,
+      Deque<Schema> processingStack,
+      Set<String> processedFields) {
+    if (typeEntity == null || typeEntity.getCustomProperties() == null) {
+      return;
+    }
+
+    for (CustomProperty customProperty : typeEntity.getCustomProperties()) {
+      String propertyName = customProperty.getName();
+      String propertyType = customProperty.getPropertyType().getName();
+      String fullFieldName = propertyName; // No parent path for custom properties
+
+      LOG.debug("Processing custom property '{}'", fullFieldName);
+
+      if (isEntityReferenceList(propertyType)) {
+        String referenceType = "array<entityReference>";
+        fieldTypesMap.putIfAbsent(fullFieldName, referenceType);
+        processedFields.add(fullFieldName);
+        LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, referenceType);
+
+        Schema itemSchema = resolveSchemaByType("entityReference", schemaUri, schemaClient);
+        if (itemSchema != null) {
+          extractFieldsFromSchema(
+              itemSchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
+        } else {
+          LOG.warn(
+              "Schema for type 'entityReference' not found. Skipping nested field extraction for '{}'.",
+              fullFieldName);
+        }
+      } else if (isEntityReference(propertyType)) {
+        String referenceType = "entityReference";
+        fieldTypesMap.putIfAbsent(fullFieldName, referenceType);
+        processedFields.add(fullFieldName);
+        LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, referenceType);
+
+        Schema referredSchema = resolveSchemaByType("entityReference", schemaUri, schemaClient);
+        if (referredSchema != null) {
+          extractFieldsFromSchema(
+              referredSchema, fullFieldName, fieldTypesMap, processingStack, processedFields);
+        } else {
+          LOG.warn(
+              "Schema for type 'entityReference' not found. Skipping nested field extraction for '{}'.",
+              fullFieldName);
+        }
+      } else {
+        fieldTypesMap.putIfAbsent(fullFieldName, propertyType);
+        processedFields.add(fullFieldName);
+        LOG.debug("Added custom property '{}', Type: '{}'", fullFieldName, propertyType);
+      }
+    }
+  }
+
+  private List<FieldDefinition> convertMapToFieldList(Map<String, String> fieldTypesMap) {
+    List<FieldDefinition> fieldsList = new ArrayList<>();
+    for (Map.Entry<String, String> entry : fieldTypesMap.entrySet()) {
+      fieldsList.add(new FieldDefinition(entry.getKey(), entry.getValue()));
+    }
+    return fieldsList;
+  }
+
+  private boolean isEntityReferenceList(String propertyType) {
+    return "entityReferenceList".equalsIgnoreCase(propertyType);
+  }
+
+  private boolean isEntityReference(String propertyType) {
+    return "entityReference".equalsIgnoreCase(propertyType);
+  }
+
+  private Schema resolveSchemaByType(String typeName, String schemaUri, SchemaClient schemaClient) {
+    String referencePath = determineReferencePath(typeName);
+    try {
+      return loadSchema(referencePath, schemaUri, schemaClient);
+    } catch (SchemaProcessingException e) {
+      LOG.error("Failed to load schema for type '{}': {}", typeName, e.getMessage());
+      return null;
+    }
+  }
+
+  private Schema loadSchema(String schemaPath, String schemaUri, SchemaClient schemaClient)
+      throws SchemaProcessingException {
+    InputStream schemaInputStream = getClass().getClassLoader().getResourceAsStream(schemaPath);
+    if (schemaInputStream == null) {
+      LOG.error("Schema file not found at path: {}", schemaPath);
+      throw new SchemaProcessingException(
+          "Schema file not found for path: " + schemaPath,
+          SchemaProcessingException.ErrorType.RESOURCE_NOT_FOUND);
+    }
+
+    JSONObject rawSchema = new JSONObject(new JSONTokener(schemaInputStream));
+    SchemaLoader schemaLoader =
+        SchemaLoader.builder()
+            .schemaJson(rawSchema)
+            .resolutionScope(schemaUri) // Base URI for resolving $ref
+            .schemaClient(schemaClient)
+            .build();
+
+    try {
+      Schema schema = schemaLoader.load().build();
+      LOG.debug("Schema '{}' loaded successfully.", schemaPath);
+      return schema;
+    } catch (Exception e) {
+      LOG.error("Error loading schema '{}': {}", schemaPath, e.getMessage());
+      throw new SchemaProcessingException(
+          "Error loading schema '" + schemaPath + "': " + e.getMessage(),
+          SchemaProcessingException.ErrorType.OTHER);
+    }
+  }
+
+  private String determineReferenceType(String refUri) {
+    // Pattern to extract the definition name if present
+    Pattern definitionPattern = Pattern.compile("^(?:.*/)?basic\\.json#/definitions/([\\w-]+)$");
+    Matcher matcher = definitionPattern.matcher(refUri);
+    if (matcher.find()) {
+      String definition = matcher.group(1);
+      return switch (definition) {
+        case "duration" -> "duration";
+        case "markdown" -> "markdown";
+        case "timestamp" -> "timestamp";
+        case "integer" -> "integer";
+        case "number" -> "number";
+        case "string" -> "string";
+        case "uuid" -> "uuid";
+        case "email" -> "email";
+        case "href" -> "href";
+        case "timeInterval" -> "timeInterval";
+        case "date" -> "date";
+        case "dateTime" -> "dateTime";
+        case "time" -> "time";
+        case "date-cp" -> "date-cp";
+        case "dateTime-cp" -> "dateTime-cp";
+        case "time-cp" -> "time-cp";
+        case "enum" -> "enum";
+        case "enumWithDescriptions" -> "enumWithDescriptions";
+        case "timezone" -> "timezone";
+        case "entityLink" -> "entityLink";
+        case "entityName" -> "entityName";
+        case "testCaseEntityName" -> "testCaseEntityName";
+        case "fullyQualifiedEntityName" -> "fullyQualifiedEntityName";
+        case "sqlQuery" -> "sqlQuery";
+        case "sqlFunction" -> "sqlFunction";
+        case "expression" -> "expression";
+        case "jsonSchema" -> "jsonSchema";
+        case "entityExtension" -> "entityExtension";
+        case "providerType" -> "providerType";
+        case "componentConfig" -> "componentConfig";
+        case "status" -> "status";
+        case "sourceUrl" -> "sourceUrl";
+        case "style" -> "style";
+        default -> {
+          LOG.warn("Unrecognized definition '{}' in refUri '{}'", definition, refUri);
+          yield "object";
+        }
+      };
+    }
+
+    // Existing file-based mappings
+    if (refUri.matches(".*basic\\.json$")) {
+      return "uuid";
+    }
+    if (refUri.matches(".*entityReference\\.json(?:#.*)?$")) {
+      return "entityReference";
+    }
+    if (refUri.matches(".*entityReferenceList\\.json(?:#.*)?$")) {
+      return "array<entityReference>";
+    }
+    if (refUri.matches(".*tagLabel\\.json(?:#.*)?$")) {
+      return "tagLabel";
+    }
+    if (refUri.matches(".*fullyQualifiedEntityName\\.json(?:#.*)?$")) {
+      return "fullyQualifiedEntityName";
+    }
+    if (refUri.matches(".*entityVersion\\.json(?:#.*)?$")) {
+      return "entityVersion";
+    }
+    if (refUri.matches(".*markdown\\.json(?:#.*)?$")) {
+      return "markdown";
+    }
+    if (refUri.matches(".*timestamp\\.json(?:#.*)?$")) {
+      return "timestamp";
+    }
+    if (refUri.matches(".*href\\.json(?:#.*)?$")) {
+      return "href";
+    }
+    if (refUri.matches(".*duration\\.json(?:#.*)?$")) {
+      return "duration";
+    }
+    return null;
+  }
+
+  private String mapSchemaTypeToSimpleType(Schema schema) {
+    if (schema == null) {
+      LOG.debug("Mapping type: null -> 'object'");
+      return "object";
+    }
+    if (schema instanceof StringSchema) {
+      LOG.debug("Mapping schema instance '{}' to 'string'", schema.getClass().getSimpleName());
+      return "string";
+    } else if (schema instanceof NumberSchema numberSchema) {
+      if (numberSchema.requiresInteger()) {
+        LOG.debug("Mapping schema instance '{}' to 'integer'", schema.getClass().getSimpleName());
+        return "integer";
+      } else {
+        LOG.debug("Mapping schema instance '{}' to 'number'", schema.getClass().getSimpleName());
+        return "number";
+      }
+    } else if (schema instanceof BooleanSchema) {
+      LOG.debug("Mapping schema instance '{}' to 'boolean'", schema.getClass().getSimpleName());
+      return "boolean";
+    } else if (schema instanceof ObjectSchema) {
+      LOG.debug("Mapping schema instance '{}' to 'object'", schema.getClass().getSimpleName());
+      return "object";
+    } else if (schema instanceof ArraySchema) {
+      LOG.debug("Mapping schema instance '{}' to 'array'", schema.getClass().getSimpleName());
+      return "array";
+    } else if (schema instanceof NullSchema) {
+      LOG.debug("Mapping schema instance '{}' to 'null'", schema.getClass().getSimpleName());
+      return "null";
+    } else {
+      LOG.debug(
+          "Mapping unknown schema instance '{}' to 'string'", schema.getClass().getSimpleName());
+      return "string";
+    }
+  }
+
+  private boolean isPrimitiveType(String type) {
+    return type.equals("string")
+        || type.equals("integer")
+        || type.equals("number")
+        || type.equals("boolean")
+        || type.equals("uuid")
+        || // Treat 'uuid' as a primitive type
+        type.equals("timestamp")
+        || type.equals("href")
+        || type.equals("duration")
+        || type.equals("date")
+        || type.equals("dateTime")
+        || type.equals("time")
+        || type.equals("date-cp")
+        || type.equals("dateTime-cp")
+        || type.equals("time-cp")
+        || type.equals("enum")
+        || type.equals("enumWithDescriptions")
+        || type.equals("timezone")
+        || type.equals("entityLink")
+        || type.equals("entityName")
+        || type.equals("testCaseEntityName")
+        || type.equals("fullyQualifiedEntityName")
+        || type.equals("sqlQuery")
+        || type.equals("sqlFunction")
+        || type.equals("expression")
+        || type.equals("jsonSchema")
+        || type.equals("entityExtension")
+        || type.equals("providerType")
+        || type.equals("componentConfig")
+        || type.equals("status")
+        || type.equals("sourceUrl")
+        || type.equals("style");
+  }
+
+  private String determineReferencePath(String typeName) {
+    String baseSchemaDirectory = "json/schema/entity/";
+    String schemaFileName = typeName + ".json";
+    return baseSchemaDirectory + schemaFileName;
+  }
+
+  private String determineSchemaPath(String entityType) {
+    String subdirectory = getEntitySubdirectory(entityType);
+    return "json/schema/entity/" + subdirectory + "/" + entityType + ".json";
+  }
+
+  private String getEntitySubdirectory(String entityType) {
+    Map<String, String> entityTypeToSubdirectory =
+        Map.of(
+            "dashboard", "data",
+            "table", "data",
+            "pipeline", "services",
+            "votes", "data");
+    return entityTypeToSubdirectory.getOrDefault(entityType, "data");
+  }
+
+  @Slf4j
+  private static class CustomSchemaClient implements SchemaClient {
+    private final String baseUri;
+
+    public CustomSchemaClient(String baseUri) {
+      this.baseUri = baseUri;
+    }
+
+    @Override
+    public InputStream get(String url) {
+      LOG.debug("SchemaClient: Resolving URL '{}' against base URI '{}'", url, baseUri);
+      String resourcePath = mapUrlToResourcePath(url);
+      LOG.debug("SchemaClient: Loading resource from path '{}'", resourcePath);
+      InputStream is = getClass().getClassLoader().getResourceAsStream(resourcePath);
+      if (is == null) {
+        LOG.error("Resource not found: {}", resourcePath);
+        throw new RuntimeException("Resource not found: " + resourcePath);
+      }
+      return is;
+    }
+
+    private String mapUrlToResourcePath(String url) {
+      if (url.startsWith("https://open-metadata.org/schema/")) {
+        String relativePath = url.substring("https://open-metadata.org/schema/".length());
+        return "json/schema/" + relativePath;
+      } else {
+        throw new RuntimeException("Unsupported URL: " + url);
+      }
+    }
+  }
+
+  @lombok.Getter
+  @lombok.Setter
+  public static class FieldDefinition {
+    private String name;
+    private String type;
+
+    public FieldDefinition(String name, String type) {
+      this.name = name;
+      this.type = type;
+    }
+  }
+}
diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/Utilities.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/Utilities.java
index 0a9b0143156f..f3ac975ac2c7 100644
--- a/openmetadata-service/src/main/java/org/openmetadata/service/util/Utilities.java
+++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/Utilities.java
@@ -38,6 +38,14 @@ public static String getDateFromEpoch(long epochTimestampInMilli) {
     return getFormattedDateFromEpoch(epochTimestampInMilli, "d");
   }
 
+  public static String cleanUpDoubleQuotes(String input) {
+    return input.replaceAll("\"", "");
+  }
+
+  public static String doubleQuoteRegexEscape(String input) {
+    return String.format("%s(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)", input);
+  }
+
   private static String getFormattedDateFromEpoch(long epochTimestamp, String format) {
     Instant instant = Instant.ofEpochMilli(epochTimestamp);
     LocalDateTime dateTime = LocalDateTime.ofInstant(instant, ZoneId.systemDefault());
diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/tag_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/tag_index_mapping.json
index f0f0ac7291a2..f05b718c4871 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/en/tag_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/en/tag_index_mapping.json
@@ -108,7 +108,7 @@
         "type": "boolean"
       },
       "disabled": {
-        "type": "text"
+        "type": "boolean"
       },
       "entityType": {
         "type": "keyword"
diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json
index c2fc82835b94..53dfc36a2477 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/en/test_case_index_mapping.json
@@ -447,6 +447,10 @@
       "deleted": {
         "type": "boolean"
       },
+      "originEntityFQN": {
+        "type": "keyword",
+        "normalizer": "lowercase_normalizer"
+      },
       "table": {
         "properties": {
           "id": {
diff --git a/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json
index a402ebfdff93..237d15572859 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/en/test_suite_index_mapping.json
@@ -439,6 +439,10 @@
             "type": "text"
           }
         }
+      },
+      "serviceType": {
+        "type": "keyword",
+        "normalizer": "lowercase_normalizer"
       }
     }
   }
diff --git a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json
index cf47f916fd05..99eef985c98c 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/indexMapping.json
@@ -101,7 +101,7 @@
     "indexName": "test_case_result_search_index",
     "indexMappingFile": "/elasticsearch/%s/test_case_result_index_mapping.json",
     "alias": "testCaseResult",
-    "parentAliases": ["testSuite", "testCase", "all"],
+    "parentAliases": ["testSuite", "testCase"],
     "childAliases": []
   },
   "testSuite": {
@@ -158,7 +158,7 @@
     "indexMappingFile": "/elasticsearch/%s/classification_index_mapping.json",
     "alias": "classification",
     "parentAliases": ["all"],
-    "childAliases": []
+    "childAliases": ["tag"]
   },
   "user": {
     "indexName": "user_search_index",
diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/tag_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/tag_index_mapping.json
index 074194a80884..9e74af72f8eb 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/jp/tag_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/jp/tag_index_mapping.json
@@ -99,6 +99,9 @@
         "deleted": {
           "type": "text"
         },
+        "disabled": {
+          "type": "boolean"
+        },
         "classification": {
           "properties": {
             "id": {
diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_index_mapping.json
index 70ab60c98ccb..0b502f4cb9cc 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/jp/test_case_index_mapping.json
@@ -47,6 +47,10 @@
   },
   "mappings": {
     "properties": {
+      "originEntityFQN": {
+        "type": "keyword",
+        "normalizer": "lowercase_normalizer"
+      },
       "id": {
         "type": "text"
       },
diff --git a/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json
index 3d89c5623e7c..bec0237dcaf8 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json
@@ -419,6 +419,10 @@
           }
         }
       },
+      "serviceType": {
+        "type": "keyword",
+        "normalizer": "lowercase_normalizer"
+      },
       "totalVotes": {
         "type": "long",
         "null_value": 0
diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/tag_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/tag_index_mapping.json
index c107c5333dfa..0171e71bab71 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/zh/tag_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/zh/tag_index_mapping.json
@@ -97,6 +97,9 @@
       "deleted": {
         "type": "boolean"
       },
+      "disabled": {
+        "type": "boolean"
+      },
       "classification": {
         "properties": {
           "id": {
diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_index_mapping.json
index 28738e686170..8cc5dee6fee2 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/zh/test_case_index_mapping.json
@@ -37,6 +37,10 @@
   },
   "mappings": {
     "properties": {
+      "originEntityFQN": {
+        "type": "keyword",
+        "normalizer": "lowercase_normalizer"
+      },
       "id": {
         "type": "text"
       },
diff --git a/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json b/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json
index fa6d86f8550c..38337ae9ce5f 100644
--- a/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json
+++ b/openmetadata-service/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json
@@ -408,6 +408,10 @@
           }
         }
       },
+      "serviceType": {
+        "type": "keyword",
+        "normalizer": "lowercase_normalizer"
+      },
       "totalVotes": {
         "type": "long",
         "null_value": 0
diff --git a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json
index 3169c3634a42..d853ea037dc9 100644
--- a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json
+++ b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json
@@ -39,6 +39,7 @@
       "storedProcedure",
       "dataProduct",
       "testCaseResolutionStatus",
+      "testCaseResult",
       "apiService",
       "apiEndpoint",
       "apiCollection"
diff --git a/openmetadata-service/src/main/resources/json/data/policy/OrganizationPolicy.json b/openmetadata-service/src/main/resources/json/data/policy/OrganizationPolicy.json
index 3ad245e38b54..5036e370e015 100644
--- a/openmetadata-service/src/main/resources/json/data/policy/OrganizationPolicy.json
+++ b/openmetadata-service/src/main/resources/json/data/policy/OrganizationPolicy.json
@@ -22,6 +22,13 @@
       "operations": ["EditOwners"],
       "effect": "allow",
       "condition": "noOwner()"
+    },
+    {
+      "name": "OrganizationPolicy-ViewAll-Rule",
+      "description" : "Allow all users to discover data assets.",
+      "resources" : ["all"],
+      "operations": ["ViewAll"],
+      "effect": "allow"
     }
   ]
 }
\ No newline at end of file
diff --git a/openmetadata-service/src/main/resources/json/data/testConnections/api/rest.json b/openmetadata-service/src/main/resources/json/data/testConnections/api/rest.json
index a2ab1060f47e..2a10215658ca 100644
--- a/openmetadata-service/src/main/resources/json/data/testConnections/api/rest.json
+++ b/openmetadata-service/src/main/resources/json/data/testConnections/api/rest.json
@@ -1,5 +1,5 @@
 {
-    "name": "REST",
+    "name": "Rest",
     "displayName": "REST Test Connection",
     "description": "This Test Connection validates the schema provided for openapi",
     "steps": [
diff --git a/openmetadata-service/src/main/resources/json/data/tests/tableDiff.json b/openmetadata-service/src/main/resources/json/data/tests/tableDiff.json
index 074284eef396..ed0b68ec70fa 100644
--- a/openmetadata-service/src/main/resources/json/data/tests/tableDiff.json
+++ b/openmetadata-service/src/main/resources/json/data/tests/tableDiff.json
@@ -42,6 +42,13 @@
       "description": "Use this where clause to filter the rows to compare.",
       "dataType": "STRING",
       "required": false
+    },
+    {
+      "name": "caseSensitiveColumns",
+      "displayName": "Case sensitive columns",
+      "description": "Use case sensitivity when comparing the columns.",
+      "dataType": "BOOLEAN",
+      "required": false
     }
   ],
   "provider": "system",
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java
index 83eaa7add77e..ba7cb071a669 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/OpenMetadataApplicationTest.java
@@ -99,7 +99,7 @@ public abstract class OpenMetadataApplicationTest {
       "org.testcontainers.containers.MySQLContainer";
   private static final String JDBC_CONTAINER_IMAGE = "mysql:8";
   private static final String ELASTIC_SEARCH_CONTAINER_IMAGE =
-      "docker.elastic.co/elasticsearch/elasticsearch:8.10.2";
+      "docker.elastic.co/elasticsearch/elasticsearch:8.11.4";
 
   private static String HOST;
   private static String PORT;
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluatorResourceTest.java
similarity index 79%
rename from openmetadata-service/src/test/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluatorTest.java
rename to openmetadata-service/src/test/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluatorResourceTest.java
index 3b9a10d7cb70..b149a70c9859 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluatorTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/events/subscription/AlertsRuleEvaluatorResourceTest.java
@@ -14,7 +14,9 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.openmetadata.schema.api.data.CreateTable;
+import org.openmetadata.schema.api.tests.CreateTestCase;
 import org.openmetadata.schema.entity.data.Table;
+import org.openmetadata.schema.tests.TestCase;
 import org.openmetadata.schema.tests.type.TestCaseResult;
 import org.openmetadata.schema.tests.type.TestCaseStatus;
 import org.openmetadata.schema.type.ChangeDescription;
@@ -23,20 +25,24 @@
 import org.openmetadata.schema.type.ColumnDataType;
 import org.openmetadata.schema.type.EventType;
 import org.openmetadata.schema.type.FieldChange;
+import org.openmetadata.schema.type.Include;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.OpenMetadataApplicationTest;
 import org.openmetadata.service.resources.EntityResourceTest;
 import org.openmetadata.service.resources.databases.TableResourceTest;
+import org.openmetadata.service.resources.dqtests.TestCaseResourceTest;
 import org.springframework.expression.EvaluationContext;
 import org.springframework.expression.spel.support.SimpleEvaluationContext;
 
 class AlertsRuleEvaluatorResourceTest extends OpenMetadataApplicationTest {
   private static TableResourceTest tableResourceTest;
+  private static TestCaseResourceTest testCaseResourceTest;
 
   @BeforeAll
   public static void setup(TestInfo test) throws URISyntaxException, IOException {
     tableResourceTest = new TableResourceTest();
     tableResourceTest.setup(test);
+    testCaseResourceTest = new TestCaseResourceTest();
   }
 
   @Test
@@ -47,6 +53,7 @@ void test_matchAnySource() {
     AlertsRuleEvaluator alertsRuleEvaluator = new AlertsRuleEvaluator(changeEvent);
     EvaluationContext evaluationContext =
         SimpleEvaluationContext.forReadOnlyDataBinding()
+            .withInstanceMethods()
             .withRootObject(alertsRuleEvaluator)
             .build();
     assertTrue(evaluateExpression("matchAnySource('alert')", evaluationContext));
@@ -73,6 +80,7 @@ void test_matchAnyOwnerName(TestInfo test) throws IOException {
     AlertsRuleEvaluator alertsRuleEvaluator = new AlertsRuleEvaluator(changeEvent);
     EvaluationContext evaluationContext =
         SimpleEvaluationContext.forReadOnlyDataBinding()
+            .withInstanceMethods()
             .withRootObject(alertsRuleEvaluator)
             .build();
     assertTrue(
@@ -81,12 +89,25 @@ void test_matchAnyOwnerName(TestInfo test) throws IOException {
     assertFalse(evaluateExpression("matchAnyOwnerName('tempName')", evaluationContext));
   }
 
-  // issue: https://github.com/open-metadata/OpenMetadata/issues/10376
-  void test_matchAnyEntityFqn(TestInfo test) throws IOException {
+  @Test
+  void test_matchAnyEntityFqn() throws IOException {
     // Create Table Entity
+    // SpEl parsing fails for non-basic UTF-8
+    // https://github.com/open-metadata/OpenMetadata/issues/10376
     List<Column> columns = List.of(TableResourceTest.getColumn(C1, ColumnDataType.INT, null));
-    CreateTable create = tableResourceTest.createRequest(test).withColumns(columns);
+    CreateTable create = tableResourceTest.createRequest("table").withColumns(columns);
     Table createdTable = tableResourceTest.createAndCheckEntity(create, ADMIN_AUTH_HEADERS);
+    createdTable.setFullyQualifiedName(
+        "ServiceName.DatabaseName.SchemaName." + createdTable.getName());
+    CreateTestCase createTestCase = testCaseResourceTest.createRequest("testCase");
+    TestCase testCase =
+        testCaseResourceTest.createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS);
+    testCase = Entity.getEntity(testCase.getEntityReference(), "testSuites", Include.ALL);
+    testCase.setFullyQualifiedName(
+        "ServiceName.DatabaseName.SchemaName.table." + testCase.getName());
+    String testSuiteFqn =
+        "ServiceName.DatabaseName.SchemaName.table." + testCase.getName() + ".testSuite";
+    testCase.getTestSuites().get(0).setFullyQualifiedName(testSuiteFqn);
 
     // Create a change Event with the Entity Table
     ChangeEvent changeEvent = new ChangeEvent();
@@ -97,11 +118,27 @@ void test_matchAnyEntityFqn(TestInfo test) throws IOException {
     AlertsRuleEvaluator alertsRuleEvaluator = new AlertsRuleEvaluator(changeEvent);
     EvaluationContext evaluationContext =
         SimpleEvaluationContext.forReadOnlyDataBinding()
+            .withInstanceMethods()
             .withRootObject(alertsRuleEvaluator)
             .build();
     String fqn = createdTable.getFullyQualifiedName();
-    assertTrue(evaluateExpression("matchAnyEntityFqn('" + fqn + "')", evaluationContext));
-    assertFalse(evaluateExpression("matchAnyEntityFqn('testFQN1')", evaluationContext));
+    assertTrue(evaluateExpression("matchAnyEntityFqn({'" + fqn + "'})", evaluationContext));
+    assertFalse(evaluateExpression("(matchAnyEntityFqn({'FOO'}))", evaluationContext));
+
+    // Create a change Event with the Entity Test Case
+    changeEvent = new ChangeEvent();
+    changeEvent.setEntityType(Entity.TEST_CASE);
+    changeEvent.setEntity(testCase);
+    // Test Entity FQN match for test case
+    alertsRuleEvaluator = new AlertsRuleEvaluator(changeEvent);
+    evaluationContext =
+        SimpleEvaluationContext.forReadOnlyDataBinding()
+            .withInstanceMethods()
+            .withRootObject(alertsRuleEvaluator)
+            .build();
+    assertTrue(
+        evaluateExpression("matchAnyEntityFqn({'" + testSuiteFqn + "'})", evaluationContext));
+    assertFalse(evaluateExpression("(matchAnyEntityFqn({'FOO'}))", evaluationContext));
   }
 
   @Test
@@ -120,6 +157,7 @@ void test_matchAnyEntityId(TestInfo test) throws IOException {
     AlertsRuleEvaluator alertsRuleEvaluator = new AlertsRuleEvaluator(changeEvent);
     EvaluationContext evaluationContext =
         SimpleEvaluationContext.forReadOnlyDataBinding()
+            .withInstanceMethods()
             .withRootObject(alertsRuleEvaluator)
             .build();
     String id = createdTable.getId().toString();
@@ -138,6 +176,7 @@ void test_matchAnyEventType() {
     AlertsRuleEvaluator alertsRuleEvaluator = new AlertsRuleEvaluator(changeEvent);
     EvaluationContext evaluationContext =
         SimpleEvaluationContext.forReadOnlyDataBinding()
+            .withInstanceMethods()
             .withRootObject(alertsRuleEvaluator)
             .build();
     assertTrue(evaluateExpression("matchAnyEventType('entityCreated')", evaluationContext));
@@ -164,6 +203,7 @@ void test_matchTestResult() {
     AlertsRuleEvaluator alertsRuleEvaluator = new AlertsRuleEvaluator(changeEvent);
     EvaluationContext evaluationContext =
         SimpleEvaluationContext.forReadOnlyDataBinding()
+            .withInstanceMethods()
             .withRootObject(alertsRuleEvaluator)
             .build();
     assertTrue(evaluateExpression("matchTestResult('Success')", evaluationContext));
@@ -180,6 +220,7 @@ void test_matchUpdatedBy() {
     AlertsRuleEvaluator alertsRuleEvaluator = new AlertsRuleEvaluator(changeEvent);
     EvaluationContext evaluationContext =
         SimpleEvaluationContext.forReadOnlyDataBinding()
+            .withInstanceMethods()
             .withRootObject(alertsRuleEvaluator)
             .build();
     assertTrue(evaluateExpression("matchUpdatedBy('test')", evaluationContext));
@@ -200,6 +241,7 @@ void test_matchAnyFieldChange() {
     AlertsRuleEvaluator alertsRuleEvaluator = new AlertsRuleEvaluator(changeEvent);
     EvaluationContext evaluationContext =
         SimpleEvaluationContext.forReadOnlyDataBinding()
+            .withInstanceMethods()
             .withRootObject(alertsRuleEvaluator)
             .build();
     assertTrue(evaluateExpression("matchAnyFieldChange('test')", evaluationContext));
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
index c4bd1aed2528..aceef19862d8 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/EntityResourceTest.java
@@ -403,8 +403,7 @@ public abstract class EntityResourceTest<T extends EntityInterface, K extends Cr
   public static Type TIMESTAMP_TYPE;
 
   public static Type ENUM_TYPE;
-
-  public static Type ENUM_WITH_DESCRIPTIONS_TYPE;
+  public static Type TABLE_TYPE;
 
   // Run webhook related tests randomly. This will ensure these tests are not run for every entity
   // evey time junit tests are run to save time. But over the course of development of a release,
@@ -2894,7 +2893,7 @@ protected void validateLatestVersion(
   /**
    * Helper function to generate JSON PATCH, submit PATCH API request and validate response.
    */
-  protected final T patchEntityAndCheck(
+  public final T patchEntityAndCheck(
       T updated,
       String originalJson,
       Map<String, String> authHeaders,
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
index 7f4c06816c32..d4c0138c3573 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java
@@ -108,6 +108,7 @@
 import org.openmetadata.service.resources.databases.TableResourceTest;
 import org.openmetadata.service.resources.feeds.FeedResourceTest;
 import org.openmetadata.service.resources.feeds.MessageParser;
+import org.openmetadata.service.search.SearchAggregation;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.SearchRepository;
 import org.openmetadata.service.search.indexes.TestCaseIndex;
@@ -3082,10 +3083,9 @@ void aggregate_testCaseResults(TestInfo testInfo) throws IOException, ParseExcep
     // Test aggregation
     String aggregationQuery =
         "bucketName=dates:aggType=date_histogram:field=timestamp&calendar_interval=1d,bucketName=dimesion:aggType=terms:field=testDefinition.dataQualityDimension";
-    Map<String, Object> aggregationString =
-        SearchIndexUtils.buildAggregationString(aggregationQuery);
+    SearchAggregation aggregation = SearchIndexUtils.buildAggregationTree(aggregationQuery);
     DataQualityReport dataQualityReport =
-        searchRepository.genericAggregation(null, "testCaseResult", aggregationString);
+        searchRepository.genericAggregation(null, "testCaseResult", aggregation);
     assertNotNull(dataQualityReport.getData());
   }
 
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java
index f7e918c2a647..b4e0cff94a6e 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java
@@ -2,10 +2,11 @@
 
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import static javax.ws.rs.core.Response.Status.NOT_FOUND;
-import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
 import static org.openmetadata.service.util.TestUtils.LONG_ENTITY_NAME;
 import static org.openmetadata.service.util.TestUtils.assertListNotNull;
@@ -13,21 +14,22 @@
 import static org.openmetadata.service.util.TestUtils.assertResponse;
 import static org.openmetadata.service.util.TestUtils.assertResponseContains;
 
-import es.org.elasticsearch.search.aggregations.AggregationBuilder;
-import es.org.elasticsearch.search.aggregations.AggregationBuilders;
+import es.org.elasticsearch.client.Request;
+import es.org.elasticsearch.client.RestClient;
 import java.io.IOException;
 import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.UUID;
 import java.util.stream.Collectors;
-import javax.json.JsonObject;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.Response;
 import org.apache.http.client.HttpResponseException;
+import org.apache.http.util.EntityUtils;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
@@ -48,7 +50,7 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.resources.EntityResourceTest;
 import org.openmetadata.service.resources.databases.TableResourceTest;
-import org.openmetadata.service.search.elasticsearch.ElasticSearchClient;
+import org.openmetadata.service.search.models.IndexMapping;
 import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.util.ResultList;
 import org.openmetadata.service.util.TestUtils;
@@ -279,6 +281,38 @@ void test_inheritOwnerFromTable(TestInfo test) throws IOException {
     assertOwners(table.getOwners(), testSuite.getOwners());
   }
 
+  @Test
+  void test_inheritDomainFromTable(TestInfo test) throws IOException {
+    TableResourceTest tableResourceTest = new TableResourceTest();
+    CreateTable tableReq =
+        tableResourceTest
+            .createRequest(test)
+            .withColumns(
+                List.of(
+                    new Column()
+                        .withName(C1)
+                        .withDisplayName("c1")
+                        .withDataType(ColumnDataType.VARCHAR)
+                        .withDataLength(10)))
+            .withDomain(DOMAIN1.getFullyQualifiedName());
+    Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS);
+    table = tableResourceTest.getEntity(table.getId(), "*", ADMIN_AUTH_HEADERS);
+    CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName());
+    TestSuite executableTestSuite =
+        createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS);
+    TestSuite testSuite = getEntity(executableTestSuite.getId(), "domain", ADMIN_AUTH_HEADERS);
+    assertEquals(DOMAIN1.getId(), testSuite.getDomain().getId());
+    ResultList<TestSuite> testSuites =
+        listEntitiesFromSearch(
+            Map.of("domain", DOMAIN1.getFullyQualifiedName(), "fields", "domain"),
+            100,
+            0,
+            ADMIN_AUTH_HEADERS);
+    assertTrue(
+        testSuites.getData().stream()
+            .allMatch(ts -> ts.getDomain().getId().equals(DOMAIN1.getId())));
+  }
+
   @Test
   void post_createLogicalTestSuiteAndAddTests_200(TestInfo test) throws IOException {
     TestCaseResourceTest testCaseResourceTest = new TestCaseResourceTest();
@@ -622,178 +656,6 @@ protected void post_entityCreateWithInvalidName_400() {
         TestUtils.getEntityNameLengthError(entityClass));
   }
 
-  @Test
-  void buildElasticsearchAggregationFromJson(TestInfo test) {
-    JsonObject aggregationJson;
-    List<AggregationBuilder> actual;
-    List<AggregationBuilder> expected = new ArrayList<>();
-    String aggregationQuery;
-
-    // Test aggregation with nested aggregation
-    aggregationQuery =
-        """
-            {
-              "aggregations": {
-                "test_case_results": {
-                  "nested": {
-                    "path": "testCaseResultSummary"
-                  },
-                  "aggs": {
-                    "status_counts": {
-                      "terms": {
-                        "field": "testCaseResultSummary.status"
-                      }
-                    }
-                  }
-                }
-              }
-            }
-            """;
-
-    expected.add(
-        AggregationBuilders.nested("testCaseResultSummary", "testCaseResultSummary")
-            .subAggregation(
-                AggregationBuilders.terms("status_counts").field("testCaseResultSummary.status")));
-
-    aggregationJson = JsonUtils.readJson(aggregationQuery).asJsonObject();
-    actual = ElasticSearchClient.buildAggregation(aggregationJson.getJsonObject("aggregations"));
-    assertThat(actual).hasSameElementsAs(expected);
-
-    // Test aggregation with multiple aggregations
-    aggregationQuery =
-        """
-            {
-              "aggregations": {
-                "my-first-agg-name": {
-                  "terms": {
-                    "field": "my-field"
-                  }
-                },
-                "my-second-agg-name": {
-                  "terms": {
-                    "field": "my-other-field"
-                  }
-                }
-              }
-            }
-            """;
-    aggregationJson = JsonUtils.readJson(aggregationQuery).asJsonObject();
-
-    expected.clear();
-    expected.addAll(
-        List.of(
-            AggregationBuilders.terms("my-second-agg-name").field("my-other-field"),
-            AggregationBuilders.terms("my-first-agg-name").field("my-field")));
-
-    actual = ElasticSearchClient.buildAggregation(aggregationJson.getJsonObject("aggregations"));
-    assertThat(actual).hasSameElementsAs(expected);
-
-    // Test aggregation with multiple aggregations including a nested one which has itself multiple
-    // aggregations
-    aggregationQuery =
-        """
-            {
-              "aggregations": {
-                "my-first-agg-name": {
-                  "terms": {
-                    "field": "my-field"
-                  }
-                },
-                "test_case_results": {
-                  "nested": {
-                    "path": "testCaseResultSummary"
-                  },
-                  "aggs": {
-                    "status_counts": {
-                      "terms": {
-                        "field": "testCaseResultSummary.status"
-                      }
-                    },
-                    "other_status_counts": {
-                      "terms": {
-                        "field": "testCaseResultSummary.status"
-                      }
-                    }
-                  }
-                }
-              }
-            }
-            """;
-    aggregationJson = JsonUtils.readJson(aggregationQuery).asJsonObject();
-
-    expected.clear();
-    expected.addAll(
-        List.of(
-            AggregationBuilders.nested("testCaseResultSummary", "testCaseResultSummary")
-                .subAggregation(
-                    AggregationBuilders.terms("status_counts")
-                        .field("testCaseResultSummary.status"))
-                .subAggregation(
-                    AggregationBuilders.terms("other_status_counts")
-                        .field("testCaseResultSummary.status")),
-            AggregationBuilders.terms("my-first-agg-name").field("my-field")));
-
-    actual = ElasticSearchClient.buildAggregation(aggregationJson.getJsonObject("aggregations"));
-    assertThat(actual).hasSameElementsAs(expected);
-
-    // Test aggregation with nested aggregation and sub-aggregation
-    aggregationQuery =
-        """
-          {
-            "aggregations": {
-              "my-first-agg-name": {
-                "terms": {
-                  "field": "my-field"
-                },
-                "aggs": {
-                  "my-nested-agg-name": {
-                    "terms": {
-                      "field": "my-other-field"
-                    }
-                  }
-                }
-              },
-              "test_case_results": {
-                "nested": {
-                  "path": "testCaseResultSummary"
-                },
-                "aggs": {
-                  "status_counts": {
-                    "terms": {
-                      "field": "testCaseResultSummary.status"
-                    }
-                  },
-                  "other_status_counts": {
-                    "terms": {
-                      "field": "testCaseResultSummary.status"
-                    }
-                  }
-                }
-              }
-            }
-          }
-          """;
-    aggregationJson = JsonUtils.readJson(aggregationQuery).asJsonObject();
-
-    expected.clear();
-    expected.addAll(
-        List.of(
-            AggregationBuilders.nested("testCaseResultSummary", "testCaseResultSummary")
-                .subAggregation(
-                    AggregationBuilders.terms("status_counts")
-                        .field("testCaseResultSummary.status"))
-                .subAggregation(
-                    AggregationBuilders.terms("other_status_counts")
-                        .field("testCaseResultSummary.status")),
-            AggregationBuilders.terms("my-first-agg-name")
-                .field("my-field")
-                .subAggregation(
-                    AggregationBuilders.terms("my-nested-agg-name").field("my-other-field"))));
-
-    actual = ElasticSearchClient.buildAggregation(aggregationJson.getJsonObject("aggregations"));
-    assertThat(actual).hasSameElementsAs(expected);
-  }
-
   @Test
   void delete_LogicalTestSuite_200(TestInfo test) throws IOException {
     TestCaseResourceTest testCaseResourceTest = new TestCaseResourceTest();
@@ -877,6 +739,56 @@ void get_listTestSuiteFromSearchWithPagination(TestInfo testInfo) throws IOExcep
     }
   }
 
+  @Test
+  void create_executableTestSuiteAndCheckSearchClient(TestInfo test) throws IOException {
+    TableResourceTest tableResourceTest = new TableResourceTest();
+    CreateTable tableReq =
+        tableResourceTest
+            .createRequest(test)
+            .withColumns(
+                List.of(
+                    new Column()
+                        .withName(C1)
+                        .withDisplayName("c1")
+                        .withDataType(ColumnDataType.VARCHAR)
+                        .withDataLength(10)));
+    Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS);
+    CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName());
+    TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS);
+    RestClient searchClient = getSearchClient();
+    IndexMapping index = Entity.getSearchRepository().getIndexMapping(Entity.TABLE);
+    es.org.elasticsearch.client.Response response;
+    Request request =
+        new Request(
+            "GET",
+            String.format(
+                "%s/_search", index.getIndexName(Entity.getSearchRepository().getClusterAlias())));
+    String query =
+        String.format(
+            "{\"size\": 10,\"query\":{\"bool\":{\"must\":[{\"term\":{\"_id\":\"%s\"}}]}}}",
+            table.getId().toString());
+    request.setJsonEntity(query);
+    try {
+      response = searchClient.performRequest(request);
+    } finally {
+      searchClient.close();
+    }
+    String jsonString = EntityUtils.toString(response.getEntity());
+    HashMap<String, Object> map =
+        (HashMap<String, Object>) JsonUtils.readOrConvertValue(jsonString, HashMap.class);
+    LinkedHashMap<String, Object> hits = (LinkedHashMap<String, Object>) map.get("hits");
+    ArrayList<LinkedHashMap<String, Object>> hitsList =
+        (ArrayList<LinkedHashMap<String, Object>>) hits.get("hits");
+    assertNotEquals(0, hitsList.size());
+    assertTrue(
+        hitsList.stream()
+            .allMatch(
+                hit ->
+                    ((LinkedHashMap<String, Object>) hit.get("_source"))
+                        .get("id")
+                        .equals(table.getId().toString())));
+  }
+
   public ResultList<TestSuite> getTestSuites(
       Integer limit,
       String fields,
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
index aa71c798e794..248bc9b51fbc 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java
@@ -48,10 +48,12 @@
 import static org.openmetadata.service.util.TestUtils.validateTagLabel;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -87,6 +89,7 @@
 import org.openmetadata.schema.type.TagLabel.TagSource;
 import org.openmetadata.schema.type.TaskStatus;
 import org.openmetadata.schema.type.csv.CsvImportResult;
+import org.openmetadata.schema.type.customProperties.TableConfig;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.CatalogExceptionMessage;
 import org.openmetadata.service.jdbi3.EntityRepository.EntityUpdater;
@@ -644,33 +647,28 @@ invalidCustomPropertyKeyRecord, invalidCustomPropertyKey(11, "invalidCustomPrope
         };
     assertRows(result, expectedRows);
 
-    // Create glossaryTerm with  Invalid custom property of type enumWithDescriptions
+    // Create glossaryTerm with  Invalid custom property of type table
+    TableConfig tableConfig =
+        new TableConfig().withColumns(Set.of("columnName1", "columnName2", "columnName3"));
     CustomProperty glossaryTermEnumCp =
         new CustomProperty()
-            .withName("glossaryTermEnumWithDescriptionsCp")
-            .withDescription("enumWithDescriptions type custom property with multiselect = true")
-            .withPropertyType(ENUM_WITH_DESCRIPTIONS_TYPE.getEntityReference())
+            .withName("glossaryTermTableCp")
+            .withDescription("table  type custom property ")
+            .withPropertyType(TABLE_TYPE.getEntityReference())
             .withCustomPropertyConfig(
                 new CustomPropertyConfig()
-                    .withConfig(
-                        Map.of(
-                            "values",
-                            List.of(
-                                Map.of("key", "key1", "description", "description1"),
-                                Map.of("key", "key2", "description", "description2")),
-                            "multiSelect",
-                            true)));
+                    .withConfig(Map.of("columns", new ArrayList<>(tableConfig.getColumns()))));
     entityType =
         typeResourceTest.getEntityByName(
             Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS);
     entityType =
         typeResourceTest.addAndCheckCustomProperty(
             entityType.getId(), glossaryTermEnumCp, OK, ADMIN_AUTH_HEADERS);
-    String invalidEnumWithDescriptionRecord =
-        ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermEnumWithDescriptionsCp:key1|key3";
-    String invalidEnumWithDescriptionValue =
-        ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,glossaryTermEnumWithDescriptionsCp:key1|key3";
-    csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidEnumWithDescriptionValue), null);
+    String invalidTableTypeRecord =
+        ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,\"glossaryTermTableCp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value,row_1_col4_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value,row_2_col4_Value\"";
+    String invalidTableTypeValue =
+        ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,\"glossaryTermTableCp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value,row_1_col4_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value,row_2_col4_Value\"";
+    csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidTableTypeValue), null);
     result = importCsv(glossaryName, csv, false);
     Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true);
     assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1);
@@ -678,12 +676,12 @@ invalidCustomPropertyKeyRecord, invalidCustomPropertyKey(11, "invalidCustomPrope
         new String[] {
           resultsHeader,
           getFailedRecord(
-              invalidEnumWithDescriptionRecord,
+              invalidTableTypeRecord,
               invalidCustomPropertyValue(
                   11,
-                  "glossaryTermEnumWithDescriptionsCp",
-                  ENUM_WITH_DESCRIPTIONS_TYPE.getDisplayName(),
-                  "key3 not found in propertyConfig of glossaryTermEnumWithDescriptionsCp"))
+                  "glossaryTermTableCp",
+                  "table",
+                  "Column count should be less than or equal to " + tableConfig.getMaxColumns()))
         };
     assertRows(result, expectedRows);
   }
@@ -812,19 +810,21 @@ void testGlossaryImportExport() throws IOException {
           .withPropertyType(ENUM_TYPE.getEntityReference())
           .withCustomPropertyConfig(enumConfig),
       new CustomProperty()
-          .withName("glossaryTermEnumWithDescriptionsCp")
-          .withDescription("enumWithDescriptions type custom property with multiselect = true")
-          .withPropertyType(ENUM_WITH_DESCRIPTIONS_TYPE.getEntityReference())
+          .withName("glossaryTermTableCol1Cp")
+          .withDescription("table type custom property with 1 column")
+          .withPropertyType(TABLE_TYPE.getEntityReference())
           .withCustomPropertyConfig(
               new CustomPropertyConfig()
                   .withConfig(
-                      Map.of(
-                          "values",
-                          List.of(
-                              Map.of("key", "key1", "description", "description1"),
-                              Map.of("key", "key2", "description", "description2")),
-                          "multiSelect",
-                          true)))
+                      Map.of("columns", List.of("columnName1", "columnName2", "columnName3")))),
+      new CustomProperty()
+          .withName("glossaryTermTableCol3Cp")
+          .withDescription("table type custom property with 3 columns")
+          .withPropertyType(TABLE_TYPE.getEntityReference())
+          .withCustomPropertyConfig(
+              new CustomPropertyConfig()
+                  .withConfig(
+                      Map.of("columns", List.of("columnName1", "columnName2", "columnName3")))),
     };
 
     for (CustomProperty customProperty : customProperties) {
@@ -845,7 +845,7 @@ void testGlossaryImportExport() throws IOException {
                 ",g2,dsp2,dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"",
                 user1, "Approved"),
             String.format(
-                "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,glossaryTermEnumWithDescriptionsCp:key1|key2",
+                "importExportTest.g1,g11,dsp2,dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,",
                 reviewerRef.get(0), team11, "Draft"));
 
     // Update terms with change in description
@@ -858,12 +858,13 @@ void testGlossaryImportExport() throws IOException {
                 ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"",
                 user1, user2, "Approved"),
             String.format(
-                "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,glossaryTermEnumWithDescriptionsCp:key1|key2",
+                "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\"",
                 reviewerRef.get(0), team11, "Draft"));
 
     // Add new row to existing rows
     List<String> newRecords =
-        listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved,");
+        listOf(
+            ",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\"");
     testImportExport(
         glossary.getName(), GlossaryCsv.HEADERS, createRecords, updateRecords, newRecords);
   }
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java
index 18a371eb20e2..1cd3e7d8264a 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/metadata/TypeResourceTest.java
@@ -25,10 +25,14 @@
 import static org.openmetadata.service.util.TestUtils.assertResponse;
 import static org.openmetadata.service.util.TestUtils.assertResponseContains;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import javax.ws.rs.client.WebTarget;
 import javax.ws.rs.core.Response.Status;
@@ -44,9 +48,8 @@
 import org.openmetadata.schema.type.ChangeDescription;
 import org.openmetadata.schema.type.CustomPropertyConfig;
 import org.openmetadata.schema.type.EntityReference;
-import org.openmetadata.schema.type.customproperties.EnumConfig;
-import org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig;
-import org.openmetadata.schema.type.customproperties.Value;
+import org.openmetadata.schema.type.customProperties.EnumConfig;
+import org.openmetadata.schema.type.customProperties.TableConfig;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.resources.EntityResourceTest;
 import org.openmetadata.service.resources.types.TypeResource;
@@ -73,7 +76,6 @@ public void setupTypes() throws HttpResponseException {
     STRING_TYPE = getEntityByName("string", "", ADMIN_AUTH_HEADERS);
     EMAIL_TYPE = getEntityByName("email", "", ADMIN_AUTH_HEADERS);
     ENUM_TYPE = getEntityByName("enum", "", ADMIN_AUTH_HEADERS);
-    ENUM_WITH_DESCRIPTIONS_TYPE = getEntityByName("enumWithDescriptions", "", ADMIN_AUTH_HEADERS);
     DATECP_TYPE = getEntityByName("date-cp", "", ADMIN_AUTH_HEADERS);
     DATETIMECP_TYPE = getEntityByName("dateTime-cp", "", ADMIN_AUTH_HEADERS);
     TIMECP_TYPE = getEntityByName("time-cp", "", ADMIN_AUTH_HEADERS);
@@ -85,6 +87,7 @@ public void setupTypes() throws HttpResponseException {
     NUMBER_TYPE = getEntityByName("number", "", ADMIN_AUTH_HEADERS);
     SQLQUERY_TYPE = getEntityByName("sqlQuery", "", ADMIN_AUTH_HEADERS);
     TIMESTAMP_TYPE = getEntityByName("timestamp", "", ADMIN_AUTH_HEADERS);
+    TABLE_TYPE = getEntityByName("table-cp", "", ADMIN_AUTH_HEADERS);
   }
 
   @Override
@@ -296,174 +299,126 @@ void put_patch_customProperty_enum_200() throws IOException {
   }
 
   @Test
-  void put_patch_customProperty_enumWithDescriptions_200() throws IOException {
+  void put_patch_customProperty_table_200() throws IOException {
     Type databaseEntity = getEntityByName("database", "customProperties", ADMIN_AUTH_HEADERS);
+    TableConfig tableConfig = new TableConfig();
 
-    // Add a custom property of type enumWithDescriptions with PUT
-    CustomProperty enumWithDescriptionsFieldA =
+    // Add a custom property of type table with PUT
+    CustomProperty tableTypeFieldA =
         new CustomProperty()
-            .withName("enumWithDescriptionsTest")
-            .withDescription("enumWithDescriptionsTest")
-            .withPropertyType(ENUM_WITH_DESCRIPTIONS_TYPE.getEntityReference());
+            .withName("tableCustomPropertyTest")
+            .withDescription("tableCustomPropertyTest description")
+            .withPropertyType(TABLE_TYPE.getEntityReference());
     ChangeDescription change = getChangeDescription(databaseEntity, MINOR_UPDATE);
-    fieldAdded(change, "customProperties", new ArrayList<>(List.of(enumWithDescriptionsFieldA)));
+    fieldAdded(change, "customProperties", new ArrayList<>(List.of(tableTypeFieldA)));
     Type finalDatabaseEntity = databaseEntity;
     ChangeDescription finalChange = change;
     assertResponseContains(
         () ->
             addCustomPropertyAndCheck(
                 finalDatabaseEntity.getId(),
-                enumWithDescriptionsFieldA,
+                tableTypeFieldA,
                 ADMIN_AUTH_HEADERS,
                 MINOR_UPDATE,
                 finalChange),
         Status.BAD_REQUEST,
-        "EnumWithDescriptions Custom Property Type must have customPropertyConfig.");
-    enumWithDescriptionsFieldA.setCustomPropertyConfig(
-        new CustomPropertyConfig().withConfig(new EnumWithDescriptionsConfig()));
+        "Table Custom Property Type must have config populated.");
+
+    tableTypeFieldA.setCustomPropertyConfig(
+        new CustomPropertyConfig().withConfig(new TableConfig()));
     ChangeDescription change1 = getChangeDescription(databaseEntity, MINOR_UPDATE);
     Type databaseEntity1 = databaseEntity;
     assertResponseContains(
         () ->
             addCustomPropertyAndCheck(
                 databaseEntity1.getId(),
-                enumWithDescriptionsFieldA,
+                tableTypeFieldA,
                 ADMIN_AUTH_HEADERS,
                 MINOR_UPDATE,
                 change1),
         Status.BAD_REQUEST,
-        "EnumWithDescriptions Custom Property Type must have customPropertyConfig populated with values.");
-
-    List<Value> valuesWithDuplicateKey =
-        List.of(
-            new Value().withKey("A").withDescription("Description A"),
-            new Value().withKey("B").withDescription("Description B"),
-            new Value().withKey("C").withDescription("Description C"),
-            new Value().withKey("C").withDescription("Description C"));
-
-    enumWithDescriptionsFieldA.setCustomPropertyConfig(
-        new CustomPropertyConfig()
-            .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithDuplicateKey)));
-    ChangeDescription change7 = getChangeDescription(databaseEntity, MINOR_UPDATE);
-    Type databaseEntity2 = databaseEntity;
+        "Custom Property table has invalid value columns size must be between "
+            + tableConfig.getMinColumns()
+            + " and "
+            + tableConfig.getMaxColumns());
+
+    ObjectMapper mapper = new ObjectMapper();
+    ObjectNode tableConfigJson = mapper.createObjectNode();
+    ArrayNode columnsArray = tableConfigJson.putArray("columns");
+    columnsArray.add("col 1");
+    columnsArray.add("col 2");
+    columnsArray.add("col");
+    columnsArray.add("col");
+
+    tableTypeFieldA.setCustomPropertyConfig(new CustomPropertyConfig().withConfig(tableConfigJson));
+    ChangeDescription change3 = getChangeDescription(databaseEntity, MINOR_UPDATE);
+    Type databaseEntity3 = databaseEntity;
     assertResponseContains(
         () ->
             addCustomPropertyAndCheck(
-                databaseEntity2.getId(),
-                enumWithDescriptionsFieldA,
+                databaseEntity3.getId(),
+                tableTypeFieldA,
                 ADMIN_AUTH_HEADERS,
                 MINOR_UPDATE,
-                change7),
+                change3),
         Status.BAD_REQUEST,
-        "EnumWithDescriptions Custom Property key cannot have duplicates.");
-    List<Value> valuesWithUniqueKey =
-        List.of(
-            new Value().withKey("A").withDescription("Description A"),
-            new Value().withKey("B").withDescription("Description B"),
-            new Value().withKey("C").withDescription("Description C"));
-    enumWithDescriptionsFieldA.setCustomPropertyConfig(
+        "Column names must be unique.");
+
+    tableTypeFieldA.setCustomPropertyConfig(
         new CustomPropertyConfig()
-            .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithUniqueKey)));
+            .withConfig(new TableConfig().withColumns(Set.of("column1", "column2", "column3"))));
     databaseEntity =
         addCustomPropertyAndCheck(
-            databaseEntity.getId(),
-            enumWithDescriptionsFieldA,
-            ADMIN_AUTH_HEADERS,
-            MINOR_UPDATE,
-            change);
+            databaseEntity.getId(), tableTypeFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
     assertCustomProperties(
-        new ArrayList<>(List.of(enumWithDescriptionsFieldA)), databaseEntity.getCustomProperties());
-    CustomPropertyConfig prevConfig = enumWithDescriptionsFieldA.getCustomPropertyConfig();
+        new ArrayList<>(List.of(tableTypeFieldA)), databaseEntity.getCustomProperties());
+
+    CustomPropertyConfig prevConfig = tableTypeFieldA.getCustomPropertyConfig();
+
     // Changing custom property description with PUT
-    enumWithDescriptionsFieldA.withDescription("updatedEnumWithDescriptionsTest");
-    ChangeDescription change2 = getChangeDescription(databaseEntity, MINOR_UPDATE);
+    tableTypeFieldA.withDescription("updated tableCustomPropertyTest description");
+    ChangeDescription change5 = getChangeDescription(databaseEntity, MINOR_UPDATE);
     fieldUpdated(
-        change2,
-        EntityUtil.getCustomField(enumWithDescriptionsFieldA, "description"),
-        "enumWithDescriptionsTest",
-        "updatedEnumWithDescriptionsTest");
+        change5,
+        EntityUtil.getCustomField(tableTypeFieldA, "description"),
+        "tableCustomPropertyTest description",
+        "updated tableCustomPropertyTest description");
     databaseEntity =
         addCustomPropertyAndCheck(
-            databaseEntity.getId(),
-            enumWithDescriptionsFieldA,
-            ADMIN_AUTH_HEADERS,
-            MINOR_UPDATE,
-            change2);
+            databaseEntity.getId(), tableTypeFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change5);
     assertCustomProperties(
-        new ArrayList<>(List.of(enumWithDescriptionsFieldA)), databaseEntity.getCustomProperties());
-
-    List<Value> valuesWithUniqueKeyAB =
-        List.of(
-            new Value().withKey("A").withDescription("Description A"),
-            new Value().withKey("B").withDescription("Description B"));
-    enumWithDescriptionsFieldA.setCustomPropertyConfig(
-        new CustomPropertyConfig()
-            .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithUniqueKeyAB)));
-    ChangeDescription change3 = getChangeDescription(databaseEntity, MINOR_UPDATE);
-    assertResponseContains(
-        () ->
-            addCustomPropertyAndCheck(
-                databaseEntity1.getId(),
-                enumWithDescriptionsFieldA,
-                ADMIN_AUTH_HEADERS,
-                MINOR_UPDATE,
-                change3),
-        Status.BAD_REQUEST,
-        "Existing EnumWithDescriptions Custom Property values cannot be removed.");
+        new ArrayList<>(List.of(tableTypeFieldA)), databaseEntity.getCustomProperties());
 
-    enumWithDescriptionsFieldA.setCustomPropertyConfig(
-        new CustomPropertyConfig()
-            .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithDuplicateKey)));
-    ChangeDescription change4 = getChangeDescription(databaseEntity, MINOR_UPDATE);
-    assertResponseContains(
-        () ->
-            addCustomPropertyAndCheck(
-                databaseEntity1.getId(),
-                enumWithDescriptionsFieldA,
-                ADMIN_AUTH_HEADERS,
-                MINOR_UPDATE,
-                change4),
-        Status.BAD_REQUEST,
-        "EnumWithDescriptions Custom Property key cannot have duplicates.");
-    valuesWithUniqueKey =
-        List.of(
-            new Value().withKey("A").withDescription("Description A"),
-            new Value().withKey("B").withDescription("Description B"),
-            new Value().withKey("C").withDescription("Description C"),
-            new Value().withKey("D").withDescription("Description D"));
-    ChangeDescription change5 = getChangeDescription(databaseEntity, MINOR_UPDATE);
-    enumWithDescriptionsFieldA.setCustomPropertyConfig(
+    ChangeDescription change6 = getChangeDescription(databaseEntity, MINOR_UPDATE);
+    tableTypeFieldA.setCustomPropertyConfig(
         new CustomPropertyConfig()
-            .withConfig(new EnumWithDescriptionsConfig().withValues(valuesWithUniqueKey)));
+            .withConfig(new TableConfig().withColumns(Set.of("column-1", "column-2", "column-3"))));
     fieldUpdated(
-        change5,
-        EntityUtil.getCustomField(enumWithDescriptionsFieldA, "customPropertyConfig"),
+        change6,
+        EntityUtil.getCustomField(tableTypeFieldA, "customPropertyConfig"),
         prevConfig,
-        enumWithDescriptionsFieldA.getCustomPropertyConfig());
+        tableTypeFieldA.getCustomPropertyConfig());
     databaseEntity =
         addCustomPropertyAndCheck(
-            databaseEntity.getId(),
-            enumWithDescriptionsFieldA,
-            ADMIN_AUTH_HEADERS,
-            MINOR_UPDATE,
-            change5);
+            databaseEntity.getId(), tableTypeFieldA, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change6);
     assertCustomProperties(
-        new ArrayList<>(List.of(enumWithDescriptionsFieldA)), databaseEntity.getCustomProperties());
+        new ArrayList<>(List.of(tableTypeFieldA)), databaseEntity.getCustomProperties());
 
     // Changing custom property description with PATCH
     // Changes from this PATCH is consolidated with the previous changes
-    enumWithDescriptionsFieldA.withDescription("updated2");
+    tableTypeFieldA.withDescription("updated tableCustomPropertyTest description 2");
     String json = JsonUtils.pojoToJson(databaseEntity);
-    databaseEntity.setCustomProperties(List.of(enumWithDescriptionsFieldA));
+    databaseEntity.setCustomProperties(List.of(tableTypeFieldA));
     change = getChangeDescription(databaseEntity, CHANGE_CONSOLIDATED);
+
     fieldUpdated(
-        change5,
-        EntityUtil.getCustomField(enumWithDescriptionsFieldA, "description"),
-        "updatedEnumWithDescriptionsTest",
-        "updated2");
+        change6,
+        EntityUtil.getCustomField(tableTypeFieldA, "description"),
+        "updated tableCustomPropertyTest description",
+        "updated tableCustomPropertyTest description 2");
 
     databaseEntity =
-        patchEntityAndCheck(databaseEntity, json, ADMIN_AUTH_HEADERS, CHANGE_CONSOLIDATED, change5);
+        patchEntityAndCheck(databaseEntity, json, ADMIN_AUTH_HEADERS, CHANGE_CONSOLIDATED, change6);
   }
 
   @Test
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java
index a29aa7d93b64..6e282b71f950 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/searchindex/SearchIndexResourceTest.java
@@ -17,6 +17,7 @@
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import static javax.ws.rs.core.Response.Status.NOT_FOUND;
 import static javax.ws.rs.core.Response.Status.OK;
+import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -33,6 +34,13 @@
 import static org.openmetadata.service.util.TestUtils.assertListNull;
 import static org.openmetadata.service.util.TestUtils.assertResponse;
 
+import es.org.elasticsearch.script.Script;
+import es.org.elasticsearch.search.aggregations.AggregationBuilders;
+import es.org.elasticsearch.search.aggregations.BaseAggregationBuilder;
+import es.org.elasticsearch.search.aggregations.PipelineAggregatorBuilders;
+import es.org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import es.org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude;
+import es.org.elasticsearch.search.sort.SortOrder;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -52,6 +60,7 @@
 import org.openmetadata.schema.entity.data.SearchIndex;
 import org.openmetadata.schema.entity.services.SearchService;
 import org.openmetadata.schema.tests.DataQualityReport;
+import org.openmetadata.schema.tests.type.DataQualityReportMetadata;
 import org.openmetadata.schema.type.ChangeDescription;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.SearchIndexDataType;
@@ -62,8 +71,12 @@
 import org.openmetadata.service.exception.CatalogExceptionMessage;
 import org.openmetadata.service.resources.EntityResourceTest;
 import org.openmetadata.service.resources.services.SearchServiceResourceTest;
+import org.openmetadata.service.search.SearchAggregation;
+import org.openmetadata.service.search.SearchAggregationNode;
 import org.openmetadata.service.search.SearchIndexUtils;
 import org.openmetadata.service.search.SearchRepository;
+import org.openmetadata.service.search.elasticsearch.aggregations.ElasticAggregations;
+import org.openmetadata.service.search.elasticsearch.aggregations.ElasticAggregationsBuilder;
 import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.util.ResultList;
 import org.openmetadata.service.util.TestUtils;
@@ -416,55 +429,6 @@ void test_fieldWithInvalidTag(TestInfo test) throws HttpResponseException {
     listEntities(null, ADMIN_AUTH_HEADERS);
   }
 
-  @Test
-  void testBuildAggregationString(TestInfo testInfo) {
-    String aggregationString = "bucketName=my-agg-name:aggType=terms:field=my-field";
-    String expectedAggregationString = "\"my-agg-name\":{\"terms\":{\"field\":\"my-field\"}}";
-    Map<String, Object> actualAggregationstring =
-        SearchIndexUtils.buildAggregationString(aggregationString);
-    assertEquals(expectedAggregationString, actualAggregationstring.get("aggregationStr"));
-
-    // Nested Aggregation (1 level)
-    aggregationString =
-        "bucketName=entityLinks:aggType=terms:field=entityLinks.nonNormalized,bucketName=status_counts:aggType=terms:field=testCaseResults.testCaseStatus";
-    expectedAggregationString =
-        "\"entityLinks\":{\"terms\":{\"field\":\"entityLinks.nonNormalized\"},\"aggs\":{\"status_counts\":{\"terms\":{\"field\":\"testCaseResults.testCaseStatus\"}}}}";
-    actualAggregationstring = SearchIndexUtils.buildAggregationString(aggregationString);
-    assertEquals(expectedAggregationString, actualAggregationstring.get("aggregationStr"));
-
-    // Nested Aggregation (2 levels)
-    aggregationString =
-        "bucketName=entityLinks:aggType=terms:field=entityLinks.nonNormalized,bucketName=statusCount:aggType=terms:field=testCaseResults.testCaseStatus,bucketName=owner:aggType=terms:field=testSuite.owner";
-    expectedAggregationString =
-        "\"entityLinks\":{\"terms\":{\"field\":\"entityLinks.nonNormalized\"},\"aggs\":{\"statusCount\":{\"terms\":{\"field\":\"testCaseResults.testCaseStatus\"},\"aggs\":{\"owner\":{\"terms\":{\"field\":\"testSuite.owner\"}}}}}}";
-    actualAggregationstring = SearchIndexUtils.buildAggregationString(aggregationString);
-    assertEquals(expectedAggregationString, actualAggregationstring.get("aggregationStr"));
-
-    // Metric Aggregation
-    aggregationString =
-        "bucketName=entityLinks:aggType=terms:field=entityLinks.nonNormalized,bucketName=minPrice:aggType=min:field=price.adjusted";
-    actualAggregationstring = SearchIndexUtils.buildAggregationString(aggregationString);
-    expectedAggregationString =
-        "\"entityLinks\":{\"terms\":{\"field\":\"entityLinks.nonNormalized\"},\"aggs\":{\"minPrice\":{\"min\":{\"field\":\"price.adjusted\"}}}}";
-    assertEquals(expectedAggregationString, actualAggregationstring.get("aggregationStr"));
-
-    // Date histogram aggregation
-    aggregationString =
-        "bucketName=dates:aggType=date_histogram:field=timestamp&calendar_interval=2d";
-    actualAggregationstring = SearchIndexUtils.buildAggregationString(aggregationString);
-    expectedAggregationString =
-        "\"dates\":{\"date_histogram\":{\"field\":\"timestamp\",\"calendar_interval\":\"2d\"}}";
-    assertEquals(expectedAggregationString, actualAggregationstring.get("aggregationStr"));
-
-    // Date histogram aggregation with sub aggregation
-    aggregationString =
-        "bucketName=dates:aggType=date_histogram:field=timestamp&calendar_interval=2d,bucketName=minPrice:aggType=min:field=price.adjusted";
-    actualAggregationstring = SearchIndexUtils.buildAggregationString(aggregationString);
-    expectedAggregationString =
-        "\"dates\":{\"date_histogram\":{\"field\":\"timestamp\",\"calendar_interval\":\"2d\"},\"aggs\":{\"minPrice\":{\"min\":{\"field\":\"price.adjusted\"}}}}";
-    assertEquals(expectedAggregationString, actualAggregationstring.get("aggregationStr"));
-  }
-
   @Test
   void testNewAggregation(TestInfo testInfo) throws IOException {
     DataQualityReport dataQualityReport = new DataQualityReport();
@@ -474,9 +438,8 @@ void testNewAggregation(TestInfo testInfo) throws IOException {
 
     String aggregationQuery =
         "bucketName=fqn:aggType=terms:field=fullyQualifiedName,bucketName=avgTime:aggType=avg:field=updatedAt";
-    Map<String, Object> aggregationString =
-        SearchIndexUtils.buildAggregationString(aggregationQuery);
-    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregationString);
+    SearchAggregation aggregation = SearchIndexUtils.buildAggregationTree(aggregationQuery);
+    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregation);
     dataQualityReport
         .getData()
         .forEach(
@@ -487,8 +450,8 @@ void testNewAggregation(TestInfo testInfo) throws IOException {
 
     aggregationQuery =
         "bucketName=fqn:aggType=terms:field=fullyQualifiedName,bucketName=owner:aggType=terms:field=owner.name";
-    aggregationString = SearchIndexUtils.buildAggregationString(aggregationQuery);
-    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregationString);
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationQuery);
+    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregation);
     dataQualityReport
         .getData()
         .forEach(
@@ -499,8 +462,8 @@ void testNewAggregation(TestInfo testInfo) throws IOException {
 
     aggregationQuery =
         "bucketName=fqn:aggType=terms:field=fullyQualifiedName,bucketName=owner:aggType=terms:field=owner.name,bucketName=avgTime:aggType=avg:field=updatedAt";
-    aggregationString = SearchIndexUtils.buildAggregationString(aggregationQuery);
-    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregationString);
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationQuery);
+    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregation);
     dataQualityReport
         .getData()
         .forEach(
@@ -511,8 +474,8 @@ void testNewAggregation(TestInfo testInfo) throws IOException {
             });
 
     aggregationQuery = "bucketName=avgTime:aggType=avg:field=updatedAt";
-    aggregationString = SearchIndexUtils.buildAggregationString(aggregationQuery);
-    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregationString);
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationQuery);
+    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregation);
     dataQualityReport
         .getData()
         .forEach(
@@ -522,8 +485,8 @@ void testNewAggregation(TestInfo testInfo) throws IOException {
             });
 
     aggregationQuery = "bucketName=fqn:aggType=terms:field=fullyQualifiedName";
-    aggregationString = SearchIndexUtils.buildAggregationString(aggregationQuery);
-    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregationString);
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationQuery);
+    dataQualityReport = searchRepository.genericAggregation(query, "table", aggregation);
     dataQualityReport
         .getData()
         .forEach(
@@ -534,9 +497,310 @@ void testNewAggregation(TestInfo testInfo) throws IOException {
 
     aggregationQuery =
         "bucketName=dates:aggType=date_histogram:field=timestamp&calendar_interval=1d,bucketName=dimesion:aggType=terms:field=testDefinition.dataQualityDimension";
-    aggregationString = SearchIndexUtils.buildAggregationString(aggregationQuery);
-    dataQualityReport =
-        searchRepository.genericAggregation(null, "testCaseResult", aggregationString);
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationQuery);
+    searchRepository.genericAggregation(null, "testCaseResult", aggregation);
+  }
+
+  @Test
+  void testAggregationGraph() {
+    // 1. Test aggregation with no children
+    List<BaseAggregationBuilder> expectedAggregations = new ArrayList<>();
+    String aggregationString = "bucketName=my-agg-name:aggType=terms:field=my-field";
+    SearchAggregation aggregation = SearchIndexUtils.buildAggregationTree(aggregationString);
+
+    SearchAggregationNode expectedTree = new SearchAggregationNode("root", "root", null);
+    expectedTree.addChild(
+        new SearchAggregationNode("terms", "my-agg-name", Map.of("field", "my-field")));
+    assertThat(aggregation.getAggregationTree()).usingRecursiveComparison().isEqualTo(expectedTree);
+
+    DataQualityReportMetadata actualMetadata = aggregation.getAggregationMetadata();
+    DataQualityReportMetadata expectedMetadata =
+        new DataQualityReportMetadata()
+            .withDimensions(List.of("my-field"))
+            .withKeys(List.of("sterms#my-agg-name"))
+            .withMetrics(List.of("document_count"));
+    assertThat(actualMetadata).usingRecursiveComparison().isEqualTo(expectedMetadata);
+
+    expectedAggregations.add(AggregationBuilders.terms("my-agg-name").field("my-field"));
+    List<ElasticAggregations> actualElasticAggregations =
+        ElasticAggregationsBuilder.buildAggregation(
+            aggregation.getAggregationTree(), null, new ArrayList<>());
+
+    validateAggregation(actualElasticAggregations, expectedAggregations);
+
+    // 2. Test aggregation with 1 nested aggregation
+    expectedAggregations = new ArrayList<>();
+    aggregationString =
+        "bucketName=entityLinks:aggType=terms:field=entityLinks.nonNormalized,"
+            + "bucketName=status_counts:aggType=terms:field=testCaseResults.testCaseStatus";
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationString);
+
+    expectedTree = new SearchAggregationNode("root", "root", null);
+    SearchAggregationNode node1 =
+        new SearchAggregationNode(
+            "terms", "entityLinks", Map.of("field", "entityLinks.nonNormalized"));
+    node1.addChild(
+        new SearchAggregationNode(
+            "terms", "status_counts", Map.of("field", "testCaseResults.testCaseStatus")));
+    expectedTree.addChild(node1);
+    assertThat(aggregation.getAggregationTree()).usingRecursiveComparison().isEqualTo(expectedTree);
+
+    actualMetadata = aggregation.getAggregationMetadata();
+    expectedMetadata =
+        new DataQualityReportMetadata()
+            .withDimensions(List.of("entityLinks.nonNormalized", "testCaseResults.testCaseStatus"))
+            .withKeys(List.of("sterms#entityLinks", "sterms#status_counts"))
+            .withMetrics(List.of("document_count"));
+    assertThat(actualMetadata).usingRecursiveComparison().isEqualTo(expectedMetadata);
+
+    expectedAggregations.add(
+        AggregationBuilders.terms("entityLinks")
+            .field("entityLinks.nonNormalized")
+            .subAggregation(
+                AggregationBuilders.terms("status_counts")
+                    .field("testCaseResults.testCaseStatus")));
+    actualElasticAggregations =
+        ElasticAggregationsBuilder.buildAggregation(
+            aggregation.getAggregationTree(), null, new ArrayList<>());
+
+    validateAggregation(actualElasticAggregations, expectedAggregations);
+
+    // 3. Test aggregation with 2 nested aggregation
+    expectedAggregations = new ArrayList<>();
+    aggregationString =
+        "bucketName=entityLinks:aggType=terms:field=entityLinks.nonNormalized,"
+            + "bucketName=statusCount:aggType=terms:field=testCaseResults.testCaseStatus,"
+            + "bucketName=owner:aggType=terms:field=testSuite.owner";
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationString);
+
+    expectedTree = new SearchAggregationNode("root", "root", null);
+    node1 =
+        new SearchAggregationNode(
+            "terms", "entityLinks", Map.of("field", "entityLinks.nonNormalized"));
+    SearchAggregationNode node2 =
+        new SearchAggregationNode(
+            "terms", "statusCount", Map.of("field", "testCaseResults.testCaseStatus"));
+    node2.addChild(new SearchAggregationNode("terms", "owner", Map.of("field", "testSuite.owner")));
+    node1.addChild(node2);
+    expectedTree.addChild(node1);
+    assertThat(aggregation.getAggregationTree()).usingRecursiveComparison().isEqualTo(expectedTree);
+
+    actualMetadata = aggregation.getAggregationMetadata();
+    expectedMetadata =
+        new DataQualityReportMetadata()
+            .withDimensions(
+                List.of(
+                    "entityLinks.nonNormalized",
+                    "testCaseResults.testCaseStatus",
+                    "testSuite.owner"))
+            .withKeys(List.of("sterms#entityLinks", "sterms#statusCount", "sterms#owner"))
+            .withMetrics(List.of("document_count"));
+    assertThat(actualMetadata).usingRecursiveComparison().isEqualTo(expectedMetadata);
+
+    expectedAggregations.add(
+        AggregationBuilders.terms("entityLinks")
+            .field("entityLinks.nonNormalized")
+            .subAggregation(
+                AggregationBuilders.terms("statusCount")
+                    .field("testCaseResults.testCaseStatus")
+                    .subAggregation(AggregationBuilders.terms("owner").field("testSuite.owner"))));
+    actualElasticAggregations =
+        ElasticAggregationsBuilder.buildAggregation(
+            aggregation.getAggregationTree(), null, new ArrayList<>());
+
+    validateAggregation(actualElasticAggregations, expectedAggregations);
+
+    // 4. Metric aggregation
+    expectedAggregations = new ArrayList<>();
+    aggregationString =
+        "bucketName=entityLinks:aggType=terms:field=entityLinks.nonNormalized,"
+            + "bucketName=minPrice:aggType=min:field=price.adjusted";
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationString);
+
+    expectedTree = new SearchAggregationNode("root", "root", null);
+    node1 =
+        new SearchAggregationNode(
+            "terms", "entityLinks", Map.of("field", "entityLinks.nonNormalized"));
+    node2 = new SearchAggregationNode("min", "minPrice", Map.of("field", "price.adjusted"));
+    node1.addChild(node2);
+    expectedTree.addChild(node1);
+    assertThat(aggregation.getAggregationTree()).usingRecursiveComparison().isEqualTo(expectedTree);
+
+    actualMetadata = aggregation.getAggregationMetadata();
+    expectedMetadata =
+        new DataQualityReportMetadata()
+            .withDimensions(List.of("entityLinks.nonNormalized"))
+            .withKeys(List.of("sterms#entityLinks", "min#minPrice"))
+            .withMetrics(List.of("price.adjusted"));
+    assertThat(actualMetadata).usingRecursiveComparison().isEqualTo(expectedMetadata);
+
+    expectedAggregations.add(
+        AggregationBuilders.terms("entityLinks")
+            .field("entityLinks.nonNormalized")
+            .subAggregation(AggregationBuilders.min("minPrice").field("price.adjusted")));
+    actualElasticAggregations =
+        ElasticAggregationsBuilder.buildAggregation(
+            aggregation.getAggregationTree(), null, new ArrayList<>());
+
+    validateAggregation(actualElasticAggregations, expectedAggregations);
+
+    // 6. Date histogram aggregation
+    expectedAggregations = new ArrayList<>();
+    aggregationString =
+        "bucketName=dates:aggType=date_histogram:field=timestamp&calendar_interval=day";
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationString);
+
+    expectedTree = new SearchAggregationNode("root", "root", null);
+    node1 =
+        new SearchAggregationNode(
+            "date_histogram", "dates", Map.of("field", "timestamp", "calendar_interval", "day"));
+    expectedTree.addChild(node1);
+    assertThat(aggregation.getAggregationTree()).usingRecursiveComparison().isEqualTo(expectedTree);
+
+    actualMetadata = aggregation.getAggregationMetadata();
+    expectedMetadata =
+        new DataQualityReportMetadata()
+            .withDimensions(new ArrayList<>())
+            .withKeys(List.of("date_histogram#dates"))
+            .withMetrics(List.of("timestamp"));
+    assertThat(actualMetadata).usingRecursiveComparison().isEqualTo(expectedMetadata);
+
+    expectedAggregations.add(
+        AggregationBuilders.dateHistogram("dates")
+            .field("timestamp")
+            .calendarInterval(new DateHistogramInterval("day")));
+    actualElasticAggregations =
+        ElasticAggregationsBuilder.buildAggregation(
+            aggregation.getAggregationTree(), null, new ArrayList<>());
+
+    validateAggregation(actualElasticAggregations, expectedAggregations);
+
+    // 7. Date histogram aggregation
+    expectedAggregations = new ArrayList<>();
+    aggregationString =
+        "bucketName=dates:aggType=date_histogram:field=timestamp&calendar_interval=day,"
+            + "bucketName=minPrice:aggType=min:field=price.adjusted";
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationString);
+
+    expectedTree = new SearchAggregationNode("root", "root", null);
+    node1 =
+        new SearchAggregationNode(
+            "date_histogram", "dates", Map.of("field", "timestamp", "calendar_interval", "day"));
+    node2 = new SearchAggregationNode("min", "minPrice", Map.of("field", "price.adjusted"));
+    node1.addChild(node2);
+    expectedTree.addChild(node1);
+    assertThat(aggregation.getAggregationTree()).usingRecursiveComparison().isEqualTo(expectedTree);
+
+    actualMetadata = aggregation.getAggregationMetadata();
+    expectedMetadata =
+        new DataQualityReportMetadata()
+            .withDimensions(List.of("timestamp"))
+            .withKeys(List.of("date_histogram#dates", "min#minPrice"))
+            .withMetrics(List.of("price.adjusted"));
+    assertThat(actualMetadata).usingRecursiveComparison().isEqualTo(expectedMetadata);
+
+    expectedAggregations.add(
+        AggregationBuilders.dateHistogram("dates")
+            .field("timestamp")
+            .calendarInterval(new DateHistogramInterval("day"))
+            .subAggregation(AggregationBuilders.min("minPrice").field("price.adjusted")));
+    actualElasticAggregations =
+        ElasticAggregationsBuilder.buildAggregation(
+            aggregation.getAggregationTree(), null, new ArrayList<>());
+
+    validateAggregation(actualElasticAggregations, expectedAggregations);
+
+    // 8. Nested aggregation with selector sibling
+    expectedAggregations = new ArrayList<>();
+    aggregationString =
+        "bucketName=entityFQN:aggType=terms:field=originEntityFQN&size=1000,"
+            + "bucketName=status:aggType=terms:field=testCaseStatus.keyword&include=\"Failed,Aborted\"::"
+            + "bucketName=statusFilter:aggType=bucket_selector:pathValues=status._bucket_count&pathKeys=status&script=\"params.status==0\"";
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationString);
+
+    expectedTree = new SearchAggregationNode("root", "root", null);
+    node1 =
+        new SearchAggregationNode(
+            "terms", "entityFQN", Map.of("field", "originEntityFQN", "size", "1000"));
+    node2 =
+        new SearchAggregationNode(
+            "terms",
+            "status",
+            Map.of("field", "testCaseStatus.keyword", "include", "Failed,Aborted"));
+    SearchAggregationNode node3 =
+        new SearchAggregationNode(
+            "bucket_selector",
+            "statusFilter",
+            Map.of(
+                "pathValues",
+                "status._bucket_count",
+                "pathKeys",
+                "status",
+                "script",
+                "params.status==0"));
+    node1.addChild(node2);
+    node1.addChild(node3);
+    expectedTree.addChild(node1);
+    assertThat(aggregation.getAggregationTree()).usingRecursiveComparison().isEqualTo(expectedTree);
+
+    actualMetadata = aggregation.getAggregationMetadata();
+    expectedMetadata =
+        new DataQualityReportMetadata()
+            .withDimensions(List.of("originEntityFQN", "testCaseStatus.keyword"))
+            .withKeys(List.of("sterms#entityFQN", "sterms#status"))
+            .withMetrics(List.of("document_count"));
+    assertThat(actualMetadata).usingRecursiveComparison().isEqualTo(expectedMetadata);
+
+    String[] include = {"Aborted", "Failed"};
+    expectedAggregations.add(
+        AggregationBuilders.terms("entityFQN")
+            .field("originEntityFQN")
+            .size(1000)
+            .subAggregation(
+                AggregationBuilders.terms("status")
+                    .field("testCaseStatus.keyword")
+                    .includeExclude(new IncludeExclude(include, null)))
+            .subAggregation(
+                PipelineAggregatorBuilders.bucketSelector(
+                    "statusFilter",
+                    Map.of("status", "status._bucket_count"),
+                    new Script("params.status==0"))));
+    actualElasticAggregations =
+        ElasticAggregationsBuilder.buildAggregation(
+            aggregation.getAggregationTree(), null, new ArrayList<>());
+
+    validateAggregation(actualElasticAggregations, expectedAggregations);
+
+    // 9. top hits query
+    expectedAggregations = new ArrayList<>();
+    aggregationString =
+        "bucketName=byTerms:aggType=terms:field=entityFqn&size=100,"
+            + "bucketName=latest:aggType=top_hits:size=1&sort_field=timestamp&sort_order=desc";
+    aggregation = SearchIndexUtils.buildAggregationTree(aggregationString);
+    expectedTree = new SearchAggregationNode("root", "root", null);
+    node1 =
+        new SearchAggregationNode("terms", "byTerms", Map.of("field", "entityFqn", "size", "100"));
+    node2 =
+        new SearchAggregationNode(
+            "top_hits",
+            "latest",
+            Map.of("size", "1", "sort_field", "timestamp", "sort_order", "desc"));
+    node1.addChild(node2);
+    expectedTree.addChild(node1);
+
+    assertThat(aggregation.getAggregationTree()).usingRecursiveComparison().isEqualTo(expectedTree);
+
+    expectedAggregations.add(
+        AggregationBuilders.terms("byTerms")
+            .field("entityFqn")
+            .size(100)
+            .subAggregation(
+                AggregationBuilders.topHits("latest").size(1).sort("timestamp", SortOrder.DESC)));
+    actualElasticAggregations =
+        ElasticAggregationsBuilder.buildAggregation(
+            aggregation.getAggregationTree(), null, new ArrayList<>());
+
+    validateAggregation(actualElasticAggregations, expectedAggregations);
   }
 
   @Override
@@ -670,4 +934,20 @@ private static void assertField(SearchIndexField expectedField, SearchIndexField
     // Check the nested columns
     assertFields(expectedField.getChildren(), actualField.getChildren());
   }
+
+  private void validateAggregation(
+      List<ElasticAggregations> actualElasticAggregations,
+      List<BaseAggregationBuilder> expectedAggregations) {
+    for (int i = 0; i < actualElasticAggregations.size(); i++) {
+      ElasticAggregations actualElasticAggregation = actualElasticAggregations.get(i);
+      if (actualElasticAggregation.isPipelineAggregation()) {
+        assertEquals(
+            expectedAggregations.get(i),
+            actualElasticAggregation.getElasticPipelineAggregationBuilder());
+      } else {
+        assertEquals(
+            expectedAggregations.get(i), actualElasticAggregation.getElasticAggregationBuilder());
+      }
+    }
+  }
 }
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/APIServiceResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/APIServiceResourceTest.java
index a1efa41da3af..d135883f4543 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/APIServiceResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/services/APIServiceResourceTest.java
@@ -22,23 +22,23 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.openmetadata.common.utils.CommonUtil;
-import org.openmetadata.schema.api.services.CreateAPIService;
-import org.openmetadata.schema.entity.services.APIService;
+import org.openmetadata.schema.api.services.CreateApiService;
+import org.openmetadata.schema.entity.services.ApiService;
 import org.openmetadata.schema.entity.services.connections.TestConnectionResult;
 import org.openmetadata.schema.entity.services.connections.TestConnectionResultStatus;
-import org.openmetadata.schema.services.connections.api.RESTConnection;
-import org.openmetadata.schema.type.APIServiceConnection;
+import org.openmetadata.schema.services.connections.api.RestConnection;
+import org.openmetadata.schema.type.ApiConnection;
 import org.openmetadata.schema.type.ChangeDescription;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.resources.services.apiservices.APIServiceResource;
 import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.util.TestUtils;
 
-public class APIServiceResourceTest extends ServiceResourceTest<APIService, CreateAPIService> {
+public class APIServiceResourceTest extends ServiceResourceTest<ApiService, CreateApiService> {
   public APIServiceResourceTest() {
     super(
         Entity.API_SERVICE,
-        APIService.class,
+        ApiService.class,
         APIServiceResource.APIServiceList.class,
         "services/apiServices",
         "owners");
@@ -47,25 +47,25 @@ public APIServiceResourceTest() {
 
   public void setupAPIService(TestInfo test) throws HttpResponseException {
     APIServiceResourceTest apiServiceResourceTest = new APIServiceResourceTest();
-    CreateAPIService createAPIService =
+    CreateApiService createApiService =
         apiServiceResourceTest
             .createRequest(test)
             .withName("openmetadata")
-            .withServiceType(CreateAPIService.APIServiceType.REST)
+            .withServiceType(CreateApiService.ApiServiceType.Rest)
             .withConnection(TestUtils.API_SERVICE_CONNECTION);
 
-    APIService omAPIService =
-        new APIServiceResourceTest().createEntity(createAPIService, ADMIN_AUTH_HEADERS);
+    ApiService omAPIService =
+        new APIServiceResourceTest().createEntity(createApiService, ADMIN_AUTH_HEADERS);
     OPENMETADATA_API_SERVICE_REFERENCE = omAPIService.getEntityReference();
     APIServiceResourceTest sampleAPIServiceResourceTest = new APIServiceResourceTest();
-    createAPIService =
+    createApiService =
         sampleAPIServiceResourceTest
             .createRequest(test)
             .withName("sampleAPI")
-            .withServiceType(CreateAPIService.APIServiceType.REST)
+            .withServiceType(CreateApiService.ApiServiceType.Rest)
             .withConnection(TestUtils.API_SERVICE_CONNECTION);
-    APIService sampleAPIService =
-        new APIServiceResourceTest().createEntity(createAPIService, ADMIN_AUTH_HEADERS);
+    ApiService sampleAPIService =
+        new APIServiceResourceTest().createEntity(createApiService, ADMIN_AUTH_HEADERS);
     SAMPLE_API_SERVICE_REFERENCE = sampleAPIService.getEntityReference();
   }
 
@@ -93,26 +93,26 @@ void post_validService_as_admin_200_ok(TestInfo test) throws IOException {
 
   @Test
   void put_updateService_as_admin_2xx(TestInfo test) throws IOException, URISyntaxException {
-    APIServiceConnection connection1 =
-        new APIServiceConnection()
+    ApiConnection connection1 =
+        new ApiConnection()
             .withConfig(
-                new RESTConnection()
+                new RestConnection()
                     .withOpenAPISchemaURL(
                         new URI("http://sandbox.open-metadata.org/swagger.json")));
-    APIService service =
+    ApiService service =
         createAndCheckEntity(
             createRequest(test).withDescription(null).withConnection(connection1),
             ADMIN_AUTH_HEADERS);
 
-    RESTConnection credentials2 =
-        new RESTConnection()
+    RestConnection credentials2 =
+        new RestConnection()
             .withOpenAPISchemaURL(new URI("https://localhost:9400"))
             .withToken("test");
-    APIServiceConnection connection2 = new APIServiceConnection().withConfig(credentials2);
+    ApiConnection connection2 = new ApiConnection().withConfig(credentials2);
 
     // Update APIService description and connection
 
-    CreateAPIService update =
+    CreateApiService update =
         createRequest(test).withDescription("description1").withConnection(connection2);
 
     ChangeDescription change = getChangeDescription(service, MINOR_UPDATE);
@@ -123,10 +123,10 @@ void put_updateService_as_admin_2xx(TestInfo test) throws IOException, URISyntax
 
   @Test
   void put_testConnectionResult_200(TestInfo test) throws IOException {
-    APIService service = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS);
+    ApiService service = createAndCheckEntity(createRequest(test), ADMIN_AUTH_HEADERS);
     // By default, we have no result logged in
     assertNull(service.getTestConnectionResult());
-    APIService updatedService =
+    ApiService updatedService =
         putTestConnectionResult(service.getId(), TEST_CONNECTION_RESULT, ADMIN_AUTH_HEADERS);
     // Validate that the data got properly stored
     assertNotNull(updatedService.getTestConnectionResult());
@@ -135,50 +135,50 @@ void put_testConnectionResult_200(TestInfo test) throws IOException {
         updatedService.getTestConnectionResult().getStatus());
     assertEquals(updatedService.getConnection(), service.getConnection());
     // Check that the stored data is also correct
-    APIService stored = getEntity(service.getId(), ADMIN_AUTH_HEADERS);
+    ApiService stored = getEntity(service.getId(), ADMIN_AUTH_HEADERS);
     assertNotNull(stored.getTestConnectionResult());
     assertEquals(
         TestConnectionResultStatus.SUCCESSFUL, stored.getTestConnectionResult().getStatus());
     assertEquals(stored.getConnection(), service.getConnection());
   }
 
-  public APIService putTestConnectionResult(
+  public ApiService putTestConnectionResult(
       UUID serviceId, TestConnectionResult testConnectionResult, Map<String, String> authHeaders)
       throws HttpResponseException {
     WebTarget target = getResource(serviceId).path("/testConnectionResult");
-    return TestUtils.put(target, testConnectionResult, APIService.class, OK, authHeaders);
+    return TestUtils.put(target, testConnectionResult, ApiService.class, OK, authHeaders);
   }
 
   @Override
-  public CreateAPIService createRequest(String name) {
-    return new CreateAPIService()
+  public CreateApiService createRequest(String name) {
+    return new CreateApiService()
         .withName(name)
-        .withServiceType(CreateAPIService.APIServiceType.REST)
+        .withServiceType(CreateApiService.ApiServiceType.Rest)
         .withConnection(
-            new APIServiceConnection()
+            new ApiConnection()
                 .withConfig(
-                    new RESTConnection()
+                    new RestConnection()
                         .withOpenAPISchemaURL(
                             CommonUtil.getUri("http://localhost:8585/swagger.json"))));
   }
 
   @Override
   public void validateCreatedEntity(
-      APIService service, CreateAPIService createRequest, Map<String, String> authHeaders) {
+      ApiService service, CreateApiService createRequest, Map<String, String> authHeaders) {
     assertEquals(createRequest.getName(), service.getName());
-    APIServiceConnection expectedConnection = createRequest.getConnection();
-    APIServiceConnection actualConnection = service.getConnection();
+    ApiConnection expectedConnection = createRequest.getConnection();
+    ApiConnection actualConnection = service.getConnection();
     validateConnection(expectedConnection, actualConnection, service.getServiceType());
   }
 
   @Override
   public void compareEntities(
-      APIService expected, APIService updated, Map<String, String> authHeaders) {
+      ApiService expected, ApiService updated, Map<String, String> authHeaders) {
     // PATCH operation is not supported by this entity
   }
 
   @Override
-  public APIService validateGetWithDifferentFields(APIService service, boolean byName)
+  public ApiService validateGetWithDifferentFields(ApiService service, boolean byName)
       throws HttpResponseException {
     String fields = "";
     service =
@@ -209,13 +209,13 @@ public void assertFieldChange(String fieldName, Object expected, Object actual)
   }
 
   private void validateConnection(
-      APIServiceConnection expectedConnection,
-      APIServiceConnection actualConnection,
-      CreateAPIService.APIServiceType serviceType) {
+      ApiConnection expectedConnection,
+      ApiConnection actualConnection,
+      CreateApiService.ApiServiceType serviceType) {
     if (expectedConnection != null && actualConnection != null) {
-      RESTConnection restConnection = (RESTConnection) expectedConnection.getConfig();
-      RESTConnection actualESConnection =
-          JsonUtils.convertValue(actualConnection.getConfig(), RESTConnection.class);
+      RestConnection restConnection = (RestConnection) expectedConnection.getConfig();
+      RestConnection actualESConnection =
+          JsonUtils.convertValue(actualConnection.getConfig(), RestConnection.class);
       assertEquals(restConnection.getOpenAPISchemaURL(), actualESConnection.getOpenAPISchemaURL());
     }
   }
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java
index d38704979fff..7a462a61b4a3 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/ClassificationResourceTest.java
@@ -16,9 +16,13 @@
 import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.openmetadata.common.utils.CommonUtil.listOf;
 import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
+import static org.openmetadata.service.util.EntityUtil.fieldAdded;
+import static org.openmetadata.service.util.EntityUtil.fieldDeleted;
 import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
 import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
+import static org.openmetadata.service.util.TestUtils.UpdateType.CHANGE_CONSOLIDATED;
 import static org.openmetadata.service.util.TestUtils.UpdateType.MINOR_UPDATE;
 import static org.openmetadata.service.util.TestUtils.assertListNotNull;
 import static org.openmetadata.service.util.TestUtils.assertListNull;
@@ -29,6 +33,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import javax.ws.rs.core.Response.Status;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.client.HttpResponseException;
@@ -37,14 +42,19 @@
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.TestMethodOrder;
 import org.openmetadata.schema.api.classification.CreateClassification;
+import org.openmetadata.schema.api.teams.CreateRole;
 import org.openmetadata.schema.entity.classification.Classification;
 import org.openmetadata.schema.entity.classification.Tag;
+import org.openmetadata.schema.entity.teams.Role;
+import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.type.ChangeDescription;
+import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.ProviderType;
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.CatalogExceptionMessage;
 import org.openmetadata.service.resources.EntityResourceTest;
 import org.openmetadata.service.resources.tags.ClassificationResource.ClassificationList;
+import org.openmetadata.service.resources.teams.UserResourceTest;
 import org.openmetadata.service.util.JsonUtils;
 import org.openmetadata.service.util.TestUtils;
 
@@ -91,6 +101,84 @@ void delete_systemClassification() throws HttpResponseException {
             classification.getName(), Entity.CLASSIFICATION));
   }
 
+  @Test
+  void test_addUpdateRemoveRolesInClassification(TestInfo test) throws IOException {
+
+    // Create a classification with roles
+    String classificationName = getEntityName(test);
+    CreateClassification createRequest =
+        createRequest(classificationName)
+            .withDescription("Test classification with roles")
+            .withRoles(
+                listOf(
+                    DATA_CONSUMER_ROLE.getEntityReference(),
+                    DATA_STEWARD_ROLE.getEntityReference()));
+
+    Classification classification = createAndCheckEntity(createRequest, ADMIN_AUTH_HEADERS);
+
+    // Validate that roles are assigned
+    assertEquals(2, classification.getRoles().size());
+    assertTrue(
+        classification.getRoles().stream()
+            .anyMatch(
+                r ->
+                    r.getId().equals(DATA_CONSUMER_ROLE.getId())
+                        && r.getType().equals(Entity.ROLE)));
+    assertTrue(
+        classification.getRoles().stream()
+            .anyMatch(
+                r ->
+                    r.getId().equals(DATA_STEWARD_ROLE.getId())
+                        && r.getType().equals(Entity.ROLE)));
+    String origJson = JsonUtils.pojoToJson(classification);
+    // Update the classification by removing one role and adding another
+    Role role3 = createRole("Role3");
+    classification.setRoles(
+        listOf(DATA_CONSUMER_ROLE.getEntityReference(), role3.getEntityReference()));
+
+    ChangeDescription change = getChangeDescription(classification, MINOR_UPDATE);
+    fieldAdded(change, "roles", listOf(role3.getEntityReference()));
+    fieldDeleted(change, "roles", listOf(DATA_STEWARD_ROLE.getEntityReference()));
+    Classification updatedClassification =
+        patchEntityAndCheck(classification, origJson, ADMIN_AUTH_HEADERS, MINOR_UPDATE, change);
+
+    // Validate that roles are updated
+    assertEquals(2, updatedClassification.getRoles().size());
+    assertTrue(
+        updatedClassification.getRoles().stream()
+            .anyMatch(
+                r ->
+                    r.getId().equals(DATA_CONSUMER_ROLE.getId())
+                        && r.getType().equals(Entity.ROLE)));
+    assertTrue(
+        updatedClassification.getRoles().stream()
+            .anyMatch(r -> r.getId().equals(role3.getId()) && r.getType().equals(Entity.ROLE)));
+
+    origJson = JsonUtils.pojoToJson(updatedClassification);
+    change = getChangeDescription(updatedClassification, CHANGE_CONSOLIDATED);
+    updatedClassification.setRoles(null);
+    fieldDeleted(
+        change,
+        "roles",
+        listOf(DATA_CONSUMER_ROLE.getEntityReference(), DATA_STEWARD_ROLE.getEntityReference()));
+    Classification classificationWithoutRoles =
+        patchEntityAndCheck(
+            updatedClassification, origJson, ADMIN_AUTH_HEADERS, CHANGE_CONSOLIDATED, change);
+
+    assertTrue(
+        classificationWithoutRoles.getRoles() == null
+            || classificationWithoutRoles.getRoles().isEmpty());
+  }
+
+  private Role createRole(String roleName) throws IOException {
+    CreateRole createRole =
+        new CreateRole()
+            .withName(roleName)
+            .withDescription("Test role")
+            .withPolicies(listOf(POLICY1.getFullyQualifiedName()));
+    return TestUtils.post(getResource("roles"), createRole, Role.class, 201, ADMIN_AUTH_HEADERS);
+  }
+
   @Override
   public CreateClassification createRequest(String name) {
     return new CreateClassification()
@@ -111,10 +199,17 @@ public void validateCreatedEntity(
   @Override
   public void compareEntities(
       Classification expected, Classification updated, Map<String, String> authHeaders) {
+    assertEquals(expected.getName(), updated.getName());
+    assertEquals(expected.getDescription(), updated.getDescription());
+    assertEquals(expected.getFullyQualifiedName(), updated.getFullyQualifiedName());
+    assertEquals(expected.getTags(), updated.getTags());
+    if (expected.getRoles() != null) {
+      assertEntityReferences(expected.getRoles(), updated.getRoles());
+    }
+    assertEquals(expected.getMutuallyExclusive(), updated.getMutuallyExclusive());
     assertEquals(
         expected.getProvider() == null ? ProviderType.USER : expected.getProvider(),
         updated.getProvider());
-    assertEquals(expected.getMutuallyExclusive(), updated.getMutuallyExclusive());
   }
 
   @Override
@@ -135,11 +230,6 @@ public Classification validateGetWithDifferentFields(
     return classification;
   }
 
-  @Override
-  public void assertFieldChange(String fieldName, Object expected, Object actual) {
-    assertCommonFieldChange(fieldName, expected, actual);
-  }
-
   public void renameClassificationAndCheck(Classification classification, String newName)
       throws IOException {
     // User PATCH operation to rename a classification
@@ -161,4 +251,20 @@ public void renameClassificationAndCheck(Classification classification, String n
       assertTrue(child.getFullyQualifiedName().startsWith(ret.getFullyQualifiedName()));
     }
   }
+
+  public Classification updateEntity(Classification classification, String json, Map<String, String> authHeaders, TestUtils.UpdateType updateType, ChangeDescription change)
+      throws IOException {
+    return patchEntityAndCheck(classification, json, authHeaders, updateType, change);
+  }
+
+  @Override
+  public void assertFieldChange(String fieldName, Object expected, Object actual) {
+    if (expected == actual) {
+      return;
+    } else if (fieldName.equals("roles")) {
+      assertEntityReferencesFieldChange(expected, actual);
+    } else {
+      assertCommonFieldChange(fieldName, expected, actual);
+    }
+  }
 }
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/TagResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/TagResourceTest.java
index b3c8b794025e..efba7438d3ed 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/TagResourceTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/tags/TagResourceTest.java
@@ -18,13 +18,16 @@
 import static javax.ws.rs.core.Response.Status.NOT_FOUND;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.openmetadata.common.utils.CommonUtil.listOf;
 import static org.openmetadata.common.utils.CommonUtil.listOrEmpty;
 import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
+import static org.openmetadata.service.util.EntityUtil.fieldDeleted;
 import static org.openmetadata.service.util.EntityUtil.fieldUpdated;
 import static org.openmetadata.service.util.TestUtils.ADMIN_AUTH_HEADERS;
+import static org.openmetadata.service.util.TestUtils.UpdateType.CHANGE_CONSOLIDATED;
 import static org.openmetadata.service.util.TestUtils.UpdateType.MINOR_UPDATE;
 import static org.openmetadata.service.util.TestUtils.UpdateType.NO_CHANGE;
 import static org.openmetadata.service.util.TestUtils.assertEntityPagination;
@@ -32,6 +35,8 @@
 import static org.openmetadata.service.util.TestUtils.assertListNull;
 import static org.openmetadata.service.util.TestUtils.assertResponse;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.flipkart.zjsonpatch.JsonDiff;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -50,10 +55,13 @@
 import org.junit.jupiter.api.TestMethodOrder;
 import org.openmetadata.schema.api.classification.CreateClassification;
 import org.openmetadata.schema.api.classification.CreateTag;
+import org.openmetadata.schema.api.data.CreateTable;
 import org.openmetadata.schema.entity.classification.Classification;
 import org.openmetadata.schema.entity.classification.Tag;
 import org.openmetadata.schema.entity.type.Style;
 import org.openmetadata.schema.type.ChangeDescription;
+import org.openmetadata.schema.type.Column;
+import org.openmetadata.schema.type.ColumnDataType;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.Include;
 import org.openmetadata.schema.type.ProviderType;
@@ -61,6 +69,7 @@
 import org.openmetadata.service.Entity;
 import org.openmetadata.service.exception.CatalogExceptionMessage;
 import org.openmetadata.service.resources.EntityResourceTest;
+import org.openmetadata.service.resources.databases.TableResourceTest;
 import org.openmetadata.service.resources.tags.TagResource.TagList;
 import org.openmetadata.service.util.EntityUtil;
 import org.openmetadata.service.util.FullyQualifiedName;
@@ -174,7 +183,7 @@ void delete_systemTag() throws HttpResponseException {
   }
 
   @Test
-  void get_TagsWithPagination_200(TestInfo test) throws IOException {
+  void get_TagsWithPagination_200() throws IOException {
     // get Pagination results for same name entities
     boolean supportsSoftDelete = true;
     int numEntities = 5;
@@ -292,6 +301,37 @@ void get_TagsWithPagination_200(TestInfo test) throws IOException {
     }
   }
 
+  @Test
+  void test_tagInheritsRolesFromClassification(TestInfo test) throws IOException {
+    // Create a classification with roles
+    String classificationName = "Classification_" + test.getDisplayName();
+    CreateClassification createClassificationRequest =
+        new CreateClassification()
+            .withName(classificationName)
+            .withDescription("Test classification")
+            .withRoles(listOf(DATA_STEWARD_ROLE_REF, DATA_CONSUMER_ROLE_REF));
+
+    ClassificationResourceTest classificationTest = new ClassificationResourceTest();
+    Classification classification =
+        classificationTest.createEntity(createClassificationRequest, ADMIN_AUTH_HEADERS);
+
+    String tagName = "Tag_" + test.getDisplayName();
+    CreateTag createTagRequest =
+        createRequest(tagName).withClassification(classification.getFullyQualifiedName());
+
+    Tag tag = createEntity(createTagRequest, ADMIN_AUTH_HEADERS);
+
+    assertNotNull(tag.getInheritedRoles());
+    assertEquals(2, tag.getInheritedRoles().size());
+    assertTrue(
+        tag.getInheritedRoles().stream()
+            .anyMatch(r -> r.getId().equals(DATA_STEWARD_ROLE_REF.getId()) && r.getType().equals(Entity.ROLE)));
+    assertTrue(
+        tag.getInheritedRoles().stream()
+            .anyMatch(r -> r.getId().equals(DATA_CONSUMER_ROLE_REF.getId()) && r.getType().equals(Entity.ROLE)));
+
+  }
+
   private Tag createOrUpdate(String classificationName, Tag parent, String name, Status status)
       throws IOException {
     String parentFqn = parent != null ? parent.getFullyQualifiedName() : null;
@@ -417,6 +457,93 @@ public Tag updateAndCheckEntity(
     return tag;
   }
 
+  @Test
+  void test_disableClassification_disablesAllTags() throws IOException {
+    String classificationName = "TestClassification";
+    CreateClassification createClassification =
+        classificationResourceTest.createRequest(classificationName);
+    Classification classification =
+        classificationResourceTest.createAndCheckEntity(createClassification, ADMIN_AUTH_HEADERS);
+
+    String tagName1 = "Tag1";
+    String tagName2 = "Tag2";
+    CreateTag createTag1 = createRequest(tagName1).withClassification(classificationName);
+    CreateTag createTag2 = createRequest(tagName2).withClassification(classificationName);
+    Tag tag1 = createEntity(createTag1, ADMIN_AUTH_HEADERS);
+    Tag tag2 = createEntity(createTag2, ADMIN_AUTH_HEADERS);
+
+    Tag getTag1 = getEntity(tag1.getId(), ADMIN_AUTH_HEADERS);
+    Tag getTag2 = getEntity(tag2.getId(), ADMIN_AUTH_HEADERS);
+    assertFalse(getTag1.getDisabled(), "Tag1 should not be disabled");
+    assertFalse(getTag2.getDisabled(), "Tag2 should not be disabled");
+
+    String classificationJson = JsonUtils.pojoToJson(classification);
+    classification.setDisabled(true);
+    ChangeDescription change = getChangeDescription(classification, MINOR_UPDATE);
+    fieldUpdated(change, "disabled", false, true);
+    classification =
+        classificationResourceTest.patchEntityAndCheck(
+            classification,
+            classificationJson,
+            ADMIN_AUTH_HEADERS,
+            UpdateType.MINOR_UPDATE,
+            change);
+
+    getTag1 = getEntity(tag1.getId(), ADMIN_AUTH_HEADERS);
+    getTag2 = getEntity(tag2.getId(), ADMIN_AUTH_HEADERS);
+    assertTrue(
+        getTag1.getDisabled(), "Tag1 should be disabled because its Classification is disabled");
+    assertTrue(
+        getTag2.getDisabled(), "Tag2 should be disabled because its Classification is disabled");
+
+    classificationJson = JsonUtils.pojoToJson(classification);
+    ObjectMapper mapper = new ObjectMapper();
+    classification.setDisabled(false);
+    classificationResourceTest.patchEntity(
+        classification.getId(),
+        JsonDiff.asJson(
+            mapper.readTree(classificationJson),
+            mapper.readTree(JsonUtils.pojoToJson(classification))),
+        ADMIN_AUTH_HEADERS);
+
+    getTag1 = getEntity(tag1.getId(), ADMIN_AUTH_HEADERS);
+    getTag2 = getEntity(tag2.getId(), ADMIN_AUTH_HEADERS);
+    assertFalse(
+        getTag1.getDisabled(), "Tag1 should not be disabled after Classification is enabled");
+    assertFalse(
+        getTag2.getDisabled(), "Tag2 should not be disabled after Classification is enabled");
+
+    CreateTag createTag = createRequest("SingleTag").withClassification(classificationName);
+    Tag getTag = createEntity(createTag, ADMIN_AUTH_HEADERS);
+
+    getTag = getEntity(getTag.getId(), ADMIN_AUTH_HEADERS);
+    assertFalse(getTag.getDisabled(), "Tag should not be disabled initially");
+
+    String tagJson = JsonUtils.pojoToJson(getTag);
+    ChangeDescription change1 = getChangeDescription(getTag, MINOR_UPDATE);
+    getTag.setDisabled(true);
+    fieldUpdated(change1, "disabled", false, true);
+    getTag =
+        patchEntityAndCheck(getTag, tagJson, ADMIN_AUTH_HEADERS, UpdateType.MINOR_UPDATE, change);
+
+    getTag = getEntity(getTag.getId(), ADMIN_AUTH_HEADERS);
+    assertTrue(getTag.getDisabled(), "Tag should be disabled after update");
+
+    CreateTable createTable =
+        new CreateTable()
+            .withName("TestTable")
+            .withDatabaseSchema(DATABASE_SCHEMA.getFullyQualifiedName())
+            .withColumns(List.of(new Column().withName("column1").withDataType(ColumnDataType.INT)))
+            .withTags(List.of(new TagLabel().withTagFQN(getTag.getFullyQualifiedName())));
+    TableResourceTest tableResourceTest = new TableResourceTest();
+
+    assertResponse(
+        () -> tableResourceTest.createEntity(createTable, ADMIN_AUTH_HEADERS),
+        BAD_REQUEST,
+        CatalogExceptionMessage.disabledTag(
+            new TagLabel().withTagFQN(getTag.getFullyQualifiedName())));
+  }
+
   public Tag createTag(
       String name, String classification, String parentFqn, String... associatedTags)
       throws IOException {
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/security/ElasticSearchRBACConditionEvaluatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/security/ElasticSearchRBACConditionEvaluatorTest.java
index 30da785f07d6..8807db6bb4bc 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/search/security/ElasticSearchRBACConditionEvaluatorTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/security/ElasticSearchRBACConditionEvaluatorTest.java
@@ -2,7 +2,6 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.*;
@@ -16,7 +15,11 @@
 import es.org.elasticsearch.index.query.QueryBuilder;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.junit.jupiter.api.AfterEach;
@@ -55,6 +58,14 @@ public void setUp() {
               return resource.toLowerCase();
             });
     Entity.setSearchRepository(mockSearchRepository);
+    mockSubjectContext = mock(SubjectContext.class);
+    mockUser = mock(User.class);
+    EntityReference mockUserReference = mock(EntityReference.class);
+    when(mockUser.getEntityReference()).thenReturn(mockUserReference);
+    when(mockUserReference.getId()).thenReturn(UUID.randomUUID());
+    when(mockUser.getId()).thenReturn(UUID.randomUUID());
+    when(mockUser.getName()).thenReturn("testUser");
+    when(mockSubjectContext.user()).thenReturn(mockUser);
   }
 
   @AfterEach
@@ -67,26 +78,15 @@ private void setupMockPolicies(
       String effect,
       List<List<String>> resourcesList,
       List<List<MetadataOperation>> operationsList) {
-    // Mock the user
-    mockUser = mock(User.class);
-    EntityReference mockUserReference = mock(EntityReference.class);
-    when(mockUser.getEntityReference()).thenReturn(mockUserReference);
-    when(mockUserReference.getId()).thenReturn(UUID.randomUUID());
-    when(mockUser.getId()).thenReturn(UUID.randomUUID());
-    when(mockUser.getName()).thenReturn("testUser");
 
-    // Mock the policy context
     SubjectContext.PolicyContext mockPolicyContext = mock(SubjectContext.PolicyContext.class);
     when(mockPolicyContext.getPolicyName()).thenReturn("TestPolicy");
 
-    // Create a list of mock rules
     List<CompiledRule> mockRules = new ArrayList<>();
     for (int i = 0; i < expressions.size(); i++) {
       CompiledRule mockRule = mock(CompiledRule.class);
       when(mockRule.getOperations()).thenReturn(List.of(MetadataOperation.VIEW_BASIC));
       when(mockRule.getCondition()).thenReturn(expressions.get(i));
-
-      // Use resources from the corresponding index in resourcesList
       List<String> resources = (resourcesList.size() > i) ? resourcesList.get(i) : List.of("All");
       when(mockRule.getResources()).thenReturn(resources);
 
@@ -100,11 +100,7 @@ private void setupMockPolicies(
     }
 
     when(mockPolicyContext.getRules()).thenReturn(mockRules);
-
-    // Mock the subject context with this policy
-    mockSubjectContext = mock(SubjectContext.class);
     when(mockSubjectContext.getPolicies(any())).thenReturn(List.of(mockPolicyContext).iterator());
-    when(mockSubjectContext.user()).thenReturn(mockUser);
   }
 
   private void setupMockPolicies(String expression, String effect, List<String> resources) {
@@ -127,7 +123,7 @@ void testIsOwner() {
     QueryBuilder elasticQuery = ((ElasticQueryBuilder) finalQuery).build();
     String generatedQuery = elasticQuery.toString();
 
-    assertTrue(generatedQuery.contains("owner.id"), "The query should contain 'owner.id'.");
+    assertTrue(generatedQuery.contains("owners.id"), "The query should contain 'owners.id'.");
     assertTrue(
         generatedQuery.contains(mockUser.getId().toString()),
         "The query should contain the user's ID.");
@@ -144,8 +140,8 @@ void testNegationWithIsOwner() {
     assertTrue(
         generatedQuery.contains("must_not"), "The query should contain 'must_not' for negation.");
     assertTrue(
-        generatedQuery.contains("owner.id"),
-        "The query should contain 'owner.id' in the negation.");
+        generatedQuery.contains("owners.id"),
+        "The query should contain 'owners.id' in the negation.");
     assertTrue(
         generatedQuery.contains(mockUser.getId().toString()),
         "The negation should contain the user's ID.");
@@ -182,7 +178,7 @@ void testComplexCondition() {
         generatedQuery.contains("PII.Sensitive"), "The query should contain 'PII.Sensitive' tag.");
     assertTrue(generatedQuery.contains("Test.Test1"), "The query should contain 'Test.Test1' tag.");
     assertTrue(generatedQuery.contains("Test.Test2"), "The query should contain 'Test.Test2' tag.");
-    assertTrue(generatedQuery.contains("owner.id"), "The query should contain 'owner.id'.");
+    assertTrue(generatedQuery.contains("owners.id"), "The query should contain 'owners.id'.");
 
     assertTrue(
         generatedQuery.contains("must_not"), "The query should contain a negation (must_not).");
@@ -221,13 +217,13 @@ void testComplexQueryStructure() throws IOException {
     // Check for the presence of owner.id in the "must_not" clause for the negation
     assertFieldExists(
         jsonContext,
-        "$.bool.should[2].bool.must_not[0].bool.should[?(@.term['owner.id'])]",
-        "owner.id in must_not");
+        "$.bool.should[2].bool.must_not[0].bool.should[?(@.term['owners.id'])]",
+        "owners.id in must_not");
 
     // Check for the presence of must_not for the case where there is no owner
     assertFieldExists(
         jsonContext,
-        "$.bool.should[3].bool.must_not[?(@.exists.field=='owner.id')]",
+        "$.bool.should[3].bool.must_not[?(@.exists.field=='owners.id')]",
         "no owner must_not clause");
 
     // Count the number of bool clauses
@@ -388,7 +384,7 @@ void testConditionUserLacksDomain() {
     // Check for owner ID and Public tag in the query
     assertFieldExists(
         jsonContext,
-        "$.bool.should[?(@.term['owner.id'].value=='" + mockUser.getId().toString() + "')]",
+        "$.bool.should[?(@.term['owners.id'].value=='" + mockUser.getId().toString() + "')]",
         "owner.id");
     assertFieldExists(
         jsonContext, "$.bool.should[?(@.term['tags.tagFQN'].value=='Public')]", "Public tag");
@@ -415,7 +411,7 @@ void testNestedLogicalOperators() {
 
     assertTrue(generatedQuery.contains("tags.tagFQN"), "The query should contain 'tags.tagFQN'.");
     assertTrue(generatedQuery.contains("Sensitive"), "The query should contain 'Sensitive' tag.");
-    assertTrue(generatedQuery.contains("owner.id"), "The query should contain 'owner.id'.");
+    assertTrue(generatedQuery.contains("owners.id"), "The query should contain 'owners.id'.");
     assertFalse(generatedQuery.contains("match_none"), "The query should not be match_none.");
   }
 
@@ -476,7 +472,7 @@ void testComplexNestedConditions() {
 
     // Ensure that the query does not check for noOwner since inAnyTeam('HR') is false
     assertFieldDoesNotExist(
-        jsonContext, "$.bool.must_not[?(@.exists.field=='owner.id')]", "noOwner clause");
+        jsonContext, "$.bool.must_not[?(@.exists.field=='owners.id')]", "noOwner clause");
 
     // Ensure the query does not contain a match_none condition
     assertFieldDoesNotExist(jsonContext, "$.bool[?(@.match_none)]", "match_none");
@@ -505,8 +501,8 @@ void testMultipleNestedNotOperators() {
         "Internal");
     assertFieldExists(
         jsonContext,
-        "$.bool.must_not[0].bool.should[1].bool.should[?(@.term['owner.id'])]",
-        "owner.id");
+        "$.bool.must_not[0].bool.should[1].bool.should[?(@.term['owners.id'])]",
+        "owners.id");
   }
 
   @Test
@@ -573,7 +569,7 @@ void testNestedAndOrWithMultipleMethods() {
     assertTrue(
         generatedQuery.contains("\"domain.id\""), "The query should contain 'domain.id' term.");
     assertTrue(
-        generatedQuery.contains("\"owner.id\""), "The query should contain 'owner.id' term.");
+        generatedQuery.contains("\"owners.id\""), "The query should contain 'owners.id' term.");
     assertFalse(
         generatedQuery.contains("\"must_not\" : [ { \"exists\""),
         "The query should not contain 'must_not' clause for 'noOwner()'.");
@@ -634,7 +630,7 @@ void testComplexConditionWithAllMethods() {
 
     // Check for the presence of owner.id in the second should block
     assertFieldExists(
-        jsonContext, "$.bool.should[1].bool.should[?(@.term['owner.id'])]", "owner.id");
+        jsonContext, "$.bool.should[1].bool.should[?(@.term['owners.id'])]", "owners.id");
   }
 
   @Test
@@ -740,7 +736,7 @@ void testComplexConditionWithMultipleNegations() {
         "Confidential");
 
     // Ownership (isOwner condition) should be in `should`
-    assertFieldExists(jsonContext, "$.bool.should[?(@.term['owner.id'])]", "owner.id");
+    assertFieldExists(jsonContext, "$.bool.should[?(@.term['owners.id'])]", "owners.id");
   }
 
   @Test
@@ -761,8 +757,8 @@ void testNotHasDomainWhenUserHasNoDomain() {
         "must_not for hasDomain");
     assertFieldExists(
         jsonContext,
-        "$.bool.should[?(@.term['owner.id'].value=='" + mockUser.getId().toString() + "')]",
-        "owner.id");
+        "$.bool.should[?(@.term['owners.id'].value=='" + mockUser.getId().toString() + "')]",
+        "owners.id");
     assertFieldDoesNotExist(jsonContext, "$.bool[?(@.match_none)]", "match_none should not exist");
   }
 
@@ -861,33 +857,33 @@ void testMultipleRulesInPolicy() {
     // Check for the "Table" index condition
     assertFieldExists(
         jsonContext,
-        "$.bool.must[0].bool.must[?(@.terms._index && @.terms._index[?(@ == 'table')])]",
+        "$.bool.should[0].bool.must[?(@.terms._index && @.terms._index[?(@ == 'table')])]",
         "Index filtering for 'Table' resource");
     assertFieldExists(
         jsonContext,
-        "$.bool.must[0].bool.must[?(@.match_all)]",
+        "$.bool.should[0].bool.must[?(@.match_all)]",
         "match_all for hasAnyRole 'Admin'");
     assertFieldExists(
         jsonContext,
-        "$.bool.must[0].bool.should[?(@.term['tags.tagFQN'].value=='Sensitive')]",
+        "$.bool.should[0].bool.should[?(@.term['tags.tagFQN'].value=='Sensitive')]",
         "Sensitive tag");
 
     // Check for the "Dashboard" index condition
     assertFieldExists(
         jsonContext,
-        "$.bool.must[1].bool.must[?(@.terms._index && @.terms._index[?(@ == 'database')])]",
+        "$.bool.should[1].bool.must[?(@.terms._index && @.terms._index[?(@ == 'database')])]",
         "Index filtering for 'Database' resource");
     assertFieldExists(
         jsonContext,
-        "$.bool.must[1].bool.must[?(@.match_all)]",
+        "$.bool.should[1].bool.must[?(@.match_all)]",
         "match_all for inAnyTeam 'Engineering'");
     assertFieldExists(
         jsonContext,
-        "$.bool.must[1].bool.must[?(@.term['tags.tagFQN'].value=='Confidential')]",
+        "$.bool.should[1].bool.must[?(@.term['tags.tagFQN'].value=='Confidential')]",
         "Confidential tag");
     assertFieldExists(
         jsonContext,
-        "$.bool.must[1].bool.must[?(@.term['tags.tagFQN'].value=='Internal')]",
+        "$.bool.should[1].bool.must[?(@.term['tags.tagFQN'].value=='Internal')]",
         "Internal tag");
   }
 
@@ -929,7 +925,7 @@ void testMultiplePoliciesInRole() {
     // Check for domain filtering in the "Table" index clause
     assertFieldExists(
         jsonContext,
-        "$.bool.must[0].bool.must[?(@.term['domain.id'].value=='"
+        "$.bool.should[0].bool.must[?(@.term['domain.id'].value=='"
             + domain.getId().toString()
             + "')]",
         "user's domain ID");
@@ -937,7 +933,7 @@ void testMultiplePoliciesInRole() {
     // Check for the matchAnyTag clause for Public tag in the "Table" index clause
     assertFieldExists(
         jsonContext,
-        "$.bool.must[0].bool.should[?(@.term['tags.tagFQN'].value=='Public')]",
+        "$.bool.should[0].bool.should[?(@.term['tags.tagFQN'].value=='Public')]",
         "Public tag");
   }
 
@@ -979,7 +975,7 @@ void testRoleAndPolicyInheritanceFromTeams() {
     // Adjust the assertion for the hasDomain clause
     assertFieldExists(
         jsonContext,
-        "$.bool.must[0].bool.must[?(@.term['domain.id'].value=='"
+        "$.bool.should[0].bool.must[?(@.term['domain.id'].value=='"
             + domain.getId().toString()
             + "')]",
         "user's domain ID");
@@ -987,13 +983,13 @@ void testRoleAndPolicyInheritanceFromTeams() {
     // Check for the inAnyTeam('Engineering') clause
     assertFieldExists(
         jsonContext,
-        "$.bool.must[1].bool.must[?(@.match_all)]",
+        "$.bool.should[1].bool.must[?(@.match_all)]",
         "match_all for inAnyTeam 'Engineering'");
 
     // Check for the matchAnyTag clause for Critical tag
     assertFieldExists(
         jsonContext,
-        "$.bool.must[1].bool.should[?(@.term['tags.tagFQN'].value=='Critical')]",
+        "$.bool.should[1].bool.should[?(@.term['tags.tagFQN'].value=='Critical')]",
         "Critical tag");
   }
 
@@ -1007,11 +1003,14 @@ void testRuleWithNonViewOperationIgnored() {
         List.of(List.of(MetadataOperation.EDIT_DESCRIPTION)));
 
     // Mock user ownership
-    when(mockUser.getId()).thenReturn(UUID.randomUUID());
+    UUID userId = UUID.randomUUID();
+    when(mockUser.getId()).thenReturn(userId);
 
     // Evaluate the condition
     OMQueryBuilder finalQuery = evaluator.evaluateConditions(mockSubjectContext);
-    assertNull(finalQuery, "The query should be null since the rule is ignored");
+    QueryBuilder elasticQuery = ((ElasticQueryBuilder) finalQuery).build();
+    String generatedQuery = elasticQuery.toString();
+    assertTrue(generatedQuery.contains("match_all"), "The query should contain 'match_all'");
   }
 
   @Test
@@ -1033,8 +1032,199 @@ void testRuleWithViewBasicOperationApplied() {
     String generatedQuery = elasticQuery.toString();
 
     // The rule should affect the search query
-    assertTrue(generatedQuery.contains("owner.id"), "The query should contain 'owner.id'");
+    assertTrue(generatedQuery.contains("owners.id"), "The query should contain 'owner.id'");
     assertTrue(
         generatedQuery.contains(userId.toString()), "The query should contain the user's ID");
   }
+
+  @Test
+  void testDenyAllOperationsOnTableResource() {
+    setupMockPolicies(
+        List.of(""), "DENY", List.of(List.of("Table")), List.of(List.of(MetadataOperation.ALL)));
+
+    UUID userId = UUID.randomUUID();
+    when(mockUser.getId()).thenReturn(userId);
+
+    OMQueryBuilder finalQuery = evaluator.evaluateConditions(mockSubjectContext);
+    QueryBuilder elasticQuery = ((ElasticQueryBuilder) finalQuery).build();
+    String generatedQuery = elasticQuery.toString();
+    DocumentContext jsonContext = JsonPath.parse(generatedQuery);
+
+    assertFieldExists(
+        jsonContext,
+        "$.bool.must_not[0].bool.must[?(@.terms._index && @.terms._index[?(@ == 'table')])]",
+        "must_not clause excluding 'table'");
+
+    // Assertions to ensure 'table' is not included in 'must' or 'should' clauses
+    assertFieldDoesNotExist(
+        jsonContext,
+        "$.bool.must[?(@.terms._index && @.terms._index[?(@ == 'table')])]",
+        "'table' should not be included in must clauses");
+    assertFieldDoesNotExist(
+        jsonContext,
+        "$.bool.should[?(@.terms._index && @.terms._index[?(@ == 'table')])]",
+        "'table' should not be included in should clauses");
+  }
+
+  @Test
+  void testUserSeesOwnedAndUnownedResourcesIncludingTeamOwnership() {
+    // Set up a policy with three rules
+    setupMockPolicies(
+        List.of(
+            "noOwner()", // Rule 1 condition (ViewBasic)
+            "isOwner()" // Rule 2 condition (All operations)
+            ),
+        "ALLOW",
+        List.of(
+            List.of("All"), // Rule 1 resources
+            List.of("All") // Rule 2 resources
+            ),
+        List.of(List.of(MetadataOperation.EDIT_OWNERS), List.of(MetadataOperation.ALL)));
+
+    UUID userId = UUID.randomUUID();
+    when(mockUser.getId()).thenReturn(userId);
+
+    UUID teamId1 = UUID.randomUUID();
+    UUID teamId2 = UUID.randomUUID();
+    EntityReference team1 = new EntityReference();
+    team1.setId(teamId1);
+    team1.setName("TeamA");
+
+    EntityReference team2 = new EntityReference();
+    team2.setId(teamId2);
+    team2.setName("TeamB");
+
+    when(mockUser.getTeams()).thenReturn(List.of(team1, team2));
+
+    OMQueryBuilder finalQuery = evaluator.evaluateConditions(mockSubjectContext);
+    QueryBuilder elasticQuery = ((ElasticQueryBuilder) finalQuery).build();
+    String generatedQuery = elasticQuery.toString();
+    DocumentContext jsonContext = JsonPath.parse(generatedQuery);
+
+    assertFieldExists(
+        jsonContext,
+        "$.bool.should[?(@.term['owners.id'].value=='" + userId + "')]",
+        "The query should allow resources where the user is the owner");
+
+    assertFieldExists(
+        jsonContext,
+        "$.bool.should[?(@.term['owners.id'].value=='" + teamId1 + "')]",
+        "The query should allow resources where TeamA is the owner");
+    assertFieldExists(
+        jsonContext,
+        "$.bool.should[?(@.term['owners.id'].value=='" + teamId2 + "')]",
+        "The query should allow resources where TeamB is the owner");
+  }
+
+  @Test
+  void testPoliciesProcessing() {
+    List<CompiledRule> compiledRules = new ArrayList<>();
+    List<Map<String, Object>> policies = getPolicies();
+
+    for (Map<String, Object> policy : policies) {
+      CompiledRule rule = createCompiledRule(policy);
+      compiledRules.add(rule);
+    }
+
+    SubjectContext.PolicyContext mockPolicyContext = mock(SubjectContext.PolicyContext.class);
+    when(mockPolicyContext.getRules()).thenReturn(compiledRules);
+    Iterator<SubjectContext.PolicyContext> policyContextIterator =
+        Collections.singletonList(mockPolicyContext).iterator();
+    when(mockSubjectContext.getPolicies(anyList())).thenReturn(policyContextIterator);
+
+    OMQueryBuilder finalQuery = evaluator.evaluateConditions(mockSubjectContext);
+
+    QueryBuilder elasticQuery = ((ElasticQueryBuilder) finalQuery).build();
+    String generatedQuery = elasticQuery.toString();
+    DocumentContext jsonContext = JsonPath.parse(generatedQuery);
+
+    assertFalse(
+        generatedQuery.contains("Create"), "The query should not contain 'Create' operation");
+    assertFalse(
+        generatedQuery.contains("Delete"), "The query should not contain 'Delete' operation");
+    assertFalse(generatedQuery.contains("Edit"), "The query should not contain 'Edit' operation");
+
+    boolean containsOwnerCondition = generatedQuery.contains("owners.id");
+    boolean containsTagCondition = generatedQuery.contains("tags.tagFQN");
+    assertTrue(
+        containsOwnerCondition || containsTagCondition,
+        "The query should include 'isOwner' or 'matchAnyTag' conditions where applicable");
+  }
+
+  // Helper methods
+  private CompiledRule createCompiledRule(Map<String, Object> policyDef) {
+    CompiledRule rule = mock(CompiledRule.class);
+    when(rule.getName()).thenReturn((String) policyDef.get("name"));
+    when(rule.getDescription()).thenReturn((String) policyDef.get("description"));
+    when(rule.getEffect())
+        .thenReturn(CompiledRule.Effect.valueOf(((String) policyDef.get("effect")).toUpperCase()));
+    when(rule.getOperations())
+        .thenReturn(mapOperations((List<String>) policyDef.get("operations")));
+    when(rule.getResources()).thenReturn((List<String>) policyDef.get("resources"));
+    when(rule.getCondition()).thenReturn((String) policyDef.getOrDefault("condition", ""));
+    return rule;
+  }
+
+  private List<MetadataOperation> mapOperations(List<String> operations) {
+    List<MetadataOperation> mappedOperations = new ArrayList<>();
+    for (String op : operations) {
+      if (op.equalsIgnoreCase("Create")
+          || op.equalsIgnoreCase("Delete")
+          || op.toLowerCase().startsWith("edit")) {
+        mappedOperations.add(MetadataOperation.VIEW_BASIC);
+      } else if (op.equalsIgnoreCase("All")) {
+        mappedOperations.add(MetadataOperation.ALL);
+      } else {
+        mappedOperations.add(MetadataOperation.fromValue(op));
+      }
+    }
+    return mappedOperations;
+  }
+
+  private List<Map<String, Object>> getPolicies() {
+    List<Map<String, Object>> policyDefs = new ArrayList<>();
+
+    Map<String, Object> policy1 = new HashMap<>();
+    policy1.put("name", "APP_ALLOW");
+    policy1.put("description", "APP MARKET PLACE DEFINITION ALLOW AS OWNER");
+    policy1.put("effect", "allow");
+    policy1.put("operations", List.of("ViewAll"));
+    policy1.put("resources", List.of("appMarketPlaceDefinition"));
+    policy1.put("condition", "matchAnyTag('tags.a') && isOwner()");
+    policyDefs.add(policy1);
+
+    Map<String, Object> policy2 = new HashMap<>();
+    policy2.put("name", "APP_ALLOW_2");
+    policy2.put("description", "APP MARKET PLACE DEFINITION ALLOW");
+    policy2.put("effect", "allow");
+    policy2.put("operations", List.of("ViewBasic"));
+    policy2.put("resources", List.of("appMarketPlaceDefinition"));
+    policyDefs.add(policy2);
+
+    Map<String, Object> policy3 = new HashMap<>();
+    policy3.put("name", "APP_DENY");
+    policy3.put("description", "APP MARKET PLACE DEFINITION DENY");
+    policy3.put("effect", "deny");
+    policy3.put("operations", List.of("Create", "Delete", "EditAll"));
+    policy3.put("resources", List.of("appMarketPlaceDefinition"));
+    policyDefs.add(policy3);
+
+    Map<String, Object> policy4 = new HashMap<>();
+    policy4.put("name", "APP_ALLOW_2");
+    policy4.put("description", "APP ALLOW");
+    policy4.put("effect", "allow");
+    policy4.put("operations", List.of("ViewBasic"));
+    policy4.put("resources", List.of("app"));
+    policyDefs.add(policy4);
+
+    Map<String, Object> policy5 = new HashMap<>();
+    policy5.put("name", "APP_ALLOW_3");
+    policy5.put("description", "APP DENY");
+    policy5.put("effect", "deny");
+    policy5.put("operations", List.of("Create", "Delete", "EditAll"));
+    policy5.put("resources", List.of("app"));
+    policyDefs.add(policy5);
+
+    return policyDefs;
+  }
 }
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/security/OpenSearchRBACConditionEvaluatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/security/OpenSearchRBACConditionEvaluatorTest.java
index 4d2c8c3ec65c..3d3e0bb19fc6 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/search/security/OpenSearchRBACConditionEvaluatorTest.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/security/OpenSearchRBACConditionEvaluatorTest.java
@@ -143,7 +143,7 @@ void testOpenSearchNegationWithDomainAndOwnerChecks() {
         "must_not for hasDomain");
     assertFieldExists(
         jsonContext,
-        "$.bool.should[?(@.term['owner.id'].value=='" + mockUser.getId().toString() + "')]",
+        "$.bool.should[?(@.term['owners.id'].value=='" + mockUser.getId().toString() + "')]",
         "owner.id");
     assertFieldDoesNotExist(jsonContext, "$.bool[?(@.match_none)]", "match_none should not exist");
   }
diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java
index 96d31db0b754..8ad7106a59b4 100644
--- a/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java
+++ b/openmetadata-service/src/test/java/org/openmetadata/service/util/TestUtils.java
@@ -13,6 +13,7 @@
 
 package org.openmetadata.service.util;
 
+import static org.assertj.core.api.Assertions.fail;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -67,7 +68,7 @@
 import org.openmetadata.schema.entity.type.CustomProperty;
 import org.openmetadata.schema.entity.type.Style;
 import org.openmetadata.schema.security.credentials.AWSCredentials;
-import org.openmetadata.schema.services.connections.api.RESTConnection;
+import org.openmetadata.schema.services.connections.api.RestConnection;
 import org.openmetadata.schema.services.connections.database.BigQueryConnection;
 import org.openmetadata.schema.services.connections.database.MysqlConnection;
 import org.openmetadata.schema.services.connections.database.RedshiftConnection;
@@ -83,7 +84,7 @@
 import org.openmetadata.schema.services.connections.search.ElasticSearchConnection;
 import org.openmetadata.schema.services.connections.search.OpenSearchConnection;
 import org.openmetadata.schema.services.connections.storage.S3Connection;
-import org.openmetadata.schema.type.APIServiceConnection;
+import org.openmetadata.schema.type.ApiConnection;
 import org.openmetadata.schema.type.EntityReference;
 import org.openmetadata.schema.type.MessagingConnection;
 import org.openmetadata.schema.type.MlModelConnection;
@@ -183,10 +184,10 @@ public final class TestUtils {
       new SearchConnection()
           .withConfig(new OpenSearchConnection().withHostPort("http://localhost:9200"));
 
-  public static final APIServiceConnection API_SERVICE_CONNECTION =
-      new APIServiceConnection()
+  public static final ApiConnection API_SERVICE_CONNECTION =
+      new ApiConnection()
           .withConfig(
-              new RESTConnection()
+              new RestConnection()
                   .withOpenAPISchemaURL(getUri("http://localhost:8585/swagger.json")));
 
   public static final MetadataConnection AMUNDSEN_CONNECTION =
@@ -723,4 +724,23 @@ public static void assertFieldDoesNotExist(
       assertTrue(true, "The path does not exist as expected: " + jsonPath);
     }
   }
+
+  public static void assertResponseStatus(Executable action, Status expectedStatus) {
+    try {
+      action.execute();
+      fail("Expected HttpResponseException with status " + expectedStatus.getStatusCode());
+    } catch (HttpResponseException e) {
+      assertEquals(
+          expectedStatus.getStatusCode(),
+          e.getStatusCode(),
+          "Expected status code "
+              + expectedStatus.getStatusCode()
+              + " but got "
+              + e.getStatusCode());
+    } catch (Exception e) {
+      fail("Expected HttpResponseException but got " + e.getClass().getName());
+    } catch (Throwable e) {
+      throw new RuntimeException(e);
+    }
+  }
 }
diff --git a/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SchemaProcessingException.java b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SchemaProcessingException.java
new file mode 100644
index 000000000000..5ecf05101e40
--- /dev/null
+++ b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/SchemaProcessingException.java
@@ -0,0 +1,24 @@
+package org.openmetadata.sdk.exception;
+
+import lombok.Getter;
+
+@Getter
+public class SchemaProcessingException extends Exception {
+  public enum ErrorType {
+    RESOURCE_NOT_FOUND,
+    UNSUPPORTED_URL,
+    OTHER
+  }
+
+  private final ErrorType errorType;
+
+  public SchemaProcessingException(String message, ErrorType errorType) {
+    super(message);
+    this.errorType = errorType;
+  }
+
+  public SchemaProcessingException(String message, Throwable cause, ErrorType errorType) {
+    super(message, cause);
+    this.errorType = errorType;
+  }
+}
diff --git a/openmetadata-spec/src/main/resources/json/schema/api/classification/createClassification.json b/openmetadata-spec/src/main/resources/json/schema/api/classification/createClassification.json
index 5316d4855482..4d86fe894c29 100644
--- a/openmetadata-spec/src/main/resources/json/schema/api/classification/createClassification.json
+++ b/openmetadata-spec/src/main/resources/json/schema/api/classification/createClassification.json
@@ -30,6 +30,13 @@
     "domain" : {
       "description": "Fully qualified name of the domain the Table belongs to.",
       "type": "string"
+    },
+    "roles": {
+      "description": "List of roles allowed to apply tags from this classification.",
+      "type": "array",
+      "items": {
+        "$ref": "../../type/entityReference.json"
+      }
     }
   },
   "required": ["name", "description"],
diff --git a/openmetadata-spec/src/main/resources/json/schema/api/services/createApiService.json b/openmetadata-spec/src/main/resources/json/schema/api/services/createApiService.json
index db1af1eb6e07..e1648fd09318 100644
--- a/openmetadata-spec/src/main/resources/json/schema/api/services/createApiService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/api/services/createApiService.json
@@ -1,10 +1,10 @@
 {
-  "$id": "https://open-metadata.org/schema/api/services/createAPIService.json",
+  "$id": "https://open-metadata.org/schema/api/services/createApiService.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "CreateApiServiceRequest",
   "description": "Create API Service entity request",
   "type": "object",
-  "javaType": "org.openmetadata.schema.api.services.CreateAPIService",
+  "javaType": "org.openmetadata.schema.api.services.CreateApiService",
   "javaInterfaces": ["org.openmetadata.schema.CreateEntity"],
 
   "properties": {
@@ -21,10 +21,10 @@
       "$ref": "../../type/basic.json#/definitions/markdown"
     },
     "serviceType": {
-      "$ref": "../../entity/services/apiService.json#/definitions/APIServiceType"
+      "$ref": "../../entity/services/apiService.json#/definitions/apiServiceType"
     },
     "connection": {
-      "$ref": "../../entity/services/apiService.json#/definitions/apiServiceConnection"
+      "$ref": "../../entity/services/apiService.json#/definitions/apiConnection"
     },
     "tags": {
       "description": "Tags for this API Service.",
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/automations/testServiceConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/automations/testServiceConnection.json
index b0992bc878a9..9b906a360143 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/automations/testServiceConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/automations/testServiceConnection.json
@@ -9,6 +9,9 @@
     "connection": {
       "description": "Connection object.",
       "oneOf": [
+        {
+          "$ref": "../services/apiService.json#/definitions/apiConnection"
+        },
         {
           "$ref": "../services/databaseService.json#/definitions/databaseConnection"
         },
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/classification/classification.json b/openmetadata-spec/src/main/resources/json/schema/entity/classification/classification.json
index 59f747a78233..62698c20d8fd 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/classification/classification.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/classification/classification.json
@@ -35,6 +35,13 @@
       "type" : "integer",
       "minimum": 0
     },
+    "roles": {
+      "description": "List of roles allowed to apply tags from this classification.",
+      "type": "array",
+      "items": {
+        "$ref": "../../type/entityReference.json"
+      }
+    },
     "updatedAt": {
       "description": "Last update time corresponding to the new version of the entity in Unix epoch time milliseconds.",
       "$ref": "../../type/basic.json#/definitions/timestamp"
@@ -65,7 +72,8 @@
     },
     "disabled" : {
       "description": "System classifications can't be deleted. Use this flag to disable them.",
-      "type": "boolean"
+      "type": "boolean",
+      "default": false
     },
     "mutuallyExclusive" : {
       "description" : "Tags under this classification are mutually exclusive. When mutually exclusive is `true` the tags from this classification are used to **classify** an entity. An entity can only be in one class - example, it can only be either `tier1` or `tier2` and not both. When mutually exclusive is `false`, the tags from this classification are used to **categorize** an entity. An entity have multiple tags simultaneously - example a customer can be `newCustomer` and `atRisk` simultaneously.",
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/classification/tag.json b/openmetadata-spec/src/main/resources/json/schema/entity/classification/tag.json
index 465863442553..fe79c226a3e8 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/classification/tag.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/classification/tag.json
@@ -45,6 +45,10 @@
       "description": "Children tags under this tag.",
       "$ref": "../../type/entityReferenceList.json"
     },
+    "inheritedRoles": {
+      "description": "Roles that a tag is inheriting through classification.",
+      "$ref": "../../type/entityReferenceList.json"
+    },
     "version": {
       "description": "Metadata version of the entity.",
       "$ref": "../../type/entityHistory.json#/definitions/entityVersion"
@@ -84,7 +88,8 @@
     },
     "disabled": {
       "description": "System tags can't be deleted. Use this flag to disable them.",
-      "type": "boolean"
+      "type": "boolean",
+      "default": false
     },
     "mutuallyExclusive": {
       "description": "Children tags under this group are mutually exclusive. When mutually exclusive is `true` the tags from this group are used to **classify** an entity. An entity can only be in one class - example, it can only be either `tier1` or `tier2` and not both. When mutually exclusive is `false`, the tags from this group are used to **categorize** an entity. An entity can be in multiple categories simultaneously - example a customer can be `newCustomer` and `atRisk` simultaneously.",
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/apiCollection.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/apiCollection.json
index 370f9ad0ce13..f39bf6481ad9 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/data/apiCollection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/apiCollection.json
@@ -74,7 +74,7 @@
     },
     "serviceType": {
       "description": "Service type where this API Collection is hosted in.",
-      "$ref": "../services/apiService.json#/definitions/APIServiceType"
+      "$ref": "../services/apiService.json#/definitions/apiServiceType"
     },
     "changeDescription": {
       "description": "Change that lead to this version of the entity.",
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/apiEndpoint.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/apiEndpoint.json
index 8c028fafaf28..dcbb280247ff 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/data/apiEndpoint.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/apiEndpoint.json
@@ -136,7 +136,7 @@
     },
     "serviceType": {
       "description": "Service type where this API Collection is hosted in.",
-      "$ref": "../services/apiService.json#/definitions/APIServiceType"
+      "$ref": "../services/apiService.json#/definitions/apiServiceType"
     },
     "changeDescription": {
       "description": "Change that lead to this version of the entity.",
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json
index a8092ac328e0..7ce119b3990e 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json
@@ -28,6 +28,12 @@
         {
           "name": "TableauDataModel"
         },
+        {
+          "name": "TableauPublishedDatasource"
+        },
+        {
+          "name": "TableauEmbeddedDatasource"
+        },
         {
           "name": "SupersetDataModel"
         },
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json b/openmetadata-spec/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json
index 76c7e415c94f..315348886eef 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/policies/accessControl/resourceDescriptor.json
@@ -21,6 +21,7 @@
         "ViewTests",
         "ViewQueries",
         "ViewDataProfile",
+        "ViewProfilerGlobalConfiguration",
         "ViewSampleData",
         "ViewTestCaseFailedRowsSample",
         "EditAll",
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json
index 4619f680a1b8..145101bc36eb 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/apiService.json
@@ -1,37 +1,37 @@
 {
   "$id": "https://open-metadata.org/schema/entity/services/apiService.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "API Service",
+  "title": "Api Service",
   "description": "This schema defines the API Service entity, to capture metadata from any REST API Services.",
   "type": "object",
-  "javaType": "org.openmetadata.schema.entity.services.APIService",
+  "javaType": "org.openmetadata.schema.entity.services.ApiService",
   "javaInterfaces": [
     "org.openmetadata.schema.EntityInterface",
     "org.openmetadata.schema.ServiceEntityInterface"
   ],
   "definitions": {
-    "APIServiceType": {
+    "apiServiceType": {
       "description": "Type of api service such as REST, Webhook,...",
       "javaInterfaces": [
         "org.openmetadata.schema.EnumInterface"
       ],
       "type": "string",
       "enum": [
-        "REST",
+        "Rest",
         "WEBHOOK"
       ],
       "javaEnums": [
         {
-          "name": "REST"
+          "name": "Rest"
         },
         {
           "name": "WEBHOOK"
         }
       ]
     },
-    "apiServiceConnection": {
+    "apiConnection": {
       "type": "object",
-      "javaType": "org.openmetadata.schema.type.APIServiceConnection",
+      "javaType": "org.openmetadata.schema.type.ApiConnection",
       "description": "API Service Connection.",
       "javaInterfaces": [
         "org.openmetadata.schema.ServiceConnectionEntityInterface"
@@ -41,7 +41,7 @@
           "mask": true,
           "oneOf": [
             {
-              "$ref": "./connections/apiService/restConnection.json"
+              "$ref": "./connections/api/restConnection.json"
             }
           ]
         }
@@ -68,14 +68,14 @@
     },
     "serviceType": {
       "description": "Type of API service such as REST, WEBHOOK..",
-      "$ref": "#/definitions/APIServiceType"
+      "$ref": "#/definitions/apiServiceType"
     },
     "description": {
       "description": "Description of a API service instance.",
       "$ref": "../../type/basic.json#/definitions/markdown"
     },
     "connection": {
-      "$ref": "#/definitions/apiServiceConnection"
+      "$ref": "#/definitions/apiConnection"
     },
     "pipelines": {
       "description": "References to pipelines deployed for this API service to extract metadata, usage, lineage etc..",
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/apiService/restConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/api/restConnection.json
similarity index 82%
rename from openmetadata-spec/src/main/resources/json/schema/entity/services/connections/apiService/restConnection.json
rename to openmetadata-spec/src/main/resources/json/schema/entity/services/connections/api/restConnection.json
index a00eb5673b9a..4df7a505c914 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/apiService/restConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/api/restConnection.json
@@ -1,24 +1,24 @@
 {
-  "$id": "https://open-metadata.org/schema/entity/services/connections/apiService/restConnection.json",
+  "$id": "https://open-metadata.org/schema/entity/services/connections/api/restConnection.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "RESTConnection",
+  "title": "RestConnection",
   "description": "REST Connection Config",
   "type": "object",
-  "javaType": "org.openmetadata.schema.services.connections.api.RESTConnection",
+  "javaType": "org.openmetadata.schema.services.connections.api.RestConnection",
   "definitions": {
-    "RESTAPIType": {
+    "restType": {
       "description": "REST API type",
       "type": "string",
-      "enum": ["REST"],
-      "default": "REST"
+      "enum": ["Rest"],
+      "default": "Rest"
     }
   },
   "properties": {
     "type": {
       "title": "REST API Type",
       "description": "REST API Type",
-      "$ref": "#/definitions/RESTAPIType",
-      "default": "REST"
+      "$ref": "#/definitions/restType",
+      "default": "Rest"
     },
     "openAPISchemaURL": {
       "expose": true,
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json
index 3f3fa95cab2d..73fb818a5c69 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/connectionBasicType.json
@@ -64,6 +64,11 @@
       "description": "Supports DBT Extraction.",
       "type": "boolean",
       "default": true
+    }, 
+    "supportsDataDiff": {
+      "description": "Supports the data diff data qualty specification.",
+      "type": "boolean",
+      "default": true
     },
     "dataStorageConfig": {
       "description": "Storage config to store sample data",
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/athenaConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/athenaConnection.json
index d6fd7c3517de..f6fa6c3bb0a5 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/athenaConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/athenaConnection.json
@@ -86,6 +86,10 @@
       "type": "boolean",
       "default": true
     },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
+    },
     "sampleDataStorageConfig": {
       "title": "Storage Config for Sample Data",
       "$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json
index ba1fc6af000b..f718cfe08d3a 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/bigQueryConnection.json
@@ -104,6 +104,10 @@
     "sampleDataStorageConfig": {
       "title": "Storage Config for Sample Data",
       "$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"
+    },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
     }
   },
   "additionalProperties": false,
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json
index b5a245456bc5..01d95a145976 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mssqlConnection.json
@@ -101,6 +101,10 @@
     "supportsQueryComment": {
       "title": "Supports Query Comment",
       "$ref": "../connectionBasicType.json#/definitions/supportsQueryComment"
+    },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
     }
   },
   "additionalProperties": false,
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json
index 7c7fda05d47a..b8455270fb9c 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json
@@ -102,6 +102,10 @@
     "sampleDataStorageConfig": {
       "title": "Storage Config for Sample Data",
       "$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"
+    },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
     }
   },
   "additionalProperties": false,
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/oracleConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/oracleConnection.json
index 9363ef5e35b1..455099d16014 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/oracleConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/oracleConnection.json
@@ -140,6 +140,10 @@
       "title": "Supports Query Comment",
       "$ref": "../connectionBasicType.json#/definitions/supportsQueryComment"
     },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
+    },
     "sampleDataStorageConfig": {
       "title": "Storage Config for Sample Data",
       "$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json
index c7d12ed100b0..71defb2941b8 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/postgresConnection.json
@@ -121,6 +121,10 @@
     "sampleDataStorageConfig": {
       "title": "Storage Config for Sample Data",
       "$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"
+    },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
     }
   },
   "additionalProperties": false,
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/redshiftConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/redshiftConnection.json
index e441a380238a..808158b04ce1 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/redshiftConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/redshiftConnection.json
@@ -105,6 +105,10 @@
     "sampleDataStorageConfig": {
       "title": "Storage Config for Sample Data",
       "$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"
+    },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
     }
   },
   "additionalProperties": false,
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHanaConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHanaConnection.json
index 9337f0e66133..7db0325126e2 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHanaConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/sapHanaConnection.json
@@ -74,6 +74,10 @@
     "sampleDataStorageConfig": {
       "title": "Storage Config for Sample Data",
       "$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"
+    },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
     }
   },
   "additionalProperties": false,
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json
index 695c321c25cc..3ba10f899eb0 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/snowflakeConnection.json
@@ -128,6 +128,10 @@
     "sampleDataStorageConfig": {
       "title": "Storage Config for Sample Data",
       "$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"
+    },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
     }
   },
   "additionalProperties": false,
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/trinoConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/trinoConnection.json
index 08b3ab4c64d0..df1b761be8f7 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/trinoConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/trinoConnection.json
@@ -118,6 +118,10 @@
     "sampleDataStorageConfig": {
       "title": "Storage Config for Sample Data",
       "$ref": "../connectionBasicType.json#/definitions/sampleDataStorageConfig"
+    },
+    "supportsDataDiff": {
+        "title": "Supports Data Diff Extraction.",
+        "$ref": "../connectionBasicType.json#/definitions/supportsDataDiff"
     }
   },
   "additionalProperties": false,
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json
index 0a7fb06d71e4..07598d15cbd8 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/matillionConnection.json
@@ -17,6 +17,7 @@
     "matillionETL": {
       "description": "Matillion ETL Auth Config",
       "type": "object",
+      "title": "Matillion ETL Auth Config",
       "properties": {
         "type": {
           "type": "string",
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/serviceConnection.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/serviceConnection.json
index 53521dd93992..9e53af62e1f1 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/serviceConnection.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/connections/serviceConnection.json
@@ -9,7 +9,7 @@
       "description": "Supported services",
       "oneOf": [
         {
-          "$ref": "../apiService.json#/definitions/apiServiceConnection"
+          "$ref": "../apiService.json#/definitions/apiConnection"
         },
         {
           "$ref": "../dashboardService.json#/definitions/dashboardConnection"
diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/services/serviceType.json b/openmetadata-spec/src/main/resources/json/schema/entity/services/serviceType.json
index 7fdbb85abcd3..395e6c635cd1 100644
--- a/openmetadata-spec/src/main/resources/json/schema/entity/services/serviceType.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/services/serviceType.json
@@ -14,7 +14,7 @@
     "Pipeline",
     "Storage",
     "Search",
-    "API"
+    "Api"
   ],
   "additionalProperties": false
 }
diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json
index 1a1b17787e6e..1703f4b5b805 100644
--- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json
+++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/workflow.json
@@ -163,6 +163,14 @@
           "$ref": "#/definitions/logLevels",
           "default": "INFO"
         },
+        "successThreshold": {
+          "title": "Success Threshold",
+          "description": "The percentage of successfully processed records that must be achieved for the pipeline to be considered successful. Otherwise, the pipeline will be marked as failed.",
+          "type": "integer",
+          "default": 90,
+          "minimum": 0,
+          "maximum": 100
+        },
         "openMetadataServerConfig": {
           "$ref": "../entity/services/connections/metadata/openMetadataConnection.json"
         },
diff --git a/openmetadata-spec/src/main/resources/json/schema/type/basic.json b/openmetadata-spec/src/main/resources/json/schema/type/basic.json
index a96aa49df4d5..4c2528391daf 100644
--- a/openmetadata-spec/src/main/resources/json/schema/type/basic.json
+++ b/openmetadata-spec/src/main/resources/json/schema/type/basic.json
@@ -105,22 +105,6 @@
         "type": "string"
       }
     },
-    "enumWithDescriptions": {
-      "$comment": "@om-field-type",
-      "description": "List of values in Enum with description for each key.",
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "key": {
-            "type": "string"
-          },
-          "description": {
-            "type": "string"
-          }
-        }
-      }
-    },
     "timezone": {
       "description": "Timezone of the user in the format `America/Los_Angeles`, `Brazil/East`, etc.",
       "type": "string",
diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/complexTypes.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/complexTypes.json
index 76737b615932..6bb1023e16d4 100644
--- a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/complexTypes.json
+++ b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/complexTypes.json
@@ -1,7 +1,6 @@
 {
   "$id": "https://open-metadata.org/schema/type/customProperties/complexTypes.json",
   "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "Basic",
   "description": "This schema defines custom properties complex types.",
   "definitions": {
     "entityReference": {
@@ -99,6 +98,45 @@
           }
         }
       }
+    },
+    "table-cp": {
+      "$comment": "@om-field-type",
+      "title": "Table",
+      "description": "A table-type custom property having rows and columns where all column data types are strings.",
+      "type": "object",
+      "javaType": "org.openmetadata.schema.type.customProperties.Table",
+      "properties": {
+        "columns": {
+          "type": "array",
+          "description": "List of column names defined at the entity type level.",
+          "items": {
+            "type": "string",
+            "description": "The name of the column."
+          },
+          "minItems": 1,
+          "maxItems": 3,
+          "uniqueItems": true
+        },
+        "rows": {
+          "type": "array",
+          "description": "List of rows added at the entity instance level. Each row contains dynamic fields based on the defined columns.",
+          "items": {
+            "type": "object",
+            "description": "A row in the table, with dynamic key-value pairs corresponding to columns.",
+            "additionalProperties": {
+              "type": "string",
+              "description": "The cell value of each column in the row."
+            }
+          }
+        }
+      },
+      "required": ["columns"],
+      "additionalProperties": false
+    }
+  },
+  "properties": {
+    "table-cp": {
+      "$ref": "#/definitions/table-cp"
     }
   }
 }
\ No newline at end of file
diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json
index 459dbac8aab6..e9ffe0df6b9a 100644
--- a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json
+++ b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumConfig.json
@@ -3,7 +3,7 @@
   "$schema": "http://json-schema.org/draft-07/schema#",
   "title": "EnumConfig",
   "type": "object",
-  "javaType": "org.openmetadata.schema.type.customproperties.EnumConfig",
+  "javaType": "org.openmetadata.schema.type.customProperties.EnumConfig",
   "description": "Applies to Enum type, this config is used to define list of enum values",
   "properties": {
     "multiSelect": {
diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumWithDescriptionsConfig.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumWithDescriptionsConfig.json
deleted file mode 100644
index e241a4b75554..000000000000
--- a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/enumWithDescriptionsConfig.json
+++ /dev/null
@@ -1,31 +0,0 @@
-{
-  "$id": "https://open-metadata.org/schema/type/customPropertyEnumWithDescriptionsConfig.json",
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "title": "EnumWithDescriptionsConfig",
-  "type": "object",
-  "javaType": "org.openmetadata.schema.type.customproperties.EnumWithDescriptionsConfig",
-  "description": "Custom property to define enum values with descriptions and extendable fields",
-  "properties": {
-    "multiSelect": {
-      "type": "boolean",
-      "default": false
-    },
-    "values": {
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "key": {
-            "type": "string"
-          },
-          "description": {
-            "type": "string"
-          }
-        },
-        "required": ["key","description"],
-        "additionalProperties": false
-      }
-    }
-  },
-  "additionalProperties": false
-}
\ No newline at end of file
diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperties/tableConfig.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/tableConfig.json
new file mode 100644
index 000000000000..55a1bdaf78a7
--- /dev/null
+++ b/openmetadata-spec/src/main/resources/json/schema/type/customProperties/tableConfig.json
@@ -0,0 +1,33 @@
+{
+  "$id": "https://open-metadata.org/schema/type/customProperties/tableConfig.json",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "TableConfig",
+  "description": "Custom property configuration for table-type property where all column data types are strings.",
+  "type": "object",
+  "javaType": "org.openmetadata.schema.type.customProperties.TableConfig",
+  "properties": {
+    "columns": {
+      "type": "array",
+      "description": "List of column names defined at the entity type level.",
+      "items": {
+        "type": "string",
+        "description": "The name of the column."
+      },
+      "minItems": 1,
+      "maxItems": 3,
+      "uniqueItems": true
+    },
+    "minColumns": {
+      "type": "integer",
+      "default": 1,
+      "$comment": "For internal use only: Minimum number of columns."
+    },
+    "maxColumns": {
+      "type": "integer",
+      "default": 3,
+      "$comment": "For internal use only: Maximum number of columns."
+    }
+  },
+  "required": ["columns"],
+  "additionalProperties": false
+}
\ No newline at end of file
diff --git a/openmetadata-spec/src/main/resources/json/schema/type/customProperty.json b/openmetadata-spec/src/main/resources/json/schema/type/customProperty.json
index 4073605c01d6..6cc2ba6624c7 100644
--- a/openmetadata-spec/src/main/resources/json/schema/type/customProperty.json
+++ b/openmetadata-spec/src/main/resources/json/schema/type/customProperty.json
@@ -35,7 +35,7 @@
               "$ref": "#/definitions/entityTypes"
             },
             {
-              "$ref": "../type/customProperties/enumWithDescriptionsConfig.json"
+              "$ref": "customProperties/tableConfig.json"
             }
           ]
         }
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/customProperty.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/customProperty.ts
index ccfbfa985c8c..a0bc78e8409e 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/constant/customProperty.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/customProperty.ts
@@ -48,25 +48,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'containers',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_dashboard: {
@@ -79,25 +69,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'dashboards',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_database: {
@@ -110,25 +90,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'databases',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_databaseSchema: {
@@ -141,25 +111,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'databaseSchemas',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_glossaryTerm: {
@@ -172,25 +132,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'glossaryTerm',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_mlmodel: {
@@ -203,25 +153,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'mlmodels',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_pipeline: {
@@ -234,25 +174,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: true,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'pipelines',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_searchIndex: {
@@ -265,25 +195,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'searchIndexes',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_storedProcedure: {
@@ -296,25 +216,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'storedProcedures',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_table: {
@@ -327,25 +237,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'tables',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_topic: {
@@ -358,25 +258,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'topics',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
   entity_apiCollection: {
     name: 'apiCollection',
@@ -388,25 +278,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'apiCollections',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 
   entity_apiEndpoint: {
@@ -419,25 +299,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'apiEndpoints',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
   entity_dataProduct: {
     name: 'dataProduct',
@@ -449,25 +319,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'dataProducts',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
   entity_dashboardDataModel: {
     name: 'dataModel',
@@ -479,25 +339,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'dashboardDataModels',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
   entity_metric: {
     name: 'metric',
@@ -509,25 +359,15 @@ export const CUSTOM_PROPERTIES_ENTITIES = {
       values: ['enum1', 'enum2', 'enum3'],
       multiSelect: false,
     },
-    enumWithDescriptionConfig: {
-      values: [
-        {
-          key: 'enumWithDescription1',
-          description: 'This is enumWithDescription1',
-        },
-        {
-          key: 'enumWithDescription2',
-          description: 'This is enumWithDescription2',
-        },
-      ],
-      multiSelect: false,
-    },
     dateFormatConfig: 'yyyy-MM-dd',
     dateTimeFormatConfig: 'yyyy-MM-dd HH:mm:ss',
     timeFormatConfig: 'HH:mm:ss',
     entityReferenceConfig: ['User', 'Team'],
     entityObj: {},
     entityApiType: 'metrics',
+    tableConfig: {
+      columns: ['pw-column1', 'pw-column2'],
+    },
   },
 };
 
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts
index 849535a71308..b07430dc15ef 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts
@@ -14,7 +14,7 @@ export const CUSTOM_PROPERTIES_TYPES = {
   STRING: 'String',
   MARKDOWN: 'Markdown',
   SQL_QUERY: 'Sql Query',
-  ENUM_WITH_DESCRIPTIONS: 'Enum With Descriptions',
+  TABLE: 'Table',
 };
 
 export const FIELD_VALUES_CUSTOM_PROPERTIES = {
@@ -30,4 +30,8 @@ This project is designed to **simplify** and *automate* daily tasks. It aims to:
 2. **Real-Time Analytics**: Get up-to-date insights on task progress.
 3. **Automation**: Automate repetitive workflows using custom scripts.`,
   SQL_QUERY: 'SELECT * FROM table_name WHERE id="20";',
+  TABLE: {
+    columns: ['pw-import-export-column1', 'pw-import-export-column2'],
+    rows: 'pw-import-export-row1-column1,pw-import-export-row1-column2',
+  },
 };
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts
index 33b31e87e7d2..037403f28ce3 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/settings.ts
@@ -147,10 +147,7 @@ export const SETTINGS_OPTIONS_PATH = {
     GlobalSettingsMenuCategory.MEMBERS,
     `${GlobalSettingsMenuCategory.MEMBERS}.${GlobalSettingOptions.ADMINS}`,
   ],
-  [GlobalSettingOptions.PERSONA]: [
-    GlobalSettingsMenuCategory.MEMBERS,
-    `${GlobalSettingsMenuCategory.MEMBERS}.${GlobalSettingOptions.PERSONA}`,
-  ],
+  [GlobalSettingOptions.PERSONA]: [GlobalSettingOptions.PERSONA],
 
   // Access Control
 
@@ -165,10 +162,6 @@ export const SETTINGS_OPTIONS_PATH = {
 
   // Open-metadata
 
-  [GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE]: [
-    GlobalSettingsMenuCategory.PREFERENCES,
-    `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE}`,
-  ],
   [GlobalSettingOptions.EMAIL]: [
     GlobalSettingsMenuCategory.PREFERENCES,
     `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.EMAIL}`,
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts
index 25d7c3179b1c..5bfe3076a09d 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ActivityFeed.spec.ts
@@ -30,6 +30,7 @@ import {
 } from '../../utils/activityFeed';
 import { performAdminLogin } from '../../utils/admin';
 import {
+  clickOutside,
   descriptionBox,
   redirectToHomePage,
   removeLandingBanner,
@@ -877,6 +878,7 @@ base.describe('Activity feed with Data Consumer User', () => {
         const dropdownValue = page2.getByTestId(`tag-PII.None`);
         await dropdownValue.hover();
         await dropdownValue.click();
+        await clickOutside(page2);
 
         await expect(page2.getByTestId('selected-tag-PII.None')).toBeVisible();
 
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/EntitySummaryPanel.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/EntitySummaryPanel.spec.ts
index 4ff9e53375d5..9606d49f9a80 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/EntitySummaryPanel.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/EntitySummaryPanel.spec.ts
@@ -43,7 +43,7 @@ test.describe('Entity Summary Panel', () => {
     await expect(
       page.locator('.ant-drawer-title > [data-testid="entity-link"]')
     ).toBeVisible();
-    await expect(page.getByTestId('Owner-value')).toBeVisible();
+    await expect(page.getByTestId('Owners-value')).toBeVisible();
     await expect(page.getByTestId('Tier-label')).toBeVisible();
     await expect(page.getByTestId('Service-label')).toBeVisible();
     await expect(page.getByTestId('Usage-label')).toBeVisible();
@@ -58,7 +58,7 @@ test.describe('Entity Summary Panel', () => {
     await expect(
       page.locator('.ant-drawer-title > [data-testid="entity-link"]')
     ).toBeVisible();
-    await expect(page.getByTestId('Owner-value')).toBeVisible();
+    await expect(page.getByTestId('Owners-value')).toBeVisible();
     await expect(page.getByTestId('Tier-label')).toBeVisible();
     await expect(page.getByTestId('Service-label')).toBeVisible();
     await expect(page.getByTestId('Database-label')).toBeVisible();
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts
index c0a0968e49c1..3c941f8d775e 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/ExploreQuickFilters.spec.ts
@@ -80,7 +80,7 @@ test('search dropdown should work properly for quick filters', async ({
 
 test('should search for empty or null filters', async ({ page }) => {
   const items = [
-    { label: 'Owner', key: 'owners.displayName.keyword' },
+    { label: 'Owners', key: 'owners.displayName.keyword' },
     { label: 'Tag', key: 'tags.tagFQN' },
     { label: 'Domain', key: 'domain.displayName.keyword' },
     { label: 'Tier', key: 'tier.tagFQN' },
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ApiServiceRest.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ApiServiceRest.spec.ts
index ec7a2f4c7887..d9b46f3d5148 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ApiServiceRest.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/ApiServiceRest.spec.ts
@@ -35,7 +35,7 @@ test.describe('API service', () => {
     await settingClick(page, GlobalSettingOptions.APIS);
 
     await page.getByTestId('add-service-button').click();
-    await page.getByTestId('REST').click();
+    await page.getByTestId('Rest').click();
     await page.getByTestId('next-button').click();
 
     // step 1
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/Metric.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/Metric.spec.ts
index 3e8647d613bd..e76384102618 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/Metric.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/Metric.spec.ts
@@ -149,7 +149,7 @@ test.describe('Listing page and add Metric flow should work', () => {
       page.getByRole('cell', { name: 'Glossary Terms', exact: true })
     ).toBeVisible();
     await expect(
-      page.getByRole('cell', { name: 'Owner', exact: true })
+      page.getByRole('cell', { name: 'Owners', exact: true })
     ).toBeVisible();
 
     // check for metric entities in table
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PersonaFlow.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PersonaFlow.spec.ts
index 720a33b3b166..50aec87f6aba 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PersonaFlow.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Flow/PersonaFlow.spec.ts
@@ -102,15 +102,7 @@ test.describe.serial('Persona operations', () => {
     // Verify created persona details
 
     await expect(
-      page
-        .getByTestId('persona-details-card')
-        .getByText(PERSONA_DETAILS.displayName)
-    ).toBeVisible();
-
-    await expect(
-      page
-        .getByTestId('persona-details-card')
-        .getByText(PERSONA_DETAILS.description)
+      page.getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`)
     ).toBeVisible();
 
     const personaResponse = page.waitForResponse(
@@ -120,8 +112,7 @@ test.describe.serial('Persona operations', () => {
     );
 
     await page
-      .locator('[data-testid="persona-details-card"]')
-      .getByText(PERSONA_DETAILS.displayName)
+      .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`)
       .click();
 
     await personaResponse;
@@ -153,8 +144,7 @@ test.describe.serial('Persona operations', () => {
     page,
   }) => {
     await page
-      .locator('[data-testid="persona-details-card"]')
-      .getByText(PERSONA_DETAILS.displayName)
+      .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`)
       .click();
 
     await page.getByTestId('edit-description').click();
@@ -180,9 +170,7 @@ test.describe.serial('Persona operations', () => {
 
   test('Persona rename flow should work properly', async ({ page }) => {
     await page
-      .locator('[data-testid="persona-details-card"]')
-      .getByText(PERSONA_DETAILS.displayName)
-
+      .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`)
       .click();
 
     await updatePersonaDisplayName({ page, displayName: 'Test Persona' });
@@ -203,8 +191,7 @@ test.describe.serial('Persona operations', () => {
 
   test('Remove users in persona should work properly', async ({ page }) => {
     await page
-      .locator('[data-testid="persona-details-card"]')
-      .getByText(PERSONA_DETAILS.displayName)
+      .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`)
       .click();
 
     await page
@@ -231,8 +218,7 @@ test.describe.serial('Persona operations', () => {
 
   test('Delete persona should work properly', async ({ page }) => {
     await page
-      .locator('[data-testid="persona-details-card"]')
-      .getByText(PERSONA_DETAILS.displayName)
+      .getByTestId(`persona-details-card-${PERSONA_DETAILS.name}`)
       .click();
 
     await page.click('[data-testid="manage-button"]');
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Customproperties-part2.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Customproperties-part2.spec.ts
index c75d77638af9..c2df89ac1aab 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Customproperties-part2.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Customproperties-part2.spec.ts
@@ -52,38 +52,29 @@ test.describe('Custom properties with custom property config', () => {
     });
   });
 
-  test.describe(
-    'Add update and delete Enum With Descriptions custom properties',
-    () => {
-      Object.values(CUSTOM_PROPERTIES_ENTITIES).forEach(async (entity) => {
-        const propertyName = `pwcustomproperty${entity.name}test${uuid()}`;
+  test.describe('Add update and delete Table custom properties', () => {
+    Object.values(CUSTOM_PROPERTIES_ENTITIES).forEach(async (entity) => {
+      const propertyName = `pwcustomproperty${entity.name}test${uuid()}`;
 
-        test(`Add Enum With Descriptions custom property for ${entity.name}`, async ({
-          page,
-        }) => {
-          test.slow(true);
+      test(`Add Table custom property for ${entity.name}`, async ({ page }) => {
+        test.slow(true);
 
-          await settingClick(page, entity.entityApiType, true);
+        await settingClick(page, entity.entityApiType, true);
 
-          await addCustomPropertiesForEntity({
-            page,
-            propertyName,
-            customPropertyData: entity,
-            customType: 'Enum With Descriptions',
-            enumWithDescriptionConfig: entity.enumWithDescriptionConfig,
-          });
+        await addCustomPropertiesForEntity({
+          page,
+          propertyName,
+          customPropertyData: entity,
+          customType: 'Table',
+          tableConfig: entity.tableConfig,
+        });
 
-          await editCreatedProperty(
-            page,
-            propertyName,
-            'Enum With Descriptions'
-          );
+        await editCreatedProperty(page, propertyName, 'Table');
 
-          await deleteCreatedProperty(page, propertyName);
-        });
+        await deleteCreatedProperty(page, propertyName);
       });
-    }
-  );
+    });
+  });
 
   test.describe(
     'Add update and delete Entity Reference custom properties',
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExploreTree.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExploreTree.spec.ts
index 9f0ba1bacd5b..9db6cfb2d1c9 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExploreTree.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ExploreTree.spec.ts
@@ -66,8 +66,8 @@ test.describe('Explore Tree scenarios ', () => {
       await expect(
         page.getByTestId('search-dropdown-Domain').locator('span')
       ).toContainText('Domain');
-      await expect(page.getByTestId('search-dropdown-Owner')).toContainText(
-        'Owner'
+      await expect(page.getByTestId('search-dropdown-Owners')).toContainText(
+        'Owners'
       );
       await expect(
         page.getByTestId('search-dropdown-Tag').locator('span')
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts
index c28eeba14f22..9d3394b4fa73 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts
@@ -12,7 +12,10 @@
  */
 import { expect, test } from '@playwright/test';
 import { CUSTOM_PROPERTIES_ENTITIES } from '../../constant/customProperty';
-import { CUSTOM_PROPERTIES_TYPES } from '../../constant/glossaryImportExport';
+import {
+  CUSTOM_PROPERTIES_TYPES,
+  FIELD_VALUES_CUSTOM_PROPERTIES,
+} from '../../constant/glossaryImportExport';
 import { GlobalSettingOptions } from '../../constant/settings';
 import { SidebarItem } from '../../constant/sidebar';
 import { Glossary } from '../../support/glossary/Glossary';
@@ -96,7 +99,9 @@ test.describe('Glossary Bulk Import Export', () => {
           propertyName,
           customPropertyData: entity,
           customType: property,
-          enumWithDescriptionConfig: entity.enumWithDescriptionConfig,
+          tableConfig: {
+            columns: FIELD_VALUES_CUSTOM_PROPERTIES.TABLE.columns,
+          },
         });
       }
     });
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ServiceIngestion.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ServiceIngestion.spec.ts
index 2bfecbae4ec9..70f21743c5ef 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ServiceIngestion.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/ServiceIngestion.spec.ts
@@ -16,6 +16,7 @@ import { PLAYWRIGHT_INGESTION_TAG_OBJ } from '../../constant/config';
 import { MYSQL, POSTGRES, REDSHIFT } from '../../constant/service';
 import { GlobalSettingOptions } from '../../constant/settings';
 import AirflowIngestionClass from '../../support/entity/ingestion/AirflowIngestionClass';
+import ApiIngestionClass from '../../support/entity/ingestion/ApiIngestionClass';
 import BigQueryIngestionClass from '../../support/entity/ingestion/BigQueryIngestionClass';
 import KafkaIngestionClass from '../../support/entity/ingestion/KafkaIngestionClass';
 import MetabaseIngestionClass from '../../support/entity/ingestion/MetabaseIngestionClass';
@@ -30,6 +31,7 @@ import { INVALID_NAMES, redirectToHomePage } from '../../utils/common';
 import { settingClick, SettingOptionsType } from '../../utils/sidebar';
 
 const services = [
+  ApiIngestionClass,
   S3IngestionClass,
   MetabaseIngestionClass,
   MysqlIngestionClass,
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts
index cbb3cbea0160..504498aa1b46 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/Tags.spec.ts
@@ -13,6 +13,8 @@
 import { expect, Page, test } from '@playwright/test';
 import { SidebarItem } from '../../constant/sidebar';
 import { TableClass } from '../../support/entity/TableClass';
+import { ClassificationClass } from '../../support/tag/ClassificationClass';
+import { TagClass } from '../../support/tag/TagClass';
 import {
   clickOutside,
   createNewPage,
@@ -21,7 +23,7 @@ import {
   uuid,
 } from '../../utils/common';
 import { sidebarClick } from '../../utils/sidebar';
-import { submitForm, validateForm } from '../../utils/tag';
+import { addTagToTableColumn, submitForm, validateForm } from '../../utils/tag';
 
 const NEW_CLASSIFICATION = {
   name: `PlaywrightClassification-${uuid()}`,
@@ -57,16 +59,26 @@ const permanentDeleteModal = async (page: Page, entity: string) => {
 test.use({ storageState: 'playwright/.auth/admin.json' });
 
 const table = new TableClass();
+const classification = new ClassificationClass({
+  provider: 'system',
+});
+const tag = new TagClass({
+  classification: classification.data.name,
+});
 
 test.beforeAll(async ({ browser }) => {
   const { apiContext, afterAction } = await createNewPage(browser);
   await table.create(apiContext);
+  await classification.create(apiContext);
+  await tag.create(apiContext);
   await afterAction();
 });
 
 test.afterAll(async ({ browser }) => {
   const { apiContext, afterAction } = await createNewPage(browser);
   await table.delete(apiContext);
+  await classification.delete(apiContext);
+  await tag.delete(apiContext);
   await afterAction();
 });
 
@@ -101,6 +113,99 @@ test('Classification Page', async ({ page }) => {
     expect(headers).toEqual(['Tag', 'Display Name', 'Description', 'Actions']);
   });
 
+  await test.step('Disabled system tags should not render', async () => {
+    const classificationResponse = page.waitForResponse(
+      `/api/v1/tags?*parent=${classification.responseData.name}*`
+    );
+    await page
+      .locator(`[data-testid="side-panel-classification"]`)
+      .filter({ hasText: classification.responseData.displayName })
+      .click();
+    await classificationResponse;
+
+    await page.click('[data-testid="manage-button"]');
+
+    const fetchTags = page.waitForResponse(
+      `/api/v1/tags?fields=usageCount&parent=${classification.responseData.name}*`
+    );
+    const disabledTag = page.waitForResponse('/api/v1/classifications/*');
+    await page.click('[data-testid="enable-disable-title"]');
+    await disabledTag;
+    await fetchTags;
+
+    await expect(
+      page.locator(
+        `[data-testid="classification-${classification.responseData.name}"] [data-testid="disabled"]`
+      )
+    ).toBeVisible();
+
+    await expect(
+      page.locator('[data-testid="add-new-tag-button"]')
+    ).toBeDisabled();
+
+    await expect(
+      page.locator('[data-testid="no-data-placeholder"]')
+    ).toBeVisible();
+
+    // Check if the disabled Classification tag is not visible in the table
+    await table.visitEntityPage(page);
+
+    await page.click(
+      '[data-testid="classification-tags-0"] [data-testid="entity-tags"] [data-testid="add-tag"]'
+    );
+
+    const tagResponse = page.waitForResponse(
+      `/api/v1/search/query?q=*${tag.responseData.displayName}***`
+    );
+    await page.fill(
+      '[data-testid="tag-selector"] input',
+      tag.responseData.displayName
+    );
+    await tagResponse;
+
+    await expect(
+      page.locator('[data-testid="tag-selector"] > .ant-select-selector')
+    ).toContainText(tag.responseData.displayName);
+
+    await expect(
+      page.getByTestId(
+        `[data-testid="tag-${tag.responseData.fullyQualifiedName}"]`
+      )
+    ).not.toBeVisible();
+
+    await expect(page.getByTestId('saveAssociatedTag')).not.toBeVisible();
+
+    // Re-enable the disabled Classification
+    await classification.visitPage(page);
+
+    await page.click('[data-testid="manage-button"]');
+
+    const enableTagResponse = page.waitForResponse('/api/v1/classifications/*');
+    await page.click('[data-testid="enable-disable-title"]');
+    await enableTagResponse;
+
+    await expect(
+      page.locator('[data-testid="add-new-tag-button"]')
+    ).not.toBeDisabled();
+
+    await expect(
+      page.locator(
+        `[data-testid="classification-${classification.responseData.name}"] [data-testid="disabled"]`
+      )
+    ).not.toBeVisible();
+
+    await table.visitEntityPage(page);
+
+    await addTagToTableColumn(page, {
+      tagName: tag.responseData.name,
+      tagFqn: tag.responseData.fullyQualifiedName,
+      tagDisplayName: tag.responseData.displayName,
+      tableId: table.entityResponseData?.['id'],
+      columnNumber: 1,
+      rowName: 'shop_id numeric',
+    });
+  });
+
   await test.step('Create classification with validation checks', async () => {
     await page.click('[data-testid="add-classification"]');
     await page.waitForSelector('.ant-modal-content', {
@@ -169,41 +274,14 @@ test('Classification Page', async ({ page }) => {
     await table.visitEntityPage(page);
     const { name, displayName } = NEW_TAG;
 
-    await page.click(
-      '[data-testid="classification-tags-0"] [data-testid="entity-tags"] [data-testid="add-tag"]'
-    );
-    await page.fill('[data-testid="tag-selector"] input', name);
-    await page.click(`[data-testid="tag-${tagFqn}"]`);
-
-    await expect(
-      page.locator('[data-testid="tag-selector"] > .ant-select-selector')
-    ).toContainText(displayName);
-
-    const saveAssociatedTag = page.waitForResponse(
-      (response) =>
-        response.request().method() === 'PATCH' &&
-        response
-          .url()
-          .includes(`/api/v1/tables/${table.entityResponseData?.['id']}`)
-    );
-    await page.click('[data-testid="saveAssociatedTag"]');
-    await saveAssociatedTag;
-
-    await page.waitForSelector('.ant-select-dropdown', {
-      state: 'detached',
+    await addTagToTableColumn(page, {
+      tagName: name,
+      tagFqn,
+      tagDisplayName: displayName,
+      tableId: table.entityResponseData?.['id'],
+      columnNumber: 0,
+      rowName: 'user_id numeric',
     });
-
-    await expect(
-      page
-        .getByRole('row', { name: 'user_id numeric Unique' })
-        .getByTestId('tags-container')
-    ).toContainText(displayName);
-
-    await expect(
-      page.locator(
-        '[data-testid="classification-tags-0"] [data-testid="tags-container"] [data-testid="icon"]'
-      )
-    ).toBeVisible();
   });
 
   await test.step(
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts
index 4390a90d24ef..bdbd9799c7ed 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiCollectionClass.ts
@@ -24,10 +24,10 @@ export class ApiCollectionClass extends EntityClass {
   private apiCollectionName = `pw-api-collection-${uuid()}`;
   service = {
     name: this.serviceName,
-    serviceType: 'REST',
+    serviceType: 'Rest',
     connection: {
       config: {
-        type: 'REST',
+        type: 'Rest',
         openAPISchemaURL: 'https://sandbox-beta.open-metadata.org/swagger.json',
       },
     },
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiEndpointClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiEndpointClass.ts
index 20cb80271321..722667be45f8 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiEndpointClass.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ApiEndpointClass.ts
@@ -23,10 +23,10 @@ export class ApiEndpointClass extends EntityClass {
   private apiCollectionName = `pw-api-collection-${uuid()}`;
   service = {
     name: this.serviceName,
-    serviceType: 'REST',
+    serviceType: 'Rest',
     connection: {
       config: {
-        type: 'REST',
+        type: 'Rest',
         openAPISchemaURL: 'https://sandbox-beta.open-metadata.org/swagger.json',
       },
     },
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ApiIngestionClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ApiIngestionClass.ts
new file mode 100644
index 000000000000..36565e388f17
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/ingestion/ApiIngestionClass.ts
@@ -0,0 +1,47 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+import { Page } from '@playwright/test';
+import { uuid } from '../../../utils/common';
+import {
+  checkServiceFieldSectionHighlighting,
+  Services,
+} from '../../../utils/serviceIngestion';
+import ServiceBaseClass from './ServiceBaseClass';
+
+class ApiIngestionClass extends ServiceBaseClass {
+  constructor() {
+    super(Services.API, `pw-api-with-%-${uuid()}`, 'Rest', 'Containers');
+  }
+
+  async createService(page: Page) {
+    await super.createService(page);
+  }
+
+  async updateService(page: Page) {
+    await super.updateService(page);
+  }
+
+  async fillConnectionDetails(page: Page) {
+    const openAPISchemaURL = 'https://docs.open-metadata.org/swagger.json';
+
+    await page.locator('#root\\/openAPISchemaURL').fill(openAPISchemaURL);
+    await checkServiceFieldSectionHighlighting(page, 'openAPISchemaURL');
+  }
+
+  async deleteService(page: Page): Promise<void> {
+    await super.deleteService(page);
+  }
+}
+
+export default ApiIngestionClass;
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/service/ApiServiceClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/service/ApiServiceClass.ts
index 39ded95079e6..aa57a1db0cdd 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/service/ApiServiceClass.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/service/ApiServiceClass.ts
@@ -21,10 +21,10 @@ import { EntityClass } from '../EntityClass';
 export class ApiServiceClass extends EntityClass {
   entity = {
     name: `pw-api-service-${uuid()}`,
-    serviceType: 'REST',
+    serviceType: 'Rest',
     connection: {
       config: {
-        type: 'REST',
+        type: 'Rest',
         openAPISchemaURL: 'https://sandbox-beta.open-metadata.org/swagger.json',
       },
     },
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts
index f2b4e53d41cb..783bc7d1dbb2 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/advancedSearch.ts
@@ -23,7 +23,7 @@ type EntityFields = {
 
 export const FIELDS: EntityFields[] = [
   {
-    id: 'Owner',
+    id: 'Owners',
     name: 'owners.displayName.keyword',
     localSearch: false,
   },
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts
index 0202119b438e..5700f34343ed 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts
@@ -17,8 +17,8 @@ import {
   ENTITY_REFERENCE_PROPERTIES,
 } from '../constant/customProperty';
 import {
-  EntityTypeEndpoint,
   ENTITY_PATH,
+  EntityTypeEndpoint,
 } from '../support/entity/Entity.interface';
 import { UserClass } from '../support/user/UserClass';
 import { clickOutside, descriptionBox, uuid } from './common';
@@ -29,6 +29,7 @@ export enum CustomPropertyType {
   MARKDOWN = 'Markdown',
 }
 export enum CustomPropertyTypeByName {
+  TABLE_CP = 'table-cp',
   STRING = 'string',
   INTEGER = 'integer',
   MARKDOWN = 'markdown',
@@ -44,7 +45,6 @@ export enum CustomPropertyTypeByName {
   TIME_CP = 'time-cp',
   DATE_CP = 'date-cp',
   DATE_TIME_CP = 'dateTime-cp',
-  ENUM_WITH_DESCRIPTION = 'enumWithDescriptions',
 }
 
 export interface CustomProperty {
@@ -57,6 +57,27 @@ export interface CustomProperty {
   };
 }
 
+export const fillTableColumnInputDetails = async (
+  page: Page,
+  text: string,
+  columnName: string
+) => {
+  await page
+    .locator(`div[data-state-props-id="${columnName}"]`)
+    .last()
+    .dblclick();
+
+  await page
+    .getByTestId('edit-table-type-property-modal')
+    .getByRole('textbox')
+    .fill(text);
+
+  await page
+    .locator(`div[data-state-props-id="${columnName}"]`)
+    .last()
+    .press('Enter', { delay: 100 });
+};
+
 export const setValueForProperty = async (data: {
   page: Page;
   propertyName: string;
@@ -123,15 +144,6 @@ export const setValueForProperty = async (data: {
 
       break;
 
-    case 'enumWithDescriptions':
-      await page.click('#enumWithDescriptionValues');
-      await page.fill('#enumWithDescriptionValues', value, { force: true });
-      await page.press('#enumWithDescriptionValues', 'Enter');
-      await clickOutside(page);
-      await container.locator('[data-testid="inline-save-btn"]').click();
-
-      break;
-
     case 'sqlQuery':
       await page.locator("pre[role='presentation']").last().click();
       await page.keyboard.type(value);
@@ -213,6 +225,19 @@ export const setValueForProperty = async (data: {
 
       break;
     }
+
+    case 'table-cp': {
+      const values = value.split(',');
+      await page.locator('[data-testid="add-new-row"]').click();
+
+      await fillTableColumnInputDetails(page, values[0], 'pw-column1');
+
+      await fillTableColumnInputDetails(page, values[1], 'pw-column2');
+
+      await page.locator('[data-testid="update-table-type-property"]').click();
+
+      break;
+    }
   }
   await patchRequest;
 };
@@ -234,7 +259,7 @@ export const validateValueForProperty = async (data: {
     .locator(`[data-testid="toggle-${propertyName}"]`)
     .isVisible();
 
-  if (toggleBtnVisibility) {
+  if (toggleBtnVisibility && propertyType !== 'table-cp') {
     await container.locator(`[data-testid="toggle-${propertyName}"]`).click();
   }
 
@@ -251,17 +276,11 @@ export const validateValueForProperty = async (data: {
     );
   } else if (propertyType === 'sqlQuery') {
     await expect(container.locator('.CodeMirror-scroll')).toContainText(value);
-  } else if (propertyType === 'enumWithDescriptions') {
-    await expect(
-      container.locator('[data-testid="enum-with-description-table"]')
-    ).toBeVisible();
+  } else if (propertyType === 'table-cp') {
+    const values = value.split(',');
 
     await expect(
-      container
-        .locator('[data-testid="enum-with-description-table"]')
-        .getByText(value, {
-          exact: true,
-        })
+      page.getByRole('row', { name: `${values[0]} ${values[1]}` })
     ).toBeVisible();
   } else if (
     ![
@@ -316,11 +335,6 @@ export const getPropertyValues = (
         value: 'small',
         newValue: 'medium',
       };
-    case 'enumWithDescriptions':
-      return {
-        value: 'enumWithDescription1',
-        newValue: 'enumWithDescription2',
-      };
     case 'sqlQuery':
       return {
         value: 'Select * from table',
@@ -368,6 +382,12 @@ export const getPropertyValues = (
         newValue: '2025-07-09 15:07:59',
       };
 
+    case 'table-cp':
+      return {
+        value: 'column1,column2',
+        newValue: 'column3,column4',
+      };
+
     default:
       return {
         value: '',
@@ -447,25 +467,6 @@ export const createCustomPropertyForEntity = async (
                 },
               }
             : {}),
-          ...(item.name === 'enumWithDescriptions'
-            ? {
-                customPropertyConfig: {
-                  config: {
-                    multiSelect: true,
-                    values: [
-                      {
-                        key: 'enumWithDescription1',
-                        description: 'This is enumWithDescription1',
-                      },
-                      {
-                        key: 'enumWithDescription2',
-                        description: 'This is enumWithDescription2',
-                      },
-                    ],
-                  },
-                },
-              }
-            : {}),
           ...(['entityReference', 'entityReferenceList'].includes(item.name)
             ? {
                 customPropertyConfig: {
@@ -497,6 +498,15 @@ export const createCustomPropertyForEntity = async (
                 },
               }
             : {}),
+          ...(item.name === 'table-cp'
+            ? {
+                customPropertyConfig: {
+                  config: {
+                    columns: ['pw-column1', 'pw-column2'],
+                  },
+                },
+              }
+            : {}),
         },
       }
     );
@@ -534,19 +544,16 @@ export const addCustomPropertiesForEntity = async ({
   enumConfig,
   formatConfig,
   entityReferenceConfig,
-  enumWithDescriptionConfig,
+  tableConfig,
 }: {
   page: Page;
   propertyName: string;
   customPropertyData: { description: string };
   customType: string;
   enumConfig?: { values: string[]; multiSelect: boolean };
-  enumWithDescriptionConfig?: {
-    values: { key: string; description: string }[];
-    multiSelect: boolean;
-  };
   formatConfig?: string;
   entityReferenceConfig?: string[];
+  tableConfig?: { columns: string[] };
 }) => {
   // Add Custom property for selected entity
   await page.click('[data-testid="add-field-button"]');
@@ -624,26 +631,14 @@ export const addCustomPropertiesForEntity = async ({
       await page.click('#root\\/multiSelect');
     }
   }
-  // Enum With Description configuration
-  if (customType === 'Enum With Descriptions' && enumWithDescriptionConfig) {
-    for await (const [
-      index,
-      val,
-    ] of enumWithDescriptionConfig.values.entries()) {
-      await page.locator('[data-testid="add-enum-description-config"]').click();
-      await page.locator(`#key-${index}`).fill(val.key);
-      await page.locator(descriptionBox).nth(index).fill(val.description);
+  // Table configuration
+  if (customType === 'Table' && tableConfig) {
+    for (const val of tableConfig.columns) {
+      await page.click('#root\\/columns');
+      await page.fill('#root\\/columns', val);
+      await page.press('#root\\/columns', 'Enter');
     }
     await clickOutside(page);
-
-    if (enumWithDescriptionConfig.multiSelect) {
-      await page.click('#root\\/multiSelect');
-    }
-
-    await page
-      .locator(descriptionBox)
-      .nth(2)
-      .fill(customPropertyData.description);
   }
 
   // Entity reference configuration
@@ -672,9 +667,8 @@ export const addCustomPropertiesForEntity = async ({
   }
 
   // Description
-  if (customType !== 'Enum With Descriptions') {
-    await page.fill(descriptionBox, customPropertyData.description);
-  }
+
+  await page.fill(descriptionBox, customPropertyData.description);
 
   const createPropertyPromise = page.waitForResponse(
     '/api/v1/metadata/types/name/*?fields=customProperties'
@@ -709,22 +703,14 @@ export const editCreatedProperty = async (
     ).toContainText('["enum1","enum2","enum3"]');
   }
 
-  if (type === 'Enum With Descriptions') {
-    await expect(
-      page
-        .getByRole('row', {
-          name: `${propertyName} enumWithDescriptions enumWithDescription1`,
-        })
-        .getByTestId('enum-with-description-config')
-    ).toBeVisible();
+  if (type === 'Table') {
+    await expect(page.getByText('Columns:pw-column1pw-column2')).toBeVisible();
   }
 
   await editButton.click();
 
-  if (type !== 'Enum With Descriptions') {
-    await page.locator(descriptionBox).fill('');
-    await page.locator(descriptionBox).fill('This is new description');
-  }
+  await page.locator(descriptionBox).fill('');
+  await page.locator(descriptionBox).fill('This is new description');
 
   if (type === 'Enum') {
     await page.click('#root\\/customPropertyConfig');
@@ -732,10 +718,6 @@ export const editCreatedProperty = async (
     await page.press('#root\\/customPropertyConfig', 'Enter');
     await clickOutside(page);
   }
-  if (type === 'Enum With Descriptions') {
-    await page.locator(descriptionBox).nth(0).fill('');
-    await page.locator(descriptionBox).nth(0).fill('This is new description');
-  }
 
   if (ENTITY_REFERENCE_PROPERTIES.includes(type ?? '')) {
     await page.click('#root\\/customPropertyConfig');
@@ -769,16 +751,6 @@ export const editCreatedProperty = async (
     ).toContainText('["enum1","enum2","enum3","updatedValue"]');
   }
 
-  if (type === 'Enum With Descriptions') {
-    await expect(
-      page
-        .getByRole('row', {
-          name: `${propertyName} enumWithDescriptions enumWithDescription1`,
-        })
-        .getByTestId('enum-with-description-config')
-    ).toBeVisible();
-  }
-
   if (ENTITY_REFERENCE_PROPERTIES.includes(type ?? '')) {
     await expect(
       page.locator(
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/customizeLandingPage.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/customizeLandingPage.ts
index a73082ed9695..cd1090fde057 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/utils/customizeLandingPage.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/customizeLandingPage.ts
@@ -21,7 +21,7 @@ export const navigateToCustomizeLandingPage = async (
 ) => {
   const getPersonas = page.waitForResponse('/api/v1/personas*');
 
-  await settingClick(page, GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE);
+  await settingClick(page, GlobalSettingOptions.PERSONA);
 
   await getPersonas;
 
@@ -32,9 +32,11 @@ export const navigateToCustomizeLandingPage = async (
   );
 
   // Navigate to the customize landing page
-  await page.click(
-    `[data-testid="persona-details-card-${personaName}"] [data-testid="customize-page-button"]`
-  );
+  await page.getByTestId(`persona-details-card-${personaName}`).click();
+
+  await page.getByRole('tab', { name: 'Customize UI' }).click();
+
+  await page.getByTestId('LandingPage').click();
 
   expect((await getCustomPageDataResponse).status()).toBe(
     customPageDataResponse
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts
index b060e62128dd..c5b7e7e2ea16 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/glossary.ts
@@ -37,6 +37,8 @@ import { addMultiOwner } from './entity';
 import { sidebarClick } from './sidebar';
 import { TaskDetails, TASK_OPEN_FETCH_LINK } from './task';
 
+const GLOSSARY_NAME_VALIDATION_ERROR = 'Name size must be between 1 and 128';
+
 export const descriptionBox =
   '.toastui-editor-md-container > .toastui-editor > .ProseMirror';
 
@@ -146,7 +148,7 @@ export const setupGlossaryAndTerms = async (page: Page) => {
   return { glossary, term1, term2, cleanup };
 };
 
-export const validateForm = async (page) => {
+export const validateForm = async (page: Page) => {
   // Error messages
   await expect(page.locator('#name_help')).toHaveText('Name is required');
   await expect(page.locator('#description_help')).toHaveText(
@@ -603,6 +605,13 @@ export const updateNameForGlossaryTerm = async (
 
   await expect(page.locator('#name')).toBeVisible();
 
+  // Max length validation
+  await page.locator('#name').fill(INVALID_NAMES.MAX_LENGTH);
+
+  await expect(page.locator('#name_help')).toHaveText(
+    GLOSSARY_NAME_VALIDATION_ERROR
+  );
+
   await page.fill('#name', name);
   const updateNameResponsePromise = page.waitForResponse(
     `/api/v1/${endPoint}/*`
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts
index 1003eda7f7bf..ced6575cb1c1 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts
@@ -11,12 +11,12 @@
  *  limitations under the License.
  */
 import { expect, Page } from '@playwright/test';
-import { CUSTOM_PROPERTIES_ENTITIES } from '../constant/customProperty';
 import {
   CUSTOM_PROPERTIES_TYPES,
   FIELD_VALUES_CUSTOM_PROPERTIES,
 } from '../constant/glossaryImportExport';
 import { descriptionBox, uuid } from './common';
+import { fillTableColumnInputDetails } from './customProperty';
 
 export const createGlossaryTermRowDetails = () => {
   return {
@@ -175,30 +175,17 @@ const editGlossaryCustomProperty = async (
     await page.getByTestId('inline-save-btn').click();
   }
 
-  if (type === CUSTOM_PROPERTIES_TYPES.ENUM_WITH_DESCRIPTIONS) {
-    await page.getByTestId('enum-with-description-select').click();
+  if (type === CUSTOM_PROPERTIES_TYPES.TABLE) {
+    const columns = FIELD_VALUES_CUSTOM_PROPERTIES.TABLE.columns;
+    const values = FIELD_VALUES_CUSTOM_PROPERTIES.TABLE.rows.split(',');
 
-    await page.waitForSelector('.ant-select-dropdown', {
-      state: 'visible',
-    });
+    await page.locator('[data-testid="add-new-row"]').click();
 
-    // await page
-    //   .getByRole('option', {
-    //     name: CUSTOM_PROPERTIES_ENTITIES.entity_glossaryTerm
-    //       .enumWithDescriptionConfig.values[0].key,
-    //   })
-    //   .click();
+    await fillTableColumnInputDetails(page, values[0], columns[0]);
 
-    await page
-      .locator('span')
-      .filter({
-        hasText:
-          CUSTOM_PROPERTIES_ENTITIES.entity_glossaryTerm
-            .enumWithDescriptionConfig.values[0].key,
-      })
-      .click();
+    await fillTableColumnInputDetails(page, values[1], columns[1]);
 
-    await page.getByTestId('inline-save-btn').click();
+    await page.locator('[data-testid="update-table-type-property"]').click();
   }
 };
 
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts
index 211222a7d8a8..7d9b1390b240 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/serviceIngestion.ts
@@ -24,6 +24,7 @@ export enum Services {
   MLModels = GlobalSettingOptions.MLMODELS,
   Storage = GlobalSettingOptions.STORAGES,
   Search = GlobalSettingOptions.SEARCH,
+  API = GlobalSettingOptions.APIS,
 }
 
 export const getEntityTypeFromService = (service: Services) => {
@@ -42,6 +43,8 @@ export const getEntityTypeFromService = (service: Services) => {
       return EntityTypeEndpoint.MlModelService;
     case Services.Pipeline:
       return EntityTypeEndpoint.PipelineService;
+    case Services.API:
+      return EntityTypeEndpoint.ApiService;
     default:
       return EntityTypeEndpoint.DatabaseService;
   }
@@ -63,6 +66,8 @@ export const getServiceCategoryFromService = (service: Services) => {
       return 'mlmodelService';
     case Services.Pipeline:
       return 'pipelineService';
+    case Services.API:
+      return 'apiService';
     default:
       return 'databaseService';
   }
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/tag.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/tag.ts
index 713cd7f02983..ffcc3c253bf5 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/utils/tag.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/tag.ts
@@ -89,3 +89,54 @@ export async function validateForm(page: Page) {
 
   await expect(page.getByText(NAME_VALIDATION_ERROR)).toBeVisible();
 }
+
+export const addTagToTableColumn = async (
+  page: Page,
+  {
+    tagName,
+    tagFqn,
+    tagDisplayName,
+    tableId,
+    columnNumber,
+    rowName,
+  }: {
+    tagName: string;
+    tagFqn: string;
+    tagDisplayName: string;
+    tableId: string;
+    columnNumber: number;
+    rowName: string;
+  }
+) => {
+  await page.click(
+    `[data-testid="classification-tags-${columnNumber}"] [data-testid="entity-tags"] [data-testid="add-tag"]`
+  );
+  await page.fill('[data-testid="tag-selector"] input', tagName);
+  await page.click(`[data-testid="tag-${tagFqn}"]`);
+
+  await expect(
+    page.locator('[data-testid="tag-selector"] > .ant-select-selector')
+  ).toContainText(tagDisplayName);
+
+  const saveAssociatedTag = page.waitForResponse(
+    (response) =>
+      response.request().method() === 'PATCH' &&
+      response.url().includes(`/api/v1/tables/${tableId}`)
+  );
+  await page.click('[data-testid="saveAssociatedTag"]');
+  await saveAssociatedTag;
+
+  await page.waitForSelector('.ant-select-dropdown', {
+    state: 'detached',
+  });
+
+  await expect(
+    page.getByRole('row', { name: rowName }).getByTestId('tags-container')
+  ).toContainText(tagDisplayName);
+
+  await expect(
+    page.locator(
+      `[data-testid="classification-tags-${columnNumber}"] [data-testid="tags-container"] [data-testid="icon"]`
+    )
+  ).toBeVisible();
+};
diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/task.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/task.ts
index 2d71b8f66cc8..07129203dde7 100644
--- a/openmetadata-ui/src/main/resources/ui/playwright/utils/task.ts
+++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/task.ts
@@ -156,7 +156,9 @@ export const checkTaskCount = async (
   openTask = 0,
   closedTask = 0
 ) => {
-  await page.waitForLoadState('networkidle');
+  await page.waitForSelector('.ant-skeleton-element ', {
+    state: 'detached',
+  });
 
   const openTaskElement = await page.getByTestId('open-task').textContent();
 
diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/API/REST.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/ApiEntity/Rest.md
similarity index 100%
rename from openmetadata-ui/src/main/resources/ui/public/locales/en-US/API/REST.md
rename to openmetadata-ui/src/main/resources/ui/public/locales/en-US/ApiEntity/Rest.md
diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/ApiEntity/workflows/metadata.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/ApiEntity/workflows/metadata.md
new file mode 100644
index 000000000000..9487e44b27e1
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/ApiEntity/workflows/metadata.md
@@ -0,0 +1,44 @@
+# Metadata
+
+API Service Metadata Pipeline Configuration.
+
+$$section
+### API Collection Filter Pattern $(id="apiCollectionFilterPattern")
+
+API Collection filter patterns to control whether to include API Collections as part of metadata ingestion.
+
+**Include**: Explicitly include API Collections by adding a list of regular expressions to the `Include` field. OpenMetadata will include all API Collections with names matching one or more of the supplied regular expressions. All other API Collections will be excluded.
+
+For example, to include only those API Collections whose name starts with the word `demo`, add the regex pattern in the include field as `^demo.*`.
+
+**Exclude**: Explicitly exclude API Collections by adding a list of regular expressions to the `Exclude` field. OpenMetadata will exclude all API Collections with names matching one or more of the supplied regular expressions. All other API Collections will be included.
+
+For example, to exclude all API Collections with the name containing the word `demo`, add the regex pattern in the exclude field as `.*demo.*`.
+
+Checkout [this](https://docs.open-metadata.org/connectors/ingestion/workflows/metadata/filter-patterns/database#database-filter-pattern) document for further examples on filter patterns.
+$$
+
+$$section
+### Enable Debug Logs $(id="enableDebugLog")
+
+Set the `Enable Debug Log` toggle to set the logging level of the process to debug. You can check these logs in the Ingestion tab of the service and dig deeper into any errors you might find.
+$$
+
+$$section
+### Mark Deleted API Collection $(id="markDeletedApiCollections")
+
+Optional configuration to soft delete `API Collections` in OpenMetadata if the source `API Collections` are deleted. After deleting, all the associated entities like lineage, etc., with that `API Collection` will be deleted.
+$$
+
+$$section
+### Override Metadata $(id="overrideMetadata")
+
+Set the `Override Metadata` toggle to control whether to override the existing metadata in the OpenMetadata server with the metadata fetched from the source.
+
+If the toggle is `enabled`, the metadata fetched from the source will override and replace the existing metadata in the OpenMetadata.
+
+If the toggle is `disabled`, the metadata fetched from the source will not override the existing metadata in the OpenMetadata server. In this case the metadata will only get updated for fields that has no value added in OpenMetadata.
+
+This is applicable for fields like description, tags, owner and displayName
+
+$$
\ No newline at end of file
diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/Alation.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/Alation.md
new file mode 100644
index 000000000000..584dc3089780
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Metadata/Alation.md
@@ -0,0 +1,374 @@
+# Alation
+
+In this section, we provide guides and references to use the Alation connector.
+
+## Requirements
+
+Follow the official documentation to generate a API Access Token from [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis#create-an-api-access-token)
+
+## Connection Details
+
+$$section
+### Host Port $(id="hostPort")
+
+Host and port of the Alation service.
+$$
+
+$$section
+### Authentication Type $(id="authType")
+
+Following authentication types are supported:
+
+1. Basic Authentication: We'll use the user credentials to generate the access token required to authenticate Alation APIs.
+- username: Username of the user.
+- password: Password of the user.
+
+2. Access Token Authentication: The access token created using the steps mentioned [here](https://developer.alation.com/dev/docs/authentication-into-alation-apis#create-via-ui) can directly be entered. We'll use that directly to authenticate the Alation APIs
+- accessToken: Generated access token
+$$
+
+## Alation backend database Connection details
+Alation APIs do not provide us with some of the metadata. This metadata we extract directly from the alation's backend database by query the tables directly.
+Note that this is a optional config and if it is not provided primary metadata will still be ingested.
+Below is the metadata fetched from alation database:
+`1. User and Group Relationships`
+
+Choose either postgres or mysql connection depending on the db:
+## Postgres Connection 
+
+You can use Postgres Connection when you have SSO enabled and your Superset is backed by Postgres database.
+
+### Connection Scheme
+
+SQLAlchemy driver scheme options.
+
+
+### Username
+
+Username to connect to Postgres. 
+
+
+$$note
+
+Make sure the user has select privileges on `dashboards`, `tables` & `slices` tables of superset schema.
+
+$$
+
+$$section
+### Auth Config $(id="authType")
+There are 2 types of auth configs:
+- Basic Auth.
+- IAM based Auth.
+
+User can authenticate the Postgres Instance with auth type as `Basic Authentication` i.e. Password **or** by using `IAM based Authentication` to connect to AWS related services.
+$$
+
+## Basic Auth
+
+### Password
+
+Password to connect to Postgres.
+
+
+## IAM Auth Config
+
+$$section
+### AWS Access Key ID $(id="awsAccessKeyId")
+
+When you interact with AWS, you specify your AWS security credentials to verify who you are and whether you have permission to access the resources that you are requesting. AWS uses the security credentials to authenticate and authorize your requests ([docs](https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html)).
+
+Access keys consist of two parts:
+1. An access key ID (for example, `AKIAIOSFODNN7EXAMPLE`),
+2. And a secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`).
+
+You must use both the access key ID and secret access key together to authenticate your requests.
+
+You can find further information on how to manage your access keys [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html)
+$$
+
+$$section
+### AWS Secret Access Key $(id="awsSecretAccessKey")
+
+Secret access key (for example, `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY`).
+$$
+
+$$section
+### AWS Region $(id="awsRegion")
+
+Each AWS Region is a separate geographic area in which AWS clusters data centers ([docs](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Concepts.RegionsAndAvailabilityZones.html)).
+
+As AWS can have instances in multiple regions, we need to know the region the service you want reach belongs to.
+
+Note that the AWS Region is the only required parameter when configuring a connection. When connecting to the services programmatically, there are different ways in which we can extract and use the rest of AWS configurations. You can find further information about configuring your credentials [here](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#configuring-credentials).
+$$
+
+$$section
+### AWS Session Token $(id="awsSessionToken")
+
+If you are using temporary credentials to access your services, you will need to inform the AWS Access Key ID and AWS Secrets Access Key. Also, these will include an AWS Session Token.
+
+You can find more information on [Using temporary credentials with AWS resources](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html).
+$$
+
+$$section
+### Endpoint URL $(id="endPointURL")
+
+To connect programmatically to an AWS service, you use an endpoint. An *endpoint* is the URL of the entry point for an AWS web service. The AWS SDKs and the AWS Command Line Interface (AWS CLI) automatically use the default endpoint for each service in an AWS Region. But you can specify an alternate endpoint for your API requests.
+
+Find more information on [AWS service endpoints](https://docs.aws.amazon.com/general/latest/gr/rande.html).
+$$
+
+$$section
+### Profile Name $(id="profileName")
+
+A named profile is a collection of settings and credentials that you can apply to an AWS CLI command. When you specify a profile to run a command, the settings and credentials are used to run that command. Multiple named profiles can be stored in the config and credentials files.
+
+You can inform this field if you'd like to use a profile other than `default`.
+
+Find here more information about [Named profiles for the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html).
+$$
+
+$$section
+### Assume Role ARN $(id="assumeRoleArn")
+
+Typically, you use `AssumeRole` within your account or for cross-account access. In this field you'll set the `ARN` (Amazon Resource Name) of the policy of the other account.
+
+A user who wants to access a role in a different account must also have permissions that are delegated from the account administrator. The administrator must attach a policy that allows the user to call `AssumeRole` for the `ARN` of the role in the other account.
+
+This is a required field if you'd like to `AssumeRole`.
+
+Find more information on [AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html).
+$$
+
+$$section
+### Assume Role Session Name $(id="assumeRoleSessionName")
+
+An identifier for the assumed role session. Use the role session name to uniquely identify a session when the same role is assumed by different principals or for different reasons.
+
+By default, we'll use the name `OpenMetadataSession`.
+
+Find more information about the [Role Session Name](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=An%20identifier%20for%20the%20assumed%20role%20session.).
+$$
+
+$$section
+### Assume Role Source Identity $(id="assumeRoleSourceIdentity")
+
+The source identity specified by the principal that is calling the `AssumeRole` operation. You can use source identity information in AWS CloudTrail logs to determine who took actions with a role.
+
+Find more information about [Source Identity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#:~:text=Required%3A%20No-,SourceIdentity,-The%20source%20identity).
+$$
+
+### Host and Port
+
+Host and port of the Postgres service.
+
+Example: `localhost:5432`
+
+$$section
+### Database $(id="database")
+
+Initial Postgres database to connect to. If you want to ingest all databases, set `ingestAllDatabases` to true.
+$$
+
+### SSL Mode
+
+SSL Mode to connect to postgres database.
+
+$$section
+### Classification Name $(id="classificationName")
+
+By default, the Postgres policy tags in OpenMetadata are classified under the name `PostgresPolicyTags`. However, you can create a custom classification name of your choice for these tags. Once you have ingested Postgres data, the custom classification name will be visible in the Classifications list on the Tags page.
+$$
+
+$$section
+### Ingest All Databases $(id="ingestAllDatabases")
+
+If ticked, the workflow will be able to ingest all database in the cluster. If not ticked, the workflow will only ingest tables from the database set above.
+$$
+
+$$section
+### SSL Mode $(id="sslMode")
+
+SSL Mode to connect to postgres database. E.g, `prefer`, `verify-ca`, `allow` etc.
+$$
+$$note
+if you are using `IAM auth`, select either `allow` (recommended) or other option based on your use case.
+$$
+
+
+### SSL CA 
+The CA certificate used for SSL validation (`sslrootcert`).
+
+$$note
+Postgres only needs CA Certificate
+$$
+
+$$section
+### Connection Arguments $(id="connectionArguments")
+
+Additional connection arguments such as security or protocol configs that can be sent to service during connection.
+$$
+
+$$section
+### Connection Options $(id="connectionOptions")
+
+Additional connection options to build the URL that can be sent to service during the connection.
+$$
+--------
+
+## Mysql Connection 
+
+You can use Mysql Connection when you have SSO enabled and your Superset is backed by Mysql database.
+
+### Scheme
+SQLAlchemy driver scheme options.
+
+### Username
+Username to connect to Mysql.
+
+$$note
+
+Make sure the user has select privileges on `dashboards`, `tables` & `slices` tables of superset schema.
+
+$$
+
+### Password
+Password to connect to Mysql.
+
+### Host Port
+Host and port of the Mysql service. This should be specified as a string in the format `hostname:port`.
+
+**Example**: `localhost:3306`, `host.docker.internal:3306`
+
+$$section
+### Database Name $(id="databaseName")
+In OpenMetadata, the Database Service hierarchy works as follows:
+```
+Database Service > Database > Schema > Table
+```
+In the case of MySQL, we won't have a Database as such. If you'd like to see your data in a database named something other than `default`, you can specify the name in this field.
+$$
+
+$$section
+### Database Schema $(id="databaseSchema")
+This is an optional parameter. When set, the value will be used to restrict the metadata reading to a single database (corresponding to the value passed in this field). When left blank, OpenMetadata will scan all the databases.
+$$
+
+$$section
+### SSL CA $(id="caCertificate")
+The CA certificate used for SSL validation (`ssl_ca`)
+$$
+
+$$section
+### SSL Certificate $(id="sslCertificate")
+The SSL certificate used for client authentication (`ssl_cert`)
+$$
+
+$$section
+### SSL Key $(id="sslKey")
+The private key associated with the SSL certificate (`ssl_key`)
+$$
+
+### Connection Options
+Additional connection options to build the URL that can be sent to the service during the connection.
+
+### Connection Arguments
+Additional connection arguments such as security or protocol configs that can be sent to the service during connection.
+
+
+$$section
+### Project Name $(id="projectName")
+
+Project name to create the refreshToken. Can be anything.
+$$
+
+
+$$section
+### Pagination Limit $(id="paginationLimit")
+
+Pagination limit used for Alation APIs pagination
+$$
+
+$$section
+### Include Undeployed Datasources $(id="includeUndeployedDatasources")
+
+Specifies if undeployed datasources should be included while ingesting. By default is set to `false`.
+$$
+
+$$section
+### Include Hidden Datasources $(id="includeHiddenDatasources")
+
+Specifies if hidden datasources should be included while ingesting. By default is set to `false`.
+$$
+
+$$section
+### Ingest Datasources $(id="ingestDatasources")
+
+Specifies if databases, schemas and tables should be included while ingesting. By default is set to `true`.
+$$
+
+$$section
+### Ingest Users And Groups $(id="ingestUsersAndGroups")
+
+Specifies if users and groups should be included while ingesting. By default is set to `true`.
+$$
+
+$$section
+### Ingest Domains $(id="ingestDomains")
+
+Specifies if domains and subdomains should be included while ingesting. By default is set to `true`.
+$$
+
+$$section
+### Ingest KnowledgeArticles $(id="ingestKnowledgeArticles")
+
+Specifies if knowledge articles should be included while ingesting. By default is set to `true`.
+$$
+
+$$section
+### Ingest Dashboards $(id="ingestDashboards")
+
+Specifies if BI sources and dashboards should be included while ingesting. By default is set to `true`.
+$$
+
+$$section
+### Alation Tag Classification Name $(id="alationTagClassificationName")
+
+Specifies if hidden datasources should be included while ingesting. By default is set to `false`.
+$$
+
+$$section
+### Ingest Datasources $(id="ingestDatasources")
+
+Specifies if hidden datasources should be included while ingesting. By default is set to `false`.
+$$
+
+$$section
+### Ingest Datasources $(id="ingestDatasources")
+
+Specifies if hidden datasources should be included while ingesting. By default is set to `false`.
+$$
+
+
+$$section
+### Connection Options $(id="connectionOptions")
+
+Additional connection options to build the URL that can be sent to service during the connection.
+$$
+
+$$section
+### Connection Arguments $(id="connectionArguments")
+
+Additional connection arguments such as security or protocol configs that can be sent to service during connection.
+
+The following arguments are intended to be used in conjunction and are specifically for Alation DataSource APIs:
+- skip: This parameter determines the count of records to bypass at the start of the dataset. When set to 0, as in this case, it means that no records will be bypassed. If set to 10, it will bypass the first 10 records.
+
+- limit: This argument specifies the maximum number of records to return. Here, it's set to 10, meaning only the first 10 records will be returned.
+
+To perform incremental ingestion, these arguments should be used together. For instance, if there are a total of 30 datasources in Alation, the ingestion can be configured to execute three times, with each execution ingesting 10 datasources. 
+- 1st execution: {"skip": 0, "limit": 10}
+- 2nd execution: {"skip": 10, "limit": 10}
+- 3rd execution: {"skip": 20, "limit": 10}
+
+$$
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/date-time.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/date-time.svg
new file mode 100644
index 000000000000..ecd706f34119
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/date-time.svg
@@ -0,0 +1,14 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M7.02299 11.7777C7.434 11.7777 7.76718 11.4594 7.76718 11.0666C7.76718 10.6739 7.434 10.3555 7.02299 10.3555C6.61199 10.3555 6.27881 10.6739 6.27881 11.0666C6.27881 11.4594 6.61199 11.7777 7.02299 11.7777Z" fill="#292929"/>
+<path d="M7.02299 14.6215C7.434 14.6215 7.76718 14.3031 7.76718 13.9104C7.76718 13.5176 7.434 13.1992 7.02299 13.1992C6.61199 13.1992 6.27881 13.5176 6.27881 13.9104C6.27881 14.3031 6.61199 14.6215 7.02299 14.6215Z" fill="#292929"/>
+<path d="M9.99956 11.7777C10.4106 11.7777 10.7437 11.4594 10.7437 11.0666C10.7437 10.6739 10.4106 10.3555 9.99956 10.3555C9.58855 10.3555 9.25537 10.6739 9.25537 11.0666C9.25537 11.4594 9.58855 11.7777 9.99956 11.7777Z" fill="#292929"/>
+<path d="M9.99956 14.6215C10.4106 14.6215 10.7437 14.3031 10.7437 13.9104C10.7437 13.5176 10.4106 13.1992 9.99956 13.1992C9.58855 13.1992 9.25537 13.5176 9.25537 13.9104C9.25537 14.3031 9.58855 14.6215 9.99956 14.6215Z" fill="#292929"/>
+<path d="M12.9766 11.7777C13.3876 11.7777 13.7208 11.4594 13.7208 11.0666C13.7208 10.6739 13.3876 10.3555 12.9766 10.3555C12.5656 10.3555 12.2324 10.6739 12.2324 11.0666C12.2324 11.4594 12.5656 11.7777 12.9766 11.7777Z" fill="#292929"/>
+<path d="M12.9766 14.6215C13.3876 14.6215 13.7208 14.3031 13.7208 13.9104C13.7208 13.5176 13.3876 13.1992 12.9766 13.1992C12.5656 13.1992 12.2324 13.5176 12.2324 13.9104C12.2324 14.3031 12.5656 14.6215 12.9766 14.6215Z" fill="#292929"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M18 6.08863V15.3334C18 16.041 17.706 16.7187 17.1829 17.2193C16.659 17.7193 15.9498 18.0002 15.2093 18.0002H4.7907C4.05023 18.0002 3.34102 17.7193 2.81712 17.2193C2.29395 16.7187 2 16.041 2 15.3334V6.08863C2 5.38105 2.29395 4.70334 2.81712 4.2027C3.34102 3.70277 4.05023 3.42188 4.7907 3.42188H15.2093C15.9498 3.42188 16.659 3.70277 17.1829 4.2027C17.706 4.70334 18 5.38105 18 6.08863ZM16.8837 6.08863C16.8837 5.66409 16.7073 5.25732 16.3933 4.95722C16.0793 4.65712 15.6536 4.48858 15.2093 4.48858H4.7907C4.34642 4.48858 3.92074 4.65712 3.6067 4.95722C3.29265 5.25732 3.11628 5.66409 3.11628 6.08863V15.3334C3.11628 15.7579 3.29265 16.1647 3.6067 16.4648C3.92074 16.7649 4.34642 16.9335 4.7907 16.9335H15.2093C15.6536 16.9335 16.0793 16.7649 16.3933 16.4648C16.7073 16.1647 16.8837 15.7579 16.8837 15.3334V6.08863Z" fill="#292929"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M17.4419 7.33398C17.75 7.33398 18 7.57293 18 7.86734C18 8.16175 17.75 8.40069 17.4419 8.40069H2.55814C2.25005 8.40069 2 8.16175 2 7.86734C2 7.57293 2.25005 7.33398 2.55814 7.33398H17.4419Z" fill="#292929"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M12.4185 2.53335C12.4185 2.23894 12.6685 2 12.9766 2C13.2847 2 13.5347 2.23894 13.5347 2.53335V5.3779C13.5347 5.67231 13.2847 5.91125 12.9766 5.91125C12.6685 5.91125 12.4185 5.67231 12.4185 5.3779V2.53335Z" fill="#292929"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M6.46533 2.53335C6.46533 2.23894 6.71538 2 7.02347 2C7.33156 2 7.58161 2.23894 7.58161 2.53335V5.3779C7.58161 5.67231 7.33156 5.91125 7.02347 5.91125C6.71538 5.91125 6.46533 5.67231 6.46533 5.3779V2.53335Z" fill="#292929"/>
+<path d="M14.9548 18.5C16.8699 18.5 18.4095 16.9258 18.4095 15C18.4095 13.0742 16.8699 11.5 14.9548 11.5C13.0396 11.5 11.5 13.0742 11.5 15C11.5 16.9258 13.0396 18.5 14.9548 18.5Z" fill="white" stroke="#292929"/>
+<path d="M14.9546 13.7129V15.4272L15.7988 16.2843" stroke="#292929" stroke-linecap="round"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/date.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/date.svg
new file mode 100644
index 000000000000..25451c0406fd
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/date.svg
@@ -0,0 +1,12 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.79998 11.7777C7.19271 11.7777 7.51109 11.4593 7.51109 11.0666C7.51109 10.6738 7.19271 10.3555 6.79998 10.3555C6.40724 10.3555 6.08887 10.6738 6.08887 11.0666C6.08887 11.4593 6.40724 11.7777 6.79998 11.7777Z" fill="#292929"/>
+<path d="M6.79998 14.6214C7.19271 14.6214 7.51109 14.3031 7.51109 13.9103C7.51109 13.5176 7.19271 13.1992 6.79998 13.1992C6.40724 13.1992 6.08887 13.5176 6.08887 13.9103C6.08887 14.3031 6.40724 14.6214 6.79998 14.6214Z" fill="#292929"/>
+<path d="M9.64422 11.7777C10.037 11.7777 10.3553 11.4593 10.3553 11.0666C10.3553 10.6738 10.037 10.3555 9.64422 10.3555C9.25148 10.3555 8.93311 10.6738 8.93311 11.0666C8.93311 11.4593 9.25148 11.7777 9.64422 11.7777Z" fill="#292929"/>
+<path d="M9.64422 14.6214C10.037 14.6214 10.3553 14.3031 10.3553 13.9103C10.3553 13.5176 10.037 13.1992 9.64422 13.1992C9.25148 13.1992 8.93311 13.5176 8.93311 13.9103C8.93311 14.3031 9.25148 14.6214 9.64422 14.6214Z" fill="#292929"/>
+<path d="M12.4889 11.7777C12.8817 11.7777 13.2001 11.4593 13.2001 11.0666C13.2001 10.6738 12.8817 10.3555 12.4889 10.3555C12.0962 10.3555 11.7778 10.6738 11.7778 11.0666C11.7778 11.4593 12.0962 11.7777 12.4889 11.7777Z" fill="#292929"/>
+<path d="M12.4889 14.6214C12.8817 14.6214 13.2001 14.3031 13.2001 13.9103C13.2001 13.5176 12.8817 13.1992 12.4889 13.1992C12.0962 13.1992 11.7778 13.5176 11.7778 13.9103C11.7778 14.3031 12.0962 14.6214 12.4889 14.6214Z" fill="#292929"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M17.2889 6.08854V15.333C17.2889 16.0405 17.008 16.7182 16.5081 17.2189C16.0075 17.7188 15.3298 17.9997 14.6222 17.9997H4.66667C3.95911 17.9997 3.28142 17.7188 2.7808 17.2189C2.28089 16.7182 2 16.0405 2 15.333V6.08854C2 5.38099 2.28089 4.7033 2.7808 4.20268C3.28142 3.70276 3.95911 3.42188 4.66667 3.42188H14.6222C15.3298 3.42188 16.0075 3.70276 16.5081 4.20268C17.008 4.7033 17.2889 5.38099 17.2889 6.08854ZM16.2222 6.08854C16.2222 5.66401 16.0537 5.25725 15.7536 4.95716C15.4535 4.65707 15.0468 4.48854 14.6222 4.48854H4.66667C4.24213 4.48854 3.83538 4.65707 3.53529 4.95716C3.2352 5.25725 3.06667 5.66401 3.06667 6.08854V15.333C3.06667 15.7575 3.2352 16.1643 3.53529 16.4644C3.83538 16.7645 4.24213 16.933 4.66667 16.933H14.6222C15.0468 16.933 15.4535 16.7645 15.7536 16.4644C16.0537 16.1643 16.2222 15.7575 16.2222 15.333V6.08854Z" fill="#292929"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M16.7556 7.33398C17.05 7.33398 17.2889 7.57292 17.2889 7.86732C17.2889 8.16172 17.05 8.40065 16.7556 8.40065H2.53333C2.23893 8.40065 2 8.16172 2 7.86732C2 7.57292 2.23893 7.33398 2.53333 7.33398H16.7556Z" fill="#292929"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.9556 2.53333C11.9556 2.23893 12.1945 2 12.4889 2C12.7833 2 13.0222 2.23893 13.0222 2.53333V5.37778C13.0222 5.67218 12.7833 5.91111 12.4889 5.91111C12.1945 5.91111 11.9556 5.67218 11.9556 5.37778V2.53333Z" fill="#292929"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M6.2666 2.53333C6.2666 2.23893 6.50553 2 6.79993 2C7.09433 2 7.33327 2.23893 7.33327 2.53333V5.37778C7.33327 5.67218 7.09433 5.91111 6.79993 5.91111C6.50553 5.91111 6.2666 5.67218 6.2666 5.37778V2.53333Z" fill="#292929"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/duration.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/duration.svg
new file mode 100644
index 000000000000..9cee0d77a681
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/duration.svg
@@ -0,0 +1,3 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M15.3179 5.5264V3.06667H16.1781C16.4731 3.06667 16.7115 2.82827 16.7115 2.53333C16.7115 2.2384 16.4731 2 16.1781 2H14.7845H5.92693H4.53333C4.2384 2 4 2.2384 4 2.53333C4 2.82827 4.2384 3.06667 4.53333 3.06667H5.3936V5.5264C5.3936 7.02453 6.2144 8.39307 7.536 9.09813L9.224 10L7.536 10.9013C6.2144 11.6064 5.3936 12.9755 5.3936 14.4731V16.9333H4.53333C4.2384 16.9333 4 17.1717 4 17.4667C4 17.7616 4.2384 18 4.53333 18H5.92693H14.784H16.1776C16.4725 18 16.7109 17.7616 16.7109 17.4667C16.7109 17.1717 16.4725 16.9333 16.1776 16.9333H15.3173V14.4736C15.3173 12.9755 14.4965 11.6069 13.1749 10.9019L11.4875 10L13.1755 9.09867C14.4971 8.39307 15.3179 7.02453 15.3179 5.5264ZM14.2512 14.4736V16.9333H6.46027V14.4736C6.46027 13.3707 7.06507 12.3621 8.03787 11.8421L10.3557 10.6048L12.6736 11.8421C13.6464 12.3621 14.2512 13.3701 14.2512 14.4736ZM10.3557 9.3952L8.03787 8.15787C7.06507 7.63787 6.46027 6.62987 6.46027 5.5264V3.06667H14.2507V5.5264C14.2507 6.62933 13.6459 7.63787 12.6731 8.15787L10.3557 9.3952Z" fill="#292929"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/email.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/email.svg
new file mode 100644
index 000000000000..bc15adf6207c
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/email.svg
@@ -0,0 +1,4 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M15.0909 16.3636H4.90909C4.1378 16.3628 3.39834 16.0561 2.85296 15.5107C2.30758 14.9653 2.00082 14.2258 2 13.4545V6.90909C2.00082 6.1378 2.30758 5.39834 2.85296 4.85296C3.39834 4.30758 4.1378 4.00082 4.90909 4H15.0909C15.8622 4.00082 16.6017 4.30758 17.147 4.85296C17.6924 5.39834 17.9992 6.1378 18 6.90909V13.4545C17.9992 14.2258 17.6924 14.9653 17.147 15.5107C16.6017 16.0561 15.8622 16.3628 15.0909 16.3636ZM4.90909 5.45455C4.52346 5.45499 4.15374 5.60837 3.88106 5.88106C3.60837 6.15374 3.45499 6.52346 3.45455 6.90909V13.4545C3.45499 13.8402 3.60837 14.2099 3.88106 14.4826C4.15374 14.7553 4.52346 14.9086 4.90909 14.9091H15.0909C15.4765 14.9086 15.8463 14.7553 16.1189 14.4826C16.3916 14.2099 16.545 13.8402 16.5455 13.4545V6.90909C16.545 6.52346 16.3916 6.15374 16.1189 5.88106C15.8463 5.60837 15.4765 5.45499 15.0909 5.45455H4.90909Z" fill="#292929"/>
+<path d="M10.0001 11.2233C9.19113 11.2236 8.40526 10.9537 7.76712 10.4566L3.00785 6.75487C2.85556 6.63639 2.75657 6.46227 2.73266 6.2708C2.70875 6.07934 2.76188 5.88621 2.88036 5.73392C2.99885 5.58163 3.17297 5.48264 3.36444 5.45873C3.5559 5.43483 3.74902 5.48796 3.90132 5.60644L8.66055 9.30814C9.04328 9.60657 9.51471 9.76863 10 9.76863C10.4854 9.76863 10.9568 9.60657 11.3395 9.30814L16.0988 5.60644C16.251 5.48796 16.4442 5.43483 16.6356 5.45873C16.8271 5.48264 17.0012 5.58163 17.1197 5.73392C17.2382 5.88621 17.2913 6.07934 17.2674 6.2708C17.2435 6.46227 17.1445 6.63639 16.9922 6.75487L12.233 10.4566C11.5948 10.9536 10.809 11.2234 10.0001 11.2233Z" fill="#292929"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/entity-list.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/entity-list.svg
new file mode 100644
index 000000000000..9457f32561bc
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/entity-list.svg
@@ -0,0 +1,11 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M4 4C3.7775 4 3.55999 4.06598 3.37498 4.1896C3.18998 4.31321 3.04578 4.48891 2.96064 4.69448C2.87549 4.90005 2.85321 5.12625 2.89662 5.34448C2.94003 5.56271 3.04717 5.76316 3.20451 5.9205C3.36184 6.07783 3.56229 6.18497 3.78052 6.22838C3.99875 6.27179 4.22495 6.24951 4.43052 6.16436C4.63609 6.07922 4.81179 5.93502 4.9354 5.75002C5.05902 5.56501 5.125 5.3475 5.125 5.125C5.125 4.82663 5.00647 4.54048 4.7955 4.3295C4.58452 4.11853 4.29837 4 4 4ZM4 5.75C3.87639 5.75 3.75555 5.71334 3.65277 5.64467C3.54999 5.57599 3.46988 5.47838 3.42258 5.36418C3.37527 5.24997 3.36289 5.12431 3.38701 5.00307C3.41113 4.88183 3.47065 4.77047 3.55806 4.68306C3.64547 4.59565 3.75683 4.53612 3.87807 4.51201C3.99931 4.48789 4.12497 4.50027 4.23918 4.54758C4.35338 4.59488 4.45099 4.67499 4.51967 4.77777C4.58834 4.88055 4.625 5.00139 4.625 5.125C4.62481 5.2907 4.55891 5.44957 4.44174 5.56674C4.32457 5.68391 4.1657 5.74981 4 5.75Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M16.625 4.625H8.625C8.34888 4.625 8.125 4.40113 8.125 4.125C8.125 3.84888 8.34888 3.625 8.625 3.625H16.625C16.9011 3.625 17.125 3.84888 17.125 4.125C17.125 4.40113 16.9011 4.625 16.625 4.625Z" fill="#292929"/>
+<path d="M13.875 6.5H8.625C8.34887 6.5 8.125 6.27613 8.125 6C8.125 5.72387 8.34887 5.5 8.625 5.5H13.875C14.1511 5.5 14.375 5.72387 14.375 6C14.375 6.27613 14.1511 6.5 13.875 6.5Z" fill="#292929"/>
+<path d="M4 8.875C3.7775 8.875 3.55999 8.94098 3.37498 9.0646C3.18998 9.18821 3.04578 9.36391 2.96064 9.56948C2.87549 9.77505 2.85321 10.0012 2.89662 10.2195C2.94003 10.4377 3.04717 10.6382 3.20451 10.7955C3.36184 10.9528 3.56229 11.06 3.78052 11.1034C3.99875 11.1468 4.22495 11.1245 4.43052 11.0394C4.63609 10.9542 4.81179 10.81 4.9354 10.625C5.05902 10.44 5.125 10.2225 5.125 10C5.125 9.70163 5.00647 9.41548 4.7955 9.2045C4.58452 8.99353 4.29837 8.875 4 8.875ZM4 10.625C3.87639 10.625 3.75555 10.5883 3.65277 10.5197C3.54999 10.451 3.46988 10.3534 3.42258 10.2392C3.37527 10.125 3.36289 9.99931 3.38701 9.87807C3.41113 9.75683 3.47065 9.64547 3.55806 9.55806C3.64547 9.47065 3.75683 9.41112 3.87807 9.38701C3.99931 9.36289 4.12497 9.37527 4.23918 9.42258C4.35338 9.46988 4.45099 9.54999 4.51967 9.65277C4.58834 9.75555 4.625 9.87639 4.625 10C4.62481 10.1657 4.55891 10.3246 4.44174 10.4417C4.32457 10.5589 4.1657 10.6248 4 10.625Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M16.625 9.5H8.625C8.34888 9.5 8.125 9.27613 8.125 9C8.125 8.72388 8.34888 8.5 8.625 8.5H16.625C16.9011 8.5 17.125 8.72388 17.125 9C17.125 9.27613 16.9011 9.5 16.625 9.5Z" fill="#292929"/>
+<path d="M13.875 11.375H8.625C8.34887 11.375 8.125 11.1511 8.125 10.875C8.125 10.5989 8.34887 10.375 8.625 10.375H13.875C14.1511 10.375 14.375 10.5989 14.375 10.875C14.375 11.1511 14.1511 11.375 13.875 11.375Z" fill="#292929"/>
+<path d="M4 13.625C3.7775 13.625 3.55999 13.691 3.37498 13.8146C3.18998 13.9382 3.04578 14.1139 2.96064 14.3195C2.87549 14.525 2.85321 14.7512 2.89662 14.9695C2.94003 15.1877 3.04717 15.3882 3.20451 15.5455C3.36184 15.7028 3.56229 15.81 3.78052 15.8534C3.99875 15.8968 4.22495 15.8745 4.43052 15.7894C4.63609 15.7042 4.81179 15.56 4.9354 15.375C5.05902 15.19 5.125 14.9725 5.125 14.75C5.125 14.4516 5.00647 14.1655 4.7955 13.9545C4.58452 13.7435 4.29837 13.625 4 13.625ZM4 15.375C3.87639 15.375 3.75555 15.3383 3.65277 15.2697C3.54999 15.201 3.46988 15.1034 3.42258 14.9892C3.37527 14.875 3.36289 14.7493 3.38701 14.6281C3.41113 14.5068 3.47065 14.3955 3.55806 14.3081C3.64547 14.2207 3.75683 14.1611 3.87807 14.137C3.99931 14.1129 4.12497 14.1253 4.23918 14.1726C4.35338 14.2199 4.45099 14.3 4.51967 14.4028C4.58834 14.5055 4.625 14.6264 4.625 14.75C4.62481 14.9157 4.55891 15.0746 4.44174 15.1917C4.32457 15.3089 4.1657 15.3748 4 15.375Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M16.625 14.375H8.625C8.34887 14.375 8.125 14.1511 8.125 13.875C8.125 13.5989 8.34887 13.375 8.625 13.375H16.625C16.9011 13.375 17.125 13.5989 17.125 13.875C17.125 14.1511 16.9011 14.375 16.625 14.375Z" fill="#292929"/>
+<path d="M13.875 16.375H8.625C8.34887 16.375 8.125 16.1511 8.125 15.875C8.125 15.5989 8.34887 15.375 8.625 15.375H13.875C14.1511 15.375 14.375 15.5989 14.375 15.875C14.375 16.1511 14.1511 16.375 13.875 16.375Z" fill="#292929"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/entity.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/entity.svg
new file mode 100644
index 000000000000..1cb51fca34d6
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/entity.svg
@@ -0,0 +1,19 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M10 4.25C10.2225 4.25 10.44 4.18402 10.625 4.0604C10.81 3.93679 10.9542 3.76109 11.0394 3.55552C11.1245 3.34995 11.1468 3.12375 11.1034 2.90552C11.06 2.6873 10.9528 2.48684 10.7955 2.32951C10.6382 2.17217 10.4377 2.06503 10.2195 2.02162C10.0012 1.97821 9.77505 2.00049 9.56948 2.08564C9.36391 2.17078 9.18821 2.31498 9.0646 2.49998C8.94098 2.68499 8.875 2.9025 8.875 3.125C8.875 3.42337 8.99353 3.70952 9.2045 3.9205C9.41548 4.13147 9.70163 4.25 10 4.25ZM10 2.5C10.1236 2.5 10.2445 2.53666 10.3472 2.60533C10.45 2.67401 10.5301 2.77162 10.5774 2.88582C10.6247 3.00003 10.6371 3.12569 10.613 3.24693C10.5889 3.36817 10.5294 3.47953 10.4419 3.56694C10.3545 3.65435 10.2432 3.71388 10.1219 3.73799C10.0007 3.76211 9.87503 3.74973 9.76082 3.70243C9.64662 3.65512 9.54901 3.57501 9.48033 3.47223C9.41166 3.36945 9.375 3.24861 9.375 3.125C9.37519 2.9593 9.44109 2.80043 9.55826 2.68327C9.67543 2.5661 9.8343 2.50019 10 2.5Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M16.875 8.875C16.6525 8.875 16.435 8.94098 16.25 9.0646C16.065 9.18821 15.9208 9.36391 15.8356 9.56948C15.7505 9.77505 15.7282 10.0012 15.7716 10.2195C15.815 10.4377 15.9222 10.6382 16.0795 10.7955C16.2368 10.9528 16.4373 11.06 16.6555 11.1034C16.8738 11.1468 17.1 11.1245 17.3055 11.0394C17.5111 10.9542 17.6868 10.81 17.8104 10.625C17.934 10.44 18 10.2225 18 10C18 9.70163 17.8815 9.41548 17.6705 9.2045C17.4595 8.99353 17.1734 8.875 16.875 8.875ZM16.875 10.625C16.7514 10.625 16.6305 10.5883 16.5278 10.5197C16.425 10.451 16.3449 10.3534 16.2976 10.2392C16.2503 10.125 16.2379 9.99931 16.262 9.87807C16.2861 9.75683 16.3457 9.64547 16.4331 9.55806C16.5205 9.47065 16.6318 9.41112 16.7531 9.38701C16.8743 9.36289 17 9.37527 17.1142 9.42258C17.2284 9.46988 17.326 9.54999 17.3947 9.65277C17.4633 9.75555 17.5 9.87639 17.5 10C17.4998 10.1657 17.4339 10.3246 17.3167 10.4417C17.1996 10.5589 17.0407 10.6248 16.875 10.625Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M3.125 8.875C2.9025 8.875 2.68499 8.94098 2.49998 9.0646C2.31498 9.18821 2.17078 9.36391 2.08564 9.56948C2.00049 9.77505 1.97821 10.0012 2.02162 10.2195C2.06503 10.4377 2.17217 10.6382 2.32951 10.7955C2.48684 10.9528 2.6873 11.06 2.90552 11.1034C3.12375 11.1468 3.34995 11.1245 3.55552 11.0394C3.76109 10.9542 3.93679 10.81 4.0604 10.625C4.18402 10.44 4.25 10.2225 4.25 10C4.25 9.70163 4.13147 9.41548 3.9205 9.2045C3.70952 8.99353 3.42337 8.875 3.125 8.875ZM3.125 10.625C3.00139 10.625 2.88055 10.5883 2.77777 10.5197C2.67499 10.451 2.59488 10.3534 2.54758 10.2392C2.50027 10.125 2.48789 9.99931 2.51201 9.87807C2.53613 9.75683 2.59565 9.64547 2.68306 9.55806C2.77047 9.47065 2.88183 9.41112 3.00307 9.38701C3.12431 9.36289 3.24997 9.37527 3.36418 9.42258C3.47838 9.46988 3.57599 9.54999 3.64467 9.65277C3.71334 9.75555 3.75 9.87639 3.75 10C3.74981 10.1657 3.68391 10.3246 3.56674 10.4417C3.44957 10.5589 3.2907 10.6248 3.125 10.625Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M5.13867 6.26367C5.36118 6.26367 5.57868 6.19769 5.76369 6.07408C5.94869 5.95046 6.09289 5.77476 6.17804 5.56919C6.26318 5.36362 6.28546 5.13742 6.24206 4.9192C6.19865 4.70097 6.0915 4.50051 5.93417 4.34318C5.77683 4.18584 5.57638 4.0787 5.35815 4.03529C5.13992 3.99188 4.91372 4.01416 4.70815 4.09931C4.50259 4.18446 4.32689 4.32865 4.20327 4.51366C4.07965 4.69866 4.01367 4.91617 4.01367 5.13867C4.01367 5.43704 4.1322 5.72319 4.34318 5.93417C4.55416 6.14515 4.8403 6.26367 5.13867 6.26367ZM5.13867 4.51367C5.26229 4.51367 5.38312 4.55033 5.4859 4.619C5.58868 4.68768 5.66879 4.78529 5.7161 4.8995C5.7634 5.0137 5.77578 5.13937 5.75166 5.2606C5.72755 5.38184 5.66802 5.49321 5.58061 5.58061C5.49321 5.66802 5.38184 5.72755 5.2606 5.75166C5.13937 5.77578 5.0137 5.7634 4.89949 5.7161C4.78529 5.66879 4.68768 5.58868 4.619 5.4859C4.55033 5.38312 4.51367 5.26229 4.51367 5.13867C4.51386 4.97297 4.57977 4.81411 4.69694 4.69694C4.81411 4.57977 4.97297 4.51386 5.13867 4.51367Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M14.8613 6.26367C15.0838 6.26367 15.3013 6.19769 15.4863 6.07408C15.6713 5.95046 15.8155 5.77476 15.9007 5.56919C15.9858 5.36362 16.0081 5.13742 15.9647 4.9192C15.9213 4.70097 15.8142 4.50051 15.6568 4.34318C15.4995 4.18584 15.299 4.0787 15.0808 4.03529C14.8626 3.99188 14.6364 4.01416 14.4308 4.09931C14.2252 4.18446 14.0495 4.32865 13.9259 4.51366C13.8023 4.69866 13.7363 4.91617 13.7363 5.13867C13.7363 5.43704 13.8549 5.72319 14.0658 5.93417C14.2768 6.14515 14.563 6.26367 14.8613 6.26367ZM14.8613 4.51367C14.9849 4.51367 15.1058 4.55033 15.2086 4.619C15.3113 4.68768 15.3914 4.78529 15.4388 4.8995C15.4861 5.0137 15.4984 5.13937 15.4743 5.2606C15.4502 5.38184 15.3907 5.49321 15.3033 5.58061C15.2159 5.66802 15.1045 5.72755 14.9833 5.75166C14.862 5.77578 14.7364 5.7634 14.6222 5.7161C14.5079 5.66879 14.4103 5.58868 14.3417 5.4859C14.273 5.38312 14.2363 5.26229 14.2363 5.13867C14.2365 4.97297 14.3024 4.81411 14.4196 4.69694C14.5368 4.57977 14.6956 4.51386 14.8613 4.51367Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M10 15.75C9.7775 15.75 9.55999 15.816 9.37498 15.9396C9.18998 16.0632 9.04578 16.2389 8.96064 16.4445C8.87549 16.65 8.85321 16.8762 8.89662 17.0945C8.94003 17.3127 9.04717 17.5132 9.20451 17.6705C9.36184 17.8278 9.5623 17.935 9.78052 17.9784C9.99875 18.0218 10.225 17.9995 10.4305 17.9144C10.6361 17.8292 10.8118 17.685 10.9354 17.5C11.059 17.315 11.125 17.0975 11.125 16.875C11.125 16.5766 11.0065 16.2905 10.7955 16.0795C10.5845 15.8685 10.2984 15.75 10 15.75ZM10 17.5C9.87639 17.5 9.75555 17.4633 9.65277 17.3947C9.54999 17.326 9.46988 17.2284 9.42258 17.1142C9.37527 17 9.36289 16.8743 9.38701 16.7531C9.41113 16.6318 9.47065 16.5205 9.55806 16.4331C9.64547 16.3456 9.75683 16.2861 9.87807 16.262C9.99931 16.2379 10.125 16.2503 10.2392 16.2976C10.3534 16.3449 10.451 16.425 10.5197 16.5278C10.5883 16.6305 10.625 16.7514 10.625 16.875C10.6248 17.0407 10.5589 17.1996 10.4417 17.3167C10.3246 17.4339 10.1657 17.4998 10 17.5Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M5.13867 13.7363C4.91617 13.7363 4.69866 13.8023 4.51366 13.9259C4.32865 14.0495 4.18446 14.2252 4.09931 14.4308C4.01416 14.6364 3.99188 14.8626 4.03529 15.0808C4.0787 15.299 4.18584 15.4995 4.34318 15.6568C4.50051 15.8142 4.70097 15.9213 4.9192 15.9647C5.13742 16.0081 5.36362 15.9858 5.56919 15.9007C5.77476 15.8155 5.95046 15.6713 6.07408 15.4863C6.19769 15.3013 6.26367 15.0838 6.26367 14.8613C6.26367 14.563 6.14515 14.2768 5.93417 14.0658C5.72319 13.8549 5.43704 13.7363 5.13867 13.7363ZM5.13867 15.4863C5.01506 15.4863 4.89422 15.4497 4.79144 15.381C4.68866 15.3123 4.60855 15.2147 4.56125 15.1005C4.51394 14.9863 4.50157 14.8606 4.52568 14.7394C4.5498 14.6182 4.60932 14.5068 4.69673 14.4194C4.78414 14.332 4.8955 14.2725 5.01674 14.2483C5.13798 14.2242 5.26365 14.2366 5.37785 14.2839C5.49205 14.3312 5.58966 14.4113 5.65834 14.5141C5.72702 14.6169 5.76367 14.7377 5.76367 14.8613C5.76349 15.027 5.69758 15.1859 5.58041 15.3031C5.46324 15.4202 5.30438 15.4861 5.13867 15.4863Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M14.8613 13.7363C14.6388 13.7363 14.4213 13.8023 14.2363 13.9259C14.0513 14.0495 13.9071 14.2252 13.822 14.4308C13.7368 14.6364 13.7145 14.8626 13.7579 15.0808C13.8014 15.299 13.9085 15.4995 14.0658 15.6568C14.2232 15.8142 14.4236 15.9213 14.6419 15.9647C14.8601 16.0081 15.0863 15.9858 15.2918 15.9007C15.4974 15.8155 15.6731 15.6713 15.7967 15.4863C15.9203 15.3013 15.9863 15.0838 15.9863 14.8613C15.9863 14.563 15.8678 14.2768 15.6568 14.0658C15.4458 13.8549 15.1597 13.7363 14.8613 13.7363ZM14.8613 15.4863C14.7377 15.4863 14.6169 15.4497 14.5141 15.381C14.4113 15.3123 14.3312 15.2147 14.2839 15.1005C14.2366 14.9863 14.2242 14.8606 14.2483 14.7394C14.2725 14.6182 14.332 14.5068 14.4194 14.4194C14.5068 14.332 14.6182 14.2725 14.7394 14.2483C14.8606 14.2242 14.9863 14.2366 15.1005 14.2839C15.2147 14.3312 15.3123 14.4113 15.381 14.5141C15.4497 14.6169 15.4863 14.7377 15.4863 14.8613C15.4861 15.027 15.4202 15.1859 15.3031 15.3031C15.1859 15.4202 15.027 15.4861 14.8613 15.4863Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M6.56825 7.39937C6.58521 7.463 6.62606 7.51764 6.68228 7.55193C6.73851 7.58621 6.80578 7.5975 6.87012 7.58345C6.91375 7.57277 6.95367 7.55048 6.98566 7.51895C7.01764 7.48741 7.0405 7.44781 7.0518 7.40434L7.97675 8.32926C8.08317 8.20053 8.20153 8.08216 8.33026 7.97573L7.40532 7.05084C7.45911 7.0368 7.50667 7.00518 7.54042 6.96102C7.57418 6.91685 7.59221 6.86266 7.59163 6.80708C7.59106 6.75149 7.57192 6.69769 7.53726 6.65422C7.5026 6.61076 7.4544 6.58014 7.40033 6.56721L6.70044 6.39766C6.65879 6.38756 6.61525 6.38835 6.57399 6.39994C6.53273 6.41152 6.49514 6.43352 6.46483 6.46382C6.43453 6.49412 6.41252 6.53171 6.40093 6.57296C6.38933 6.61422 6.38854 6.65777 6.39862 6.69942L6.56825 7.39937Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M13.4319 12.5998C13.419 12.5457 13.3884 12.4976 13.3449 12.4629C13.3015 12.4282 13.2477 12.4091 13.1921 12.4085C13.1365 12.408 13.0823 12.426 13.0382 12.4597C12.994 12.4935 12.9624 12.5411 12.9484 12.5948L12.0234 11.6699C11.917 11.7986 11.7986 11.917 11.6699 12.0234L12.5947 12.9482C12.541 12.9623 12.4934 12.994 12.4597 13.0381C12.426 13.0823 12.408 13.1365 12.4086 13.1921C12.4092 13.2477 12.4283 13.3015 12.463 13.3449C12.4976 13.3884 12.5458 13.419 12.5999 13.432C12.5999 13.432 13.2992 13.6014 13.2997 13.6015C13.3414 13.6111 13.3848 13.61 13.426 13.5984C13.4671 13.5869 13.5047 13.5652 13.5353 13.5354C13.5656 13.5051 13.5877 13.4675 13.5993 13.4262C13.6109 13.385 13.6116 13.3414 13.6016 13.2998L13.4319 12.5998Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M13.9217 10.5054C13.9571 10.561 14.0127 10.6007 14.0768 10.6162C14.1408 10.6317 14.2084 10.6219 14.2653 10.5886L14.8802 10.2136C14.9167 10.1913 14.947 10.16 14.968 10.1226C14.9889 10.0853 15 10.0431 15 10.0003C15 9.95741 14.9889 9.91527 14.968 9.87791C14.947 9.84054 14.9167 9.8092 14.8802 9.78689L14.2653 9.41189C14.218 9.38278 14.1623 9.37035 14.107 9.37656C14.0518 9.38278 14.0002 9.40729 13.9605 9.44619C13.9208 9.48509 13.8952 9.53615 13.8879 9.59125C13.8806 9.64635 13.8919 9.70233 13.92 9.75027H12.6123C12.6201 9.83264 12.625 9.91589 12.625 10.0003C12.625 10.0846 12.6201 10.1679 12.6123 10.2503H13.92C13.8972 10.289 13.8853 10.3332 13.8857 10.3781C13.886 10.423 13.8984 10.467 13.9217 10.5054Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M9.99977 7.375C10.0842 7.375 10.1674 7.37983 10.2498 7.38766V6.08065C10.2978 6.10878 10.3538 6.12005 10.409 6.11266C10.4641 6.10527 10.5152 6.07966 10.5541 6.03989C10.593 6.00012 10.6175 5.94848 10.6237 5.89319C10.6299 5.83789 10.6174 5.78212 10.5882 5.73474L10.2132 5.11987C10.1909 5.08326 10.1596 5.05301 10.1222 5.03202C10.0848 5.01103 10.0426 5 9.99978 5C9.95691 5 9.91476 5.01103 9.87738 5.03202C9.84 5.05301 9.80865 5.08326 9.78634 5.11987L9.41134 5.73474C9.38226 5.7821 9.36985 5.83783 9.37608 5.89305C9.38232 5.94828 9.40683 5.99984 9.44573 6.03953C9.48464 6.07922 9.53569 6.10477 9.59078 6.11211C9.64587 6.11945 9.70183 6.10816 9.74977 6.08004V7.38766C9.83215 7.37983 9.91538 7.375 9.99977 7.375Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M12.0234 8.32925L12.9484 7.40433C12.9597 7.4478 12.9825 7.4874 13.0145 7.51894C13.0465 7.55047 13.0864 7.57276 13.1301 7.58344C13.1945 7.59842 13.2622 7.58752 13.3186 7.5531C13.3751 7.51867 13.4158 7.46347 13.4319 7.39936L13.6016 6.69941C13.6116 6.65776 13.6108 6.61422 13.5993 6.57296C13.5877 6.5317 13.5656 6.49412 13.5353 6.46382C13.505 6.43352 13.4675 6.41152 13.4262 6.39994C13.3849 6.38835 13.3414 6.38756 13.2997 6.39765L12.5998 6.56721C12.5458 6.58016 12.4976 6.61079 12.463 6.65425C12.4283 6.69771 12.4092 6.7515 12.4086 6.80708C12.408 6.86267 12.426 6.91685 12.4597 6.96103C12.4934 7.00521 12.541 7.03686 12.5947 7.05094L11.6699 7.97573C11.7986 8.08215 11.917 8.20052 12.0234 8.32925Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M10 7.875C9.57972 7.875 9.16887 7.99963 8.81941 8.23313C8.46996 8.46663 8.19759 8.7985 8.03676 9.1868C7.87592 9.57509 7.83384 10.0024 7.91583 10.4146C7.99783 10.8268 8.20021 11.2054 8.4974 11.5026C8.79459 11.7998 9.17322 12.0022 9.58543 12.0842C9.99764 12.1662 10.4249 12.1241 10.8132 11.9632C11.2015 11.8024 11.5334 11.53 11.7669 11.1806C12.0004 10.8311 12.125 10.4203 12.125 10C12.125 9.43641 11.9011 8.89591 11.5026 8.4974C11.1041 8.09888 10.5636 7.875 10 7.875ZM10 11.625C9.67861 11.625 9.36443 11.5297 9.0972 11.3511C8.82997 11.1726 8.62169 10.9188 8.4987 10.6219C8.3757 10.3249 8.34352 9.9982 8.40622 9.68298C8.46893 9.36776 8.62369 9.07821 8.85095 8.85095C9.07821 8.62369 9.36776 8.46892 9.68298 8.40622C9.9982 8.34352 10.3249 8.3757 10.6219 8.4987C10.9188 8.62169 11.1726 8.82997 11.3511 9.0972C11.5297 9.36443 11.625 9.67861 11.625 10C11.6245 10.4308 11.4532 10.8439 11.1485 11.1485C10.8439 11.4532 10.4308 11.6245 10 11.625Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M6.07998 9.75026C6.10811 9.70231 6.1194 9.64634 6.11206 9.59125C6.10471 9.53615 6.07916 9.48509 6.03945 9.44619C5.99975 9.40729 5.94818 9.38278 5.89294 9.37656C5.83771 9.37035 5.78198 9.38278 5.73462 9.41188L5.11981 9.78688C5.08322 9.80919 5.05298 9.84054 5.032 9.8779C5.01102 9.91527 5 9.95741 5 10.0003C5 10.0431 5.01102 10.0853 5.032 10.1226C5.05298 10.16 5.08322 10.1913 5.11981 10.2136L5.73462 10.5886C5.79148 10.6222 5.85916 10.6322 5.92331 10.6167C5.98747 10.6012 6.04304 10.5612 6.07824 10.5054C6.10154 10.467 6.11401 10.423 6.11431 10.3781C6.11462 10.3331 6.10276 10.289 6.07998 10.2503H7.38766C7.37983 10.1679 7.37499 10.0846 7.37499 10.0003C7.37499 9.91588 7.37983 9.83262 7.38766 9.75026H6.07998Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M7.97675 11.6699L7.0518 12.5948C7.03776 12.5411 7.00616 12.4935 6.962 12.4597C6.91784 12.426 6.86366 12.408 6.80808 12.4085C6.7525 12.4091 6.6987 12.4282 6.65524 12.4629C6.61179 12.4976 6.58116 12.5457 6.56824 12.5998L6.39862 13.2998C6.38854 13.3414 6.38933 13.385 6.40093 13.4262C6.41252 13.4675 6.43453 13.5051 6.46484 13.5354C6.49544 13.5652 6.53304 13.5869 6.57421 13.5985C6.61537 13.61 6.65877 13.6111 6.70044 13.6015L7.40033 13.432C7.4544 13.419 7.50258 13.3884 7.53724 13.3449C7.5719 13.3015 7.59104 13.2477 7.59162 13.1921C7.5922 13.1365 7.57418 13.0823 7.54044 13.0382C7.5067 12.994 7.45916 12.9624 7.40538 12.9483L8.33025 12.0234C8.20153 11.917 8.08317 11.7986 7.97675 11.6699Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+<path d="M9.99975 12.6259C9.91536 12.6259 9.83212 12.6211 9.74975 12.6133V13.9208C9.7018 13.8927 9.64583 13.8815 9.59075 13.8888C9.53567 13.8962 9.48462 13.9217 9.44573 13.9614C9.40684 14.0011 9.38232 14.0527 9.37609 14.1079C9.36985 14.1631 9.38225 14.2188 9.41131 14.2662L9.78631 14.8811C9.80862 14.9177 9.83997 14.9479 9.87735 14.9689C9.91472 14.9899 9.95687 15.0009 9.99974 15.0009C10.0426 15.0009 10.0848 14.9899 10.1221 14.9689C10.1595 14.9479 10.1909 14.9177 10.2132 14.8811L10.5882 14.2662C10.6172 14.2188 10.6296 14.1631 10.6234 14.1079C10.6172 14.0527 10.5927 14.0011 10.5538 13.9614C10.5149 13.9217 10.4638 13.8962 10.4087 13.8888C10.3537 13.8814 10.2977 13.8927 10.2497 13.9208V12.6133C10.1674 12.6211 10.0841 12.6259 9.99975 12.6259Z" fill="#292929" stroke="#292929" stroke-width="0.3"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/enum.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/enum.svg
new file mode 100644
index 000000000000..3744491c26a7
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/enum.svg
@@ -0,0 +1,8 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M6.41422 9.5H3.08578C2.48706 9.5 2 9.01294 2 8.41422V5.08578C2 4.48706 2.48706 4 3.08578 4H6.41422C7.01294 4 7.5 4.48706 7.5 5.08578V8.41422C7.5 9.01294 7.01294 9.5 6.41422 9.5ZM3.08578 5C3.03847 5 3 5.03847 3 5.08578V8.41422C3 8.46153 3.03847 8.5 3.08578 8.5H6.41422C6.46153 8.5 6.5 8.46153 6.5 8.41422V5.08578C6.5 5.03847 6.46153 5 6.41422 5H3.08578Z" fill="#292929"/>
+<path d="M17.5 6.5H9.5C9.22387 6.5 9 6.27613 9 6C9 5.72387 9.22387 5.5 9.5 5.5H17.5C17.7761 5.5 18 5.72387 18 6C18 6.27613 17.7761 6.5 17.5 6.5Z" fill="#292929"/>
+<path d="M14.75 8.375H9.5C9.22387 8.375 9 8.15113 9 7.875C9 7.59887 9.22387 7.375 9.5 7.375H14.75C15.0261 7.375 15.25 7.59887 15.25 7.875C15.25 8.15113 15.0261 8.375 14.75 8.375Z" fill="#292929"/>
+<path d="M6.41422 16.75H3.08578C2.48706 16.75 2 16.2629 2 15.6642V12.3358C2 11.7371 2.48706 11.25 3.08578 11.25H6.41422C7.01294 11.25 7.5 11.7371 7.5 12.3358V15.6642C7.5 16.2629 7.01294 16.75 6.41422 16.75ZM3.08578 12.25C3.03847 12.25 3 12.2885 3 12.3358V15.6642C3 15.7115 3.03847 15.75 3.08578 15.75H6.41422C6.46153 15.75 6.5 15.7115 6.5 15.6642V12.3358C6.5 12.2885 6.46153 12.25 6.41422 12.25H3.08578Z" fill="#292929"/>
+<path d="M17.5 13.625H9.5C9.22387 13.625 9 13.4011 9 13.125C9 12.8489 9.22387 12.625 9.5 12.625H17.5C17.7761 12.625 18 12.8489 18 13.125C18 13.4011 17.7761 13.625 17.5 13.625Z" fill="#292929"/>
+<path d="M14.75 15.625H9.5C9.22387 15.625 9 15.4011 9 15.125C9 14.8489 9.22387 14.625 9.5 14.625H14.75C15.0261 14.625 15.25 14.8489 15.25 15.125C15.25 15.4011 15.0261 15.625 14.75 15.625Z" fill="#292929"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/integer.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/integer.svg
new file mode 100644
index 000000000000..2e73065c979a
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/integer.svg
@@ -0,0 +1,10 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16.5882 4H3.41176C2.63207 4 2 4.63207 2 5.41176V14.8235C2 15.6032 2.63207 16.2353 3.41176 16.2353H16.5882C17.3679 16.2353 18 15.6032 18 14.8235V5.41176C18 4.63207 17.3679 4 16.5882 4Z" fill="white" stroke="#292929"/>
+<mask id="path-2-inside-1_10935_271490" fill="white">
+<path d="M7.15338 12.4995V9.57216C6.60865 9.99032 6.24192 10.1995 6.05319 10.1995C5.96312 10.1995 5.8827 10.1641 5.81192 10.0933C5.7433 10.0203 5.70898 9.93677 5.70898 9.84237C5.70898 9.73301 5.7433 9.65254 5.81192 9.60105C5.88055 9.54967 6.00172 9.48312 6.17543 9.40162C6.43493 9.27945 6.64189 9.1507 6.7963 9.01564C6.95285 8.88049 7.09119 8.72934 7.21128 8.56199C7.33138 8.39475 7.40965 8.29178 7.44611 8.2532C7.48258 8.21461 7.5512 8.19531 7.652 8.19531C7.76566 8.19531 7.85681 8.23927 7.92544 8.32717C7.99407 8.41517 8.02837 8.5363 8.02837 8.69075V12.374C8.02837 12.8051 7.88147 13.0206 7.58766 13.0206C7.45683 13.0206 7.35175 12.9767 7.2724 12.8888C7.19305 12.8009 7.15338 12.6711 7.15338 12.4995Z"/>
+</mask>
+<path d="M7.15338 12.4995V9.57216C6.60865 9.99032 6.24192 10.1995 6.05319 10.1995C5.96312 10.1995 5.8827 10.1641 5.81192 10.0933C5.7433 10.0203 5.70898 9.93677 5.70898 9.84237C5.70898 9.73301 5.7433 9.65254 5.81192 9.60105C5.88055 9.54967 6.00172 9.48312 6.17543 9.40162C6.43493 9.27945 6.64189 9.1507 6.7963 9.01564C6.95285 8.88049 7.09119 8.72934 7.21128 8.56199C7.33138 8.39475 7.40965 8.29178 7.44611 8.2532C7.48258 8.21461 7.5512 8.19531 7.652 8.19531C7.76566 8.19531 7.85681 8.23927 7.92544 8.32717C7.99407 8.41517 8.02837 8.5363 8.02837 8.69075V12.374C8.02837 12.8051 7.88147 13.0206 7.58766 13.0206C7.45683 13.0206 7.35175 12.9767 7.2724 12.8888C7.19305 12.8009 7.15338 12.6711 7.15338 12.4995Z" fill="#292929"/>
+<path d="M7.15338 9.57216H8.15338V7.54384L6.54446 8.77893L7.15338 9.57216ZM5.81192 10.0933L5.08357 10.7785L5.09403 10.7896L5.10481 10.8004L5.81192 10.0933ZM5.81192 9.60105L5.21255 8.80058L5.2118 8.80115L5.81192 9.60105ZM6.17543 9.40162L6.6002 10.3069L6.60136 10.3064L6.17543 9.40162ZM6.7963 9.01564L6.14281 8.25867L6.13793 8.26295L6.7963 9.01564ZM7.21128 8.56199L6.399 7.97873L6.39885 7.97894L7.21128 8.56199ZM7.92544 8.32717L8.71398 7.71219L8.71366 7.71178L7.92544 8.32717ZM8.15338 12.4995V9.57216H6.15338V12.4995H8.15338ZM6.54446 8.77893C6.28883 8.97516 6.1016 9.10142 5.97445 9.17391C5.94429 9.19111 5.92062 9.20349 5.90297 9.21208C5.89427 9.21631 5.88759 9.21935 5.88285 9.22142C5.87807 9.2235 5.87577 9.22437 5.87577 9.22437C5.87581 9.22436 5.87657 9.22407 5.87801 9.22358C5.87944 9.22309 5.88168 9.22234 5.88468 9.22142C5.89052 9.21963 5.90027 9.21685 5.91357 9.21387C5.93818 9.20835 5.98677 9.19945 6.05319 9.19945V11.1995C6.43379 11.1995 6.77464 11.0199 6.96511 10.9113C7.20469 10.7747 7.47319 10.5873 7.7623 10.3654L6.54446 8.77893ZM6.05319 9.19945C6.14326 9.19945 6.23813 9.21803 6.32808 9.25761C6.41601 9.2963 6.47884 9.34599 6.51903 9.38618L5.10481 10.8004C5.34026 11.0358 5.66723 11.1995 6.05319 11.1995V9.19945ZM6.54027 9.40808C6.58058 9.45093 6.62586 9.51339 6.65986 9.59626C6.6943 9.68019 6.70898 9.7652 6.70898 9.84237H4.70898C4.70898 10.2083 4.85236 10.5327 5.08357 10.7785L6.54027 9.40808ZM6.70898 9.84237C6.70898 9.88986 6.70176 9.98164 6.65454 10.0923C6.60421 10.2103 6.52114 10.3191 6.41204 10.401L5.2118 8.80115C4.82381 9.09223 4.70898 9.51373 4.70898 9.84237H6.70898ZM6.41129 10.4015C6.39248 10.4156 6.39444 10.4118 6.43536 10.3893C6.47202 10.3692 6.52579 10.3418 6.6002 10.3069L5.75066 8.49632C5.57134 8.58045 5.3694 8.68314 5.21255 8.80058L6.41129 10.4015ZM6.60136 10.3064C6.91468 10.1589 7.2088 9.98339 7.45467 9.76834L6.13793 8.26295C6.07497 8.31801 5.95519 8.40003 5.7495 8.49686L6.60136 10.3064ZM7.44977 9.77259C7.66556 9.58631 7.85735 9.37687 8.02372 9.14505L6.39885 7.97894C6.32503 8.0818 6.24015 8.17467 6.14283 8.25869L7.44977 9.77259ZM8.02356 9.14526C8.08101 9.06525 8.1247 9.00589 8.15606 8.96464C8.17174 8.94403 8.182 8.93102 8.18775 8.92393C8.19569 8.91413 8.18919 8.92282 8.17297 8.93999L6.71926 7.56641C6.62633 7.66475 6.50506 7.83102 6.399 7.97873L8.02356 9.14526ZM8.17297 8.93999C8.03131 9.08991 7.87774 9.14936 7.79559 9.17246C7.71381 9.19545 7.65855 9.19531 7.652 9.19531V7.19531C7.49803 7.19531 7.05627 7.20974 6.71926 7.56641L8.17297 8.93999ZM7.652 9.19531C7.57709 9.19531 7.47702 9.18002 7.37124 9.12902C7.26487 9.07772 7.18759 9.00709 7.13721 8.94256L8.71366 7.71178C8.45132 7.37576 8.06671 7.19531 7.652 7.19531V9.19531ZM7.13689 8.94215C7.02445 8.79797 7.02837 8.675 7.02837 8.69075H9.02837C9.02837 8.39761 8.96369 8.03237 8.71398 7.71219L7.13689 8.94215ZM7.02837 8.69075V12.374H9.02837V8.69075H7.02837ZM7.02837 12.374C7.02837 12.4282 7.02346 12.4451 7.02568 12.4354C7.02787 12.4257 7.04176 12.3693 7.09188 12.2958C7.14648 12.2157 7.23044 12.136 7.34115 12.0819C7.4473 12.03 7.53862 12.0206 7.58766 12.0206V14.0206C8.01649 14.0206 8.4589 13.8412 8.74451 13.4222C8.98328 13.0719 9.02837 12.672 9.02837 12.374H7.02837ZM7.58766 12.0206C7.63521 12.0206 7.71011 12.0289 7.79659 12.0651C7.88545 12.1023 7.95965 12.1577 8.01471 12.2187L6.5301 13.5588C6.82013 13.8801 7.20691 14.0206 7.58766 14.0206V12.0206ZM8.01471 12.2187C8.16243 12.3824 8.15338 12.5333 8.15338 12.4995H6.15338C6.15338 12.8089 6.22367 13.2194 6.5301 13.5588L8.01471 12.2187Z" fill="#292929" mask="url(#path-2-inside-1_10935_271490)"/>
+<path d="M13.0896 8.67556L13.7649 8.00034L14.4401 8.67556H13.0896Z" fill="#292929" stroke="#292929"/>
+<path d="M13.7649 12.2338L13.0896 11.5586H14.4401L13.7649 12.2338Z" fill="#292929" stroke="#292929"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/markdown.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/markdown.svg
new file mode 100644
index 000000000000..0786a39254e2
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/markdown.svg
@@ -0,0 +1,8 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M9.375 8.125V11.875" stroke="#292929" stroke-linecap="round"/>
+<path d="M4.375 8.125V11.875" stroke="#292929" stroke-linecap="round"/>
+<path d="M13.125 3.125H6.875C4.11358 3.125 1.875 5.36358 1.875 8.125V11.875C1.875 14.6364 4.11358 16.875 6.875 16.875H13.125C15.8864 16.875 18.125 14.6364 18.125 11.875V8.125C18.125 5.36358 15.8864 3.125 13.125 3.125Z" stroke="#292929" stroke-linejoin="round"/>
+<path d="M13.75 8.125V11.5625" stroke="#292929" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M11.5 10.5L13.5 12H14L16 10.5" stroke="#292929" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M5 8.5L6.6 11.5H7L9 8.5" stroke="#292929" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/number.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/number.svg
new file mode 100644
index 000000000000..715ebe96b0c9
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/number.svg
@@ -0,0 +1,13 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g clip-path="url(#clip0_10949_271911)">
+<path d="M14.5 15.4428C13.2385 16.4872 11.6403 17.0622 10 17.0622C8.11347 17.0622 6.34003 16.3275 5.00597 14.9937C3.67222 13.6597 2.93753 11.8862 2.93753 9.99969C2.93753 9.69531 2.95691 9.39 2.99566 9.09156C3.05191 8.65125 3.15034 8.21469 3.28847 7.79406C3.29206 7.78312 3.29591 7.77213 3.29956 7.76116C3.33481 7.76381 3.37034 7.76562 3.40625 7.76562C4.18156 7.76562 4.8125 7.13469 4.8125 6.35937C4.8125 5.58406 4.18156 4.95312 3.40625 4.95312C2.63094 4.95312 2 5.58406 2 6.35937C2 6.76034 2.16903 7.12244 2.43925 7.37888C2.42503 7.41981 2.41119 7.46075 2.39781 7.50159C2.24125 7.97878 2.12937 8.47378 2.06562 8.97222C2.02222 9.30937 2.00003 9.655 2.00003 9.99969C2.00003 12.1366 2.83222 14.1456 4.34316 15.6566C5.85409 17.1675 7.86316 17.9997 10 17.9997C11.8582 17.9997 13.6688 17.3481 15.0978 16.1647C15.2975 15.9997 15.3253 15.7044 15.16 15.505C14.995 15.3053 14.6997 15.2775 14.5 15.4428Z" fill="black"/>
+<path d="M17.5605 12.6208C17.5748 12.5799 17.5886 12.539 17.602 12.4981C17.7585 12.0209 17.8704 11.5259 17.9342 11.0275C17.9776 10.6903 17.9998 10.3447 17.9998 10C17.9998 7.86313 17.1676 5.85406 15.6567 4.34313C14.1457 2.83219 12.1367 2 9.99979 2C8.14167 2 6.33104 2.65156 4.90198 3.835C4.70229 4 4.67448 4.29531 4.83979 4.49469C5.00479 4.69438 5.3001 4.72219 5.49979 4.55687C6.76135 3.5125 8.35948 2.9375 9.99979 2.9375C11.8864 2.9375 13.6598 3.67219 14.9939 5.00594C16.3276 6.34 17.0623 8.11344 17.0623 10C17.0623 10.3044 17.0429 10.6097 17.0042 10.9081C16.9479 11.3484 16.8495 11.785 16.7114 12.2056C16.7078 12.2166 16.7039 12.2276 16.7003 12.2385C16.665 12.2359 16.6295 12.2341 16.5935 12.2341C15.8182 12.2341 15.1873 12.865 15.1873 13.6403C15.1873 14.4156 15.8182 15.0466 16.5935 15.0466C17.3689 15.0466 17.9998 14.4156 17.9998 13.6403C17.9998 13.2393 17.8308 12.8773 17.5605 12.6208Z" fill="black"/>
+<path d="M12.6612 13.2035C12.4023 13.2035 12.1924 12.9937 12.1924 12.7348V8.74167L11.8931 9.08874C11.724 9.28477 11.428 9.30664 11.2319 9.13755C11.0359 8.96845 11.014 8.67249 11.1831 8.47642L12.3062 7.17433C12.4341 7.02599 12.6408 6.97286 12.8245 7.04111C13.0081 7.10936 13.1299 7.28461 13.1299 7.48049V12.7348C13.1299 12.9937 12.92 13.2035 12.6612 13.2035Z" fill="black"/>
+<path d="M8.17032 13.2031C7.07403 13.2031 6.18213 12.3112 6.18213 11.215V8.78503C6.18213 7.68875 7.07403 6.79688 8.17032 6.79688C9.2666 6.79688 10.1585 7.68878 10.1585 8.78503V11.215C10.1585 12.3112 9.26657 13.2031 8.17032 13.2031ZM8.17032 7.73438C7.59097 7.73438 7.11963 8.20572 7.11963 8.78503V11.215C7.11963 11.7943 7.59097 12.2656 8.17032 12.2656C8.74966 12.2656 9.22097 11.7943 9.22097 11.215V8.78503C9.22097 8.20572 8.74963 7.73438 8.17032 7.73438Z" fill="black"/>
+</g>
+<defs>
+<clipPath id="clip0_10949_271911">
+<rect width="16" height="16" fill="white" transform="translate(2 2)"/>
+</clipPath>
+</defs>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/sql-query.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/sql-query.svg
new file mode 100644
index 000000000000..54a3072f160b
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/sql-query.svg
@@ -0,0 +1,5 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16.6776 5.53223L13.1274 2H5.30924C4.76766 2 4.32275 2.41477 4.32275 2.95632V8.00794C4.32275 8.00697 4.35575 8 4.37769 8H4.80662V2.95632C4.80662 2.68597 5.03888 2.48387 5.30924 2.48387H12.9087L13.0647 2.63016V4.61703C13.0647 5.15635 13.5033 5.58065 14.0426 5.58065H16.0295L16.1615 5.73532V17.0112C16.1615 17.2816 15.9737 17.5161 15.7033 17.5161H5.30924C5.03888 17.5161 4.80662 17.2815 4.80662 17.0112V15.0645H4.37769C4.35575 15.0645 4.32275 15.0657 4.32275 15.0647V17.0112C4.32275 17.5528 4.76766 18 5.30924 18H15.7033C16.2449 18 16.6776 17.5528 16.6776 17.0112V5.53223ZM14.0426 5.09677C13.7722 5.09677 13.5486 4.88739 13.5486 4.61703V3.16877V3.15939L15.5002 5.09677H14.0426Z" fill="#333333"/>
+<path d="M8.7975 10.666C8.3765 10.666 8.16602 10.9694 8.16602 11.576C8.16602 12.1771 8.37508 12.4776 8.79324 12.4776C9.00585 12.4776 9.16353 12.4046 9.26634 12.2586C9.36911 12.1126 9.4205 11.8851 9.4205 11.576C9.4205 11.2656 9.3684 11.0364 9.26421 10.8882C9.16005 10.7401 9.00447 10.666 8.7975 10.666Z" fill="#333333"/>
+<path d="M13.6323 8.48242H3.98058C3.43903 8.48242 3 8.92145 3 9.463V13.5664C3 14.1079 3.43903 14.5469 3.98058 14.5469H13.6323C14.1739 14.5469 14.6129 14.1079 14.6129 13.5664V9.463C14.6129 8.92145 14.1739 8.48242 13.6323 8.48242ZM5.68813 11.0167C5.75119 11.0649 5.92236 11.1528 6.20161 11.2804C6.46952 11.4008 6.65555 11.5302 6.75974 11.6684C6.8639 11.8066 6.91603 11.9806 6.91603 12.1904C6.91603 12.3832 6.86713 12.5543 6.76932 12.7039C6.67152 12.8535 6.53045 12.9696 6.34619 13.0526C6.1619 13.1355 5.94571 13.177 5.69768 13.177C5.49068 13.177 5.31868 13.1624 5.17832 13.1334C5.038 13.1044 4.90319 13.0536 4.7419 12.9813V12.2329C4.90319 12.3151 5.06703 12.3793 5.24 12.4253C5.4129 12.4714 5.5709 12.4945 5.71548 12.4945C5.84019 12.4945 5.93123 12.4728 5.98939 12.4296C6.04748 12.3864 6.07636 12.3307 6.07636 12.2627C6.07636 12.2202 6.06458 12.1829 6.04119 12.151C6.01781 12.1192 5.98016 12.0869 5.92845 12.0543C5.87671 12.0217 5.73884 11.9551 5.51487 11.8545C5.31213 11.7623 5.1601 11.673 5.05877 11.5865C4.95742 11.5001 4.88229 11.4008 4.83339 11.2888C4.78448 11.1769 4.76003 11.0443 4.76003 10.8912C4.76003 10.6049 4.86423 10.3817 5.07258 10.2215C5.28094 10.0613 5.56726 9.9812 5.93158 9.9812C6.25336 9.9812 6.58148 10.0556 6.91603 10.2044L6.65874 10.8529C6.36813 10.7197 6.11723 10.6531 5.90606 10.6531C5.7969 10.6531 5.71752 10.6722 5.66794 10.7105C5.61829 10.7487 5.59352 10.7962 5.59352 10.8529C5.59352 10.914 5.62503 10.9685 5.68813 11.0167ZM9.34632 13.8695L8.80316 13.1809L8.80645 13.1812V13.1791L8.7989 13.177C8.31126 13.177 7.93471 13.0402 7.67461 12.7666C7.41448 12.4931 7.2831 12.0947 7.2831 11.5717C7.2831 11.0543 7.41174 10.6592 7.67045 10.3863C7.92913 10.1135 8.30481 9.977 8.79807 9.977C9.29274 9.977 9.66752 10.1124 9.92271 10.3831C10.1778 10.6539 10.3053 11.0515 10.3053 11.5759C10.3053 12.2805 10.0813 12.7529 9.63342 12.9981L10.3925 13.8695H9.34632ZM12.9032 13.1276H10.871V10.0308H11.7097V12.4502H12.9032V13.1276Z" fill="#333333"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/string.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/string.svg
new file mode 100644
index 000000000000..f6b7526b5e86
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/string.svg
@@ -0,0 +1,6 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="2.5" y="4.5" width="15" height="12" rx="1.5" stroke="black"/>
+<path d="M7.29977 11.7276C7.2965 11.7153 7.29261 11.7034 7.28809 11.6916L6.01547 7.7728C5.96296 7.61094 5.80074 7.5 5.61687 7.5C5.43276 7.5 5.27078 7.61094 5.21804 7.7728L3.96032 11.6452C3.93747 11.6862 3.92213 11.7314 3.91677 11.7793L3.51751 13.0086C3.45173 13.2108 3.577 13.4236 3.79714 13.4838C4.0175 13.5444 4.24913 13.4296 4.31471 13.227L4.64801 12.201H6.58522L6.91836 13.227C6.97221 13.3928 7.13761 13.5 7.31673 13.5C7.35616 13.5 7.39628 13.4948 7.43617 13.484C7.6563 13.4236 7.7816 13.2108 7.71579 13.0086L7.29977 11.7276ZM4.89627 11.4369L5.61687 9.21872L6.33711 11.4369H4.89627Z" fill="black"/>
+<path d="M16.0838 9.63366C16.3137 9.63366 16.5 9.46271 16.5 9.2516V9.02083C16.5 8.18217 15.7278 7.5 14.779 7.5H14.662C13.7134 7.5 12.9414 8.18219 12.9414 9.02083V11.9789C12.9414 12.8175 13.7134 13.4997 14.662 13.4997H14.779C15.7278 13.4997 16.4998 12.8175 16.5 11.9791V11.6989C16.5 11.4877 16.3137 11.3168 16.0838 11.3168C15.854 11.3168 15.6679 11.4877 15.6679 11.6989V11.9791C15.6679 12.3961 15.2691 12.7358 14.779 12.7358H14.662C14.1722 12.7358 13.7736 12.3963 13.7736 11.9791V9.02081C13.7736 8.60376 14.172 8.26412 14.662 8.26412H14.779C15.2688 8.26412 15.6679 8.60355 15.6679 9.02081V9.25158C15.6679 9.46269 15.854 9.63366 16.0838 9.63366Z" fill="black"/>
+<path d="M11.2426 10.1269C11.4997 9.85867 11.6559 9.50681 11.6559 9.12226V9.02083C11.6559 8.18217 10.9128 7.5 9.9995 7.5H8.84359C8.61398 7.5 8.42765 7.67095 8.42765 7.88206V13.1176C8.42765 13.3288 8.61376 13.4997 8.84359 13.4997H10.2805C11.3163 13.4997 12.1588 12.7155 12.1588 11.7516V11.6283C12.1588 10.9912 11.791 10.4328 11.2426 10.1269ZM9.2595 8.26415H9.99947C10.454 8.26415 10.8237 8.60358 10.8237 9.02083V9.12226C10.8237 9.53972 10.4542 9.87978 9.99947 9.87978H9.2595V8.26415ZM11.3269 11.7514C11.3269 12.2941 10.8575 12.7358 10.2805 12.7358H9.2595V10.6437H10.2805C10.8575 10.6437 11.3269 11.0854 11.3269 11.628V11.7514Z" fill="black"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/table.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/table.svg
new file mode 100644
index 000000000000..caaff5cfbc57
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/table.svg
@@ -0,0 +1,9 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M16.3333 17.6667H3.66667C2.748 17.6667 2 16.9187 2 16V4.66667C2 3.748 2.748 3 3.66667 3H16.3333C17.252 3 18 3.748 18 4.66667V16C18 16.9187 17.252 17.6667 16.3333 17.6667ZM3.66667 3.66667C3.11533 3.66667 2.66667 4.11533 2.66667 4.66667V16C2.66667 16.5513 3.11533 17 3.66667 17H16.3333C16.8847 17 17.3333 16.5513 17.3333 16V4.66667C17.3333 4.11533 16.8847 3.66667 16.3333 3.66667H3.66667Z" fill="#292929"/>
+<path d="M17.6667 7.66667H2.33333C2.14933 7.66667 2 7.51733 2 7.33333C2 7.14933 2.14933 7 2.33333 7H17.6667C17.8507 7 18 7.14933 18 7.33333C18 7.51733 17.8507 7.66667 17.6667 7.66667Z" fill="#292929"/>
+<path d="M17.6667 10.9987H2.33333C2.14933 10.9987 2 10.8494 2 10.6654C2 10.4814 2.14933 10.332 2.33333 10.332H17.6667C17.8507 10.332 18 10.4814 18 10.6654C18 10.8494 17.8507 10.9987 17.6667 10.9987Z" fill="#292929"/>
+<path d="M17.6667 14.3327H2.33333C2.14933 14.3327 2 14.1833 2 13.9993C2 13.8153 2.14933 13.666 2.33333 13.666H17.6667C17.8507 13.666 18 13.8153 18 13.9993C18 14.1833 17.8507 14.3327 17.6667 14.3327Z" fill="#292929"/>
+<path d="M6.33333 17.6667C6.14933 17.6667 6 17.5173 6 17.3333V7.33333C6 7.14933 6.14933 7 6.33333 7C6.51733 7 6.66667 7.14933 6.66667 7.33333V17.3333C6.66667 17.5173 6.51733 17.6667 6.33333 17.6667Z" fill="#292929"/>
+<path d="M10.0003 17.6667C9.81633 17.6667 9.66699 17.5173 9.66699 17.3333V7.33333C9.66699 7.14933 9.81633 7 10.0003 7C10.1843 7 10.3337 7.14933 10.3337 7.33333V17.3333C10.3337 17.5173 10.1843 17.6667 10.0003 17.6667Z" fill="#292929"/>
+<path d="M13.6663 17.6667C13.4823 17.6667 13.333 17.5173 13.333 17.3333V7.33333C13.333 7.14933 13.4823 7 13.6663 7C13.8503 7 13.9997 7.14933 13.9997 7.33333V17.3333C13.9997 17.5173 13.8503 17.6667 13.6663 17.6667Z" fill="#292929"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/time-interval.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/time-interval.svg
new file mode 100644
index 000000000000..70424ee73e19
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/time-interval.svg
@@ -0,0 +1,3 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M15.4602 6.53285L16.1659 5.82712C16.4402 5.55287 16.4402 5.1082 16.1659 4.83409C15.8917 4.55984 15.4471 4.55984 15.1729 4.83409L14.4671 5.53983C13.2831 4.55325 11.8443 3.95724 10.3095 3.81772V2.40446H10.9883C11.3761 2.40446 11.6905 2.08998 11.6905 1.70216C11.6905 1.31435 11.3761 1 10.9883 1H8.22623C7.83841 1 7.52406 1.31435 7.52406 1.70216C7.52406 2.08998 7.83841 2.40446 8.22623 2.40446H8.90504V3.81772C5.06313 4.16708 2 7.39459 2 11.3927C2 15.5971 5.40233 19 9.60735 19C13.8117 19 17.2146 15.5977 17.2146 11.3927C17.2146 9.59666 16.596 7.89584 15.4602 6.53285ZM9.60721 17.5955C6.18703 17.5955 3.40433 14.813 3.40433 11.3927C3.40433 7.97247 6.18703 5.18991 9.60721 5.18991C13.0275 5.18991 15.8101 7.97247 15.8101 11.3927C15.8101 14.813 13.0275 17.5955 9.60721 17.5955ZM12.8615 8.1385C13.1357 8.41275 13.1357 8.85742 12.8615 9.13153L10.1038 11.8892C9.82954 12.1635 9.38487 12.1635 9.11076 11.8892C8.83652 11.615 8.83652 11.1703 9.11076 10.8962L11.8683 8.1385C12.1426 7.86426 12.5872 7.86426 12.8615 8.1385Z" fill="black"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/time.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/time.svg
new file mode 100644
index 000000000000..a7517f3aac9a
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/time.svg
@@ -0,0 +1,4 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M12.8511 11.4139L10.6202 9.74077V6.33258C10.6202 5.98989 10.3432 5.71289 10.0005 5.71289C9.65786 5.71289 9.38086 5.98989 9.38086 6.33258V10.0506C9.38086 10.2458 9.47258 10.4299 9.62873 10.5464L12.1074 12.4054C12.219 12.4891 12.3491 12.5294 12.4786 12.5294C12.6676 12.5294 12.8535 12.4445 12.975 12.2809C13.1808 12.0075 13.125 11.619 12.8511 11.4139Z" fill="black"/>
+<path d="M10 2C5.58853 2 2 5.58853 2 10C2 14.4115 5.58853 18 10 18C14.4115 18 18 14.4115 18 10C18 5.58853 14.4115 2 10 2ZM10 16.7607C6.27266 16.7607 3.23934 13.7273 3.23934 10C3.23934 6.27266 6.27266 3.23934 10 3.23934C13.728 3.23934 16.7607 6.27266 16.7607 10C16.7607 13.7273 13.7273 16.7607 10 16.7607Z" fill="black"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/timestamp.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/timestamp.svg
new file mode 100644
index 000000000000..7e72e98bd405
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/customproperties/timestamp.svg
@@ -0,0 +1,6 @@
+<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M14 6.42773C11.7949 6.42773 10 8.22202 10 10.4277C10 12.6334 11.7949 14.4277 14 14.4277C16.2051 14.4277 18 12.6334 18 10.4277C18 8.22202 16.2051 6.42773 14 6.42773ZM14 13.2849C12.4246 13.2849 11.1429 12.0032 11.1429 10.4277C11.1429 8.85231 12.4246 7.57059 14 7.57059C15.5754 7.57059 16.8571 8.85231 16.8571 10.4277C16.8571 12.0032 15.5754 13.2849 14 13.2849Z" fill="black"/>
+<path d="M14.5714 8.71401V10.4283C14.5714 10.5797 14.5114 10.7254 14.404 10.8323L13.5469 11.6894C13.3234 11.9129 12.9623 11.9129 12.7389 11.6894C12.5154 11.466 12.5154 11.1049 12.7389 10.8814L13.4286 10.1917V8.71401C13.4286 8.39858 13.684 8.14258 14 8.14258C14.316 8.14258 14.5714 8.39858 14.5714 8.71401Z" fill="black"/>
+<path d="M2.57143 16.1436H11.7143C12.0303 16.1436 12.2857 15.8876 12.2857 15.5722V15.0008C12.2857 14.6853 12.0303 14.4293 11.7143 14.4293C11.3983 14.4293 11.1429 14.6853 11.1429 15.0008H3.14286V14.4293C3.14286 13.799 3.65543 13.2865 4.28571 13.2865C4.476 13.2865 9.68743 13.2865 9.42857 13.2865C9.74457 13.2865 10 13.0305 10 12.715C10 12.3996 9.74457 12.1436 9.42857 12.1436H8.85714V9.10133C8.85714 8.54475 9.02114 8.04704 9.31886 7.69904C10.3583 6.48418 10.196 4.63333 8.91886 3.61961C7.30114 2.33561 4.86286 3.14475 4.37257 5.14875C4.15257 6.04875 4.35886 6.96533 4.93829 7.66475C5.25429 8.04647 5.42857 8.59104 5.42857 9.1979V12.1436H4.28571C3.02514 12.1436 2 13.1688 2 14.4293V15.5722C2 15.8876 2.25543 16.1436 2.57143 16.1436ZM5.48286 5.42018C5.77257 4.23561 7.23429 3.74133 8.20857 4.51504C8.96571 5.11618 9.07943 6.2219 8.45086 6.95618C7.97543 7.51047 7.71429 8.27275 7.71429 9.10133V12.1436H6.57143V9.1979C6.57143 8.31275 6.31086 7.5299 5.81771 6.93504C5.46629 6.51104 5.34686 5.97333 5.48286 5.42018Z" fill="black"/>
+<path d="M11.1427 16.7148C11.4587 16.7148 11.7141 16.9708 11.7141 17.2863C11.7141 17.6017 11.4587 17.8577 11.1427 17.8577H3.14272C2.82672 17.8577 2.57129 17.6017 2.57129 17.2863C2.57129 16.9708 2.82672 16.7148 3.14272 16.7148H11.1427Z" fill="black"/>
+</svg>
diff --git a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-search.svg b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-search.svg
index 206c4dcc02c9..9be36acac921 100644
--- a/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-search.svg
+++ b/openmetadata-ui/src/main/resources/ui/src/assets/svg/ic-search.svg
@@ -1 +1,7 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none"><path fill="#CCC" d="m19.45 16.8-4.589-4.59a8.842 8.842 0 0 1-2.651 2.652l4.59 4.589c.732.732 1.919.732 2.65 0a1.873 1.873 0 0 0 0-2.652ZM15 7.5a7.5 7.5 0 1 0-15 0 7.5 7.5 0 0 0 15 0Zm-7.5 5.625A5.63 5.63 0 0 1 1.875 7.5 5.63 5.63 0 0 1 7.5 1.875 5.631 5.631 0 0 1 13.125 7.5 5.63 5.63 0 0 1 7.5 13.125Z"/><path fill="#CCC" d="M3.125 7.5h1.25A3.128 3.128 0 0 1 7.5 4.375v-1.25A4.38 4.38 0 0 0 3.125 7.5Z"/></svg>
\ No newline at end of file
+<svg viewBox="0 0 20 21" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.2403 5.18031C9.53367 4.01467 7.28685 4.01466 5.58013 5.18031L5.58004 5.18037C4.73566 5.75686 4.08044 6.56996 3.69665 7.5176L3.6964 7.51822C3.30832 8.47365 3.20686 9.52139 3.40444 10.5335L3.40456 10.5342C3.60024 11.5421 4.08751 12.4704 4.80592 13.204L4.80704 13.2051C5.75403 14.1769 7.05336 14.725 8.41021 14.725C9.76717 14.725 11.0665 14.1769 12.0134 13.2052L12.0146 13.204C12.733 12.4704 13.2203 11.5421 13.4159 10.5342L13.416 10.5335C13.6137 9.52139 13.5122 8.47367 13.1241 7.51824L13.1238 7.51757C12.7401 6.56996 12.0849 5.75687 11.2404 5.18039L11.2403 5.18031ZM11.9768 4.1017C9.82599 2.63276 6.99449 2.63277 4.84361 4.10171ZM11.977 4.10178C13.0338 4.82332 13.8539 5.84098 14.3342 7.027C14.8177 8.2173 14.9441 9.5226 14.6979 10.7836C14.4536 12.0422 13.8451 13.2014 12.948 14.1175C11.7553 15.341 10.119 16.0311 8.41021 16.0311C6.70136 16.0311 5.06498 15.3409 3.87222 14.1173C2.97528 13.2012 2.36688 12.042 2.12248 10.7834C1.87643 9.52252 2.00281 8.21732 2.48618 7.02709C2.96658 5.84103 3.78668 4.82333 4.84351 4.10178" fill="#292929"/>
+<path d="M13.0612 13.0752C12.7874 12.8019 12.3438 12.8023 12.0704 13.0762C11.797 13.3499 11.7975 13.7936 12.0713 14.0669L13.0612 13.0752ZM16.6956 18.6856C16.9695 18.9591 17.413 18.9587 17.6865 18.6848C17.9598 18.4109 17.9594 17.9674 17.6855 17.694L16.6956 18.6856ZM12.0713 14.0669L16.6956 18.6856L17.6855 17.694L13.0612 13.0752L12.0713 14.0669Z" fill="#292929"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M11.9934 12.9993C12.3092 12.6829 12.8217 12.6825 13.138 12.9983L17.7624 17.617C17.7624 17.617 17.7624 17.617 17.7624 17.617C18.0788 17.9329 18.0792 18.4453 17.7634 18.7618C17.4476 19.0782 16.9351 19.0786 16.6187 18.7627L11.9943 14.144C11.9943 14.144 11.9943 14.144 11.9943 14.144C11.678 13.8282 11.6775 13.3157 11.9934 12.9993ZM17.5316 17.6942L13.0612 13.2292L12.2251 14.0668L16.6955 18.5318L17.5316 17.6942ZM17.6094 18.608C17.4047 18.813 17.0872 18.8366 16.8565 18.6785ZM16.8565 18.6785L17.6785 17.855C17.837 18.0854 17.814 18.403 17.6094 18.608M12.0781 13.906L12.9001 13.0825C12.6696 12.9245 12.352 12.9481 12.1474 13.1531C11.9428 13.358 11.9198 13.6757 12.0781 13.906Z" fill="#292929"/>
+<path d="M7.79393 6.48132C7.7532 6.09655 8.03204 5.75158 8.4168 5.71081C9.70533 5.57428 10.9315 6.29663 11.4365 7.48979C11.5874 7.8461 11.4208 8.25723 11.0646 8.40807C10.7082 8.55893 10.2971 8.39237 10.1462 8.03604C9.88169 7.41104 9.23939 7.03268 8.56448 7.10419C8.17972 7.14496 7.83475 6.8661 7.79393 6.48132Z" fill="#757575"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M8.40535 5.60241C9.74153 5.46083 11.013 6.2099 11.5368 7.4472C11.7111 7.85886 11.5186 8.33385 11.107 8.50813C10.6953 8.68242 10.2203 8.49 10.046 8.0783C9.80016 7.49745 9.20322 7.1458 8.57597 7.21226C8.13143 7.25936 7.73288 6.93718 7.68572 6.49264C7.63866 6.04811 7.9608 5.64951 8.40535 5.60241ZM8.53674 5.80948C8.50065 5.81191 8.4645 5.81505 8.42829 5.81888L8.41682 5.71065L8.52505 5.69918L8.53674 5.80948ZM11.292 7.43263C10.7963 6.37588 9.70138 5.73082 8.53674 5.80948L8.66164 6.98787C8.62556 6.98932 8.58934 6.99195 8.55304 6.99579L8.5645 7.10403L8.45627 7.1155L8.44375 6.99733C8.47952 7.00013 8.51606 6.99971 8.55304 6.99579M7.90218 6.46968C7.87171 6.18165 8.05322 5.92004 8.32111 5.84028L8.44375 6.99733C8.16508 6.97547 7.93275 6.7577 7.90218 6.46968ZM10.2008 7.8946C9.91028 7.31708 9.30759 6.96205 8.66164 6.98787L8.67274 7.09256L8.5645 7.10403M10.2008 7.8946L10.1038 7.93564L10.1887 8.1361L10.2981 8.08977C10.4523 8.32296 10.7554 8.4206 11.0221 8.30767C11.0221 8.30767 11.0221 8.30768 11.0221 8.30767C11.2888 8.19476 11.4297 7.90918 11.3696 7.63618L11.479 7.58985L11.4365 7.48962L11.3941 7.3894L11.292 7.43263M11.292 7.43263L10.2008 7.8946C10.217 7.92686 10.2322 7.95982 10.2465 7.99344C10.261 8.02768 10.2783 8.05984 10.2981 8.08977L11.3696 7.63618C11.3618 7.60113 11.3508 7.5663 11.3363 7.53206L11.4365 7.48962" fill="#757575"/>
+</svg>
\ No newline at end of file
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedDrawer/ActivityFeedDrawer.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedDrawer/ActivityFeedDrawer.tsx
index 6f5880b74a80..e6e7fd49c601 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedDrawer/ActivityFeedDrawer.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedDrawer/ActivityFeedDrawer.tsx
@@ -16,7 +16,6 @@ import classNames from 'classnames';
 import React, { FC } from 'react';
 import { useTranslation } from 'react-i18next';
 import { Thread, ThreadType } from '../../../generated/entity/feed/thread';
-import { useReserveSidebar } from '../../../hooks/useReserveSidebar';
 import Loader from '../../common/Loader/Loader';
 import ActivityFeedEditor from '../ActivityFeedEditor/ActivityFeedEditor';
 import FeedPanelBodyV1 from '../ActivityFeedPanel/FeedPanelBodyV1';
@@ -34,7 +33,6 @@ const ActivityFeedDrawer: FC<ActivityFeedDrawerProps> = ({
   className,
 }) => {
   const { t } = useTranslation();
-  const { isSidebarReserve } = useReserveSidebar();
   const {
     focusReplyEditor,
     isDrawerLoading,
@@ -73,12 +71,7 @@ const ActivityFeedDrawer: FC<ActivityFeedDrawerProps> = ({
       {isDrawerLoading ? (
         <Loader />
       ) : (
-        <Row
-          className={classNames({
-            ['reserve-right-sidebar']: isSidebarReserve,
-          })}
-          gutter={[0, 16]}
-          id="feed-panel">
+        <Row gutter={[0, 16]} id="feed-panel">
           <Col span={24}>
             <FeedPanelBodyV1
               isOpenInDrawer
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component.tsx
index 8e7016e67bc4..3bd9325efaef 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/ActivityFeedTab.component.tsx
@@ -10,12 +10,13 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-import { Menu, Space, Typography } from 'antd';
+import { Menu, Skeleton, Space, Typography } from 'antd';
 import { AxiosError } from 'axios';
 import classNames from 'classnames';
 import { noop } from 'lodash';
 import {
   default as React,
+  ReactNode,
   RefObject,
   useCallback,
   useEffect,
@@ -108,7 +109,13 @@ export const ActivityFeedTab = ({
   const [taskFilter, setTaskFilter] = useState<ThreadTaskStatus>(
     ThreadTaskStatus.Open
   );
-  const [count, setCount] = useState<FeedCounts>(FEED_COUNT_INITIAL_DATA);
+  const [countData, setCountData] = useState<{
+    loading: boolean;
+    data: FeedCounts;
+  }>({
+    loading: false,
+    data: FEED_COUNT_INITIAL_DATA,
+  });
 
   const {
     postFeed,
@@ -139,6 +146,17 @@ export const ActivityFeedTab = ({
     [activeTab]
   );
 
+  const getElementWithCountLoader = useCallback(
+    (element: ReactNode) => {
+      if (countData.loading) {
+        return <Skeleton.Button active className="count-loader" size="small" />;
+      }
+
+      return element;
+    },
+    [countData.loading]
+  );
+
   const handleTabChange = (subTab: string) => {
     history.push(
       entityUtilClassBase.getEntityLink(
@@ -169,29 +187,37 @@ export const ActivityFeedTab = ({
     }
   }, [activeTab]);
 
-  const handleFeedCount = useCallback((data: FeedCounts) => {
-    setCount(data);
-    onUpdateFeedCount?.(data);
-  }, []);
+  const handleFeedCount = useCallback(
+    (data: FeedCounts) => {
+      setCountData((prev) => ({ ...prev, data }));
+      onUpdateFeedCount?.(data);
+    },
+    [setCountData]
+  );
 
   const fetchFeedsCount = async () => {
+    setCountData((prev) => ({ ...prev, loading: true }));
     if (isUserEntity) {
       try {
         const res = await getFeedCount(getEntityUserLink(fqn));
-        setCount({
-          conversationCount: res[0].conversationCount ?? 0,
-          totalTasksCount: res[0].totalTaskCount,
-          openTaskCount: res[0].openTaskCount ?? 0,
-          closedTaskCount: res[0].closedTaskCount ?? 0,
-          totalCount: res[0].conversationCount ?? 0 + res[0].totalTaskCount,
-          mentionCount: res[0].mentionCount ?? 0,
-        });
+        setCountData((prev) => ({
+          ...prev,
+          data: {
+            conversationCount: res[0].conversationCount ?? 0,
+            totalTasksCount: res[0].totalTaskCount,
+            openTaskCount: res[0].openTaskCount ?? 0,
+            closedTaskCount: res[0].closedTaskCount ?? 0,
+            totalCount: res[0].conversationCount ?? 0 + res[0].totalTaskCount,
+            mentionCount: res[0].mentionCount ?? 0,
+          },
+        }));
       } catch (err) {
         showErrorToast(err as AxiosError, t('server.entity-feed-fetch-error'));
       }
     } else {
-      getFeedCounts(entityType, fqn, handleFeedCount);
+      await getFeedCounts(entityType, fqn, handleFeedCount);
     }
+    setCountData((prev) => ({ ...prev, loading: false }));
   };
 
   const getThreadType = useCallback((activeTab) => {
@@ -239,7 +265,7 @@ export const ActivityFeedTab = ({
 
   const refetchFeedData = useCallback(() => {
     if (
-      entityFeedTotalCount !== count.totalCount &&
+      entityFeedTotalCount !== countData.data.totalCount &&
       isActivityFeedTab &&
       refetchFeed
     ) {
@@ -259,7 +285,7 @@ export const ActivityFeedTab = ({
     threadType,
     entityType,
     refetchFeed,
-    count.totalCount,
+    countData.data.totalCount,
     entityFeedTotalCount,
     isActivityFeedTab,
   ]);
@@ -338,7 +364,7 @@ export const ActivityFeedTab = ({
                 <span>
                   {!isUserEntity &&
                     getCountBadge(
-                      count.conversationCount,
+                      countData.data.conversationCount,
                       '',
                       activeTab === ActivityFeedTabs.ALL
                     )}
@@ -362,7 +388,7 @@ export const ActivityFeedTab = ({
 
                       <span>
                         {getCountBadge(
-                          count.mentionCount,
+                          countData.data.mentionCount,
                           '',
                           activeTab === ActivityFeedTabs.MENTIONS
                         )}
@@ -384,7 +410,11 @@ export const ActivityFeedTab = ({
                   <span>{t('label.task-plural')}</span>
                 </Space>
                 <span>
-                  {getCountBadge(count.openTaskCount, '', isTaskActiveTab)}
+                  {getCountBadge(
+                    countData.data.openTaskCount,
+                    '',
+                    isTaskActiveTab
+                  )}
                 </span>
               </div>
             ),
@@ -400,33 +430,37 @@ export const ActivityFeedTab = ({
       <div className="center-container" id="center-container">
         {isTaskActiveTab && (
           <div className="d-flex gap-4 p-sm p-x-lg activity-feed-task">
-            <Typography.Text
-              className={classNames(
-                'cursor-pointer p-l-xss d-flex items-center',
-                {
-                  'font-medium': taskFilter === ThreadTaskStatus.Open,
-                }
-              )}
-              data-testid="open-task"
-              onClick={() => {
-                handleUpdateTaskFilter(ThreadTaskStatus.Open);
-                setActiveThread();
-              }}>
-              <TaskIcon className="m-r-xss" width={14} /> {count.openTaskCount}{' '}
-              {t('label.open')}
-            </Typography.Text>
-            <Typography.Text
-              className={classNames('cursor-pointer d-flex items-center', {
-                'font-medium': taskFilter === ThreadTaskStatus.Closed,
-              })}
-              data-testid="closed-task"
-              onClick={() => {
-                handleUpdateTaskFilter(ThreadTaskStatus.Closed);
-                setActiveThread();
-              }}>
-              <CheckIcon className="m-r-xss" width={14} />{' '}
-              {count.closedTaskCount} {t('label.closed')}
-            </Typography.Text>
+            {getElementWithCountLoader(
+              <Typography.Text
+                className={classNames(
+                  'cursor-pointer p-l-xss d-flex items-center',
+                  {
+                    'font-medium': taskFilter === ThreadTaskStatus.Open,
+                  }
+                )}
+                data-testid="open-task"
+                onClick={() => {
+                  handleUpdateTaskFilter(ThreadTaskStatus.Open);
+                  setActiveThread();
+                }}>
+                <TaskIcon className="m-r-xss" width={14} />{' '}
+                {countData.data.openTaskCount} {t('label.open')}
+              </Typography.Text>
+            )}
+            {getElementWithCountLoader(
+              <Typography.Text
+                className={classNames('cursor-pointer d-flex items-center', {
+                  'font-medium': taskFilter === ThreadTaskStatus.Closed,
+                })}
+                data-testid="closed-task"
+                onClick={() => {
+                  handleUpdateTaskFilter(ThreadTaskStatus.Closed);
+                  setActiveThread();
+                }}>
+                <CheckIcon className="m-r-xss" width={14} />{' '}
+                {countData.data.closedTaskCount} {t('label.closed')}
+              </Typography.Text>
+            )}
           </div>
         )}
         <ActivityFeedListV1
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/activity-feed-tab.less b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/activity-feed-tab.less
index 5221925a1270..7864cdd383d6 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/activity-feed-tab.less
+++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedTab/activity-feed-tab.less
@@ -51,3 +51,7 @@
 .activity-feed-task {
   background-color: @grey-1;
 }
+
+.count-loader.ant-skeleton-element .ant-skeleton-button-sm {
+  width: 72px;
+}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.test.tsx
index e5c82195eea3..2b03600155b8 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.test.tsx
@@ -25,6 +25,17 @@ jest.mock('../../hooks/useApplicationStore', () => {
   };
 });
 
+jest.mock(
+  '../Settings/Applications/ApplicationsProvider/ApplicationsProvider',
+  () => {
+    return {
+      useApplicationsProvider: jest.fn(() => ({
+        applications: [],
+      })),
+    };
+  }
+);
+
 jest.mock('../../components/MyData/LeftSidebar/LeftSidebar.component', () =>
   jest.fn().mockReturnValue(<p>Sidebar</p>)
 );
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx
index dcc016a28e7e..58d344f6af99 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/AppContainer.tsx
@@ -17,20 +17,20 @@ import React, { useCallback, useEffect, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useLimitStore } from '../../context/LimitsProvider/useLimitsStore';
 import { useApplicationStore } from '../../hooks/useApplicationStore';
-import { useReserveSidebar } from '../../hooks/useReserveSidebar';
 import { getLimitConfig } from '../../rest/limitsAPI';
 import applicationRoutesClass from '../../utils/ApplicationRoutesClassBase';
 import Appbar from '../AppBar/Appbar';
 import { LimitBanner } from '../common/LimitBanner/LimitBanner';
 import LeftSidebar from '../MyData/LeftSidebar/LeftSidebar.component';
 import applicationsClassBase from '../Settings/Applications/AppDetails/ApplicationsClassBase';
+import { useApplicationsProvider } from '../Settings/Applications/ApplicationsProvider/ApplicationsProvider';
 import './app-container.less';
 
 const AppContainer = () => {
   const { i18n } = useTranslation();
   const { Header, Sider, Content } = Layout;
   const { currentUser } = useApplicationStore();
-  const { isSidebarReserve } = useReserveSidebar();
+  const { applications } = useApplicationsProvider();
   const AuthenticatedRouter = applicationRoutesClass.getRouteElements();
   const ApplicationExtras = applicationsClassBase.getApplicationExtension();
   const isDirectionRTL = useMemo(() => i18n.dir() === 'rtl', [i18n]);
@@ -46,19 +46,29 @@ const AppContainer = () => {
     }
   }, []);
 
+  const appendReserveRightSidebarClass = useCallback(() => {
+    const element = document.getElementsByTagName('body');
+    element[0].classList.add('reserve-right-sidebar');
+  }, []);
+
   useEffect(() => {
     if (currentUser?.id) {
       fetchLimitConfig();
     }
   }, [currentUser?.id]);
 
+  useEffect(() => {
+    if (applicationsClassBase.isFloatingButtonPresent(applications)) {
+      appendReserveRightSidebarClass();
+    }
+  }, [applications]);
+
   return (
     <Layout>
       <LimitBanner />
       <Layout
         className={classNames('app-container', {
           ['extra-banner']: Boolean(bannerDetails),
-          ['reserve-right-sidebar']: isSidebarReserve,
         })}>
         <Sider
           className={classNames('left-sidebar-col', {
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/app-container.less b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/app-container.less
index e0d632e2ab77..e8eb7eda3898 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/app-container.less
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AppContainer/app-container.less
@@ -24,6 +24,7 @@
   background-color: white;
   .ant-layout-header {
     line-height: inherit;
+    background: @white;
   }
   .navbar-container {
     border-bottom: 1px solid @border-color;
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.test.tsx
index 7071e335ffad..a95c77337676 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.test.tsx
@@ -412,7 +412,7 @@ describe('SettingsRouter', () => {
 
   it('should render PersonaPage component for persona list route', async () => {
     render(
-      <MemoryRouter initialEntries={[`/settings/members/persona`]}>
+      <MemoryRouter initialEntries={[`/settings/persona`]}>
         <SettingsRouter />
       </MemoryRouter>
     );
@@ -422,7 +422,7 @@ describe('SettingsRouter', () => {
 
   it('should render PersonaDetailsPage component for persona details route', async () => {
     render(
-      <MemoryRouter initialEntries={[`/settings/members/persona/testPersona`]}>
+      <MemoryRouter initialEntries={[`/settings/persona/testPersona`]}>
         <SettingsRouter />
       </MemoryRouter>
     );
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx
index c41c19f6e9f4..86548c1f67a8 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/SettingsRouter.tsx
@@ -160,6 +160,12 @@ const SettingsRouter = () => {
 
       {/* Setting Page Routes with categories */}
 
+      <AdminProtectedRoute
+        exact
+        component={PersonaPage}
+        path={getSettingPath(GlobalSettingOptions.PERSONA)}
+      />
+
       <Route
         exact
         component={GlobalSettingCategoryPage}
@@ -203,22 +209,10 @@ const SettingsRouter = () => {
         )}>
         <Redirect to={getTeamsWithFqnPath(TeamType.Organization)} />
       </Route>
-      <AdminProtectedRoute
-        exact
-        component={PersonaPage}
-        path={getSettingPath(
-          GlobalSettingsMenuCategory.MEMBERS,
-          GlobalSettingOptions.PERSONA
-        )}
-      />
       <AdminProtectedRoute
         exact
         component={PersonaDetailsPage}
-        path={getSettingPath(
-          GlobalSettingsMenuCategory.MEMBERS,
-          GlobalSettingOptions.PERSONA,
-          true
-        )}
+        path={getSettingPath(GlobalSettingOptions.PERSONA, '', true)}
       />
       {/* Roles route start
        * Do not change the order of these route
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withAdvanceSearch.tsx b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withAdvanceSearch.tsx
index 3098df133874..19b05f55ec23 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withAdvanceSearch.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/AppRouter/withAdvanceSearch.tsx
@@ -14,8 +14,8 @@ import React, { FC } from 'react';
 import { AdvanceSearchProvider } from '../../components/Explore/AdvanceSearchProvider/AdvanceSearchProvider.component';
 
 export const withAdvanceSearch =
-  (Component: FC) =>
-  (props: JSX.IntrinsicAttributes & { children?: React.ReactNode }) => {
+  <P extends Record<string, unknown>>(Component: FC<P>) =>
+  (props: P) => {
     return (
       <AdvanceSearchProvider>
         <Component {...props} />
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx
index b578c2fd66b5..cea014035d0b 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Auth/AuthProviders/AuthProvider.tsx
@@ -769,7 +769,6 @@ export const AuthProvider = ({
       updateAxiosInterceptors: initializeAxiosInterceptors,
     });
 
-    return cleanup;
   }, [handleSuccessfulLogin]);
 
   const isConfigLoading =
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.component.tsx
index 41903907db34..8ebf9c75d90e 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.component.tsx
@@ -107,10 +107,12 @@ export const ExtraInfoLink = ({
   label,
   value,
   href,
+  newTab = false,
 }: {
   label: string;
   value: string | number;
   href: string;
+  newTab?: boolean;
 }) => (
   <>
     <Divider className="self-center" type="vertical" />
@@ -118,7 +120,11 @@ export const ExtraInfoLink = ({
       {!isEmpty(label) && (
         <span className="text-grey-muted m-r-xss">{`${label}: `}</span>
       )}
-      <Typography.Link href={href} style={{ fontSize: '12px' }}>
+      <Typography.Link
+        href={href}
+        rel={newTab ? 'noopener noreferrer' : undefined}
+        style={{ fontSize: '12px' }}
+        target={newTab ? '_blank' : undefined}>
         {value}{' '}
       </Typography.Link>
       <Icon
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.test.tsx
index 84d2d0e3083e..d8b911e28014 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/DataAssets/DataAssetsHeader/DataAssetsHeader.test.tsx
@@ -21,7 +21,7 @@ import { Container } from '../../../generated/entity/data/container';
 import { MOCK_TIER_DATA } from '../../../mocks/TableData.mock';
 import { getContainerByName } from '../../../rest/storageAPI';
 import { DEFAULT_ENTITY_PERMISSION } from '../../../utils/PermissionsUtils';
-import { DataAssetsHeader } from './DataAssetsHeader.component';
+import { DataAssetsHeader, ExtraInfoLink } from './DataAssetsHeader.component';
 import { DataAssetsHeaderProps } from './DataAssetsHeader.interface';
 
 const mockProps: DataAssetsHeaderProps = {
@@ -103,6 +103,37 @@ jest.mock('../../../rest/storageAPI', () => ({
     .mockImplementation(() => Promise.resolve({ name: 'test' })),
 }));
 
+describe('ExtraInfoLink component', () => {
+  const mockProps = {
+    label: 'myLabel',
+    value: 'example',
+    href: 'http://example.com/',
+  };
+
+  it('should not have target and rel attributes when newTab is false (default)', () => {
+    const { container } = render(<ExtraInfoLink {...mockProps} />);
+
+    const elm = container.querySelector('a');
+
+    expect(elm).toHaveAttribute('href', mockProps.href);
+    expect(elm).not.toHaveAttribute('target');
+    expect(elm).not.toHaveAttribute('rel');
+  });
+
+  it('should have target and rel attributes when newTab is true', () => {
+    const { container } = render(<ExtraInfoLink {...mockProps} newTab />);
+
+    const elm = container.querySelector('a');
+
+    expect(elm).toHaveAttribute('href', mockProps.href);
+    expect(elm).toHaveAttribute('target', '_blank');
+
+    const rel = (elm?.getAttribute('rel') || '').split(/\s+/g);
+
+    expect(rel.sort()).toStrictEqual(['noopener', 'noreferrer'].sort());
+  });
+});
+
 describe('DataAssetsHeader component', () => {
   it('should call getContainerByName API on Page load for container assets', () => {
     const mockGetContainerByName = getContainerByName as jest.Mock;
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.component.tsx
index 18800dd105b5..20a328fc28c7 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.component.tsx
@@ -182,7 +182,7 @@ export const TestSuites = ({ summaryPanel }: { summaryPanel: ReactNode }) => {
         },
       },
       {
-        title: t('label.owner'),
+        title: t('label.owner-plural'),
         dataIndex: 'owners',
         key: 'owners',
         render: (owners: EntityReference[]) => <OwnerLabel owners={owners} />,
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx
index b049a6be7f6c..babbd838659f 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx
@@ -129,7 +129,7 @@ describe('TestSuites component', () => {
       'label.name',
       'label.test-plural',
       'label.success %',
-      'label.owner',
+      'label.owner-plural',
     ]);
     expect(await screen.findByTestId('test-suite-table')).toBeInTheDocument();
     expect(
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/TableQueryRightPanel/TableQueryRightPanel.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/TableQueryRightPanel/TableQueryRightPanel.component.tsx
index 209f82e9ec56..d9b58e217783 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/TableQueryRightPanel/TableQueryRightPanel.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Database/TableQueries/TableQueryRightPanel/TableQueryRightPanel.component.tsx
@@ -86,7 +86,7 @@ const TableQueryRightPanel = ({
             <Space className="relative" direction="vertical" size={4}>
               <Space align="center" className="w-full" size={0}>
                 <Typography.Text className="right-panel-label">
-                  {t('label.owner')}
+                  {t('label.owner-plural')}
                 </Typography.Text>
 
                 {(EditAll || EditOwners) && (
@@ -99,7 +99,7 @@ const TableQueryRightPanel = ({
                     }>
                     <Tooltip
                       title={t('label.edit-entity', {
-                        entity: t('label.owner-lowercase'),
+                        entity: t('label.owner-lowercase-plural'),
                       })}>
                       <Button
                         className="cursor-pointer flex-center"
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/AddDomainForm/AddDomainForm.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/AddDomainForm/AddDomainForm.component.tsx
index 5a1820abfc00..357da5b3fa85 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/AddDomainForm/AddDomainForm.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/AddDomainForm/AddDomainForm.component.tsx
@@ -143,7 +143,7 @@ const AddDomainForm = ({
     name: 'owners',
     id: 'root/owner',
     required: false,
-    label: t('label.owner'),
+    label: t('label.owner-plural'),
     type: FieldTypes.USER_TEAM_SELECT,
     props: {
       hasPermission: true,
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/AddDomainForm/AddDomainForm.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/AddDomainForm/AddDomainForm.test.tsx
index e548070156fb..6406e78fcdb5 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/AddDomainForm/AddDomainForm.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/AddDomainForm/AddDomainForm.test.tsx
@@ -80,7 +80,7 @@ describe('Test Add Domain component', () => {
     expect(screen.getByTestId('color-color-input')).toBeInTheDocument();
     expect(screen.getByText('label.domain-type')).toBeInTheDocument();
     expect(screen.getAllByTestId('helper-icon')).toHaveLength(2);
-    expect(screen.getByText('label.owner')).toBeInTheDocument();
+    expect(screen.getByText('label.owner-plural')).toBeInTheDocument();
     expect(screen.getByTestId('add-owner')).toBeInTheDocument();
     expect(screen.getByText('label.expert-plural')).toBeInTheDocument();
     expect(screen.getByTestId('add-experts')).toBeInTheDocument();
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx
index 2378134a3d78..d6bce2a20276 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/DomainTabs/DocumentationTab/DocumentationTab.component.tsx
@@ -200,7 +200,7 @@ const DocumentationTab = ({
             <Col data-testid="domain-owner-name" span="24">
               <div className="d-flex items-center m-b-xss">
                 <Typography.Text className="right-panel-label">
-                  {t('label.owner')}
+                  {t('label.owner-plural')}
                 </Typography.Text>
                 {editOwnerPermission && domain.owners && (
                   <UserTeamSelectableList
@@ -210,7 +210,7 @@ const DocumentationTab = ({
                     onUpdate={(updatedUser) => handleUpdatedOwner(updatedUser)}>
                     <Tooltip
                       title={t('label.edit-entity', {
-                        entity: t('label.owner'),
+                        entity: t('label.owner-plural'),
                       })}>
                       <Button
                         className="cursor-pointer flex-center m-l-xss"
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Domain/SubDomainsTable/SubDomainsTable.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Domain/SubDomainsTable/SubDomainsTable.component.tsx
index 098270d20103..81d341d72aac 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Domain/SubDomainsTable/SubDomainsTable.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Domain/SubDomainsTable/SubDomainsTable.component.tsx
@@ -73,7 +73,7 @@ const SubDomainsTable = ({
           ),
       },
       {
-        title: t('label.owner'),
+        title: t('label.owner-plural'),
         dataIndex: 'owners',
         key: 'owners',
         render: (owners: EntityReference[]) => <OwnerLabel owners={owners} />,
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx
index 705ffd5b7c1a..06fe575e1a35 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Entity/Task/TaskTab/TaskTab.component.tsx
@@ -956,9 +956,9 @@ export const TaskTab = ({
       {isTaskTestCaseResult ? (
         <Modal
           destroyOnClose
-          maskClosable
           closable={false}
           closeIcon={null}
+          maskClosable={false}
           okButtonProps={{
             loading: isActionLoading,
           }}
@@ -1019,9 +1019,9 @@ export const TaskTab = ({
       ) : (
         <Modal
           destroyOnClose
-          maskClosable
           closable={false}
           closeIcon={null}
+          maskClosable={false}
           open={showEditTaskModel}
           title={`${t('label.edit-entity', {
             entity: t('label.task-lowercase'),
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.test.tsx
index 054e3607d83c..4ef19c3b6b16 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.test.tsx
@@ -65,7 +65,7 @@ describe('CommonEntitySummaryInfo component', () => {
     );
 
     expect(screen.queryByTestId('label.owner-label')).not.toBeInTheDocument();
-    expect(screen.getByTestId('label.owner-value')).toHaveTextContent(
+    expect(screen.getByTestId('label.owner-plural-value')).toHaveTextContent(
       'No Owner'
     );
     expect(screen.getByTestId('Type-label')).toHaveTextContent('Type');
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.tsx
index 4a32df36b2c4..de4466a7e112 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/CommonEntitySummaryInfo/CommonEntitySummaryInfo.tsx
@@ -30,7 +30,7 @@ function CommonEntitySummaryInfo({
   return (
     <Row gutter={[0, 4]}>
       {entityInfo.map((info) => {
-        const isOwner = info.name === t('label.owner');
+        const isOwner = info.name === t('label.owner-plural');
 
         return info.visible?.includes(componentType) ? (
           <Col key={info.name} span={24}>
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/DataProductSummary/DataProductSummary.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/DataProductSummary/DataProductSummary.component.tsx
index 3783c5226127..5a19a085866d 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/DataProductSummary/DataProductSummary.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/DataProductSummary/DataProductSummary.component.tsx
@@ -82,7 +82,7 @@ const DataProductSummary = ({
             <Typography.Text
               className="summary-panel-section-title"
               data-testid="owner-header">
-              {t('label.owner')}
+              {t('label.owner-plural')}
             </Typography.Text>
           </Col>
           <Col span={24}>
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/mocks/CommonEntitySummaryInfo.mock.ts b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/mocks/CommonEntitySummaryInfo.mock.ts
index 691336c45017..d4fdb24457c4 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/mocks/CommonEntitySummaryInfo.mock.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Explore/EntitySummaryPanel/mocks/CommonEntitySummaryInfo.mock.ts
@@ -13,7 +13,7 @@
 
 export const mockCommonEntityInfo = [
   {
-    name: 'label.owner',
+    name: 'label.owner-plural',
     value: 'No Owner',
     url: '',
     isLink: false,
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossary/AddGlossary.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossary/AddGlossary.component.tsx
index dda3df2d9712..1681b3d3ad4f 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossary/AddGlossary.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossary/AddGlossary.component.tsx
@@ -196,7 +196,7 @@ const AddGlossary = ({
     name: 'owners',
     id: 'root/owner',
     required: false,
-    label: t('label.owner'),
+    label: t('label.owner-plural'),
     type: FieldTypes.USER_TEAM_SELECT,
     props: {
       hasPermission: true,
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx
index 06e409d3bea6..ca248a75e6b1 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/AddGlossaryTermForm/AddGlossaryTermForm.component.tsx
@@ -314,7 +314,7 @@ const AddGlossaryTermForm = ({
     name: 'owners',
     id: 'root/owner',
     required: false,
-    label: t('label.owner'),
+    label: t('label.owner-plural'),
     type: FieldTypes.USER_TEAM_SELECT,
     props: {
       hasPermission: true,
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx
index c753cc350695..25e64c16ae50 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.component.tsx
@@ -158,7 +158,7 @@ const GlossaryDetailsRightPanel = ({
       <Col data-testid="glossary-right-panel-owner-link" span="24">
         <div className="d-flex items-center m-b-xs">
           <Typography.Text className="right-panel-label">
-            {t('label.owner')}
+            {t('label.owner-plural')}
           </Typography.Text>
           {(permissions.EditOwners || permissions.EditAll) &&
             selectedData.owners &&
@@ -171,7 +171,7 @@ const GlossaryDetailsRightPanel = ({
                 onUpdate={(updatedUser) => handleUpdatedOwner(updatedUser)}>
                 <Tooltip
                   title={t('label.edit-entity', {
-                    entity: t('label.owner'),
+                    entity: t('label.owner-plural'),
                   })}>
                   <Button
                     className="cursor-pointer flex-center m-l-xss"
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx
index bf399b6f8e6d..adfca2cf6cd9 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryDetailsRightPanel/GlossaryDetailsRightPanel.test.tsx
@@ -78,7 +78,7 @@ describe('GlossaryDetailsRightPanel', () => {
     );
 
     expect(getByTestId('glossary-right-panel-owner-link')).toHaveTextContent(
-      'label.owner'
+      'label.owner-plural'
     );
     expect(getByTestId('glossary-reviewer-heading-name')).toHaveTextContent(
       'label.reviewer-plural'
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx
index fa77762831cf..9c0851a77ab9 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/GlossaryHeader.component.tsx
@@ -657,6 +657,17 @@ const GlossaryHeader = ({
       <EntityNameModal
         allowRename
         entity={selectedData as EntityReference}
+        nameValidationRules={[
+          {
+            min: 1,
+            max: 128,
+            message: t('message.entity-size-in-between', {
+              entity: t('label.name'),
+              min: 1,
+              max: 128,
+            }),
+          },
+        ]}
         title={t('label.edit-entity', {
           entity: t('label.name'),
         })}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.component.tsx
index 353dedacb500..6897cd75d051 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTermTab/GlossaryTermTab.component.tsx
@@ -163,7 +163,7 @@ const GlossaryTermTab = ({
           ),
       },
       {
-        title: t('label.owner'),
+        title: t('label.owner-plural'),
         dataIndex: 'owners',
         key: 'owners',
         width: '17%',
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx
index 4da2ba649e69..fddce22633e6 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryTerms/GlossaryTermsV1.component.tsx
@@ -159,7 +159,11 @@ const GlossaryTermsV1 = ({
 
   const onTermUpdate = async (data: GlossaryTerm | Glossary) => {
     await handleGlossaryTermUpdate(data as GlossaryTerm);
-    getEntityFeedCount();
+    // For name change, do not update the feed. It will be updated when the page is redirected to
+    // have the new value.
+    if (glossaryTerm.name === data.name) {
+      getEntityFeedCount();
+    }
   };
 
   const tabItems = useMemo(() => {
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.component.tsx
index 3be5a50d8dc7..ce65ff03214d 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.component.tsx
@@ -25,6 +25,7 @@ const EntityNameModal: React.FC<EntityNameModalProps> = ({
   // re-name will update actual name of the entity, it will impact across application
   // By default its disabled, send allowRename true to get the functionality
   allowRename = false,
+  nameValidationRules = [],
 }) => {
   const { t } = useTranslation();
   const [form] = Form.useForm<{ name: string; displayName: string }>();
@@ -83,6 +84,7 @@ const EntityNameModal: React.FC<EntityNameModalProps> = ({
               pattern: ENTITY_NAME_REGEX,
               message: t('message.entity-name-validation'),
             },
+            ...nameValidationRules,
           ]}>
           <Input
             disabled={!allowRename}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.interface.ts
index ff5521432a31..76beef687ce3 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/EntityNameModal/EntityNameModal.interface.ts
@@ -10,6 +10,8 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
+import { Rule } from 'antd/lib/form';
+
 export type EntityName = { name: string; displayName: string };
 
 export interface EntityNameModalProps {
@@ -19,4 +21,5 @@ export interface EntityNameModalProps {
   onSave: (obj: EntityName) => void | Promise<void>;
   entity: Partial<EntityName>;
   title: string;
+  nameValidationRules?: Rule[];
 }
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx
index e98d9324e1c8..abd5b9b0810a 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx
@@ -11,14 +11,13 @@
  *  limitations under the License.
  */
 import { Button, Modal, Typography } from 'antd';
-import React, { useCallback, useEffect, useMemo, useState } from 'react';
+import React, { useEffect, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 
 import { AxiosError } from 'axios';
-import { isObject } from 'lodash';
 import { EntityType } from '../../../enums/entity.enum';
 import { GlossaryTerm } from '../../../generated/entity/data/glossaryTerm';
-import { EnumConfig, Type, ValueClass } from '../../../generated/entity/type';
+import { Type } from '../../../generated/entity/type';
 import { getTypeByFQN } from '../../../rest/metadataTypeAPI';
 import {
   convertCustomPropertyStringToEntityExtension,
@@ -41,32 +40,21 @@ export const ModalWithCustomPropertyEditor = ({
   visible,
 }: ModalWithCustomPropertyEditorProps) => {
   const { t } = useTranslation();
+
   const [isLoading, setIsLoading] = useState<boolean>(false);
   const [isSaveLoading, setIsSaveLoading] = useState<boolean>(false);
-  const [customPropertyValue, setCustomPropertyValue] =
-    useState<ExtensionDataProps>();
-  const [customPropertyTypes, setCustomPropertyTypes] = useState<Type>();
-
-  const enumWithDescriptionsKeyPairValues = useMemo(() => {
-    const valuesWithEnumKey: Record<string, ValueClass[]> = {};
 
-    customPropertyTypes?.customProperties?.forEach((property) => {
-      if (property.propertyType.name === 'enumWithDescriptions') {
-        valuesWithEnumKey[property.name] = (
-          property.customPropertyConfig?.config as EnumConfig
-        ).values as ValueClass[];
-      }
-    });
+  const [extensionObject, setExtensionObject] = useState<ExtensionDataProps>();
 
-    return valuesWithEnumKey;
-  }, [customPropertyTypes]);
+  const [customPropertyEntityRecord, setCustomPropertyEntityRecord] =
+    useState<Type>();
 
   const fetchTypeDetail = async () => {
     setIsLoading(true);
     try {
       const response = await getTypeByFQN(entityType);
-      setCustomPropertyTypes(response);
-      setCustomPropertyValue(
+      setCustomPropertyEntityRecord(response);
+      setExtensionObject(
         convertCustomPropertyStringToEntityExtension(value ?? '', response)
       );
     } catch (err) {
@@ -80,49 +68,15 @@ export const ModalWithCustomPropertyEditor = ({
     setIsSaveLoading(true);
     await onSave(
       convertEntityExtensionToCustomPropertyString(
-        customPropertyValue,
-        customPropertyTypes
+        extensionObject,
+        customPropertyEntityRecord
       )
     );
     setIsSaveLoading(false);
   };
 
-  // EnumWithDescriptions values are change only contain keys,
-  // so we need to modify the extension data to include descriptions for them to display in the table
-  const modifyExtensionData = useCallback(
-    (extension: ExtensionDataProps) => {
-      const modifiedExtension = Object.entries(extension).reduce(
-        (acc, [key, value]) => {
-          if (enumWithDescriptionsKeyPairValues[key]) {
-            return {
-              ...acc,
-              [key]: (value as string[] | ValueClass[]).map((item) => {
-                if (isObject(item)) {
-                  return item;
-                }
-
-                return {
-                  key: item,
-                  description: enumWithDescriptionsKeyPairValues[key].find(
-                    (val) => val.key === item
-                  )?.description,
-                };
-              }),
-            };
-          }
-
-          return { ...acc, [key]: value };
-        },
-        {}
-      );
-
-      return modifiedExtension;
-    },
-    [enumWithDescriptionsKeyPairValues]
-  );
-
   const onExtensionUpdate = async (data: GlossaryTerm) => {
-    setCustomPropertyValue(modifyExtensionData(data.extension));
+    setExtensionObject(data.extension);
   };
 
   useEffect(() => {
@@ -166,7 +120,7 @@ export const ModalWithCustomPropertyEditor = ({
           hasEditAccess
           hasPermission
           isRenderedInRightPanel
-          entityDetails={{ extension: customPropertyValue } as GlossaryTerm}
+          entityDetails={{ extension: extensionObject } as GlossaryTerm}
           entityType={EntityType.GLOSSARY_TERM}
           handleExtensionUpdate={onExtensionUpdate}
         />
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts
index 5fa33f3b60b4..ae801b2e3969 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts
@@ -11,15 +11,16 @@
  *  limitations under the License.
  */
 import { EntityType } from '../../../enums/entity.enum';
-import { EntityReference, ValueClass } from '../../../generated/entity/type';
+import { EntityReference } from '../../../generated/entity/type';
+import { TableTypePropertyValueType } from '../../common/CustomPropertyTable/CustomPropertyTable.interface';
 
 export type ExtensionDataTypes =
   | string
   | string[]
   | EntityReference
   | EntityReference[]
-  | ValueClass[]
-  | { start: string; end: string };
+  | { start: number; end: number }
+  | Partial<TableTypePropertyValueType>;
 
 export interface ExtensionDataProps {
   [key: string]: ExtensionDataTypes;
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whatsNewData.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whatsNewData.ts
index 1088e007e54c..370084de3614 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whatsNewData.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/WhatsNewModal/whatsNewData.ts
@@ -17,7 +17,7 @@ import incidentManagerSampleData from '../../../assets/img/incidentManagerSample
 import profilerConfigPage from '../../../assets/img/profilerConfigPage.png';
 import collateIcon from '../../../assets/svg/ic-collate.svg';
 
-export const COOKIE_VERSION = 'VERSION_1_5_6'; // To be changed with each release.
+export const COOKIE_VERSION = 'VERSION_1_5_7'; // To be changed with each release.
 
 // for youtube video make isImage = false and path = {video embed id}
 // embed:- youtube video => share => click on embed and take {url with id} from it
@@ -1032,4 +1032,50 @@ To continue pursuing this objective, the application was completely refactored t
 -   **Improvement**: Collate API with Knowledge Center routes ${CollateIconWithLinkMD}.`,
     },
   },
+  {
+    id: 49,
+    version: 'v1.5.7',
+    description: 'Released on 17th October 2024.',
+    features: [],
+    changeLogs: {
+      Improvements: `-   **Feature**: Add table-type custom property.
+-   **Feature**: support Persian language option
+-   **Feature**: Postgres stored procedures support.
+-   **Feature**: Allow Custom Property Update in Glossary Bulk Import/export.
+-   **Improvement**: Remove table details from table level Import/Export, allowing updates only for column details.
+-   **MINOR**: looker exclude version.
+-   **MINOR**: Add deleteEntityByFQNPrefix.
+-   **MINOR**: Reduce lineage response size.
+-   **MINOR**: Updated pyiceberg version to 0.5.1
+-   **MINOR**: Fix dark border shown in navbar on UI.
+-   **MINOR**: Add column case sensitivity parameter.    
+-   **MINOR**: Pagination with search on service page.
+-   **MINOR**: Added loader in activity feed open and closed count.
+-   **MINOR**: Superset get primitive datatype in case of array, struct.
+-   **MINOR**: fix term references validation msg on glossary import.
+-   **MINOR**: supported search filter and only all show in case of all node value selected.
+-   **Fix**: Fix PinotDB Ingestion.
+-   **Fix**: MSAL popup auth issue.
+-   **Fix**: Fix Alerts for Test Suites.
+-   **Fix**: Added Glue Pipeline Lineage.
+-   **Fix**: ClassGraph performance issue.
+-   **Fix**: Superset query for mysql con. 
+-   **Fix**: Empty Connection Overwrite Logic.
+-   **Fix**: Couchbase columns not fetched fix.
+-   **Fix**: Quicksight Ingestion Error handled.
+-   **Fix**: DBT Manifest and Run results parsing.
+-   **Fix**: Increase MAX_AGGREGATE_SIZE in search.
+-   **Fix**: Add display name field in the advanced search filter.
+-   **Fix**: On dashboard soft delete, chart should not be visible.
+-   **Fix**: Fix the automator page breaking when no source is selected.
+-   **Fix**: Salesforce table description from label if not through query.
+-   **Fix**: Add Import/export support for table type custom property in glossary.
+-   **Fix**: Fix exception in search due to exception in database.displayName and databaseSchema.aggregation.
+-   **MINOR**: Knowledge Center publicationDate mismatch error ${CollateIconWithLinkMD}
+-   **MINOR**: Add owner label for knowledge center right panel ${CollateIconWithLinkMD}
+-   **Fix**: Automator pagination & improvments ${CollateIconWithLinkMD}
+-   **Fix**: ArchiveLog to FALSE for test connection ${CollateIconWithLinkMD}
+-   **Fix**: Knowledge Page deletion is not deleting from the search index ${CollateIconWithLinkMD}`,
+    },
+  },
 ];
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/left-sidebar.less b/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/left-sidebar.less
index 09318aee17bd..716c3962a13b 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/left-sidebar.less
+++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/LeftSidebar/left-sidebar.less
@@ -261,6 +261,12 @@
       color: initial;
     }
   }
+
+  .left-sidebar-menu.ant-menu > .ant-menu-item {
+    &:first-child {
+      margin-top: 0;
+    }
+  }
 }
 
 .left-panel-item.active,
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.test.tsx
index dae6a2919924..d298dcaca2cb 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.test.tsx
@@ -48,7 +48,9 @@ describe('PersonaDetailsCard Component', () => {
     });
 
     expect(
-      await screen.findByTestId('persona-details-card')
+      await screen.findByTestId(
+        `persona-details-card-${personaWithDescription.name}`
+      )
     ).toBeInTheDocument();
   });
 
@@ -81,7 +83,7 @@ describe('PersonaDetailsCard Component', () => {
       userEvent.click(personaCardTitle);
     });
 
-    expect(mockPush).toHaveBeenCalledWith('/settings/members/persona/john-doe');
+    expect(mockPush).toHaveBeenCalledWith('/settings/persona/john-doe');
   });
 
   it('should not navigate when persona.fullyQualifiedName is missing', async () => {
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.tsx b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.tsx
index adaa39a25b7b..e6153d7f14d2 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/MyData/Persona/PersonaDetailsCard/PersonaDetailsCard.tsx
@@ -37,7 +37,7 @@ export const PersonaDetailsCard = ({ persona }: PersonaDetailsCardProps) => {
     <Card
       bodyStyle={{ height: '100%' }}
       className="h-full cursor-pointer"
-      data-testid="persona-details-card"
+      data-testid={`persona-details-card-${persona.name}`}
       onClick={handleCardClick}>
       <Space className="justify-between h-full" direction="vertical">
         <Card.Meta
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx
index 81644d1c8c95..30f7650c262b 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/NavBar/NavBar.tsx
@@ -311,7 +311,8 @@ const NavBar = ({
         return;
       }
       const newVersion = await getVersion();
-      if (version !== newVersion.version) {
+      // Compare version only if version is set previously to have fair comparison
+      if (version && version !== newVersion.version) {
         setShowVersionMissMatchAlert(true);
       }
     };
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx
index 7ea1244a54d9..65f9e95da19c 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Pipeline/PipelineDetails/PipelineDetails.component.tsx
@@ -434,7 +434,7 @@ const PipelineDetails = ({
         ),
       },
       {
-        title: t('label.owner'),
+        title: t('label.owner-plural'),
         dataIndex: 'owners',
         key: 'owners',
         width: 120,
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.ts
index fee9640dbc40..404691d3c370 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Applications/AppDetails/ApplicationsClassBase.ts
@@ -12,7 +12,7 @@
  */
 
 import { FC } from 'react';
-import { AppType } from '../../../../generated/entity/applications/app';
+import { App, AppType } from '../../../../generated/entity/applications/app';
 import { getScheduleOptionsFromSchedules } from '../../../../utils/ScheduleUtils';
 
 class ApplicationsClassBase {
@@ -34,6 +34,16 @@ class ApplicationsClassBase {
     return null;
   }
 
+  public getFloatingApplicationEntityList(): string[] {
+    return [];
+  }
+
+  public isFloatingButtonPresent(applications: App[]) {
+    return applications.some((app) =>
+      this.getFloatingApplicationEntityList().includes(app.name)
+    );
+  }
+
   public importAppScreenshot(screenshotName: string) {
     return import(`../../../../assets/img/appScreenshots/${screenshotName}`);
   }
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/AddCustomProperty/AddCustomProperty.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/AddCustomProperty/AddCustomProperty.tsx
index 90eedab4e0ff..11a0cb93dfaf 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/AddCustomProperty/AddCustomProperty.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/AddCustomProperty/AddCustomProperty.tsx
@@ -11,11 +11,10 @@
  *  limitations under the License.
  */
 
-import { PlusOutlined } from '@ant-design/icons';
-import { Button, Col, Form, Input, Row } from 'antd';
+import { Button, Col, Form, Row } from 'antd';
 import { AxiosError } from 'axios';
 import { t } from 'i18next';
-import { isUndefined, map, omit, omitBy, startCase } from 'lodash';
+import { isArray, isUndefined, map, omit, omitBy, startCase } from 'lodash';
 import React, {
   FocusEvent,
   useCallback,
@@ -24,13 +23,13 @@ import React, {
   useState,
 } from 'react';
 import { useHistory, useParams } from 'react-router-dom';
-import { ReactComponent as DeleteIcon } from '../../../../assets/svg/ic-delete.svg';
 import {
+  CUSTOM_PROPERTIES_ICON_MAP,
   ENTITY_REFERENCE_OPTIONS,
-  ENUM_WITH_DESCRIPTION,
   PROPERTY_TYPES_WITH_ENTITY_REFERENCE,
   PROPERTY_TYPES_WITH_FORMAT,
   SUPPORTED_FORMAT_MAP,
+  TABLE_TYPE_CUSTOM_PROPERTY,
 } from '../../../../constants/CustomProperty.constants';
 import { GlobalSettingsMenuCategory } from '../../../../constants/GlobalSettings.constants';
 import { CUSTOM_PROPERTY_NAME_REGEX } from '../../../../constants/regex.constants';
@@ -41,7 +40,6 @@ import {
 import { EntityType } from '../../../../enums/entity.enum';
 import { ServiceCategory } from '../../../../enums/service.enum';
 import { Category, Type } from '../../../../generated/entity/type';
-import { EnumWithDescriptionsConfig } from '../../../../generated/type/customProperties/enumWithDescriptionsConfig';
 import { CustomProperty } from '../../../../generated/type/customProperty';
 import {
   FieldProp,
@@ -53,12 +51,12 @@ import {
   getTypeByFQN,
   getTypeListByCategory,
 } from '../../../../rest/metadataTypeAPI';
+import { getEntityName } from '../../../../utils/EntityUtils';
 import { generateFormFields } from '../../../../utils/formUtils';
 import { getSettingOptionByEntityType } from '../../../../utils/GlobalSettingsUtils';
 import { getSettingPath } from '../../../../utils/RouterUtils';
 import { showErrorToast } from '../../../../utils/ToastUtils';
 import ResizablePanels from '../../../common/ResizablePanels/ResizablePanels';
-import RichTextEditor from '../../../common/RichTextEditor/RichTextEditor';
 import ServiceDocPanel from '../../../common/ServiceDocPanel/ServiceDocPanel';
 import TitleBreadcrumb from '../../../common/TitleBreadcrumb/TitleBreadcrumb.component';
 
@@ -99,12 +97,27 @@ const AddCustomProperty = () => {
   );
 
   const propertyTypeOptions = useMemo(() => {
-    return map(propertyTypes, (type) => ({
-      key: type.name,
+    return map(propertyTypes, (type) => {
+      const Icon =
+        CUSTOM_PROPERTIES_ICON_MAP[
+          type.name as keyof typeof CUSTOM_PROPERTIES_ICON_MAP
+        ];
+
       // Remove -cp from the name and convert to start case
-      label: startCase((type.displayName ?? type.name).replace(/-cp/g, '')),
-      value: type.id,
-    }));
+      const title = startCase(getEntityName(type).replace(/-cp/g, ''));
+
+      return {
+        searchField: title,
+        key: type.name,
+        label: (
+          <div className="d-flex gap-2 items-center" title={title}>
+            {Icon && <Icon width={20} />}
+            <span>{title}</span>
+          </div>
+        ),
+        value: type.id,
+      };
+    });
   }, [propertyTypes]);
 
   const {
@@ -112,7 +125,7 @@ const AddCustomProperty = () => {
     hasFormatConfig,
     hasEntityReferenceConfig,
     watchedOption,
-    hasEnumWithDescriptionConfig,
+    hasTableTypeConfig,
   } = useMemo(() => {
     const watchedOption = propertyTypeOptions.find(
       (option) => option.value === watchedPropertyType
@@ -121,8 +134,7 @@ const AddCustomProperty = () => {
 
     const hasEnumConfig = watchedOptionKey === 'enum';
 
-    const hasEnumWithDescriptionConfig =
-      watchedOptionKey === ENUM_WITH_DESCRIPTION;
+    const hasTableTypeConfig = watchedOptionKey === TABLE_TYPE_CUSTOM_PROPERTY;
 
     const hasFormatConfig =
       PROPERTY_TYPES_WITH_FORMAT.includes(watchedOptionKey);
@@ -135,7 +147,7 @@ const AddCustomProperty = () => {
       hasFormatConfig,
       hasEntityReferenceConfig,
       watchedOption,
-      hasEnumWithDescriptionConfig,
+      hasTableTypeConfig,
     };
   }, [watchedPropertyType, propertyTypeOptions]);
 
@@ -176,7 +188,7 @@ const AddCustomProperty = () => {
       formatConfig: string;
       entityReferenceConfig: string[];
       multiSelect?: boolean;
-      enumWithDescriptionsConfig?: EnumWithDescriptionsConfig['values'];
+      columns: string[];
     }
   ) => {
     if (isUndefined(typeDetail)) {
@@ -208,11 +220,10 @@ const AddCustomProperty = () => {
         };
       }
 
-      if (hasEnumWithDescriptionConfig) {
+      if (hasTableTypeConfig) {
         customPropertyConfig = {
           config: {
-            multiSelect: Boolean(data?.multiSelect),
-            values: data.enumWithDescriptionsConfig,
+            columns: data.columns,
           },
         };
       }
@@ -224,7 +235,7 @@ const AddCustomProperty = () => {
             'formatConfig',
             'entityReferenceConfig',
             'enumConfig',
-            'enumWithDescriptionsConfig',
+            'columns',
           ]),
           propertyType: {
             id: data.propertyType,
@@ -284,8 +295,8 @@ const AddCustomProperty = () => {
           field: t('label.type'),
         })}`,
         showSearch: true,
-        filterOption: (input: string, option: { label: string }) => {
-          return (option?.label ?? '')
+        filterOption: (input: string, option: { searchField: string }) => {
+          return (option?.searchField ?? '')
             .toLowerCase()
             .includes(input.toLowerCase());
         },
@@ -388,6 +399,46 @@ const AddCustomProperty = () => {
     },
   };
 
+  const tableTypePropertyConfig: FieldProp[] = [
+    {
+      name: 'columns',
+      required: true,
+      label: t('label.column-plural'),
+      id: 'root/columns',
+      type: FieldTypes.SELECT,
+      props: {
+        'data-testid': 'columns',
+        mode: 'tags',
+        placeholder: t('label.column-plural'),
+      },
+      rules: [
+        {
+          required: true,
+          validator: async (_, value) => {
+            if (isArray(value)) {
+              if (value.length > 3) {
+                return Promise.reject(
+                  t('message.maximum-count-allowed', {
+                    count: 3,
+                    label: t('label.column-plural'),
+                  })
+                );
+              }
+
+              return Promise.resolve();
+            } else {
+              return Promise.reject(
+                t('label.field-required', {
+                  field: t('label.column-plural'),
+                })
+              );
+            }
+          },
+        },
+      ],
+    },
+  ];
+
   const firstPanelChildren = (
     <div className="max-width-md w-9/10 service-form-container">
       <TitleBreadcrumb titleLinks={slashedBreadcrumb} />
@@ -415,94 +466,8 @@ const AddCustomProperty = () => {
             generateFormFields([entityReferenceConfigField])
         }
 
-        {hasEnumWithDescriptionConfig && (
-          <>
-            <Form.List name="enumWithDescriptionsConfig">
-              {(fields, { add, remove }) => (
-                <>
-                  <Form.Item
-                    className="form-item-horizontal"
-                    colon={false}
-                    label={t('label.property')}>
-                    <Button
-                      data-testid="add-enum-description-config"
-                      icon={
-                        <PlusOutlined
-                          style={{ color: 'white', fontSize: '12px' }}
-                        />
-                      }
-                      size="small"
-                      type="primary"
-                      onClick={() => {
-                        add();
-                      }}
-                    />
-                  </Form.Item>
-
-                  {fields.map((field, index) => (
-                    <Row gutter={[8, 0]} key={field.key}>
-                      <Col span={23}>
-                        <Row gutter={[8, 0]}>
-                          <Col span={24}>
-                            <Form.Item
-                              name={[field.name, 'key']}
-                              rules={[
-                                {
-                                  required: true,
-                                  message: `${t(
-                                    'message.field-text-is-required',
-                                    {
-                                      fieldText: t('label.key'),
-                                    }
-                                  )}`,
-                                },
-                              ]}>
-                              <Input
-                                id={`key-${index}`}
-                                placeholder={t('label.key')}
-                              />
-                            </Form.Item>
-                          </Col>
-                          <Col span={24}>
-                            <Form.Item
-                              name={[field.name, 'description']}
-                              rules={[
-                                {
-                                  required: true,
-                                  message: `${t(
-                                    'message.field-text-is-required',
-                                    {
-                                      fieldText: t('label.description'),
-                                    }
-                                  )}`,
-                                },
-                              ]}
-                              trigger="onTextChange"
-                              valuePropName="initialValue">
-                              <RichTextEditor height="200px" />
-                            </Form.Item>
-                          </Col>
-                        </Row>
-                      </Col>
-                      <Col span={1}>
-                        <Button
-                          data-testid={`remove-enum-description-config-${index}`}
-                          icon={<DeleteIcon width={16} />}
-                          size="small"
-                          type="text"
-                          onClick={() => {
-                            remove(field.name);
-                          }}
-                        />
-                      </Col>
-                    </Row>
-                  ))}
-                </>
-              )}
-            </Form.List>
-            {generateFormFields([multiSelectField])}
-          </>
-        )}
+        {hasTableTypeConfig && generateFormFields(tableTypePropertyConfig)}
+
         {generateFormFields([descriptionField])}
         <Row justify="end">
           <Col>
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.tsx
index 68b7f6d91bff..8bf877de0942 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/CustomPropertyTable.tsx
@@ -10,25 +10,25 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-import { Button, Space, Tag, Tooltip, Typography } from 'antd';
+import { Button, Space, Tooltip, Typography } from 'antd';
 import { ColumnsType } from 'antd/lib/table';
-import { isArray, isEmpty, isString, isUndefined } from 'lodash';
+import { isArray, isEmpty, isString, isUndefined, startCase } from 'lodash';
 import React, { FC, Fragment, useEffect, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { ReactComponent as IconEdit } from '../../../assets/svg/edit-new.svg';
 import { ReactComponent as IconDelete } from '../../../assets/svg/ic-delete.svg';
-import { ENUM_WITH_DESCRIPTION } from '../../../constants/CustomProperty.constants';
+import { CUSTOM_PROPERTIES_ICON_MAP } from '../../../constants/CustomProperty.constants';
 import { ADD_CUSTOM_PROPERTIES_DOCS } from '../../../constants/docs.constants';
 import { NO_PERMISSION_FOR_ACTION } from '../../../constants/HelperTextUtil';
 import { TABLE_SCROLL_VALUE } from '../../../constants/Table.constants';
 import { ERROR_PLACEHOLDER_TYPE, OPERATION } from '../../../enums/common.enum';
-import { EnumWithDescriptionsConfig } from '../../../generated/type/customProperties/enumWithDescriptionsConfig';
 import { CustomProperty } from '../../../generated/type/customProperty';
 import { columnSorter, getEntityName } from '../../../utils/EntityUtils';
 import ErrorPlaceHolder from '../../common/ErrorWithPlaceholder/ErrorPlaceHolder';
 import RichTextEditorPreviewer from '../../common/RichTextEditor/RichTextEditorPreviewer';
 import Table from '../../common/Table/Table';
 import ConfirmationModal from '../../Modals/ConfirmationModal/ConfirmationModal';
+import './custom-property-table.less';
 import { CustomPropertyTableProp } from './CustomPropertyTable.interface';
 import EditCustomPropertyModal, {
   FormData,
@@ -70,9 +70,7 @@ export const CustomPropertyTable: FC<CustomPropertyTableProp> = ({
     const updatedProperties = customProperties.map((property) => {
       if (property.name === selectedProperty.name) {
         const config = data.customPropertyConfig;
-        const isEnumType =
-          selectedProperty.propertyType.name === 'enum' ||
-          selectedProperty.propertyType.name === ENUM_WITH_DESCRIPTION;
+        const isEnumType = selectedProperty.propertyType.name === 'enum';
 
         return {
           ...property,
@@ -120,7 +118,21 @@ export const CustomPropertyTable: FC<CustomPropertyTableProp> = ({
         title: t('label.type'),
         dataIndex: 'propertyType',
         key: 'propertyType',
-        render: (propertyType) => getEntityName(propertyType),
+        render: (propertyType: CustomProperty['propertyType']) => {
+          const Icon =
+            CUSTOM_PROPERTIES_ICON_MAP[
+              propertyType.name as keyof typeof CUSTOM_PROPERTIES_ICON_MAP
+            ];
+
+          return (
+            <div className="d-flex gap-2 custom-property-type-chip items-center">
+              {Icon && <Icon width={20} />}
+              <span>
+                {startCase(getEntityName(propertyType).replace(/-cp/g, ''))}
+              </span>
+            </div>
+          );
+        },
       },
       {
         title: t('label.config'),
@@ -144,36 +156,18 @@ export const CustomPropertyTable: FC<CustomPropertyTableProp> = ({
 
           // If config is an object, then it is a enum config
           if (!isString(config) && !isArray(config)) {
-            if (record.propertyType.name === ENUM_WITH_DESCRIPTION) {
-              const values =
-                (config?.values as EnumWithDescriptionsConfig['values']) ?? [];
-
+            if (config?.columns) {
               return (
-                <div
-                  className="w-full d-flex gap-2 flex-column"
-                  data-testid="enum-with-description-config">
-                  <div className="w-full d-flex gap-2 flex-column">
-                    {values.map((value) => (
-                      <Tooltip
-                        key={value.key}
-                        title={value.description}
-                        trigger="hover">
-                        <Tag
-                          style={{
-                            width: 'max-content',
-                            margin: '0px',
-                            border: 'none',
-                            padding: '4px',
-                            background: 'rgba(0, 0, 0, 0.03)',
-                          }}>
-                          {value.key}
-                        </Tag>
-                      </Tooltip>
-                    ))}
-                  </div>
+                <div className="w-full d-flex gap-2 flex-column">
                   <Typography.Text>
-                    {t('label.multi-select')}:{' '}
-                    {config?.multiSelect ? t('label.yes') : t('label.no')}
+                    <span className="font-medium">{`${t(
+                      'label.column-plural'
+                    )}:`}</span>
+                    <ul className="m-b-0">
+                      {config.columns.map((column) => (
+                        <li key={column}>{column}</li>
+                      ))}
+                    </ul>
                   </Typography.Text>
                 </div>
               );
@@ -275,6 +269,7 @@ export const CustomPropertyTable: FC<CustomPropertyTableProp> = ({
     <Fragment>
       <Table
         bordered
+        className="entity-custom-properties-table"
         columns={tableColumn}
         data-testid="entity-custom-properties-table"
         dataSource={customProperties}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/EditCustomPropertyModal/EditCustomPropertyModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/EditCustomPropertyModal/EditCustomPropertyModal.tsx
index 425d246f4179..d3821557f429 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/EditCustomPropertyModal/EditCustomPropertyModal.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/EditCustomPropertyModal/EditCustomPropertyModal.tsx
@@ -10,30 +10,17 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-import { InfoCircleOutlined, PlusOutlined } from '@ant-design/icons';
-import {
-  Button,
-  Col,
-  Form,
-  Input,
-  Modal,
-  Row,
-  Tooltip,
-  Typography,
-} from 'antd';
-import { get, isUndefined, uniq } from 'lodash';
+import { Form, Modal, Typography } from 'antd';
+import { isUndefined, uniq } from 'lodash';
 import React, { FC, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
-import { ReactComponent as DeleteIcon } from '../../../../assets/svg/ic-delete.svg';
 import {
   ENTITY_REFERENCE_OPTIONS,
-  ENUM_WITH_DESCRIPTION,
   PROPERTY_TYPES_WITH_ENTITY_REFERENCE,
 } from '../../../../constants/CustomProperty.constants';
 import {
+  Config,
   CustomProperty,
-  EnumConfig,
-  ValueClass,
 } from '../../../../generated/type/customProperty';
 import {
   FieldProp,
@@ -41,11 +28,10 @@ import {
   FormItemLayout,
 } from '../../../../interface/FormUtils.interface';
 import { generateFormFields } from '../../../../utils/formUtils';
-import RichTextEditor from '../../../common/RichTextEditor/RichTextEditor';
 
 export interface FormData {
   description: string;
-  customPropertyConfig: string[] | ValueClass[];
+  customPropertyConfig: string[];
   multiSelect?: boolean;
 }
 
@@ -72,21 +58,15 @@ const EditCustomPropertyModal: FC<EditCustomPropertyModalProps> = ({
     setIsSaving(false);
   };
 
-  const {
-    hasEnumConfig,
-    hasEntityReferenceConfig,
-    hasEnumWithDescriptionConfig,
-  } = useMemo(() => {
+  const { hasEnumConfig, hasEntityReferenceConfig } = useMemo(() => {
     const propertyName = customProperty.propertyType.name ?? '';
     const hasEnumConfig = propertyName === 'enum';
-    const hasEnumWithDescriptionConfig = propertyName === ENUM_WITH_DESCRIPTION;
     const hasEntityReferenceConfig =
       PROPERTY_TYPES_WITH_ENTITY_REFERENCE.includes(propertyName);
 
     return {
       hasEnumConfig,
       hasEntityReferenceConfig,
-      hasEnumWithDescriptionConfig,
     };
   }, [customProperty]);
 
@@ -116,7 +96,7 @@ const EditCustomPropertyModal: FC<EditCustomPropertyModalProps> = ({
       placeholder: t('label.enum-value-plural'),
       onChange: (value: string[]) => {
         const enumConfig = customProperty.customPropertyConfig
-          ?.config as EnumConfig;
+          ?.config as Config;
         const updatedValues = uniq([...value, ...(enumConfig?.values ?? [])]);
         form.setFieldsValue({ customPropertyConfig: updatedValues });
       },
@@ -175,9 +155,8 @@ const EditCustomPropertyModal: FC<EditCustomPropertyModalProps> = ({
   };
 
   const initialValues = useMemo(() => {
-    if (hasEnumConfig || hasEnumWithDescriptionConfig) {
-      const enumConfig = customProperty.customPropertyConfig
-        ?.config as EnumConfig;
+    if (hasEnumConfig) {
+      const enumConfig = customProperty.customPropertyConfig?.config as Config;
 
       return {
         description: customProperty.description,
@@ -190,7 +169,7 @@ const EditCustomPropertyModal: FC<EditCustomPropertyModalProps> = ({
       description: customProperty.description,
       customPropertyConfig: customProperty.customPropertyConfig?.config,
     };
-  }, [customProperty, hasEnumConfig, hasEnumWithDescriptionConfig]);
+  }, [customProperty, hasEnumConfig]);
 
   const note = (
     <Typography.Text
@@ -244,125 +223,6 @@ const EditCustomPropertyModal: FC<EditCustomPropertyModalProps> = ({
               </>
             )}
 
-            {hasEnumWithDescriptionConfig && (
-              <>
-                <Form.List name="customPropertyConfig">
-                  {(fields, { add, remove }) => {
-                    const config =
-                      (initialValues?.customPropertyConfig as ValueClass[]) ??
-                      [];
-
-                    return (
-                      <>
-                        <Form.Item
-                          className="form-item-horizontal"
-                          colon={false}
-                          label={
-                            <div className="d-flex gap-2 items-center">
-                              <span>{t('label.property')}</span>
-                              <Tooltip
-                                title={t(
-                                  'message.enum-with-description-update-note'
-                                )}>
-                                <InfoCircleOutlined
-                                  className="m-x-xss"
-                                  style={{ color: '#C4C4C4' }}
-                                />
-                              </Tooltip>
-                            </div>
-                          }>
-                          <Button
-                            data-testid="add-enum-description-config"
-                            icon={
-                              <PlusOutlined
-                                style={{ color: 'white', fontSize: '12px' }}
-                              />
-                            }
-                            size="small"
-                            type="primary"
-                            onClick={() => {
-                              add();
-                            }}
-                          />
-                        </Form.Item>
-
-                        {fields.map((field, index) => {
-                          const isExisting = Boolean(get(config, index, false));
-
-                          return (
-                            <Row
-                              className="m-t-md"
-                              gutter={[8, 0]}
-                              key={field.key}>
-                              <Col span={23}>
-                                <Row gutter={[8, 0]}>
-                                  <Col span={24}>
-                                    <Form.Item
-                                      name={[field.name, 'key']}
-                                      rules={[
-                                        {
-                                          required: true,
-                                          message: `${t(
-                                            'message.field-text-is-required',
-                                            {
-                                              fieldText: t('label.key'),
-                                            }
-                                          )}`,
-                                        },
-                                      ]}>
-                                      <Input
-                                        disabled={isExisting}
-                                        id={`key-${index}`}
-                                        placeholder={t('label.key')}
-                                      />
-                                    </Form.Item>
-                                  </Col>
-                                  <Col span={24}>
-                                    <Form.Item
-                                      name={[field.name, 'description']}
-                                      rules={[
-                                        {
-                                          required: true,
-                                          message: `${t(
-                                            'message.field-text-is-required',
-                                            {
-                                              fieldText: t('label.description'),
-                                            }
-                                          )}`,
-                                        },
-                                      ]}
-                                      trigger="onTextChange"
-                                      valuePropName="initialValue">
-                                      <RichTextEditor height="200px" />
-                                    </Form.Item>
-                                  </Col>
-                                </Row>
-                              </Col>
-                              {!isExisting && (
-                                <Col span={1}>
-                                  <Button
-                                    data-testid={`remove-enum-description-config-${index}`}
-                                    icon={<DeleteIcon width={16} />}
-                                    size="small"
-                                    type="text"
-                                    onClick={() => {
-                                      remove(field.name);
-                                    }}
-                                  />
-                                </Col>
-                              )}
-                            </Row>
-                          );
-                        })}
-                      </>
-                    );
-                  }}
-                </Form.List>
-
-                {generateFormFields([multiSelectField])}
-              </>
-            )}
-
             {hasEntityReferenceConfig && (
               <>
                 {generateFormFields([entityReferenceConfigField])}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/custom-property-table.less b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/custom-property-table.less
new file mode 100644
index 000000000000..5ce97d8bee8a
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/CustomProperty/custom-property-table.less
@@ -0,0 +1,21 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+@import (reference) url('../../../styles/variables.less');
+
+.custom-property-type-chip {
+  background: @grey-1;
+  border: 1px solid @text-color;
+  padding: 4px 8px;
+  border-radius: 15px;
+  width: fit-content;
+}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddService/AddService.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddService/AddService.component.tsx
index e444533a4273..d6e5eb6b2ec1 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddService/AddService.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/AddService/AddService.component.tsx
@@ -243,13 +243,10 @@ const AddService = ({
 
         {activeServiceStep > 3 && (
           <SuccessScreen
+            showIngestionButton
             handleIngestionClick={() => handleAddIngestion(true)}
             handleViewServiceClick={handleViewServiceClick}
             name={serviceConfig.serviceName}
-            // API Service does not support ingestion workflows
-            showIngestionButton={
-              serviceCategory !== ServiceCategory.API_SERVICES
-            }
             state={FormSubmitType.ADD}
             suffix={getServiceCreatedLabel(serviceCategory)}
           />
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/ServiceConnectionDetails/ServiceConnectionDetails.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/ServiceConnectionDetails/ServiceConnectionDetails.component.tsx
index d3694ea7d561..9a37f0cb9e1d 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/ServiceConnectionDetails/ServiceConnectionDetails.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/ServiceConnectionDetails/ServiceConnectionDetails.component.tsx
@@ -64,8 +64,11 @@ const ServiceConnectionDetails = ({
       if (isObject(value)) {
         if (
           serviceCategory.slice(0, -1) === EntityType.PIPELINE_SERVICE &&
-          key === 'connection'
+          key === 'connection' &&
+          value.type &&
+          value.type.toLowerCase() === 'airflow'
         ) {
+          // Specific to Airflow
           const newSchemaPropertyObject = schemaPropertyObject[
             key
           ].oneOf.filter((item) => item.title === `${value.type}Connection`)[0]
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.test.tsx
index 602aa50d296f..c19344bfe717 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.test.tsx
@@ -354,7 +354,7 @@ describe('Services', () => {
       await screen.findByTestId('service-name-pipelineServices')
     ).toHaveTextContent('Glue');
 
-    expect(await screen.findByText('label.owner')).toBeInTheDocument();
+    expect(await screen.findByText('label.owner-plural')).toBeInTheDocument();
     expect(await screen.findByText('OwnerLabel')).toBeInTheDocument();
   });
 
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.tsx
index dafd55996d76..f7a9ca2a29b9 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/Settings/Services/Services.tsx
@@ -23,6 +23,7 @@ import { Link, useHistory } from 'react-router-dom';
 import {
   DISABLED,
   getServiceDetailsPath,
+  INITIAL_PAGING_VALUE,
   pagingObject,
 } from '../../../constants/constants';
 import { CONNECTORS_DOCS } from '../../../constants/docs.constants';
@@ -194,15 +195,27 @@ const Services = ({ serviceName }: ServicesProps) => {
     [searchIndex, serviceName, deleted]
   );
 
-  const handleServicePageChange = ({
-    cursorType,
-    currentPage,
-  }: PagingHandlerParams) => {
-    if (cursorType) {
-      getServiceDetails({ [cursorType]: paging[cursorType] });
-    }
-    handlePageChange(currentPage);
-  };
+  const handleServicePageChange = useCallback(
+    ({ cursorType, currentPage }: PagingHandlerParams) => {
+      if (searchTerm) {
+        handlePageChange(currentPage);
+        getServiceDetails({
+          currentPage,
+          search: searchTerm,
+          limit: pageSize,
+          filters: serviceTypeFilter?.length
+            ? `(${serviceTypeFilter
+                .map((type) => `serviceType:${type}`)
+                .join(' ')})`
+            : undefined,
+        });
+      } else if (cursorType) {
+        handlePageChange(currentPage);
+        getServiceDetails({ [cursorType]: paging[cursorType] });
+      }
+    },
+    [getServiceDetails, searchTerm, serviceTypeFilter, paging, pageSize]
+  );
 
   const addServicePermission = useMemo(
     () =>
@@ -344,7 +357,7 @@ const Services = ({ serviceName }: ServicesProps) => {
       ),
     },
     {
-      title: t('label.owner'),
+      title: t('label.owner-plural'),
       dataIndex: 'owners',
       key: 'owners',
       width: 200,
@@ -414,6 +427,7 @@ const Services = ({ serviceName }: ServicesProps) => {
 
   const handleServiceSearch = useCallback(
     async (search: string) => {
+      handlePageChange(INITIAL_PAGING_VALUE);
       setSearchTerm(search);
     },
     [getServiceDetails]
@@ -511,6 +525,7 @@ const Services = ({ serviceName }: ServicesProps) => {
         {showPagination && (
           <NextPrevious
             currentPage={currentPage}
+            isNumberBased={!isEmpty(searchTerm)}
             pageSize={pageSize}
             paging={paging}
             pagingHandler={handleServicePageChange}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.interface.ts
index 170cd6e53531..67355dc43336 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.interface.ts
@@ -79,10 +79,16 @@ export type TimeIntervalType = {
   end: number;
 };
 
+export type TableTypePropertyValueType = {
+  columns: string[];
+  rows: Record<string, string>[];
+};
+
 export type PropertyValueType =
   | string
   | number
   | string[]
   | EntityReference
   | EntityReference[]
-  | TimeIntervalType;
+  | TimeIntervalType
+  | TableTypePropertyValueType;
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx
index d7f46421c35d..391d9062e945 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/CustomPropertyTable.tsx
@@ -178,12 +178,20 @@ export const CustomPropertyTable = <T extends ExtentionEntitiesKeys>({
     maxDataCap,
   ]);
 
-  const dataSource = useMemo(() => {
+  const { dataSource, dataSourceColumns } = useMemo(() => {
     const customProperties = entityTypeDetail?.customProperties ?? [];
 
-    return Array.isArray(customProperties)
+    const dataSource = Array.isArray(customProperties)
       ? customProperties.slice(0, maxDataCap)
       : [];
+
+    // Split dataSource into three equal parts
+    const columnCount = 3;
+    const columns = Array.from({ length: columnCount }, (_, i) =>
+      dataSource.filter((_, index) => index % columnCount === i)
+    );
+
+    return { dataSource, dataSourceColumns: columns };
   }, [maxDataCap, entityTypeDetail?.customProperties]);
 
   useEffect(() => {
@@ -286,21 +294,27 @@ export const CustomPropertyTable = <T extends ExtentionEntitiesKeys>({
               </div>
             </>
           ) : (
-            <Row data-testid="custom-properties-card" gutter={[16, 16]}>
-              {dataSource.map((record) => (
-                <Col key={record.name} span={8}>
-                  <PropertyValue
-                    extension={extensionObject.extensionObject}
-                    hasEditPermissions={hasEditAccess}
-                    isRenderedInRightPanel={isRenderedInRightPanel}
-                    isVersionView={isVersionView}
-                    property={record}
-                    versionDataKeys={extensionObject.addedKeysList}
-                    onExtensionUpdate={onExtensionUpdate}
-                  />
-                </Col>
-              ))}
-            </Row>
+            <div className="custom-properties-card">
+              <Row data-testid="custom-properties-card" gutter={[16, 16]}>
+                {dataSourceColumns.map((columns, colIndex) => (
+                  <Col key={colIndex} span={8}>
+                    {columns.map((record) => (
+                      <div key={record.name} style={{ marginBottom: '16px' }}>
+                        <PropertyValue
+                          extension={extensionObject.extensionObject}
+                          hasEditPermissions={hasEditAccess}
+                          isRenderedInRightPanel={isRenderedInRightPanel}
+                          isVersionView={isVersionView}
+                          property={record}
+                          versionDataKeys={extensionObject.addedKeysList}
+                          onExtensionUpdate={onExtensionUpdate}
+                        />
+                      </div>
+                    ))}
+                  </Col>
+                ))}
+              </Row>
+            </div>
           )}
         </>
       )}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx
index b31a1f2cfe6a..af4d82a1a389 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/PropertyValue.tsx
@@ -26,7 +26,6 @@ import {
   Tooltip,
   Typography,
 } from 'antd';
-import { ColumnsType } from 'antd/lib/table';
 import { AxiosError } from 'axios';
 import classNames from 'classnames';
 import { t } from 'i18next';
@@ -60,12 +59,12 @@ import {
   ICON_DIMENSION,
   VALIDATION_MESSAGES,
 } from '../../../constants/constants';
-import { ENUM_WITH_DESCRIPTION } from '../../../constants/CustomProperty.constants';
+import { TABLE_TYPE_CUSTOM_PROPERTY } from '../../../constants/CustomProperty.constants';
 import { TIMESTAMP_UNIX_IN_MILLISECONDS_REGEX } from '../../../constants/regex.constants';
 import { CSMode } from '../../../enums/codemirror.enum';
 import { SearchIndex } from '../../../enums/search.enum';
 import { EntityReference } from '../../../generated/entity/type';
-import { EnumConfig, ValueClass } from '../../../generated/type/customProperty';
+import { Config } from '../../../generated/type/customProperty';
 import { calculateInterval } from '../../../utils/date-time/DateTimeUtils';
 import entityUtilClassBase from '../../../utils/EntityUtilClassBase';
 import { getEntityName } from '../../../utils/EntityUtils';
@@ -78,7 +77,6 @@ import { ModalWithMarkdownEditor } from '../../Modals/ModalWithMarkdownEditor/Mo
 import InlineEdit from '../InlineEdit/InlineEdit.component';
 import ProfilePicture from '../ProfilePicture/ProfilePicture';
 import RichTextEditorPreviewer from '../RichTextEditor/RichTextEditorPreviewer';
-import Table from '../Table/Table';
 import {
   PropertyValueProps,
   PropertyValueType,
@@ -86,6 +84,8 @@ import {
 } from './CustomPropertyTable.interface';
 import './property-value.less';
 import { PropertyInput } from './PropertyInput';
+import EditTableTypePropertyModal from './TableTypeProperty/EditTableTypePropertyModal';
+import TableTypePropertyView from './TableTypeProperty/TableTypePropertyView';
 
 export const PropertyValue: FC<PropertyValueProps> = ({
   isVersionView,
@@ -96,9 +96,10 @@ export const PropertyValue: FC<PropertyValueProps> = ({
   property,
   isRenderedInRightPanel = false,
 }) => {
-  const { propertyName, propertyType, value } = useMemo(() => {
+  const { propertyName, propertyType, value, isTableType } = useMemo(() => {
     const propertyName = property.name;
     const propertyType = property.propertyType;
+    const isTableType = propertyType.name === TABLE_TYPE_CUSTOM_PROPERTY;
 
     const value = extension?.[propertyName];
 
@@ -106,13 +107,15 @@ export const PropertyValue: FC<PropertyValueProps> = ({
       propertyName,
       propertyType,
       value,
+      isTableType,
     };
   }, [property, extension]);
 
   const [showInput, setShowInput] = useState<boolean>(false);
   const [isLoading, setIsLoading] = useState<boolean>(false);
 
-  const [isExpanded, setIsExpanded] = useState(false);
+  // expand the property value by default if it is a "table-type" custom property
+  const [isExpanded, setIsExpanded] = useState(isTableType);
   const [isOverflowing, setIsOverflowing] = useState(false);
   const contentRef = useRef<HTMLDivElement>(null);
 
@@ -135,16 +138,14 @@ export const PropertyValue: FC<PropertyValueProps> = ({
 
   const onInputSave = async (updatedValue: PropertyValueType) => {
     const isEnum = propertyType.name === 'enum';
-    const isEnumWithDescription = propertyType.name === ENUM_WITH_DESCRIPTION;
 
     const isArrayType = isArray(updatedValue);
 
     const enumValue = isArrayType ? updatedValue : [updatedValue];
 
-    const propertyValue =
-      isEnum || isEnumWithDescription
-        ? (enumValue as string[]).filter(Boolean)
-        : updatedValue;
+    const propertyValue = isEnum
+      ? (enumValue as string[]).filter(Boolean)
+      : updatedValue;
 
     try {
       // Omit undefined and empty values
@@ -227,7 +228,7 @@ export const PropertyValue: FC<PropertyValueProps> = ({
       }
 
       case 'enum': {
-        const enumConfig = property.customPropertyConfig?.config as EnumConfig;
+        const enumConfig = property.customPropertyConfig?.config as Config;
 
         const isMultiSelect = Boolean(enumConfig?.multiSelect);
 
@@ -272,60 +273,6 @@ export const PropertyValue: FC<PropertyValueProps> = ({
         );
       }
 
-      case ENUM_WITH_DESCRIPTION: {
-        const enumConfig = property.customPropertyConfig?.config as EnumConfig;
-
-        const isMultiSelect = Boolean(enumConfig?.multiSelect);
-
-        const values = (enumConfig?.values as ValueClass[]) ?? [];
-
-        const options = values.map((option) => ({
-          label: (
-            <Tooltip title={option.description}>
-              <span>{option.key}</span>
-            </Tooltip>
-          ),
-          value: option.key,
-        }));
-
-        const initialValues = {
-          enumWithDescriptionValues: (isArray(value) ? value : [value]).filter(
-            Boolean
-          ),
-        };
-
-        return (
-          <InlineEdit
-            isLoading={isLoading}
-            saveButtonProps={{
-              disabled: isLoading,
-              htmlType: 'submit',
-              form: 'enum-with-description-form',
-            }}
-            onCancel={onHideInput}
-            onSave={noop}>
-            <Form
-              id="enum-with-description-form"
-              initialValues={initialValues}
-              layout="vertical"
-              onFinish={(values: {
-                enumWithDescriptionValues: string | string[];
-              }) => onInputSave(values.enumWithDescriptionValues)}>
-              <Form.Item name="enumWithDescriptionValues" style={commonStyle}>
-                <Select
-                  allowClear
-                  data-testid="enum-with-description-select"
-                  disabled={isLoading}
-                  mode={isMultiSelect ? 'multiple' : undefined}
-                  options={options}
-                  placeholder={t('label.enum-value-plural')}
-                />
-              </Form.Item>
-            </Form>
-          </InlineEdit>
-        );
-      }
-
       case 'date-cp':
       case 'dateTime-cp': {
         // Default format is 'yyyy-mm-dd'
@@ -748,6 +695,30 @@ export const PropertyValue: FC<PropertyValueProps> = ({
         );
       }
 
+      case TABLE_TYPE_CUSTOM_PROPERTY: {
+        const config = property.customPropertyConfig?.config as Config;
+
+        const columns = config?.columns ?? [];
+        const rows = value?.rows ?? [];
+
+        return (
+          <>
+            {showInput && (
+              <TableTypePropertyView columns={columns} rows={rows} />
+            )}
+            <EditTableTypePropertyModal
+              columns={columns}
+              isUpdating={isLoading}
+              isVisible={showInput}
+              property={property}
+              rows={value?.rows ?? []}
+              onCancel={onHideInput}
+              onSave={onInputSave}
+            />
+          </>
+        );
+      }
+
       default:
         return null;
     }
@@ -791,42 +762,6 @@ export const PropertyValue: FC<PropertyValueProps> = ({
           </>
         );
 
-      case ENUM_WITH_DESCRIPTION: {
-        const enumWithDescriptionValues = (value as ValueClass[]) ?? [];
-
-        const columns: ColumnsType<ValueClass> = [
-          {
-            title: 'Key',
-            dataIndex: 'key',
-            key: 'key',
-            render: (key: string) => <Typography>{key}</Typography>,
-          },
-          {
-            title: 'Description',
-            dataIndex: 'description',
-            key: 'description',
-            render: (description: string) => (
-              <RichTextEditorPreviewer markdown={description || ''} />
-            ),
-          },
-        ];
-
-        return (
-          <Table
-            bordered
-            resizableColumns
-            className="w-full"
-            columns={columns}
-            data-testid="enum-with-description-table"
-            dataSource={enumWithDescriptionValues}
-            pagination={false}
-            rowKey="name"
-            scroll={isRenderedInRightPanel ? { x: true } : undefined}
-            size="small"
-          />
-        );
-      }
-
       case 'sqlQuery':
         return (
           <SchemaEditor
@@ -981,6 +916,14 @@ export const PropertyValue: FC<PropertyValueProps> = ({
         );
       }
 
+      case TABLE_TYPE_CUSTOM_PROPERTY: {
+        const columns =
+          (property.customPropertyConfig?.config as Config)?.columns ?? [];
+        const rows = value?.rows ?? [];
+
+        return <TableTypePropertyView columns={columns} rows={rows} />;
+      }
+
       case 'string':
       case 'integer':
       case 'number':
@@ -1004,7 +947,8 @@ export const PropertyValue: FC<PropertyValueProps> = ({
   const getValueElement = () => {
     const propertyValue = getPropertyValue();
 
-    return !isUndefined(value) ? (
+    // if value is not undefined or property is a table type(at least show the columns), return the property value
+    return !isUndefined(value) || isTableType ? (
       propertyValue
     ) : (
       <span className="text-grey-muted" data-testid="no-data">
@@ -1089,7 +1033,7 @@ export const PropertyValue: FC<PropertyValueProps> = ({
             ref={contentRef}
             style={{
               height: containerStyleFlag ? 'auto' : '30px',
-              overflow: containerStyleFlag ? 'visible' : 'hidden',
+              overflow: 'hidden',
             }}>
             {showInput ? getPropertyInput() : getValueElement()}
           </div>
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.tsx
new file mode 100644
index 000000000000..a453c7ef389c
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/EditTableTypePropertyModal.tsx
@@ -0,0 +1,229 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import ReactDataGrid from '@inovua/reactdatagrid-community';
+import '@inovua/reactdatagrid-community/index.css';
+import { TypeComputedProps } from '@inovua/reactdatagrid-community/types';
+import { Button, Modal, Typography } from 'antd';
+import { isEmpty, omit } from 'lodash';
+import React, { FC, MutableRefObject, useCallback, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { CustomProperty } from '../../../../generated/type/customProperty';
+import { TableTypePropertyValueType } from '../CustomPropertyTable.interface';
+import './edit-table-type-property.less';
+import TableTypePropertyView from './TableTypePropertyView';
+
+interface EditTableTypePropertyModalProps {
+  isVisible: boolean;
+  isUpdating: boolean;
+  property: CustomProperty;
+  columns: string[];
+  rows: Record<string, string>[];
+  onCancel: () => void;
+  onSave: (data: TableTypePropertyValueType) => Promise<void>;
+}
+
+let inEdit = false;
+
+const EditTableTypePropertyModal: FC<EditTableTypePropertyModalProps> = ({
+  isVisible,
+  isUpdating,
+  property,
+  columns,
+  rows,
+  onCancel,
+  onSave,
+}) => {
+  const { t } = useTranslation();
+
+  const [dataSource, setDataSource] = useState<
+    TableTypePropertyValueType['rows']
+  >(() => rows.map((row, index) => ({ ...row, id: index + '' })));
+
+  const [gridRef, setGridRef] = useState<
+    MutableRefObject<TypeComputedProps | null>
+  >({ current: null });
+
+  const filterColumns = columns.map((column) => ({
+    name: column,
+    header: column,
+    defaultFlex: 1,
+    sortable: false,
+    minWidth: 180,
+  }));
+
+  const onEditComplete = useCallback(
+    ({ value, columnId, rowId }) => {
+      const data = [...dataSource];
+
+      data[rowId][columnId] = value;
+
+      setDataSource(data);
+    },
+    [dataSource]
+  );
+
+  const onEditStart = () => {
+    inEdit = true;
+  };
+
+  const onEditStop = () => {
+    requestAnimationFrame(() => {
+      inEdit = false;
+      gridRef.current?.focus();
+    });
+  };
+
+  const onKeyDown = (event: KeyboardEvent) => {
+    if (inEdit) {
+      if (event.key === 'Escape') {
+        const [rowIndex, colIndex] = gridRef.current?.computedActiveCell ?? [
+          0, 0,
+        ];
+        const column = gridRef.current?.getColumnBy(colIndex);
+
+        gridRef.current?.cancelEdit?.({
+          rowIndex,
+          columnId: column?.name ?? '',
+        });
+      }
+
+      return;
+    }
+    const grid = gridRef.current;
+    if (!grid) {
+      return;
+    }
+    let [rowIndex, colIndex] = grid.computedActiveCell ?? [0, 0];
+
+    if (event.key === ' ' || event.key === 'Enter') {
+      const column = grid.getColumnBy(colIndex);
+      grid.startEdit?.({ columnId: column.name ?? '', rowIndex });
+      event.preventDefault();
+
+      return;
+    }
+    if (event.key !== 'Tab') {
+      return;
+    }
+    event.preventDefault();
+    event.stopPropagation();
+
+    const direction = event.shiftKey ? -1 : 1;
+
+    const columns = grid.visibleColumns;
+    const rowCount = grid.count;
+
+    colIndex += direction;
+    if (colIndex === -1) {
+      colIndex = columns.length - 1;
+      rowIndex -= 1;
+    }
+    if (colIndex === columns.length) {
+      rowIndex += 1;
+      colIndex = 0;
+    }
+    if (rowIndex < 0 || rowIndex === rowCount) {
+      return;
+    }
+
+    grid?.setActiveCell([rowIndex, colIndex]);
+  };
+
+  const handleAddRow = useCallback(() => {
+    setDataSource((data) => {
+      setTimeout(() => {
+        gridRef.current?.scrollToId(data.length + '');
+        gridRef.current?.focus();
+      }, 1);
+
+      return [...data, { id: data.length + '' }];
+    });
+  }, [gridRef]);
+
+  const handleUpdate = useCallback(async () => {
+    const modifiedRows = dataSource
+      .map((row) => omit(row, 'id'))
+      // if the row is empty, filter it out
+      .filter((row) => !isEmpty(row) && Object.values(row).some(Boolean));
+    await onSave({ rows: modifiedRows, columns });
+  }, [onSave, dataSource, columns]);
+
+  return (
+    <Modal
+      centered
+      destroyOnClose
+      closable={false}
+      data-testid="edit-table-type-property-modal"
+      footer={
+        <div className="d-flex justify-between">
+          <Button
+            data-testid="add-new-row"
+            disabled={isUpdating}
+            type="primary"
+            onClick={handleAddRow}>
+            {t('label.add-entity', { entity: t('label.row') })}
+          </Button>
+
+          <div className="d-flex gap-2">
+            <Button
+              data-testid="cancel-update-table-type-property"
+              disabled={isUpdating}
+              onClick={onCancel}>
+              {t('label.cancel')}
+            </Button>
+            <Button
+              data-testid="update-table-type-property"
+              disabled={isUpdating}
+              loading={isUpdating}
+              type="primary"
+              onClick={handleUpdate}>
+              {t('label.update')}
+            </Button>
+          </div>
+        </div>
+      }
+      maskClosable={false}
+      open={isVisible}
+      title={
+        <Typography.Text>
+          {t('label.edit-entity-name', {
+            entityType: t('label.property'),
+            entityName: property.name,
+          })}
+        </Typography.Text>
+      }
+      width={800}>
+      {isEmpty(dataSource) ? (
+        <TableTypePropertyView columns={columns} rows={rows} />
+      ) : (
+        <ReactDataGrid
+          editable
+          className="edit-table-type-property"
+          columns={filterColumns}
+          dataSource={dataSource}
+          handle={setGridRef}
+          idProperty="id"
+          minRowHeight={30}
+          showZebraRows={false}
+          style={{ height: '350px' }}
+          onEditComplete={onEditComplete}
+          onEditStart={onEditStart}
+          onEditStop={onEditStop}
+          onKeyDown={onKeyDown}
+        />
+      )}
+    </Modal>
+  );
+};
+
+export default EditTableTypePropertyModal;
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyView.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyView.tsx
new file mode 100644
index 000000000000..612ce6dfaf57
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/TableTypePropertyView.tsx
@@ -0,0 +1,68 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import { isArray } from 'lodash';
+import React, { FC } from 'react';
+import { useTranslation } from 'react-i18next';
+import { NO_DATA_PLACEHOLDER } from '../../../../constants/constants';
+import Table from '../../Table/Table';
+import './table-type-property-view.less';
+
+interface TableTypePropertyViewProps {
+  columns: string[];
+  rows: Record<string, string>[];
+}
+
+const TableTypePropertyView: FC<TableTypePropertyViewProps> = ({
+  columns,
+  rows,
+}) => {
+  const { t } = useTranslation();
+
+  if (!isArray(columns) || !isArray(rows)) {
+    return (
+      <span className="text-grey-muted" data-testid="invalid-data">
+        {t('label.field-invalid', {
+          field: `${t('label.column-plural')} or ${t('label.row-plural')}`,
+        })}
+      </span>
+    );
+  }
+
+  const tableColumns = columns.map((column: string) => ({
+    title: column,
+    dataIndex: column,
+    key: column,
+    render: (text: string) => text || NO_DATA_PLACEHOLDER,
+  }));
+
+  return (
+    <Table
+      bordered
+      resizableColumns
+      className="w-full table-type-custom-property"
+      columns={tableColumns}
+      data-testid="table-type-property-value"
+      dataSource={rows}
+      pagination={{
+        position: ['bottomCenter'],
+        pageSize: 10,
+        hideOnSinglePage: true,
+      }}
+      rowKey="name"
+      scroll={{ x: true }}
+      size="small"
+    />
+  );
+};
+
+export default TableTypePropertyView;
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/edit-table-type-property.less b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/edit-table-type-property.less
new file mode 100644
index 000000000000..ec17df5001c7
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/edit-table-type-property.less
@@ -0,0 +1,46 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+@import url('../../../../styles/variables.less');
+
+.edit-table-type-property {
+  .InovuaReactDataGrid__column-header:hover {
+    .InovuaReactDataGrid__column-header__menu-tool {
+      display: none;
+    }
+  }
+
+  .InovuaReactDataGrid__header {
+    border-top-right-radius: 6px;
+    border-top-left-radius: 6px;
+    font-weight: 500;
+    color: @grey-4;
+    text-transform: uppercase;
+    background: @grey-1;
+    .InovuaReactDataGrid__column-header__content {
+      font-weight: 500;
+    }
+  }
+
+  .InovuaReactDataGrid__virtual-list {
+    border-bottom-left-radius: 6px;
+    border-bottom-right-radius: 6px;
+  }
+
+  .InovuaReactDataGrid__row-cell-wrap {
+    color: @text-color;
+  }
+
+  border: 1px solid @border-color;
+  border-radius: 6px;
+}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/table-type-property-view.less b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/table-type-property-view.less
new file mode 100644
index 000000000000..fb30e275f921
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/TableTypeProperty/table-type-property-view.less
@@ -0,0 +1,38 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+@border-right-color: #0000000f;
+
+.table-type-custom-property.ant-table-wrapper {
+  .ant-table-bordered {
+    .ant-table-container {
+      .ant-table-content > table {
+        tbody > tr {
+          td {
+            border-right: 1px solid @border-right-color;
+            &:last-child {
+              border-right: none;
+            }
+          }
+        }
+        thead > tr {
+          th {
+            border-right: 1px solid @border-right-color;
+            &:last-child {
+              border-right: none;
+            }
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less
index 859e899a852c..be546896cc06 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less
@@ -56,6 +56,7 @@
   .ant-space-item:first-child {
     width: 100%;
   }
+  overflow-x: scroll;
 }
 .custom-property-card {
   .ant-card-body {
@@ -86,3 +87,9 @@
   font-weight: 500;
   line-height: 18px;
 }
+
+.property-description {
+  .markdown-parser .toastui-editor-contents {
+    font-size: 13px;
+  }
+}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx
index b51fa5dc4859..1ddebca5885a 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/EntitySummaryDetails/EntitySummaryDetails.tsx
@@ -120,7 +120,7 @@ const EntitySummaryDetails = ({ data }: GetInfoElementsProps) => {
             <span
               className="d-flex gap-1 items-center"
               data-testid="owner-link">
-              {t('label.no-entity', { entity: t('label.owner') })}
+              {t('label.no-entity', { entity: t('label.owner-plural') })}
             </span>
           );
       }
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.interface.ts
new file mode 100644
index 000000000000..68543369c9bb
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.interface.ts
@@ -0,0 +1,16 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+export enum QueryBuilderOutputType {
+  ELASTICSEARCH = 'elasticsearch',
+  JSON_LOGIC = 'jsonlogic',
+}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.test.tsx
new file mode 100644
index 000000000000..44b0b3895c49
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.test.tsx
@@ -0,0 +1,75 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import { Registry } from '@rjsf/utils';
+import { render, screen } from '@testing-library/react';
+import React from 'react';
+import { BasicConfig } from 'react-awesome-query-builder';
+import AntdConfig from 'react-awesome-query-builder/lib/config/antd';
+import QueryBuilderWidget from './QueryBuilderWidget';
+
+const mockOnFocus = jest.fn();
+const mockOnBlur = jest.fn();
+const mockOnChange = jest.fn();
+const baseConfig = AntdConfig as BasicConfig;
+
+jest.mock(
+  '../../../../../Explore/AdvanceSearchProvider/AdvanceSearchProvider.component',
+  () => ({
+    AdvanceSearchProvider: ({ children }: { children: React.ReactNode }) => (
+      <div data-testid="advance-search-provider-mock">{children}</div>
+    ),
+    useAdvanceSearch: jest.fn().mockImplementation(() => ({
+      toggleModal: jest.fn(),
+      sqlQuery: '',
+      onResetAllFilters: jest.fn(),
+      onChangeSearchIndex: jest.fn(),
+      config: {
+        ...baseConfig,
+        fields: {},
+      },
+    })),
+  })
+);
+
+jest.mock('react-router-dom', () => ({
+  useLocation: jest.fn(),
+}));
+
+const mockProps = {
+  onFocus: mockOnFocus,
+  onBlur: mockOnBlur,
+  onChange: mockOnChange,
+  registry: {} as Registry,
+  schema: {
+    description: 'this is query builder field',
+    title: 'rules',
+    format: 'queryBuilder',
+    entityType: 'table',
+  },
+  value: '',
+  id: 'root/queryBuilder',
+  label: 'Query Builder',
+  name: 'queryBuilder',
+  options: {
+    enumOptions: [],
+  },
+};
+
+describe('QueryBuilderWidget', () => {
+  it('should render the query builder', () => {
+    render(<QueryBuilderWidget {...mockProps} />);
+    const builder = screen.getByTestId('query-builder-form-field');
+
+    expect(builder).toBeInTheDocument();
+  });
+});
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.tsx
new file mode 100644
index 000000000000..0e1b04275e7e
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget.tsx
@@ -0,0 +1,159 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+import { InfoCircleOutlined } from '@ant-design/icons';
+import { WidgetProps } from '@rjsf/utils';
+import { Alert, Button, Col, Typography } from 'antd';
+import { t } from 'i18next';
+import { debounce, isEmpty, isUndefined } from 'lodash';
+import Qs from 'qs';
+import React, { FC, useCallback, useEffect, useMemo, useState } from 'react';
+import {
+  Builder,
+  Config,
+  ImmutableTree,
+  Query,
+  Utils as QbUtils,
+} from 'react-awesome-query-builder';
+import { getExplorePath } from '../../../../../../constants/constants';
+import { EntityType } from '../../../../../../enums/entity.enum';
+import { SearchIndex } from '../../../../../../enums/search.enum';
+import { searchQuery } from '../../../../../../rest/searchAPI';
+import searchClassBase from '../../../../../../utils/SearchClassBase';
+import { withAdvanceSearch } from '../../../../../AppRouter/withAdvanceSearch';
+import { useAdvanceSearch } from '../../../../../Explore/AdvanceSearchProvider/AdvanceSearchProvider.component';
+import './query-builder-widget.less';
+import { QueryBuilderOutputType } from './QueryBuilderWidget.interface';
+
+const QueryBuilderWidget: FC<WidgetProps> = ({
+  onChange,
+  schema,
+  value,
+  ...props
+}: WidgetProps) => {
+  const { config, treeInternal, onTreeUpdate, onChangeSearchIndex } =
+    useAdvanceSearch();
+  const [searchResults, setSearchResults] = useState<number>(0);
+  const entityType =
+    (props.formContext?.entityType ?? props?.entityType) || EntityType.ALL;
+  const searchIndexMapping = searchClassBase.getEntityTypeSearchIndexMapping();
+  const searchIndex = searchIndexMapping[entityType as string];
+  const outputType = props?.outputType ?? QueryBuilderOutputType.ELASTICSEARCH;
+
+  const fetchEntityCount = useCallback(
+    async (queryFilter: Record<string, unknown>) => {
+      try {
+        const res = await searchQuery({
+          query: '',
+          pageNumber: 0,
+          pageSize: 0,
+          queryFilter,
+          searchIndex: SearchIndex.ALL,
+          includeDeleted: false,
+          trackTotalHits: true,
+          fetchSource: false,
+        });
+        setSearchResults(res.hits.total.value ?? 0);
+      } catch (_) {
+        // silent fail
+      }
+    },
+    []
+  );
+
+  const debouncedFetchEntityCount = useMemo(
+    () => debounce(fetchEntityCount, 300),
+    [fetchEntityCount]
+  );
+
+  const queryURL = useMemo(() => {
+    const queryFilterString = !isEmpty(treeInternal)
+      ? Qs.stringify({ queryFilter: JSON.stringify(treeInternal) })
+      : '';
+
+    return `${getExplorePath({})}${queryFilterString}`;
+  }, [treeInternal]);
+
+  const handleChange = (nTree: ImmutableTree, nConfig: Config) => {
+    onTreeUpdate(nTree, nConfig);
+
+    if (outputType === QueryBuilderOutputType.ELASTICSEARCH) {
+      const data = QbUtils.elasticSearchFormat(nTree, config) ?? {};
+      const qFilter = {
+        query: data,
+      };
+      if (data) {
+        debouncedFetchEntityCount(qFilter);
+      }
+
+      onChange(JSON.stringify(qFilter));
+    } else {
+      const data = QbUtils.jsonLogicFormat(nTree, config);
+      onChange(JSON.stringify(data.logic ?? '{}'));
+    }
+  };
+
+  useEffect(() => {
+    onChangeSearchIndex(searchIndex);
+  }, [searchIndex]);
+
+  return (
+    <div
+      className="query-builder-form-field"
+      data-testid="query-builder-form-field">
+      <Query
+        {...config}
+        renderBuilder={(props) => (
+          <div className="query-builder-container query-builder qb-lite">
+            <Builder {...props} />
+          </div>
+        )}
+        value={treeInternal}
+        onChange={handleChange}
+      />
+      {outputType === QueryBuilderOutputType.ELASTICSEARCH &&
+        !isUndefined(value) && (
+          <Col span={24}>
+            <Button
+              className="w-full p-0 text-left"
+              data-testid="view-assets-banner-button"
+              disabled={false}
+              href={queryURL}
+              target="_blank"
+              type="link">
+              <Alert
+                closable
+                showIcon
+                icon={<InfoCircleOutlined height={16} />}
+                message={
+                  <>
+                    <Typography.Text>
+                      {t('message.search-entity-count', {
+                        count: searchResults,
+                      })}
+                    </Typography.Text>
+
+                    <Typography.Text className="m-l-sm text-xs text-grey-muted">
+                      {t('message.click-here-to-view-assets-on-explore')}
+                    </Typography.Text>
+                  </>
+                }
+                type="info"
+              />
+            </Button>
+          </Col>
+        )}
+    </div>
+  );
+};
+
+export default withAdvanceSearch(QueryBuilderWidget);
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/query-builder-widget.less b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/query-builder-widget.less
new file mode 100644
index 000000000000..43d768729e2a
--- /dev/null
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/query-builder-widget.less
@@ -0,0 +1,42 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+.query-builder-form-field {
+  .hide--line.one--child {
+    margin-top: 0;
+    padding-top: 16px;
+  }
+
+  .group.rule_group {
+    border: none !important;
+    padding: 0;
+
+    .group--children {
+      padding-top: 0;
+      padding-bottom: 0;
+      margin: 0;
+    }
+  }
+
+  .group--field {
+    width: 180px;
+
+    .ant-select {
+      width: 100% !important;
+    }
+
+    label {
+      font-weight: normal;
+      margin-bottom: 6px;
+    }
+  }
+}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx
index 23f624f7fa26..8f4cb2b608f6 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/FormBuilder/FormBuilder.tsx
@@ -27,6 +27,7 @@ import { FieldErrorTemplate } from '../Form/JSONSchema/JSONSchemaTemplate/FieldE
 import { ObjectFieldTemplate } from '../Form/JSONSchema/JSONSchemaTemplate/ObjectFieldTemplate';
 import AsyncSelectWidget from '../Form/JSONSchema/JsonSchemaWidgets/AsyncSelectWidget';
 import PasswordWidget from '../Form/JSONSchema/JsonSchemaWidgets/PasswordWidget';
+import QueryBuilderWidget from '../Form/JSONSchema/JsonSchemaWidgets/QueryBuilderWidget/QueryBuilderWidget';
 import SelectWidget from '../Form/JSONSchema/JsonSchemaWidgets/SelectWidget';
 import Loader from '../Loader/Loader';
 
@@ -70,6 +71,7 @@ const FormBuilder: FunctionComponent<Props> = forwardRef(
     const widgets = {
       PasswordWidget: PasswordWidget,
       autoComplete: AsyncSelectWidget,
+      queryBuilder: QueryBuilderWidget,
       ...(useSelectWidget && { SelectWidget: SelectWidget }),
     };
 
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/OwnerLabel/OwnerLabel.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/OwnerLabel/OwnerLabel.component.tsx
index 884febb15cb4..6af04681b655 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/OwnerLabel/OwnerLabel.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/OwnerLabel/OwnerLabel.component.tsx
@@ -119,7 +119,7 @@ export const OwnerLabel = ({
               const inheritedIcon = owner?.inherited ? (
                 <Tooltip
                   title={t('label.inherited-entity', {
-                    entity: t('label.owner'),
+                    entity: t('label.owner-plural'),
                   })}>
                   <InheritIcon
                     className="inherit-icon cursor-pointer"
@@ -165,7 +165,7 @@ export const OwnerLabel = ({
               className={classNames('no-owner font-medium text-xs', className)}
               data-testid="owner-link">
               {placeHolder ??
-                t('label.no-entity', { entity: t('label.owner') })}
+                t('label.no-entity', { entity: t('label.owner-plural') })}
             </Typography.Text>
           </div>
         )}
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/ServiceDocPanel/ServiceDocPanel.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/ServiceDocPanel/ServiceDocPanel.tsx
index 42f5f07ae408..a33a4502f429 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/ServiceDocPanel/ServiceDocPanel.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/ServiceDocPanel/ServiceDocPanel.tsx
@@ -83,14 +83,16 @@ const ServiceDocPanel: FC<ServiceDocPanelProp> = ({
   const fetchRequirement = async () => {
     setIsLoading(true);
     try {
+      const supportedServiceType =
+        serviceType === 'Api' ? 'ApiEntity' : serviceType;
       let response = '';
       const isEnglishLanguage = i18n.language === SupportedLocales.English;
-      let filePath = `${i18n.language}/${serviceType}/${serviceName}.md`;
-      let fallbackFilePath = `${SupportedLocales.English}/${serviceType}/${serviceName}.md`;
+      let filePath = `${i18n.language}/${supportedServiceType}/${serviceName}.md`;
+      let fallbackFilePath = `${SupportedLocales.English}/${supportedServiceType}/${serviceName}.md`;
 
       if (isWorkflow && workflowType) {
-        filePath = `${i18n.language}/${serviceType}/workflows/${workflowType}.md`;
-        fallbackFilePath = `${SupportedLocales.English}/${serviceType}/workflows/${workflowType}.md`;
+        filePath = `${i18n.language}/${supportedServiceType}/workflows/${workflowType}.md`;
+        fallbackFilePath = `${SupportedLocales.English}/${supportedServiceType}/workflows/${workflowType}.md`;
       }
 
       const [translation, fallbackTranslation] = await Promise.allSettled([
diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/UserTeamSelectableList/UserTeamSelectableList.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/common/UserTeamSelectableList/UserTeamSelectableList.component.tsx
index de127964266a..5a16b7025363 100644
--- a/openmetadata-ui/src/main/resources/ui/src/components/common/UserTeamSelectableList/UserTeamSelectableList.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/components/common/UserTeamSelectableList/UserTeamSelectableList.component.tsx
@@ -267,7 +267,7 @@ export const UserTeamSelectableList = ({
               size={8}>
               <Typography.Text className="text-grey-muted">
                 {t('label.selected-entity', {
-                  entity: label ?? t('label.owner'),
+                  entity: label ?? t('label.owner-plural'),
                 })}
               </Typography.Text>
               <div className="user-team-popover-header-content">
@@ -369,7 +369,7 @@ export const UserTeamSelectableList = ({
               !popupVisible &&
               (tooltipText ??
                 t('label.edit-entity', {
-                  entity: t('label.owner'),
+                  entity: t('label.owner-plural'),
                 }))
             }>
             <Button
diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts
index f827da99dd3d..92454b208bf5 100644
--- a/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts
@@ -22,7 +22,7 @@ export const COMMON_DROPDOWN_ITEMS = [
     key: EntityFields.DOMAIN,
   },
   {
-    label: t('label.owner'),
+    label: t('label.owner-plural'),
     key: EntityFields.OWNERS,
   },
   {
@@ -53,7 +53,7 @@ export const DATA_ASSET_DROPDOWN_ITEMS = [
     key: EntityFields.DOMAIN,
   },
   {
-    label: t('label.owner'),
+    label: t('label.owner-plural'),
     key: EntityFields.OWNERS,
   },
   {
@@ -174,7 +174,7 @@ export const GLOSSARY_DROPDOWN_ITEMS = [
     key: EntityFields.DOMAIN,
   },
   {
-    label: t('label.owner'),
+    label: t('label.owner-plural'),
     key: EntityFields.OWNERS,
   },
   {
@@ -208,7 +208,7 @@ export const DATA_PRODUCT_DROPDOWN_ITEMS = [
     key: EntityFields.DOMAIN,
   },
   {
-    label: t('label.owner'),
+    label: t('label.owner-plural'),
     key: EntityFields.OWNERS,
   },
 ];
@@ -221,7 +221,7 @@ export const DOMAIN_DATAPRODUCT_DROPDOWN_ITEMS = [
     key: EntityFields.ENTITY_TYPE,
   },
   {
-    label: t('label.owner'),
+    label: t('label.owner-plural'),
     key: EntityFields.OWNERS,
   },
   {
@@ -254,7 +254,7 @@ export const GLOSSARY_ASSETS_DROPDOWN_ITEMS = [
     key: EntityFields.DOMAIN,
   },
   {
-    label: t('label.owner'),
+    label: t('label.owner-plural'),
     key: EntityFields.OWNERS,
   },
   {
diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/CustomProperty.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/CustomProperty.constants.ts
index 01cb17d167cb..3705ca3b49ea 100644
--- a/openmetadata-ui/src/main/resources/ui/src/constants/CustomProperty.constants.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/constants/CustomProperty.constants.ts
@@ -10,6 +10,24 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
+
+import { ReactComponent as DateTimeIcon } from '../assets/svg/customproperties/date-time.svg';
+import { ReactComponent as DateIcon } from '../assets/svg/customproperties/date.svg';
+import { ReactComponent as DurationIcon } from '../assets/svg/customproperties/duration.svg';
+import { ReactComponent as EmailIcon } from '../assets/svg/customproperties/email.svg';
+import { ReactComponent as EntityReferenceListIcon } from '../assets/svg/customproperties/entity-list.svg';
+import { ReactComponent as EntityReferenceIcon } from '../assets/svg/customproperties/entity.svg';
+import { ReactComponent as EnumIcon } from '../assets/svg/customproperties/enum.svg';
+import { ReactComponent as IntegerIcon } from '../assets/svg/customproperties/integer.svg';
+import { ReactComponent as MarkDownIcon } from '../assets/svg/customproperties/markdown.svg';
+import { ReactComponent as NumberIcon } from '../assets/svg/customproperties/number.svg';
+import { ReactComponent as SQLQueryIcon } from '../assets/svg/customproperties/sql-query.svg';
+import { ReactComponent as StringIcon } from '../assets/svg/customproperties/string.svg';
+import { ReactComponent as TableIcon } from '../assets/svg/customproperties/table.svg';
+import { ReactComponent as TimeIntervalIcon } from '../assets/svg/customproperties/time-interval.svg';
+import { ReactComponent as TimeIcon } from '../assets/svg/customproperties/time.svg';
+import { ReactComponent as TimestampIcon } from '../assets/svg/customproperties/timestamp.svg';
+
 export const PROPERTY_TYPES_WITH_FORMAT = ['date-cp', 'dateTime-cp', 'time-cp'];
 
 export const PROPERTY_TYPES_WITH_ENTITY_REFERENCE = [
@@ -127,4 +145,23 @@ export const SUPPORTED_FORMAT_MAP = {
   'time-cp': SUPPORTED_TIME_FORMATS,
 };
 
-export const ENUM_WITH_DESCRIPTION = 'enumWithDescriptions';
+export const TABLE_TYPE_CUSTOM_PROPERTY = 'table-cp';
+
+export const CUSTOM_PROPERTIES_ICON_MAP = {
+  'date-cp': DateIcon,
+  'dateTime-cp': DateTimeIcon,
+  duration: DurationIcon,
+  email: EmailIcon,
+  entityReference: EntityReferenceIcon,
+  entityReferenceList: EntityReferenceListIcon,
+  enum: EnumIcon,
+  integer: IntegerIcon,
+  markdown: MarkDownIcon,
+  number: NumberIcon,
+  sqlQuery: SQLQueryIcon,
+  string: StringIcon,
+  'table-cp': TableIcon,
+  'time-cp': TimeIcon,
+  timeInterval: TimeIntervalIcon,
+  timestamp: TimestampIcon,
+};
diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts
index fe96dc62fb9d..798dd4b39df8 100644
--- a/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/constants/GlobalSettings.constants.ts
@@ -22,6 +22,7 @@ export enum GlobalSettingsMenuCategory {
   SERVICES = 'services',
   BOTS = 'bots',
   APPLICATIONS = 'apps',
+  PERSONA = 'persona',
 }
 
 export enum GlobalSettingOptions {
diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/ResizablePanel.constants.ts b/openmetadata-ui/src/main/resources/ui/src/constants/ResizablePanel.constants.ts
index 2787d162999c..9e93903e5d54 100644
--- a/openmetadata-ui/src/main/resources/ui/src/constants/ResizablePanel.constants.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/constants/ResizablePanel.constants.ts
@@ -12,11 +12,11 @@
  */
 export const COMMON_RESIZABLE_PANEL_CONFIG = {
   LEFT_PANEL: {
-    minWidth: 700,
-    flex: 0.7,
+    minWidth: 800,
+    flex: 0.85,
   },
   RIGHT_PANEL: {
     minWidth: 320,
-    flex: 0.3,
+    flex: 0.15,
   },
 };
diff --git a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts
index c4742fb2fff1..6c435278f489 100644
--- a/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts
@@ -417,7 +417,6 @@ export const BETA_SERVICES = [
   PipelineServiceType.OpenLineage,
   PipelineServiceType.Flink,
   DatabaseServiceType.Teradata,
-  APIServiceType.REST,
   StorageServiceType.Gcs,
   DatabaseServiceType.SapERP,
   PipelineServiceType.Flink,
diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts
index 41ce4bffd6c3..aeb7dfcc8195 100644
--- a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts
@@ -67,6 +67,7 @@ export enum EntityType {
   WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData',
   WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData',
   TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index',
+  TEST_CASE_RESULT = 'test_case_result_search_index',
   EVENT_SUBSCRIPTION = 'eventsubscription',
   LINEAGE_EDGE = 'lineageEdge',
   API_SERVICE = 'apiService',
diff --git a/openmetadata-ui/src/main/resources/ui/src/hooks/useReserveSidebar.test.ts b/openmetadata-ui/src/main/resources/ui/src/hooks/useReserveSidebar.test.ts
deleted file mode 100644
index 147239547181..000000000000
--- a/openmetadata-ui/src/main/resources/ui/src/hooks/useReserveSidebar.test.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- *  Copyright 2024 Collate.
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *  http://www.apache.org/licenses/LICENSE-2.0
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-import { renderHook } from '@testing-library/react-hooks';
-import { useReserveSidebar } from './useReserveSidebar';
-
-describe('useReserveSidebar', () => {
-  it('should return false when .floating-button-container is not present', () => {
-    const { result } = renderHook(() => useReserveSidebar());
-
-    expect(result.current.isSidebarReserve).toBe(false);
-  });
-
-  it('should return true when .floating-button-container is present', () => {
-    // Create a mock element and append it to the document body
-    const floatingButtonContainer = document.createElement('div');
-    floatingButtonContainer.className = 'floating-button-container';
-    document.body.appendChild(floatingButtonContainer);
-
-    const { result } = renderHook(() => useReserveSidebar());
-
-    expect(result.current.isSidebarReserve).toBe(true);
-
-    // Clean up the mock element
-    document.body.removeChild(floatingButtonContainer);
-  });
-});
diff --git a/openmetadata-ui/src/main/resources/ui/src/hooks/useReserveSidebar.ts b/openmetadata-ui/src/main/resources/ui/src/hooks/useReserveSidebar.ts
deleted file mode 100644
index 5455ba51535c..000000000000
--- a/openmetadata-ui/src/main/resources/ui/src/hooks/useReserveSidebar.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- *  Copyright 2024 Collate.
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *  http://www.apache.org/licenses/LICENSE-2.0
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-import { useEffect, useState } from 'react';
-
-type ReserverSidebar = { isSidebarReserve: boolean };
-
-/**
- * @description Hook to get the if the sidebar should have some reserved space in case of floating button
- * @returns {isSidebarReserve: boolean} - boolean value to check if the sidebar should have some reserved space
- */
-export const useReserveSidebar = (): ReserverSidebar => {
-  const [isSidebarReserve, setIsSidebarReserve] = useState<boolean>(false);
-
-  useEffect(() => {
-    const element = document.querySelector('.floating-button-container');
-    setIsSidebarReserve(Boolean(element));
-  }, []);
-
-  return {
-    isSidebarReserve,
-  };
-};
diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts
index b4cb53d84b53..4dc9971edb48 100644
--- a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts
@@ -45,7 +45,7 @@ import { PipelineService } from '../generated/entity/services/pipelineService';
 import { SearchService } from '../generated/entity/services/searchService';
 import { Team } from '../generated/entity/teams/team';
 import { User } from '../generated/entity/teams/user';
-import { TestCase } from '../generated/tests/testCase';
+import { TestCase, TestCaseResult } from '../generated/tests/testCase';
 import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus';
 import { TestSuite } from '../generated/tests/testSuite';
 import { TagLabel } from '../generated/type/tagLabel';
@@ -145,6 +145,15 @@ export interface TestCaseResolutionStatusSearchSource
   serviceType: string;
   description: string;
 }
+export interface TestCaseResultSearchSource
+  extends SearchSourceBase,
+    TestCaseResult {
+  name: string;
+  displayName: string;
+  fullyQualifiedName: string;
+  serviceType: string;
+  description: string;
+}
 
 export interface IngestionPipelineSearchSource
   extends SearchSourceBase,
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json
index f0af3b4587ea..7ede75ac2c73 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/de-de.json
@@ -247,6 +247,7 @@
     "custom-theme": "Custom Theme",
     "customise": "Customise",
     "customize-entity": "Customize {{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "DAG",
     "dag-view": "DAG-Ansicht",
     "daily-active-users-on-the-platform": "Täglich aktive Benutzer auf der Plattform",
@@ -821,6 +822,7 @@
     "owner": "Besitzer",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "besitzer",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "Besitzer",
     "page-not-found": "Seite nicht gefunden",
     "page-views-by-data-asset-plural": "Seitenaufrufe nach Datenanlage",
@@ -989,6 +991,7 @@
     "role-name": "Rollenname",
     "role-plural": "Rollen",
     "row": "Zeile",
+    "row-count": "Row Count",
     "row-count-lowercase": "Anzahl der Zeilen",
     "row-plural": "Zeilen",
     "rtl-ltr-direction": "RTL/LTR direction",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "Can not add the widget to this section due to size restrictions.",
     "can-you-add-a-description": "Können Sie eine Beschreibung hinzufügen?",
     "checkout-service-connectors-doc": "Es gibt viele Konnektoren hier, um Daten von Ihren Diensten zu indizieren. Bitte schauen Sie sich unsere Konnektoren an.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "Klicken Sie auf <0>{{text}}</0>, um Details anzuzeigen.",
     "closed-this-task": "hat diese Aufgabe geschlossen",
     "collaborate-with-other-user": "um mit anderen Benutzern zusammenzuarbeiten.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "Dies ist eine optionale Konfiguration zur Aktivierung des sanften Löschens von Tabellen. Wenn diese Option aktiviert ist, werden nur Tabellen, die aus der Quelle gelöscht wurden, sanft gelöscht, und dies gilt ausschließlich für das Schema, das gerade über die Pipeline erfasst wird. Alle zugehörigen Entitäten wie Test-Suiten oder Verbindungsdaten, die mit diesen Tabellen verknüpft waren, werden ebenfalls gelöscht.",
     "markdown-editor-placeholder": "Verwenden Sie @Erwähnen, um einen Benutzer oder ein Team zu kennzeichnen.\nVerwenden Sie #Erwähnen, um eine Datenressource zu kennzeichnen.",
     "marketplace-verify-msg": "OpenMetadata has verified that the publisher controls the domain and meets other requirements.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "The maximum value should be greater than the minimum value.",
     "member-description": "Streamline access to users and teams in OpenMetadata.",
     "mentioned-you-on-the-lowercase": "hat dich auf der",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "Die Planung kann im stündlichen, täglichen oder wöchentlichen Rhythmus eingerichtet werden. Die Zeitzone ist UTC.",
     "scheduled-run-every": "Geplant, alle auszuführen",
     "scopes-comma-separated": "Fügen Sie den Wert der Bereiche hinzu, getrennt durch Kommata",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "Suche nach Pipeline, StoredProcedures",
     "search-for-entity-types": "Suche nach Tabellen, Themen, Dashboards, Pipelines, ML-Modellen, Glossar und Tags.",
     "search-for-ingestion": "Suche nach Ingestion",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
index cd8f9bc90878..7cf7279dd766 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/en-us.json
@@ -247,6 +247,7 @@
     "custom-theme": "Custom Theme",
     "customise": "Customise",
     "customize-entity": "Customize {{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "Dag",
     "dag-view": "DAG view",
     "daily-active-users-on-the-platform": "Daily Active Users on the Platform",
@@ -821,6 +822,7 @@
     "owner": "Owner",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "owner",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "Owners",
     "page-not-found": "Page Not Found",
     "page-views-by-data-asset-plural": "Page Views by Data Assets",
@@ -989,6 +991,7 @@
     "role-name": "Role Name",
     "role-plural": "Roles",
     "row": "Row",
+    "row-count": "Row Count",
     "row-count-lowercase": "row count",
     "row-plural": "Rows",
     "rtl-ltr-direction": "RTL/LTR direction",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "Can not add the widget to this section due to size restrictions.",
     "can-you-add-a-description": "Can you add a description?",
     "checkout-service-connectors-doc": "There are a lot of connectors available here to index data from your services. Please checkout our connectors.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "Click <0>{{text}}</0> to view details.",
     "closed-this-task": "closed this task",
     "collaborate-with-other-user": "to collaborate with other users.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "This is an optional configuration for enabling soft deletion of tables. When this option is enabled, only tables that have been deleted from the source will be soft deleted, and this will apply solely to the schema that is currently being ingested via the pipeline. Any related entities such as test suites or lineage information that were associated with those tables will also be deleted.",
     "markdown-editor-placeholder": "Use @mention to tag a user or a team.\nUse #mention to tag a data asset.",
     "marketplace-verify-msg": "OpenMetadata has verified that the publisher controls the domain and meets other requirements.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "The maximum value should be greater than the minimum value.",
     "member-description": "Streamline access to users and teams in OpenMetadata.",
     "mentioned-you-on-the-lowercase": "mentioned you on the",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "Scheduling can be set up at an hourly, daily, or weekly cadence. The timezone is in UTC.",
     "scheduled-run-every": "Scheduled to run every",
     "scopes-comma-separated": "Add the Scopes value, separated by commas",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "Search for Pipeline, StoredProcedures",
     "search-for-entity-types": "Search for Tables, Topics, Dashboards, Pipelines, ML Models, Glossary and Tags.",
     "search-for-ingestion": "Search for ingestion",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
index 638008e22bf0..a7b6c3eb2ea1 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/es-es.json
@@ -247,6 +247,7 @@
     "custom-theme": "Custom Theme",
     "customise": "Personalizar",
     "customize-entity": "Personalizar {{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "DAG",
     "dag-view": "Vista DAG",
     "daily-active-users-on-the-platform": "Usuarios activos diarios en la plataforma",
@@ -821,6 +822,7 @@
     "owner": "Propietario",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "propietario",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "Propietarios",
     "page-not-found": "Página no encontrada",
     "page-views-by-data-asset-plural": "Vistas de página por activos de datos",
@@ -989,6 +991,7 @@
     "role-name": "Nombre del Rol",
     "role-plural": "Roles",
     "row": "Fila",
+    "row-count": "Row Count",
     "row-count-lowercase": "número de filas",
     "row-plural": "Filas",
     "rtl-ltr-direction": "dirección RTL/LTR",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "No se puede agregar el widget a esta sección debido a restricciones de tamaño.",
     "can-you-add-a-description": "¿Puedes agregar una descripción?",
     "checkout-service-connectors-doc": "Hay muchos conectores disponibles para ingesta de datos de tus servicios. Por favor, revisa nuestra documentación sobre conectores.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "Haz clic en <0>{{text}}</0> para ver detalles.",
     "closed-this-task": "cerró esta tarea",
     "collaborate-with-other-user": "para colaborar con otros usuarios.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "Esta es una configuración opcional para habilitar la eliminación suave de tablas. Cuando se habilita esta opción, sólo se eliminarán suavemente las tablas que se hayan eliminado de la fuente, y esto se aplicará únicamente al esquema que se esté ingiriendo actualmente a través de la canalización. Cualquier entidad relacionada, como tests o información de linaje que estuvieran asociados con esas tablas, también se eliminarán.",
     "markdown-editor-placeholder": "Usa @mención para etiquetar a un usuario o equipo.\n Usa #mención para etiquetar un activo de datos.",
     "marketplace-verify-msg": "OpenMetadata ha verificado que el editor controla el dominio y cumple con otros requisitos.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "The maximum value should be greater than the minimum value.",
     "member-description": "Optimiza el acceso a usuarios y equipos en OpenMetadata.",
     "mentioned-you-on-the-lowercase": "te mencionó en",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "La programación se puede configurar en una cadencia horaria, diaria o semanal.",
     "scheduled-run-every": "Programado para ejecutarse cada",
     "scopes-comma-separated": "Agrega el valor de ámbitos, separados por comas",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "Buscar Pipeline, StoredProcedures",
     "search-for-entity-types": "Buscar Tablas, Temas, Paneles, Pipelines, Modelos de ML, Glosarios y Etiquetas.",
     "search-for-ingestion": "Buscar orígenes de datos",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
index 49c63fd5725a..f7c323534b53 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/fr-fr.json
@@ -247,6 +247,7 @@
     "custom-theme": "Thème personnalisé",
     "customise": "Personnaliser",
     "customize-entity": "Personnaliser {{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "DAG",
     "dag-view": "Vue DAG",
     "daily-active-users-on-the-platform": "Utilisateurs Actifs Quotidiens sur la Plateforme",
@@ -821,6 +822,7 @@
     "owner": "Propriétaire",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "propriétaire",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "Propriétaires",
     "page-not-found": "Page Non Trouvée",
     "page-views-by-data-asset-plural": "Vues de Page par Actif de Données",
@@ -989,6 +991,7 @@
     "role-name": "Nom du Rôle",
     "role-plural": "Rôles",
     "row": "Ligne",
+    "row-count": "Row Count",
     "row-count-lowercase": "Nombre de Ligne",
     "row-plural": "Lignes",
     "rtl-ltr-direction": "RTL/LTR direction",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "Le widget ne peut être ajouté à cette section à cause des restrictions de taille.$",
     "can-you-add-a-description": "Pouvez-vous ajouter une description?",
     "checkout-service-connectors-doc": "Il y a de nombreux connecteurs disponibles ici pour indexer les données de vos services. Veuillez consulter nos connecteurs.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "Cliquez sur <0>{{text}}</0> pour voir les détails.",
     "closed-this-task": "fermer cette tâche",
     "collaborate-with-other-user": "pour collaborer avec d'autres utilisateurs.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "Il s'agit d'une configuration optionnelle permettant de permettre la suppression douce des tables. Lorsque cette option est activée, seules les tables supprimées de la source seront supprimées en douceur, et cela s'appliquera uniquement au schéma actuellement ingéré via le pipeline. Toute entité associée telle que les suites de tests ou les informations de lignée qui étaient associées à ces tables sera également supprimée.",
     "markdown-editor-placeholder": "Utilisez @mention pour identifier un utilisateur ou une équipe.\nUtilisez #mention pour identifier un actif de données.",
     "marketplace-verify-msg": "OpenMetadata has verified that the publisher controls the domain and meets other requirements.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "La valeur maximum doit être plus grande que la valeur minimum.",
     "member-description": "Gérez les accès des utilisateurs et équipes à OpenMetadata.",
     "mentioned-you-on-the-lowercase": "vous a mentionné dans",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "La programmation peut être configurée à une cadence horaire, quotidienne ou hebdomadaire. Le fuseau horaire est en UTC.",
     "scheduled-run-every": "Programmer pour être exécuté tous les",
     "scopes-comma-separated": "Liste de scopes séparée par une virgule.",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "Rechercher Pipeline, Procédures Stockées",
     "search-for-entity-types": "Rechercher Tables, Topics, Tableaux de Bord, Pipelines et Modèles d'IA",
     "search-for-ingestion": "Rechercher une ingestion",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json
index dc5a895f3f5b..7e6b4580c659 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/he-he.json
@@ -247,6 +247,7 @@
     "custom-theme": "Custom Theme",
     "customise": "התאמה אישית",
     "customize-entity": "התאם אישית את {{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "גרף מופע",
     "dag-view": "תצוגת גרף מופע",
     "daily-active-users-on-the-platform": "משתמשים פעילים יומיים בפלטפורמה",
@@ -821,6 +822,7 @@
     "owner": "אחראי נתונים",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "אחראי נתונים",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "אחראי נתונים",
     "page-not-found": "הדף לא נמצא",
     "page-views-by-data-asset-plural": "צפיות בדפים לפי נכסי נתונים",
@@ -989,6 +991,7 @@
     "role-name": "שם התפקיד",
     "role-plural": "תפקידים",
     "row": "שורה",
+    "row-count": "Row Count",
     "row-count-lowercase": "מספר שורות",
     "row-plural": "שורות",
     "rtl-ltr-direction": "RTL/LTR direction",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "לא ניתן להוסיף את הווידג'ט לסקשן זה עקב הגבלות בגודל.",
     "can-you-add-a-description": "האם תוכל להוסיף תיאור?",
     "checkout-service-connectors-doc": "ישנם חיבורים רבים כאן זמינים לאינדקסציה של נתונים מהשירותים שלך. בדוק את החיבורים שלנו.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "לחץ על <0>{{text}}</0> לצפייה בפרטים.",
     "closed-this-task": "סגר משימה זו",
     "collaborate-with-other-user": "לשתף פעולה עם משתמשים אחרים.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "זו הגדרת אופציונלית לאפשר מחיקה רכה של טבלאות. כאשר האפשרות מופעלת, רק טבלאות שנמחקו ממקור הנתונים יימחקו רכה, וזה יחול באופן בלעדי על הסכמה הנמצאת כעת בתהליך קליטת הנתונים דרך הצנרת נתונים. יימחקו גם יישויות קשורות כמו סוויטי ניסויים או מידע של שקפיות המשוייכים לטבלאות אלו.",
     "markdown-editor-placeholder": "השתמש ב-@תייג לתייג משתמש או צוות.\nהשתמש ב-#תייג לתייג נכס נתונים.",
     "marketplace-verify-msg": "OpenMetadata אישרה שהמוציא לפועל שולט בדומיין ועומד בדרישות האחרות.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "The maximum value should be greater than the minimum value.",
     "member-description": "Streamline access to users and teams in OpenMetadata.",
     "mentioned-you-on-the-lowercase": "ציין אותך ב",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "ניתן להגדיר שימוש כל שעה, יומית או שבועית. אזור הזמן הוא UTC.",
     "scheduled-run-every": "מתוזמן לרוץ כל",
     "scopes-comma-separated": "הוסף את ערכי הניקוד, מופרדים בפסיקים",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "חיפוש לפי תהליך טעינה, פונקציות מאוחסנות",
     "search-for-entity-types": "חיפוש לפי טבלאות, נושאים, דשבורדים, תהליכי טעינה, מודלי למידת מכונה, מילוני מונחים ותגיות",
     "search-for-ingestion": "חיפוש תהליך טעינה (Ingestion)",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
index fb40004a74e9..b810ae443db0 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ja-jp.json
@@ -247,6 +247,7 @@
     "custom-theme": "Custom Theme",
     "customise": "Customise",
     "customize-entity": "Customize {{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "Dag",
     "dag-view": "DAGビュー",
     "daily-active-users-on-the-platform": "このプラットフォームのアクティブなユーザー",
@@ -821,6 +822,7 @@
     "owner": "所有者",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "所有者",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "所有者",
     "page-not-found": "ページが存在しません",
     "page-views-by-data-asset-plural": "データアセットごとのページビュー",
@@ -989,6 +991,7 @@
     "role-name": "ロール名",
     "role-plural": "ロール",
     "row": "行",
+    "row-count": "Row Count",
     "row-count-lowercase": "行数",
     "row-plural": "行",
     "rtl-ltr-direction": "RTL/LTR direction",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "Can not add the widget to this section due to size restrictions.",
     "can-you-add-a-description": "説明を追加できますか?",
     "checkout-service-connectors-doc": "There are a lot of connectors available here to index data from your services. Please checkout our connectors.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "Click <0>{{text}}</0> to view details.",
     "closed-this-task": "このタスクを閉じました",
     "collaborate-with-other-user": "to collaborate with other users.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "This is an optional configuration for enabling soft deletion of tables. When this option is enabled, only tables that have been deleted from the source will be soft deleted, and this will apply solely to the schema that is currently being ingested via the pipeline. Any related entities such as test suites or lineage information that were associated with those tables will also be deleted.",
     "markdown-editor-placeholder": "@mention を使ってユーザやチームをタグ付けしましょう。\n#mentionを使うとデータアセットにタグ付けできます。",
     "marketplace-verify-msg": "OpenMetadata has verified that the publisher controls the domain and meets other requirements.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "The maximum value should be greater than the minimum value.",
     "member-description": "Streamline access to users and teams in OpenMetadata.",
     "mentioned-you-on-the-lowercase": "mentioned you on the",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "Scheduling can be set up at an hourly, daily, or weekly cadence. The timezone is in UTC.",
     "scheduled-run-every": "Scheduled to run every",
     "scopes-comma-separated": "スコープの値をカンマで区切って追加",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "Search for Pipeline, StoredProcedures",
     "search-for-entity-types": "テーブル、トピック、ダッシュボード、パイプライン、MLモデルの検索。",
     "search-for-ingestion": "Search for ingestion",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json
index 2e351da44d6d..9a8df974dc79 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/nl-nl.json
@@ -247,6 +247,7 @@
     "custom-theme": "Custom Theme",
     "customise": "Aanpassen",
     "customize-entity": "{{entity}} aanpassen",
+    "customize-ui": "Customize UI",
     "dag": "Dag",
     "dag-view": "DAG-weergave",
     "daily-active-users-on-the-platform": "Dagelijks actieve gebruikers op het platform",
@@ -821,6 +822,7 @@
     "owner": "Eigenaar",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "eigenaar",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "Eigenaren",
     "page-not-found": "Pagina niet gevonden",
     "page-views-by-data-asset-plural": "Paginaweergaven per data-asset",
@@ -989,6 +991,7 @@
     "role-name": "Rolnaam",
     "role-plural": "Rollen",
     "row": "Rij",
+    "row-count": "Row Count",
     "row-count-lowercase": "aantal rijen",
     "row-plural": "Rijen",
     "rtl-ltr-direction": "RTL/LTR-richting",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "Kan de widget niet toevoegen aan deze sectie vanwege beperkingen in grootte.",
     "can-you-add-a-description": "Kunt u een beschrijving toevoegen?",
     "checkout-service-connectors-doc": "Er zijn hier veel connectors beschikbaar om data van uw services te indexeren. Bekijk onze connectors.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "Klik op <0>{{text}}</0> om details te bekijken.",
     "closed-this-task": "heeft deze taak gesloten",
     "collaborate-with-other-user": "om samen te werken met andere gebruikers.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "Dit is een optionele configuratie voor het inschakelen van zachte verwijdering van tabellen. Wanneer deze optie is ingeschakeld, worden alleen tabellen die zijn verwijderd uit de bron zacht verwijderd, en dit is uitsluitend van toepassing op het schema dat momenteel wordt ingenomen via de pipeline. Elke gerelateerde entiteit, zoals testsuites of lineage-informatie, die met die tabellen was geassocieerd, wordt ook verwijderd.",
     "markdown-editor-placeholder": "Gebruik @mention om een gebruiker of een team te taggen.\nGebruik #mention om een data-asset te taggen.",
     "marketplace-verify-msg": "OpenMetadata heeft geverifieerd dat de publisher het domein beheert en aan andere eisen voldoet.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "The maximum value should be greater than the minimum value.",
     "member-description": "Stroomlijn toegang voor gebruikers en teams in OpenMetadata.",
     "mentioned-you-on-the-lowercase": "noemde je op de",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "Planning kan worden ingesteld op een uurlijkse, dagelijkse of wekelijkse cadans. De tijdzone is in UTC.",
     "scheduled-run-every": "Gepland om elke keer uit te voeren",
     "scopes-comma-separated": "Voeg de herkomstwaarde toe, gescheiden door komma's",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "Zoeken naar Pipeline, Stored Procedures",
     "search-for-entity-types": "Zoeken naar Tabellen, Onderwerpen, Dashboards, Pipelines, ML-modellen, Woordenlijst en Tags.",
     "search-for-ingestion": "Zoeken naar ingestie",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json
index d99729cb5dba..219d7edd97d7 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pr-pr.json
@@ -247,6 +247,7 @@
     "custom-theme": "تم سفارشی",
     "customise": "سفارشی‌سازی",
     "customize-entity": "سفارشی‌سازی {{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "دگ",
     "dag-view": "نمای دگ",
     "daily-active-users-on-the-platform": "کاربران فعال روزانه در پلتفرم",
@@ -821,6 +822,7 @@
     "owner": "مالک",
     "owner-kpi": "KPI مالک",
     "owner-lowercase": "مالک",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "مالکان",
     "page-not-found": "صفحه یافت نشد",
     "page-views-by-data-asset-plural": "بازدیدهای صفحه توسط دارایی‌های داده",
@@ -989,6 +991,7 @@
     "role-name": "نام نقش",
     "role-plural": "نقش‌ها",
     "row": "ردیف",
+    "row-count": "Row Count",
     "row-count-lowercase": "تعداد ردیف",
     "row-plural": "ردیف‌ها",
     "rtl-ltr-direction": "جهت RTL/LTR",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "به دلیل محدودیت‌های اندازه، نمی‌توان ویجت را به این بخش اضافه کرد.",
     "can-you-add-a-description": "آیا می‌توانید توضیحی اضافه کنید؟",
     "checkout-service-connectors-doc": "اینجا تعدادی از اتصالات برای ایندکس کردن داده‌ها از خدمات خود موجود است. لطفاً مستندات ما را بررسی کنید.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "برای مشاهده جزئیات، روی <0>{{text}}</0> کلیک کنید.",
     "closed-this-task": "این وظیفه را بست.",
     "collaborate-with-other-user": "برای همکاری با کاربران دیگر.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "این یک پیکربندی اختیاری برای فعال کردن حذف نرم جدول‌ها است. زمانی که این گزینه فعال شود، تنها جدول‌هایی که از منبع حذف شده‌اند، به‌طور نرم حذف می‌شوند و این امر تنها برای طرحی که در حال حاضر از طریق لوله داده استخراج می‌شود اعمال خواهد شد. هرگونه موجودیت مرتبط مانند مجموعه تست‌ها یا اطلاعات lineage که با آن جدول‌ها مرتبط بودند نیز حذف خواهند شد.",
     "markdown-editor-placeholder": "از @mention برای تگ کردن یک کاربر یا تیم استفاده کنید.\nاز #mention برای تگ کردن یک دارایی داده استفاده کنید.",
     "marketplace-verify-msg": "OpenMetadata تأیید کرده که ناشر کنترل دامنه را دارد و سایر الزامات را برآورده کرده است.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "مقدار حداکثر باید بزرگتر از مقدار حداقل باشد.",
     "member-description": "دسترسی به کاربران و تیم‌ها را در OpenMetadata ساده‌سازی کنید.",
     "mentioned-you-on-the-lowercase": "شما را در {{entity}} ذکر کرده است.",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "برنامه‌ریزی می‌تواند به صورت ساعتی، روزانه یا هفتگی تنظیم شود. منطقه زمانی UTC است.",
     "scheduled-run-every": "برنامه‌ریزی شده برای اجرا هر",
     "scopes-comma-separated": "مقدار Scopes را با کاما جدا کنید.",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "جستجوی خطوط لوله، StoredProcedures.",
     "search-for-entity-types": "جستجوی جداول، موضوعات، داشبوردها، خطوط لوله، مدل‌های ML، واژه‌نامه و برچسب‌ها.",
     "search-for-ingestion": "جستجوی ورود داده.",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
index 452661ee96ff..d10839ef589b 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/pt-br.json
@@ -247,6 +247,7 @@
     "custom-theme": "Custom Theme",
     "customise": "Personalizar",
     "customize-entity": "Personalizar {{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "DAG",
     "dag-view": "Visualização DAG",
     "daily-active-users-on-the-platform": "Usuários Ativos Diários na Plataforma",
@@ -821,6 +822,7 @@
     "owner": "Proprietário",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "proprietário",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "Proprietários",
     "page-not-found": "Página Não Encontrada",
     "page-views-by-data-asset-plural": "Visualizações de Página por Ativos de Dados",
@@ -989,6 +991,7 @@
     "role-name": "Nome da Função",
     "role-plural": "Funções",
     "row": "Linha",
+    "row-count": "Row Count",
     "row-count-lowercase": "contagem de linhas",
     "row-plural": "Linhas",
     "rtl-ltr-direction": "Direção RTL/LTR",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "Não é possível adicionar o widget a esta seção devido a restrições de tamanho.",
     "can-you-add-a-description": "Você pode adicionar uma descrição?",
     "checkout-service-connectors-doc": "Há muitos conectores disponíveis aqui para indexar dados de seus serviços. Por favor, confira nossos conectores.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "Clique em <0>{{text}}</0> para ver detalhes.",
     "closed-this-task": "fechou esta tarefa",
     "collaborate-with-other-user": "para colaborar com outros usuários.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "Esta é uma configuração opcional para habilitar a exclusão suave de tabelas. Quando esta opção está habilitada, apenas as tabelas que foram excluídas da fonte serão excluídas suavemente, e isso se aplicará somente ao esquema que está sendo ingerido atualmente pelo pipeline. Quaisquer entidades relacionadas, como conjuntos de teste ou informações de linhagem que estavam associadas a essas tabelas, também serão excluídas.",
     "markdown-editor-placeholder": "Use @mention para marcar um usuário ou uma equipe.\nUse #mention para marcar um ativo de dados.",
     "marketplace-verify-msg": "O OpenMetadata verificou que o editor controla o domínio e atende a outros requisitos.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "The maximum value should be greater than the minimum value.",
     "member-description": "Streamline access to users and teams in OpenMetadata.",
     "mentioned-you-on-the-lowercase": "mencionou você no",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "O agendamento pode ser configurado em uma frequência horária, diária ou semanal. O fuso horário é em UTC.",
     "scheduled-run-every": "Agendado para rodar a cada",
     "scopes-comma-separated": "Adicione o valor dos Escopos, separados por vírgulas",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "Pesquisar por Pipeline, StoredProcedures",
     "search-for-entity-types": "Pesquisar por Tabelas, Tópicos, Painéis, Pipelines, Modelos de ML, Glossário e Tags.",
     "search-for-ingestion": "Pesquisar por ingestão",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json
index 412f2d6674b4..c057687dc212 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/ru-ru.json
@@ -247,6 +247,7 @@
     "custom-theme": "Custom Theme",
     "customise": "Customise",
     "customize-entity": "Customize {{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "Dag",
     "dag-view": "Просмотр DAG",
     "daily-active-users-on-the-platform": "Количество активных пользователей на платформе",
@@ -821,6 +822,7 @@
     "owner": "Владелец",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "владелец",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "Владельцы",
     "page-not-found": "Страница не найдена",
     "page-views-by-data-asset-plural": "Просмотры объектов данных",
@@ -989,6 +991,7 @@
     "role-name": "Наименование роли",
     "role-plural": "Роли",
     "row": "Строка",
+    "row-count": "Row Count",
     "row-count-lowercase": "количество строк",
     "row-plural": "Строки",
     "rtl-ltr-direction": "RTL/LTR direction",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "Can not add the widget to this section due to size restrictions.",
     "can-you-add-a-description": "Можете добавить описание?",
     "checkout-service-connectors-doc": "Здесь доступно множество коннекторов для индексации данных из ваших сервисов. Пожалуйста, ознакомьтесь с нашими коннекторами.",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "Нажмите <0>{{text}}</0>, чтобы просмотреть подробности.",
     "closed-this-task": "задача закрыта",
     "collaborate-with-other-user": "для совместной работы с другими пользователями.",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "Это необязательная конфигурация для включения обратимого удаления таблиц. Если этот параметр включен, обратимому удалению подлежат только таблицы, которые были удалены из источника, и это будет применяться исключительно к схеме, которая в данный момент загружается через конвейер. Любые связанные объекты, такие как наборы тестов или информация о происхождении, которые были связаны с этими таблицами, также будут удалены.",
     "markdown-editor-placeholder": "Используйте @упоминание, чтобы отметить пользователя или команду.\nИспользуйте #упоминание, чтобы пометить объект данных.",
     "marketplace-verify-msg": "OpenMetadata has verified that the publisher controls the domain and meets other requirements.",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "The maximum value should be greater than the minimum value.",
     "member-description": "Streamline access to users and teams in OpenMetadata.",
     "mentioned-you-on-the-lowercase": "упоминул вас в",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "Планирование может быть настроено на почасовой, ежедневной или еженедельной частоте. Часовой пояс указан в формате UTC.",
     "scheduled-run-every": "Запланирован запуск каждый",
     "scopes-comma-separated": "Добавьте значение областей, разделенное запятыми",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "Search for Pipeline, StoredProcedures",
     "search-for-entity-types": "Поиск по таблицам, топикам, дашбордам, пайплайнам, моделям машинного обучения, глоссарию и тегам.",
     "search-for-ingestion": "Поиск для получения",
diff --git a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
index 6b0c7df0f7f3..b3d2e527c351 100644
--- a/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
+++ b/openmetadata-ui/src/main/resources/ui/src/locale/languages/zh-cn.json
@@ -247,6 +247,7 @@
     "custom-theme": "自定义主题",
     "customise": "自定义",
     "customize-entity": "自定义{{entity}}",
+    "customize-ui": "Customize UI",
     "dag": "DAG",
     "dag-view": "DAG 视图",
     "daily-active-users-on-the-platform": "平台上的每日活跃用户",
@@ -821,6 +822,7 @@
     "owner": "所有者",
     "owner-kpi": "Owner KPI",
     "owner-lowercase": "所有者",
+    "owner-lowercase-plural": "owners",
     "owner-plural": "所有者",
     "page-not-found": "没有找到页面",
     "page-views-by-data-asset-plural": "数据资产页面浏览量",
@@ -989,6 +991,7 @@
     "role-name": "角色名",
     "role-plural": "角色",
     "row": "行",
+    "row-count": "Row Count",
     "row-count-lowercase": "行计数",
     "row-plural": "行",
     "rtl-ltr-direction": "RTL/LTR direction",
@@ -1390,6 +1393,7 @@
     "can-not-add-widget": "Can not add the widget to this section due to size restrictions.",
     "can-you-add-a-description": "您能添加一些描述吗？",
     "checkout-service-connectors-doc": "我们已经预置了很多连接器, 可用于索引您的各项服务数据",
+    "click-here-to-view-assets-on-explore": "(Click to view the filtered assets on Explore page.)",
     "click-text-to-view-details": "点击<0>{{text}}</0>查看详情",
     "closed-this-task": "关闭了此任务",
     "collaborate-with-other-user": "与其他用户协作",
@@ -1599,6 +1603,7 @@
     "mark-deleted-table-message": "可选配置, 用于启用数据表的软删除。只有在数据源中被删除的表才会被软删除, 而且仅会作用到当前被工作流提取进来的 Schema(模式)。所有这些数据表的关联实体都将被删除, 例如测试集和血缘信息等。",
     "markdown-editor-placeholder": "使用 @提及 标记用户或团队。\n使用 #提及 标记数据资产",
     "marketplace-verify-msg": "OpenMetadata 已核实发布者控制该域并符合其他要求",
+    "maximum-count-allowed": "Maximum {{count}} {{label}} are allowed",
     "maximum-value-error": "最大值应大于最小值",
     "member-description": "简化对 OpenMetadata 用户和团队的访问",
     "mentioned-you-on-the-lowercase": "在以下地方提到了您",
@@ -1778,6 +1783,7 @@
     "schedule-for-ingestion-description": "可设置每小时、每天或每周的计划, 时区为 UTC",
     "scheduled-run-every": "计划每次运行",
     "scopes-comma-separated": "范围值逗号分隔",
+    "search-entity-count": "{{count}} assets have been found with this filter.",
     "search-for-edge": "搜索工作流、存储过程",
     "search-for-entity-types": "搜索数据表、消息主题、仪表板、工作流和机械学习模型",
     "search-for-ingestion": "搜索提取作业",
diff --git a/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts b/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts
index 1aa529cd30d4..848c75d4b19f 100644
--- a/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts
@@ -123,8 +123,8 @@ export const MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_OBJECT: ExtensionDat
     stringCp: 'select * from table where id="23";;',
     timeCp: '03:04:06',
     timerIntervalCp: {
-      end: '1727532820197',
-      start: '1727532807278',
+      end: 1727532820197,
+      start: 1727532807278,
     },
     timeStampCp: '1727532807278',
   };
diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.tsx
index dca8ec058a79..0c2454b77a2a 100644
--- a/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/KPIList.tsx
@@ -234,6 +234,7 @@ const KPIList = () => {
       <Col span={24}>
         <Table
           bordered
+          className="kpi-table"
           columns={columns}
           data-testid="kpi-table"
           dataSource={kpiList}
diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryPage/GlossaryPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryPage/GlossaryPage.component.tsx
index 427311974d1a..ab2d3592eb7a 100644
--- a/openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryPage/GlossaryPage.component.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/Glossary/GlossaryPage/GlossaryPage.component.tsx
@@ -280,10 +280,7 @@ const GlossaryPage = () => {
         return;
       }
       try {
-        const response = await patchGlossaryTerm(
-          activeGlossary?.id as string,
-          jsonPatch
-        );
+        const response = await patchGlossaryTerm(activeGlossary?.id, jsonPatch);
         if (response) {
           setActiveGlossary(response as ModifiedGlossary);
           if (activeGlossary?.name !== updatedData.name) {
diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/MetricsPage/MetricListPage/MetricListPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/MetricsPage/MetricListPage/MetricListPage.tsx
index 0eda60d862fd..5c2826d26f88 100644
--- a/openmetadata-ui/src/main/resources/ui/src/pages/MetricsPage/MetricListPage/MetricListPage.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/MetricsPage/MetricListPage/MetricListPage.tsx
@@ -214,7 +214,7 @@ const MetricListPage = () => {
         ),
       },
       {
-        title: t('label.owner'),
+        title: t('label.owner-plural'),
         dataIndex: 'owners',
         key: 'owners',
         width: 200,
diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.tsx
index 9ccda19b8761..a72d544d4dbd 100644
--- a/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaDetailsPage/PersonaDetailsPage.tsx
@@ -18,6 +18,7 @@ import { isUndefined } from 'lodash';
 import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useHistory } from 'react-router-dom';
+import { ReactComponent as CustomDashboardLogoIcon } from '../../../assets/svg/customize-landing-page-colored.svg';
 import { ReactComponent as IconPersona } from '../../../assets/svg/ic-personas.svg';
 import DescriptionV1 from '../../../components/common/EntityDescription/DescriptionV1';
 import ManageButton from '../../../components/common/EntityPageInfos/ManageButton/ManageButton';
@@ -28,6 +29,7 @@ import { UserSelectableList } from '../../../components/common/UserSelectableLis
 import EntityHeaderTitle from '../../../components/Entity/EntityHeaderTitle/EntityHeaderTitle.component';
 import { EntityName } from '../../../components/Modals/EntityNameModal/EntityNameModal.interface';
 import PageLayoutV1 from '../../../components/PageLayoutV1/PageLayoutV1';
+import SettingItemCard from '../../../components/Settings/SettingItemCard/SettingItemCard.component';
 import { UsersTab } from '../../../components/Settings/Users/UsersTab/UsersTabs.component';
 import {
   GlobalSettingOptions,
@@ -38,9 +40,11 @@ import { ResourceEntity } from '../../../context/PermissionProvider/PermissionPr
 import { SIZE } from '../../../enums/common.enum';
 import { EntityType } from '../../../enums/entity.enum';
 import { Persona } from '../../../generated/entity/teams/persona';
+import { PageType } from '../../../generated/system/ui/page';
 import { useFqn } from '../../../hooks/useFqn';
 import { getPersonaByName, updatePersona } from '../../../rest/PersonaAPI';
 import { getEntityName } from '../../../utils/EntityUtils';
+import { getCustomizePagePath } from '../../../utils/GlobalSettingsUtils';
 import { DEFAULT_ENTITY_PERMISSION } from '../../../utils/PermissionsUtils';
 import { getSettingPath } from '../../../utils/RouterUtils';
 import { showErrorToast } from '../../../utils/ToastUtils';
@@ -172,6 +176,15 @@ export const PersonaDetailsPage = () => {
     );
   };
 
+  const handleCustomizeItemClick = (category: string) => {
+    history.push(
+      getCustomizePagePath(
+        personaDetails?.fullyQualifiedName as string,
+        category as PageType
+      )
+    );
+  };
+
   if (isLoading) {
     return <Loader />;
   }
@@ -232,7 +245,7 @@ export const PersonaDetailsPage = () => {
             defaultActiveKey="users"
             items={[
               {
-                label: 'Users',
+                label: t('label.user-plural'),
                 key: 'users',
                 children: (
                   <UsersTab
@@ -241,6 +254,29 @@ export const PersonaDetailsPage = () => {
                   />
                 ),
               },
+              {
+                label: t('label.customize-ui'),
+                key: 'customize-ui',
+                children: (
+                  <Row gutter={[16, 16]}>
+                    <Col span={8}>
+                      <SettingItemCard
+                        data={{
+                          label: t('label.customize-entity', {
+                            entity: t('label.landing-page'),
+                          }),
+                          description: t(
+                            'message.page-sub-header-for-customize-landing-page'
+                          ),
+                          key: PageType.LandingPage,
+                          icon: CustomDashboardLogoIcon,
+                        }}
+                        onClick={handleCustomizeItemClick}
+                      />
+                    </Col>
+                  </Row>
+                ),
+              },
             ]}
             tabBarExtraContent={
               <UserSelectableList
diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaListPage/PersonaPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaListPage/PersonaPage.tsx
index 37eb446b4763..dedd5bbaa63f 100644
--- a/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaListPage/PersonaPage.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/Persona/PersonaListPage/PersonaPage.tsx
@@ -55,11 +55,7 @@ export const PersonaPage = () => {
   } = usePaging();
 
   const breadcrumbs: TitleBreadcrumbProps['titleLinks'] = useMemo(
-    () =>
-      getSettingPageEntityBreadCrumb(
-        GlobalSettingsMenuCategory.MEMBERS,
-        t('label.persona-plural')
-      ),
+    () => getSettingPageEntityBreadCrumb(GlobalSettingsMenuCategory.PERSONA),
     []
   );
 
diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ServiceDetailsPage/ServiceDetailsPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ServiceDetailsPage/ServiceDetailsPage.tsx
index d1c90a7be1ae..ef9e3e98bd41 100644
--- a/openmetadata-ui/src/main/resources/ui/src/pages/ServiceDetailsPage/ServiceDetailsPage.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/ServiceDetailsPage/ServiceDetailsPage.tsx
@@ -668,7 +668,7 @@ const ServiceDetailsPage: FunctionComponent = () => {
         showErrorToast(
           error as AxiosError,
           t('server.entity-updating-error', {
-            entity: t('label.owner-lowercase'),
+            entity: t('label.owner-lowercase-plural'),
           })
         );
       }
@@ -1021,22 +1021,21 @@ const ServiceDetailsPage: FunctionComponent = () => {
       });
     }
 
-    if (serviceCategory !== ServiceCategory.API_SERVICES) {
-      tabs.push({
+    tabs.push(
+      {
         name: t('label.ingestion-plural'),
         key: EntityTabs.INGESTIONS,
         isHidden: !showIngestionTab,
         count: ingestionPaging.total,
         children: ingestionTab,
-      });
-    }
-
-    tabs.push({
-      name: t('label.connection'),
-      isHidden: !servicePermission.EditAll,
-      key: EntityTabs.CONNECTION,
-      children: testConnectionTab,
-    });
+      },
+      {
+        name: t('label.connection'),
+        isHidden: !servicePermission.EditAll,
+        key: EntityTabs.CONNECTION,
+        children: testConnectionTab,
+      }
+    );
 
     return tabs
       .filter((tab) => !tab.isHidden)
diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/ServicesPage/ServicesPage.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/ServicesPage/ServicesPage.tsx
index 6eaf765c4cc4..7b123a14f937 100644
--- a/openmetadata-ui/src/main/resources/ui/src/pages/ServicesPage/ServicesPage.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/pages/ServicesPage/ServicesPage.tsx
@@ -105,8 +105,7 @@ const ServicesPage = () => {
                       label: 'Services',
                     },
                   ]),
-              // pipelines are not supported for apiServices so don't show pipelines tab for apiServices
-              ...(isAdminUser && serviceName !== 'apiServices'
+              ...(isAdminUser
                 ? [
                     {
                       key: 'pipelines',
diff --git a/openmetadata-ui/src/main/resources/ui/src/styles/app.less b/openmetadata-ui/src/main/resources/ui/src/styles/app.less
index 5a327fe1f3fc..ddf1c6944f16 100644
--- a/openmetadata-ui/src/main/resources/ui/src/styles/app.less
+++ b/openmetadata-ui/src/main/resources/ui/src/styles/app.less
@@ -827,7 +827,10 @@ a[href].link-text-grey,
   .activity-feed-editor-drawer,
   .policies-list-table,
   .roles-list-table,
-  .list-table {
+  .list-table,
+  .kpi-table,
+  .custom-properties-card,
+  .entity-custom-properties-table {
     margin-right: 64px;
   }
 }
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/APIServiceUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/APIServiceUtils.ts
index fe256ab5300a..9e6a2b6b2067 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/APIServiceUtils.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/APIServiceUtils.ts
@@ -13,14 +13,14 @@
 import { cloneDeep } from 'lodash';
 import { COMMON_UI_SCHEMA } from '../constants/Services.constant';
 import { APIServiceType } from '../generated/entity/services/apiService';
-import RESTConnection from '../jsons/connectionSchemas/connections/apiService/restConnection.json';
+import restConnection from '../jsons/connectionSchemas/connections/api/restConnection.json';
 
 export const getAPIConfig = (type: APIServiceType) => {
   let schema = {};
   const uiSchema = { ...COMMON_UI_SCHEMA };
   switch (type) {
     case APIServiceType.REST:
-      schema = RESTConnection;
+      schema = restConnection;
 
       break;
 
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts
index c7c57bc8e472..5a53ea1eccf3 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/AdvancedSearchClassBase.ts
@@ -471,7 +471,7 @@ class AdvancedSearchClassBase {
       },
 
       [EntityFields.OWNERS]: {
-        label: t('label.owner'),
+        label: t('label.owner-plural'),
         type: 'select',
         mainWidgetProps: this.mainWidgetProps,
 
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.test.tsx
index 585c30eb1c04..e234caa4822f 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.test.tsx
@@ -61,7 +61,9 @@ describe('AlertsUtil tests', () => {
   });
 
   it('getFunctionDisplayName should return correct text for matchAnyOwnerName', () => {
-    expect(getFunctionDisplayName('matchAnyOwnerName')).toBe('label.owner');
+    expect(getFunctionDisplayName('matchAnyOwnerName')).toBe(
+      'label.owner-plural'
+    );
   });
 
   it('getFunctionDisplayName should return correct text for matchAnyEventType', () => {
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.tsx
index a2a92f0dbddc..b4eb7f883ae2 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/Alerts/AlertsUtil.tsx
@@ -83,7 +83,7 @@ export const getFunctionDisplayName = (func: string): string => {
     case 'matchAnyEntityFqn':
       return i18next.t('label.fqn-uppercase');
     case 'matchAnyOwnerName':
-      return i18next.t('label.owner');
+      return i18next.t('label.owner-plural');
     case 'matchAnyEventType':
       return i18next.t('label.event-type');
     case 'matchTestResult':
@@ -477,7 +477,7 @@ export const getMessageFromArgumentName = (argumentName: string) => {
       return t('message.field-text-is-required', {
         fieldText: t('label.entity-list', {
           entity: t('label.entity-name', {
-            entity: t('label.owner'),
+            entity: t('label.owner-plural'),
           }),
         }),
       });
@@ -614,7 +614,7 @@ export const getFieldByArgumentType = (
           data-testid="owner-name-select"
           mode="multiple"
           placeholder={t('label.search-by-type', {
-            type: t('label.owner-lowercase'),
+            type: t('label.owner-lowercase-plural'),
           })}
         />
       );
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
index 69e218615f5a..928d0c1a2707 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
@@ -59,6 +59,7 @@
           "storedProcedure",
           "dataProduct",
           "testCaseResolutionStatus",
+          "testCaseResult",
           "apiService",
           "apiEndpoint",
           "apiCollection",
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx
index 4bb7a40a4373..fd24dcee802e 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx
@@ -10,6 +10,7 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
+import { ExtensionDataProps } from '../../components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface';
 import { EntityType } from '../../enums/entity.enum';
 import {
   MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES,
@@ -163,5 +164,14 @@ describe('CSVUtils', () => {
         MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_CONVERTED_EXTENSION_CSV_STRING
       );
     });
+
+    it('should return string correctly which contains undefined as value for property', () => {
+      const convertedCSVEntities = convertEntityExtensionToCustomPropertyString(
+        { dateCp: undefined } as unknown as ExtensionDataProps,
+        MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES
+      );
+
+      expect(convertedCSVEntities).toStrictEqual(`dateCp:undefined`);
+    });
   });
 });
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx
index 28d4d1bc9f81..c6f23d1d1987 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx
@@ -11,23 +11,30 @@
  *  limitations under the License.
  */
 import { TypeColumn } from '@inovua/reactdatagrid-community/types';
-import { compact, get, isEmpty, isUndefined, startCase } from 'lodash';
+import {
+  compact,
+  get,
+  isEmpty,
+  isString,
+  isUndefined,
+  startCase,
+} from 'lodash';
 import React from 'react';
 import { ReactComponent as SuccessBadgeIcon } from '../..//assets/svg/success-badge.svg';
 import { ReactComponent as FailBadgeIcon } from '../../assets/svg/fail-badge.svg';
+import { TableTypePropertyValueType } from '../../components/common/CustomPropertyTable/CustomPropertyTable.interface';
 import {
   ExtensionDataProps,
   ExtensionDataTypes,
 } from '../../components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface';
-import { NO_DATA_PLACEHOLDER } from '../../constants/constants';
+import { TABLE_TYPE_CUSTOM_PROPERTY } from '../../constants/CustomProperty.constants';
 import { SEMICOLON_SPLITTER } from '../../constants/regex.constants';
 import { EntityType } from '../../enums/entity.enum';
 import {
+  Config,
   CustomProperty,
   EntityReference,
-  EnumConfig,
   Type,
-  ValueClass,
 } from '../../generated/entity/type';
 import { Status } from '../../generated/type/csvImportResult';
 import { removeOuterEscapes } from '../CommonUtils';
@@ -157,6 +164,12 @@ export const getCSVStringFromColumnsAndDataSource = (
   return [header, ...compact(rows)].join('\n');
 };
 
+/**
+ *
+ * @param value  The value of the custom property in string format
+ * @param customProperty The custom property object
+ * @returns  The value of the custom property in the correct type
+ */
 const convertCustomPropertyStringToValueExtensionBasedOnType = (
   value: string,
   customProperty?: CustomProperty
@@ -185,6 +198,7 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = (
         } as EntityReference;
       });
     }
+
     case 'enum': {
       if (value.includes('|')) {
         return value.split('|');
@@ -193,34 +207,60 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = (
       }
     }
 
-    case 'enumWithDescriptions': {
-      const propertyEnumValues =
-        ((customProperty?.customPropertyConfig?.config as EnumConfig)
-          .values as ValueClass[]) ?? [];
+    case 'timeInterval': {
+      const [start, end] = value.split(':');
 
-      const keyAndValue: Record<string, ValueClass> = {};
+      return {
+        start: Number(start),
+        end: Number(end),
+      };
+    }
 
-      propertyEnumValues.forEach((cp) => (keyAndValue[cp.key] = cp));
+    case TABLE_TYPE_CUSTOM_PROPERTY: {
+      // step 1: get the columns from the custom property config
+      const columns =
+        (customProperty?.customPropertyConfig?.config as Config)?.columns ?? [];
 
-      return value.split('|').map((item) => ({
-        key: item,
-        description: keyAndValue[item].description ?? NO_DATA_PLACEHOLDER,
-      }));
-    }
+      // step 2: split the value by row
+      const rowStringList = value.split('|');
 
-    case 'timeInterval': {
-      const [start, end] = value.split(':');
+      // step 3: convert the rowStringList into objects with column names as keys
+      const rows = rowStringList.map((row) => {
+        // Step 1: Replace commas inside double quotes with a placeholder
+        const preprocessedInput = row.replace(/"([^"]*)"/g, (_, p1) => {
+          return `${p1.replace(/,/g, '__COMMA__')}`;
+        });
 
+        // Step 2: Split the row by comma
+        const rowValues = preprocessedInput.split(',');
+
+        // create an object with column names as keys
+        return columns.reduce((acc: Record<string, string>, column, index) => {
+          // replace the placeholder with comma
+          acc[column] = rowValues[index].replaceAll('__COMMA__', ',');
+
+          return acc;
+        }, {} as Record<string, string>);
+      });
+
+      // return the columns and rows
       return {
-        start,
-        end,
+        columns: columns,
+        rows: rows,
       };
     }
+
     default:
       return value;
   }
 };
 
+/**
+ *
+ * @param value  The value of the custom property in object format
+ * @param customProperty The custom property object
+ * @returns The value of the custom property in string format
+ */
 const convertCustomPropertyValueExtensionToStringBasedOnType = (
   value: ExtensionDataTypes,
   customProperty: CustomProperty
@@ -246,17 +286,36 @@ const convertCustomPropertyValueExtensionToStringBasedOnType = (
     case 'enum':
       return (value as unknown as string[]).map((item) => item).join('|');
 
-    case 'enumWithDescriptions':
-      return (value as unknown as ValueClass[])
-        .map((item) => item.key)
-        .join('|');
-
     case 'timeInterval': {
       const interval = value as { start: string; end: string };
 
       return `${interval.start}:${interval.end}`;
     }
 
+    case TABLE_TYPE_CUSTOM_PROPERTY: {
+      const tableTypeValue = value as TableTypePropertyValueType;
+
+      // step 1: get the columns from the custom property config
+      const columns = tableTypeValue?.columns ?? [];
+
+      // step 2: get the rows from the value
+      const rows = tableTypeValue?.rows ?? [];
+
+      // step 3: convert the rows into a string
+      const rowStringList = rows.map((row) => {
+        return columns
+          .map((column) => {
+            const value = row[column] ?? '';
+
+            // if value contains comma, wrap it in quotes
+            return value.includes(',') ? `"${value}"` : value;
+          })
+          .join(',');
+      });
+
+      return `${rowStringList.join('|')}`;
+    }
+
     default:
       return value;
   }
@@ -270,29 +329,34 @@ export const convertCustomPropertyStringToEntityExtension = (
     return {};
   }
 
-  const keyAndValueTypes: Record<string, CustomProperty> = {};
-
-  const result: ExtensionDataProps = {};
+  // Step 1: Create a map of custom properties by name
+  const customPropertiesMapByName: Record<string, CustomProperty> = {};
 
   customPropertyType.customProperties?.forEach(
-    (cp) => (keyAndValueTypes[cp.name] = cp)
+    (cp) => (customPropertiesMapByName[cp.name] = cp)
   );
 
-  // Split the input into pairs using `;` and handle quoted strings properly
+  // Step 2: Split the input into pairs using `;` and handle quoted strings properly
   const pairs = value.split(SEMICOLON_SPLITTER);
 
+  // Step 3: Create a map of key-value pairs
+  const result: ExtensionDataProps = {};
+
+  // Step 4: Iterate over the pairs and convert them to key-value pairs
   pairs.forEach((pair) => {
     const cleanedText = removeOuterEscapes(pair);
 
-    const [key, ...valueParts] = cleanedText.split(':');
-    const value = valueParts.join(':').trim(); // Join back in case of multiple `:`
+    const [propertyName, ...propertyValueParts] = cleanedText.split(':');
+    const propertyValue = propertyValueParts.join(':').trim(); // Join back in case of multiple `:`
+
+    const trimmedPropertyName = propertyName.trim();
 
     // Clean up quotes if they are around the value
-    if (key && value) {
-      result[key.trim()] =
+    if (trimmedPropertyName && propertyValue) {
+      result[trimmedPropertyName] =
         convertCustomPropertyStringToValueExtensionBasedOnType(
-          value,
-          keyAndValueTypes[key]
+          propertyValue,
+          customPropertiesMapByName[trimmedPropertyName]
         );
     }
   });
@@ -300,6 +364,12 @@ export const convertCustomPropertyStringToEntityExtension = (
   return result;
 };
 
+/**
+ *
+ * @param value  The value of the custom property in object format
+ * @param customPropertyType  The custom property object
+ * @returns  The value of the custom property in string format
+ */
 export const convertEntityExtensionToCustomPropertyString = (
   value?: ExtensionDataProps,
   customPropertyType?: Type
@@ -308,37 +378,54 @@ export const convertEntityExtensionToCustomPropertyString = (
     return;
   }
 
-  const keyAndValueTypes: Record<string, CustomProperty> = {};
+  // Step 1: Create a map of custom properties by name
+  const customPropertiesMapByName: Record<string, CustomProperty> = {};
+
   customPropertyType?.customProperties?.forEach(
-    (cp) => (keyAndValueTypes[cp.name] = cp)
+    (cp) => (customPropertiesMapByName[cp.name] = cp)
   );
 
-  let convertedString = '';
-
+  // Step 2: Convert the object into an array of key-value pairs
   const objectArray = Object.entries(value ?? {});
 
+  // Step 3: Convert the key-value pairs into a string
+  let convertedString = '';
   objectArray.forEach(([key, value], index) => {
     const isLastElement = objectArray.length - 1 === index;
-    if (keyAndValueTypes[key]) {
+    // Check if the key exists in the custom properties map
+    if (customPropertiesMapByName[key]) {
+      // Convert the value to a string based on the type
       const stringValue =
         convertCustomPropertyValueExtensionToStringBasedOnType(
           value,
-          keyAndValueTypes[key]
+          customPropertiesMapByName[key]
         );
 
+      const endValue = isLastElement ? '' : ';';
+
+      const hasSeparator =
+        isString(stringValue) &&
+        (stringValue.includes(',') || stringValue.includes(';'));
+
+      // Check if the property type is markdown or sqlQuery or string and add quotes around the value
       if (
         ['markdown', 'sqlQuery', 'string'].includes(
-          keyAndValueTypes[key].propertyType.name ?? ''
-        )
+          customPropertiesMapByName[key]?.propertyType?.name ?? ''
+        ) &&
+        hasSeparator
       ) {
-        convertedString += `"${`${key}:${stringValue}`}"${
-          isLastElement ? '' : ';'
-        }`;
+        convertedString += `"${`${key}:${stringValue}`}"${endValue}`;
+      } else if (
+        // Check if the property type is table and add quotes around the value
+        customPropertiesMapByName[key]?.propertyType?.name ===
+        TABLE_TYPE_CUSTOM_PROPERTY
+      ) {
+        convertedString += `"${`${key}:${stringValue}`}"${endValue}`;
       } else {
-        convertedString += `${key}:${stringValue}${isLastElement ? '' : ';'}`;
+        convertedString += `${key}:${stringValue}${endValue}`;
       }
     }
   });
 
-  return convertedString;
+  return `${convertedString}`;
 };
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx
index ac20fe39f757..d22b25fa816b 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/CommonUtils.tsx
@@ -517,52 +517,51 @@ export const replaceAllSpacialCharWith_ = (text: string) => {
  * @param onDataFetched - callback function which return FeedCounts object
  */
 
-export const getFeedCounts = (
+export const getFeedCounts = async (
   entityType: string,
   entityFQN: string,
   feedCountCallback: (countValue: FeedCounts) => void
 ) => {
-  getFeedCount(getEntityFeedLink(entityType, entityFQN))
-    .then((res) => {
-      if (res) {
-        const {
-          conversationCount,
-          openTaskCount,
-          closedTaskCount,
-          totalTasksCount,
-          totalCount,
-          mentionCount,
-        } = res.reduce((acc, item) => {
-          const conversationCount =
-            acc.conversationCount + (item.conversationCount || 0);
-          const totalTasksCount =
-            acc.totalTasksCount + (item.totalTaskCount || 0);
-
-          return {
-            conversationCount,
-            totalTasksCount,
-            openTaskCount: acc.openTaskCount + (item.openTaskCount || 0),
-            closedTaskCount: acc.closedTaskCount + (item.closedTaskCount || 0),
-            totalCount: conversationCount + totalTasksCount,
-            mentionCount: acc.mentionCount + (item.mentionCount || 0),
-          };
-        }, FEED_COUNT_INITIAL_DATA);
-
-        feedCountCallback({
+  try {
+    const res = await getFeedCount(getEntityFeedLink(entityType, entityFQN));
+    if (res) {
+      const {
+        conversationCount,
+        openTaskCount,
+        closedTaskCount,
+        totalTasksCount,
+        totalCount,
+        mentionCount,
+      } = res.reduce((acc, item) => {
+        const conversationCount =
+          acc.conversationCount + (item.conversationCount || 0);
+        const totalTasksCount =
+          acc.totalTasksCount + (item.totalTaskCount || 0);
+
+        return {
           conversationCount,
           totalTasksCount,
-          openTaskCount,
-          closedTaskCount,
-          totalCount,
-          mentionCount,
-        });
-      } else {
-        throw t('server.entity-feed-fetch-error');
-      }
-    })
-    .catch((err: AxiosError) => {
-      showErrorToast(err, t('server.entity-feed-fetch-error'));
-    });
+          openTaskCount: acc.openTaskCount + (item.openTaskCount || 0),
+          closedTaskCount: acc.closedTaskCount + (item.closedTaskCount || 0),
+          totalCount: conversationCount + totalTasksCount,
+          mentionCount: acc.mentionCount + (item.mentionCount || 0),
+        };
+      }, FEED_COUNT_INITIAL_DATA);
+
+      feedCountCallback({
+        conversationCount,
+        totalTasksCount,
+        openTaskCount,
+        closedTaskCount,
+        totalCount,
+        mentionCount,
+      });
+    } else {
+      throw t('server.entity-feed-fetch-error');
+    }
+  } catch (err) {
+    showErrorToast(err as AxiosError, t('server.entity-feed-fetch-error'));
+  }
 };
 
 /**
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/DataAssetsHeader.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/DataAssetsHeader.utils.tsx
index 613ce2955c79..0907319428ad 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/DataAssetsHeader.utils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/DataAssetsHeader.utils.tsx
@@ -105,6 +105,7 @@ export const getDataAssetsHeaderInfo = (
         <>
           {dashboardDetails.sourceUrl && (
             <ExtraInfoLink
+              newTab
               href={dashboardDetails.sourceUrl}
               label=""
               value={getEntityName(dashboardDetails)}
@@ -147,6 +148,7 @@ export const getDataAssetsHeaderInfo = (
         <>
           {pipelineDetails.sourceUrl && (
             <ExtraInfoLink
+              newTab
               href={pipelineDetails.sourceUrl}
               label=""
               value={getEntityName(pipelineDetails)}
@@ -178,6 +180,7 @@ export const getDataAssetsHeaderInfo = (
           )}
           {mlModelDetail.server && (
             <ExtraInfoLink
+              newTab
               href={mlModelDetail.server}
               label={t('label.server')}
               value={mlModelDetail.server}
@@ -388,6 +391,7 @@ export const getDataAssetsHeaderInfo = (
         <>
           {storedProcedureDetails.sourceUrl && (
             <ExtraInfoLink
+              newTab
               href={storedProcedureDetails.sourceUrl}
               label=""
               value={getEntityName(storedProcedureDetails)}
@@ -422,6 +426,7 @@ export const getDataAssetsHeaderInfo = (
         <>
           {apiCollection.endpointURL && (
             <ExtraInfoLink
+              newTab
               href={apiCollection.endpointURL}
               label={t('label.endpoint-url')}
               value={apiCollection.endpointURL}
@@ -451,6 +456,7 @@ export const getDataAssetsHeaderInfo = (
           )}
           {apiEndpoint.endpointURL && (
             <ExtraInfoLink
+              newTab
               href={apiEndpoint.endpointURL}
               label={t('label.endpoint-url')}
               value={apiEndpoint.endpointURL}
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Database/Database.util.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/Database/Database.util.test.tsx
index 8f1e96c6224f..af7bbab48b85 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/Database/Database.util.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/Database/Database.util.test.tsx
@@ -92,7 +92,7 @@ describe('Database Util', () => {
           render: expect.any(Function),
         },
         {
-          title: 'label.owner',
+          title: 'label.owner-plural',
           dataIndex: 'owners',
           key: 'owners',
           width: 120,
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/Database/Database.util.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/Database/Database.util.tsx
index 786a07e7ec8a..01653c089067 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/Database/Database.util.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/Database/Database.util.tsx
@@ -91,7 +91,7 @@ export const schemaTableColumns: ColumnsType<DatabaseSchema> = [
       ),
   },
   {
-    title: t('label.owner'),
+    title: t('label.owner-plural'),
     dataIndex: 'owners',
     key: 'owners',
     width: 120,
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.test.tsx
index 1960dc44b047..f3eaf2e84300 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.test.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.test.tsx
@@ -118,7 +118,7 @@ describe('EntityUtils unit tests', () => {
         getEntityOverview(ExplorePageTabs.CHARTS, MOCK_CHART_DATA)
       );
 
-      expect(result).toContain('label.owner');
+      expect(result).toContain('label.owner-plural');
       expect(result).toContain('label.chart');
       expect(result).toContain('label.url-uppercase');
       expect(result).toContain('Are you an ethnic minority in your city?');
@@ -140,7 +140,7 @@ describe('EntityUtils unit tests', () => {
         })
       );
 
-      expect(result).toContain('label.owner');
+      expect(result).toContain('label.owner-plural');
       expect(result).toContain('label.type');
       expect(result).toContain('label.service');
       expect(result).toContain('label.database');
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx
index e4e96703cd81..ae0ba4b39903 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx
@@ -265,7 +265,7 @@ const getTableOverview = (
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.lineage],
     },
@@ -376,7 +376,7 @@ const getPipelineOverview = (pipelineDetails: Pipeline) => {
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.lineage],
     },
@@ -424,7 +424,7 @@ const getDashboardOverview = (dashboardDetails: Dashboard) => {
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.lineage],
     },
@@ -481,7 +481,7 @@ export const getSearchIndexOverview = (
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.lineage],
     },
@@ -513,7 +513,7 @@ const getMlModelOverview = (mlModelDetails: Mlmodel) => {
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.lineage],
     },
@@ -609,7 +609,7 @@ const getChartOverview = (chartDetails: Chart) => {
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.lineage],
     },
@@ -674,7 +674,7 @@ const getDataModelOverview = (dataModelDetails: DashboardDataModel) => {
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.lineage],
     },
@@ -748,7 +748,7 @@ const getStoredProcedureOverview = (
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.lineage],
     },
@@ -826,7 +826,7 @@ const getDatabaseOverview = (databaseDetails: Database) => {
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.explore],
     },
@@ -866,7 +866,7 @@ const getDatabaseSchemaOverview = (databaseSchemaDetails: DatabaseSchema) => {
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.explore],
     },
@@ -915,7 +915,7 @@ const getEntityServiceOverview = (serviceDetails: EntityServiceUnion) => {
 
   const overview = [
     {
-      name: i18next.t('label.owner'),
+      name: i18next.t('label.owner-plural'),
       value: <OwnerLabel hasPermission={false} owners={owners} />,
       visible: [DRAWER_NAVIGATION_OPTIONS.explore],
     },
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/FeedUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/FeedUtils.tsx
index 91bec7c6668b..e16897e93a94 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/FeedUtils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/FeedUtils.tsx
@@ -617,7 +617,7 @@ export const getFeedChangeFieldLabel = (fieldName?: EntityField) => {
     [EntityField.TASKS]: i18next.t('label.task-plural'),
     [EntityField.ML_FEATURES]: i18next.t('label.ml-feature-plural'),
     [EntityField.SCHEMA_TEXT]: i18next.t('label.schema-text'),
-    [EntityField.OWNER]: i18next.t('label.owner'),
+    [EntityField.OWNER]: i18next.t('label.owner-plural'),
     [EntityField.REVIEWERS]: i18next.t('label.reviewer-plural'),
     [EntityField.SYNONYMS]: i18next.t('label.synonym-plural'),
     [EntityField.RELATEDTERMS]: i18next.t('label.related-term-plural'),
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsClassBase.ts
index 871e287e2371..3153f2814f73 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsClassBase.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/GlobalSettingsClassBase.ts
@@ -17,7 +17,6 @@ import { ReactComponent as AdminIcon } from '../assets/svg/admin-colored.svg';
 import { ReactComponent as ApplicationIcon } from '../assets/svg/application-colored.svg';
 import { ReactComponent as BotIcon } from '../assets/svg/bot-colored.svg';
 import { ReactComponent as AppearanceIcon } from '../assets/svg/custom-logo-colored.svg';
-import { ReactComponent as CustomDashboardLogoIcon } from '../assets/svg/customize-landing-page-colored.svg';
 import { ReactComponent as DashboardIcon } from '../assets/svg/dashboard-colored.svg';
 import { ReactComponent as DatabaseIcon } from '../assets/svg/database-colored.svg';
 import { ReactComponent as EmailIcon } from '../assets/svg/email-colored.svg';
@@ -95,6 +94,10 @@ class GlobalSettingsClassBase {
       name: t('label.application-plural'),
       url: GlobalSettingsMenuCategory.APPLICATIONS,
     },
+    [GlobalSettingsMenuCategory.PERSONA]: {
+      name: t('label.persona'),
+      url: GlobalSettingsMenuCategory.PERSONA,
+    },
   };
 
   protected updateSettingCategories(
@@ -117,6 +120,16 @@ class GlobalSettingsClassBase {
         icon: ServiceIcon,
         description: t('message.service-description'),
         items: [
+          {
+            label: t('label.api-uppercase-plural'),
+            description: t('message.page-sub-header-for-apis'),
+            isProtected: userPermissions.hasViewPermissions(
+              ResourceEntity.API_SERVICE,
+              permissions
+            ),
+            key: `${GlobalSettingsMenuCategory.SERVICES}.${GlobalSettingOptions.APIS}`,
+            icon: IconAPI,
+          },
           {
             label: t('label.database-plural'),
             description: t('message.page-sub-header-for-databases'),
@@ -197,17 +210,6 @@ class GlobalSettingsClassBase {
             key: `${GlobalSettingsMenuCategory.SERVICES}.${GlobalSettingOptions.METADATA}`,
             icon: OpenMetadataIcon,
           },
-          {
-            label: t('label.api-uppercase-plural'),
-            description: t('message.page-sub-header-for-apis'),
-            isProtected: userPermissions.hasViewPermissions(
-              ResourceEntity.API_SERVICE,
-              permissions
-            ),
-            key: `${GlobalSettingsMenuCategory.SERVICES}.${GlobalSettingOptions.APIS}`,
-            icon: IconAPI,
-            isBeta: true,
-          },
           {
             label: t('label.data-observability'),
             description: t('message.page-sub-header-for-data-observability'),
@@ -267,14 +269,6 @@ class GlobalSettingsClassBase {
             key: `${GlobalSettingsMenuCategory.MEMBERS}.${GlobalSettingOptions.ADMINS}`,
             icon: AdminIcon,
           },
-
-          {
-            label: t('label.persona-plural'),
-            description: t('message.page-sub-header-for-persona'),
-            isProtected: Boolean(isAdminUser),
-            key: `${GlobalSettingsMenuCategory.MEMBERS}.${GlobalSettingOptions.PERSONA}`,
-            icon: PersonasIcon,
-          },
         ],
       },
       {
@@ -312,17 +306,6 @@ class GlobalSettingsClassBase {
             key: `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.APPEARANCE}`,
             icon: AppearanceIcon,
           },
-          {
-            label: t('label.customize-entity', {
-              entity: t('label.landing-page'),
-            }),
-            description: t(
-              'message.page-sub-header-for-customize-landing-page'
-            ),
-            isProtected: Boolean(isAdminUser),
-            key: `${GlobalSettingsMenuCategory.PREFERENCES}.${GlobalSettingOptions.CUSTOMIZE_LANDING_PAGE}`,
-            icon: CustomDashboardLogoIcon,
-          },
           {
             label: t('label.email'),
             description: t('message.email-configuration-message'),
@@ -516,6 +499,13 @@ class GlobalSettingsClassBase {
         key: GlobalSettingOptions.BOTS,
         icon: BotIcon,
       },
+      {
+        category: t('label.persona-plural'),
+        description: t('message.page-sub-header-for-persona'),
+        isProtected: Boolean(isAdminUser),
+        key: GlobalSettingOptions.PERSONA,
+        icon: PersonasIcon,
+      },
     ];
   }
 }
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/QueryBuilderElasticsearchFormatUtils.js b/openmetadata-ui/src/main/resources/ui/src/utils/QueryBuilderElasticsearchFormatUtils.js
index 0c8e83f85886..c45326e1ce95 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/QueryBuilderElasticsearchFormatUtils.js
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/QueryBuilderElasticsearchFormatUtils.js
@@ -344,48 +344,52 @@ function buildEsGroup(children, conjunction, properties, recursiveFxn, config) {
 }
 
 export function elasticSearchFormat(tree, config) {
-  // -- format the es dsl here
-  if (!tree) return undefined;
-  const type = tree.get('type');
-  const properties = tree.get('properties') || new Map();
-
-  if (type === 'rule' && properties.get('field')) {
-    // -- field is null when a new blank rule is added
-    const operator = properties.get('operator');
-    const field = properties.get('field');
-    const value = properties.get('value').toJS();
-    const _valueType = properties.get('valueType')?.get(0);
-    const valueSrc = properties.get('valueSrc')?.get(0);
-
-    if (valueSrc === 'func') {
-      // -- elastic search doesn't support functions (that is post processing)
-      return;
+  try {
+    // -- format the es dsl here
+    if (!tree) return undefined;
+    const type = tree.get('type');
+    const properties = tree.get('properties') || new Map();
+
+    if (type === 'rule' && properties.get('field')) {
+      // -- field is null when a new blank rule is added
+      const operator = properties.get('operator');
+      const field = properties.get('field');
+      const value = properties.get('value').toJS();
+      const _valueType = properties.get('valueType')?.get(0);
+      const valueSrc = properties.get('valueSrc')?.get(0);
+
+      if (valueSrc === 'func') {
+        // -- elastic search doesn't support functions (that is post processing)
+        return;
+      }
+
+      if (value && Array.isArray(value[0])) {
+        return {
+          bool: {
+            should: value[0].map((val) =>
+              buildEsRule(field, [val], operator, config, valueSrc)
+            ),
+          },
+        };
+      } else {
+        return buildEsRule(field, value, operator, config, valueSrc);
+      }
     }
 
-    if (value && Array.isArray(value[0])) {
-      return {
-        bool: {
-          should: value[0].map((val) =>
-            buildEsRule(field, [val], operator, config, valueSrc)
-          ),
-        },
-      };
-    } else {
-      return buildEsRule(field, value, operator, config, valueSrc);
+    if (type === 'group' || type === 'rule_group') {
+      let conjunction = properties.get('conjunction');
+      if (!conjunction) conjunction = defaultConjunction(config);
+      const children = tree.get('children1');
+
+      return buildEsGroup(
+        children,
+        conjunction,
+        properties,
+        elasticSearchFormat,
+        config
+      );
     }
-  }
-
-  if (type === 'group' || type === 'rule_group') {
-    let conjunction = properties.get('conjunction');
-    if (!conjunction) conjunction = defaultConjunction(config);
-    const children = tree.get('children1');
-
-    return buildEsGroup(
-      children,
-      conjunction,
-      properties,
-      elasticSearchFormat,
-      config
-    );
+  } catch {
+    return {};
   }
 }
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts b/openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts
index 90e5a2846d97..e5587912b278 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/RouterUtils.ts
@@ -569,11 +569,10 @@ export const getClassificationVersionsPath = (
 };
 
 export const getPersonaDetailsPath = (fqn: string) => {
-  let path = ROUTES.SETTINGS_WITH_TAB_FQN;
+  let path = ROUTES.SETTINGS_WITH_CATEGORY_FQN;
 
   path = path
-    .replace(PLACEHOLDER_SETTING_CATEGORY, GlobalSettingsMenuCategory.MEMBERS)
-    .replace(PLACEHOLDER_ROUTE_TAB, GlobalSettingOptions.PERSONA)
+    .replace(PLACEHOLDER_SETTING_CATEGORY, GlobalSettingOptions.PERSONA)
     .replace(PLACEHOLDER_ROUTE_FQN, getEncodedFqn(fqn));
 
   return path;
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceMainTabContentUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceMainTabContentUtils.tsx
index 44d134d1853f..2284f499a95d 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceMainTabContentUtils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceMainTabContentUtils.tsx
@@ -89,7 +89,7 @@ export const getServiceMainTabColumns = (
       ]
     : []),
   {
-    title: t('label.owner'),
+    title: t('label.owner-plural'),
     dataIndex: 'owners',
     key: 'owners',
     render: (owners: ServicePageData['owners']) =>
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx
index 8f67293e3db1..0dc5e7279e0f 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/ServiceUtils.tsx
@@ -29,7 +29,6 @@ import { ServiceCategory } from '../enums/service.enum';
 import { StorageServiceType } from '../generated/entity/data/container';
 import { Database } from '../generated/entity/data/database';
 import { MlModelServiceType } from '../generated/entity/data/mlmodel';
-import { APIServiceType } from '../generated/entity/services/apiService';
 import {
   DashboardService,
   DashboardServiceType,
@@ -100,8 +99,7 @@ export const shouldTestConnection = (serviceType: string) => {
     serviceType !== DashboardServiceType.CustomDashboard &&
     serviceType !== MlModelServiceType.CustomMlModel &&
     serviceType !== PipelineServiceType.CustomPipeline &&
-    serviceType !== StorageServiceType.CustomStorage &&
-    serviceType !== APIServiceType.REST
+    serviceType !== StorageServiceType.CustomStorage
   );
 };
 
@@ -348,7 +346,7 @@ export const getServiceRouteFromServiceType = (type: ServiceTypes) => {
     return GlobalSettingOptions.SEARCH;
   }
 
-  if (type === 'apiServices') {
+  if (type === ServiceCategory.API_SERVICES) {
     return GlobalSettingOptions.APIS;
   }
 
@@ -387,7 +385,6 @@ export const getResourceEntityFromServiceCategory = (
     case ServiceCategory.STORAGE_SERVICES:
       return ResourceEntity.STORAGE_SERVICE;
 
-    case 'apiServices':
     case ServiceCategory.API_SERVICES:
       return ResourceEntity.API_SERVICE;
   }
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.test.ts
index 61efade41071..a393d968c229 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.test.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.test.ts
@@ -16,6 +16,12 @@ import tagClassBase, { TagClassBase } from './TagClassBase';
 
 jest.mock('../rest/searchAPI');
 
+
+jest.mock('./StringsUtils', () => ({
+  getEncodedFqn: jest.fn().mockReturnValue('test'),
+  escapeESReservedCharacters: jest.fn().mockReturnValue('test'),
+}));
+
 describe('TagClassBase', () => {
   beforeEach(() => {
     (searchQuery as jest.Mock).mockClear();
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.ts
index f8cdf0d45b6c..a69c641afa8c 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.ts
@@ -13,11 +13,14 @@
 import { PAGE_SIZE } from '../constants/constants';
 import { SearchIndex } from '../enums/search.enum';
 import { searchQuery } from '../rest/searchAPI';
+import { escapeESReservedCharacters, getEncodedFqn } from './StringsUtils';
 
 class TagClassBase {
   public async getTags(searchText: string, page: number) {
+    // this is to esacpe and encode any chars which is known by ES search internally
+    const encodedValue = getEncodedFqn(escapeESReservedCharacters(searchText));
     const res = await searchQuery({
-      query: `*${searchText}*`,
+      query: `*${encodedValue}*`,
       filters: 'disabled:false',
       pageNumber: page,
       pageSize: PAGE_SIZE,
diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts
index 359f7c8cf87c..ffbcefbaad73 100644
--- a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts
@@ -193,6 +193,11 @@ export const MOCK_APPLICATION_ENTITY_STATS = {
     failedRecords: 0,
     successRecords: 4,
   },
+  [EntityType.TEST_CASE_RESULT]: {
+    totalRecords: 4,
+    failedRecords: 0,
+    successRecords: 4,
+  },
 };
 
 export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
@@ -412,4 +417,10 @@ export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
     failedRecords: 0,
     successRecords: 4,
   },
+  {
+    name: EntityType.TEST_CASE_RESULT,
+    totalRecords: 4,
+    failedRecords: 0,
+    successRecords: 4,
+  },
 ];
diff --git a/openmetadata-ui/src/main/resources/ui/yarn.lock b/openmetadata-ui/src/main/resources/ui/yarn.lock
index dc6f992e8cd6..7b298461f3dd 100644
--- a/openmetadata-ui/src/main/resources/ui/yarn.lock
+++ b/openmetadata-ui/src/main/resources/ui/yarn.lock
@@ -10372,9 +10372,9 @@ markdown-it@^14.0.0:
     uc.micro "^2.1.0"
 
 markdown-to-jsx@^7.3.2:
-  version "7.3.2"
-  resolved "https://registry.yarnpkg.com/markdown-to-jsx/-/markdown-to-jsx-7.3.2.tgz#f286b4d112dad3028acc1e77dfe1f653b347e131"
-  integrity sha512-B+28F5ucp83aQm+OxNrPkS8z0tMKaeHiy0lHJs3LqCyDQFtWuenaIrkaVTgAm1pf1AU85LXltva86hlaT17i8Q==
+  version "7.5.0"
+  resolved "https://registry.yarnpkg.com/markdown-to-jsx/-/markdown-to-jsx-7.5.0.tgz#42ece0c71e842560a7d8bd9f81e7a34515c72150"
+  integrity sha512-RrBNcMHiFPcz/iqIj0n3wclzHXjwS7mzjBNWecKKVhNTIxQepIix6Il/wZCn2Cg5Y1ow2Qi84+eJrryFRWBEWw==
 
 marked@4.3.0:
   version "4.3.0"
diff --git a/pom.xml b/pom.xml
index 9be45980f522..54d7c4207c71 100644
--- a/pom.xml
+++ b/pom.xml
@@ -156,6 +156,7 @@
     <logback-core.version>1.2.13</logback-core.version>
     <logback-classic.version>1.2.13</logback-classic.version>
     <jsonpath.version>2.9.0</jsonpath.version>
+    <everit.version>1.14.4</everit.version>
   </properties>
   <dependencyManagement>
     <dependencies>
@@ -517,7 +518,11 @@
         <artifactId>picocli</artifactId>
         <version>${picocli.version}</version>
       </dependency>
-
+      <dependency>
+        <groupId>com.github.erosb</groupId>
+        <artifactId>everit-json-schema</artifactId>
+        <version>${everit.version}</version>
+      </dependency>
       <!-- avoid security issue https://security.snyk.io/vuln/SNYK-JAVA-ORGECLIPSEJETTY-1090340 -->
       <dependency>
         <groupId>org.eclipse.jetty</groupId>