Merge branch 'main' into GEN-658

open-metadata · Oct 18, 2024 · 591f1bc · 591f1bc
2 parents 6c5e87f + 141caf4
commit 591f1bc
Show file tree

Hide file tree

Showing 45 changed files with 711 additions and 116 deletions.
diff --git a/.run/Template JUnit.run.xml b/.run/Template JUnit.run.xml
@@ -5,7 +5,7 @@
     <option name="MAIN_CLASS_NAME" value="" />
     <option name="METHOD_NAME" value="" />
     <option name="TEST_OBJECT" value="package" />
-    <option name="VM_PARAMETERS" value="-ea -DjdbcContainerClassName=org.testcontainers.containers.MySQLContainer -DjdbcContainerImage=mysql:8 -DelasticSearchContainerClassName=docker.elastic.co/elasticsearch/elasticsearch:8.10.2 -DopenSearchContainerClassName=opensearchproject/opensearch:1.3.0 -DrunESTestCases=false"/>
+    <option name="VM_PARAMETERS" value="-ea -DjdbcContainerClassName=org.testcontainers.containers.MySQLContainer -DjdbcContainerImage=mysql:8 -DelasticSearchContainerClassName=docker.elastic.co/elasticsearch/elasticsearch:8.11.4 -DopenSearchContainerClassName=opensearchproject/opensearch:1.3.0 -DrunESTestCases=false"/>
     <method v="2">
       <option name="Make" enabled="true" />
     </method>

diff --git a/docker/development/docker-compose-gcp.yml b/docker/development/docker-compose-gcp.yml
@@ -46,7 +46,7 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node

diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml
@@ -42,7 +42,7 @@ services:
      - ./docker-volume/db-data:/var/lib/mysql
 
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     container_name: openmetadata_elasticsearch
     environment:
       - discovery.type=single-node

diff --git a/docker/docker-compose-quickstart/docker-compose-postgres.yml b/docker/docker-compose-quickstart/docker-compose-postgres.yml
@@ -41,7 +41,7 @@ services:
 
   elasticsearch:
     container_name: openmetadata_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m

diff --git a/docker/docker-compose-quickstart/docker-compose.yml b/docker/docker-compose-quickstart/docker-compose.yml
@@ -39,7 +39,7 @@ services:
 
   elasticsearch:
     container_name: openmetadata_elasticsearch
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.4
     environment:
       - discovery.type=single-node
       - ES_JAVA_OPTS=-Xms1024m -Xmx1024m

diff --git a/ingestion/src/metadata/ingestion/source/pipeline/airflow/metadata.py b/ingestion/src/metadata/ingestion/source/pipeline/airflow/metadata.py
@@ -15,7 +15,7 @@
 from collections import Counter
 from datetime import datetime
 from enum import Enum
-from typing import Iterable, List, Optional, cast
+from typing import Dict, Iterable, List, Optional, cast
 
 from airflow.models import BaseOperator, DagRun, TaskInstance
 from airflow.models.dag import DagModel
@@ -142,6 +142,16 @@ def session(self) -> Session:
 
         return self._session
 
+    @staticmethod
+    def _extract_serialized_task(task: Dict) -> Dict:
+        """
+        Given the serialization changes introduced in Airflow 2.10,
+        ensure compatibility with all versions.
+        """
+        if task.keys() == {"__var", "__type"}:
+            return task["__var"]
+        return task
+
     def get_pipeline_status(self, dag_id: str) -> List[DagRun]:
         """
         Return the DagRuns of given dag
@@ -328,7 +338,9 @@ def get_pipelines_list(self) -> Iterable[AirflowDagDetails]:
                     max_active_runs=data.get("max_active_runs", None),
                     description=data.get("_description", None),
                     start_date=data.get("start_date", None),
-                    tasks=data.get("tasks", []),
+                    tasks=list(
+                        map(self._extract_serialized_task, data.get("tasks", []))
+                    ),
                     schedule_interval=get_schedule_interval(data),
                     owner=self.fetch_dag_owners(data),
                 )

diff --git a/openmetadata-docs/content/v1.4.x/deployment/bare-metal/index.md b/openmetadata-docs/content/v1.4.x/deployment/bare-metal/index.md
@@ -53,7 +53,7 @@ You can refer a sample script [here](https://github.com/open-metadata/OpenMetada
 
 ## Elasticsearch (version 8.X)
 
-OpenMetadata supports ElasticSearch version up to 8.10.2. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
+OpenMetadata supports ElasticSearch version up to 8.11.4. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
 [Installing ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
 
 Please follow the instructions here to [install ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/setup.html).
@@ -169,7 +169,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support 
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version between 12 or higher
 
 For Production Systems, we recommend Amazon RDS to be in Multiple Availability Zones. For Amazon OpenSearch (or ElasticSearch) Service, we recommend Multiple Availability Zones with minimum 3 Master Nodes.

diff --git a/openmetadata-docs/content/v1.4.x/deployment/docker/index.md b/openmetadata-docs/content/v1.4.x/deployment/docker/index.md
@@ -243,7 +243,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version 12 or higher
 
 Note:-

diff --git a/openmetadata-docs/content/v1.4.x/deployment/kubernetes/aks.md b/openmetadata-docs/content/v1.4.x/deployment/kubernetes/aks.md
@@ -15,7 +15,7 @@ We support
 
 - Azure SQL (MySQL) engine version 8 or higher
 - Azure SQL (PostgreSQL) engine version 12 or higher
-- Elastic Cloud (ElasticSearch version 8.10.2)
+- Elastic Cloud (ElasticSearch version 8.11.4)
 
 Once you have the Azure SQL and Elastic Cloud on Azure configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 

diff --git a/openmetadata-docs/content/v1.4.x/deployment/kubernetes/on-prem.md b/openmetadata-docs/content/v1.4.x/deployment/kubernetes/on-prem.md
@@ -22,7 +22,7 @@ We support
 
 - MySQL engine version 8 or higher
 - PostgreSQL engine version 12 or higher
-- ElasticSearch version 8.X (upto 8.10.2) or OpenSearch Version 2.X (upto 2.7)
+- ElasticSearch version 8.X (upto 8.11.4) or OpenSearch Version 2.X (upto 2.7)
 
 Once you have the External Database and Search Engine configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 

diff --git a/openmetadata-docs/content/v1.4.x/deployment/upgrade/versions/110-to-120.md b/openmetadata-docs/content/v1.4.x/deployment/upgrade/versions/110-to-120.md
@@ -160,7 +160,7 @@ For Kubernetes Deployment, `openmetadata.config.database.dbParams` is available
 ### Version Upgrades
 
 - The OpenMetadata Server is now based on **JDK 17**
-- OpenMetadata now **requires** **Elasticsearch** version **8.10.2** or **Opensearch** version **2.7**
+- OpenMetadata now **requires** **Elasticsearch** version **8.11.4** or **Opensearch** version **2.7**
 
 There is no direct migration to bump the indexes to the new supported versions. You might see errors like:
 

diff --git a/openmetadata-docs/content/v1.5.x/deployment/bare-metal/index.md b/openmetadata-docs/content/v1.5.x/deployment/bare-metal/index.md
@@ -58,7 +58,7 @@ If you are facing an error `ERROR: could not compute MD5 hash: disabled for FIPS
 
 ## Elasticsearch (version 8.X)
 
-OpenMetadata supports ElasticSearch version up to 8.10.2. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
+OpenMetadata supports ElasticSearch version up to 8.11.4. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
 [Installing ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
 
 Please follow the instructions here to [install ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/setup.html).
@@ -174,7 +174,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support 
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version between 12 or higher
 
 For Production Systems, we recommend Amazon RDS to be in Multiple Availability Zones. For Amazon OpenSearch (or ElasticSearch) Service, we recommend Multiple Availability Zones with minimum 3 Master Nodes.

diff --git a/openmetadata-docs/content/v1.5.x/deployment/docker/index.md b/openmetadata-docs/content/v1.5.x/deployment/docker/index.md
@@ -244,7 +244,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version 12 or higher
 
 Note:-

diff --git a/openmetadata-docs/content/v1.5.x/deployment/kubernetes/aks.md b/openmetadata-docs/content/v1.5.x/deployment/kubernetes/aks.md
@@ -16,7 +16,7 @@ We support
 
 - Azure SQL (MySQL) engine version 8 or higher
 - Azure SQL (PostgreSQL) engine version 12 or higher
-- Elastic Cloud (ElasticSearch version 8.10.2)
+- Elastic Cloud (ElasticSearch version 8.11.4)
 
 Once you have the Azure SQL and Elastic Cloud on Azure configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 

diff --git a/openmetadata-docs/content/v1.5.x/deployment/kubernetes/on-prem.md b/openmetadata-docs/content/v1.5.x/deployment/kubernetes/on-prem.md
@@ -23,7 +23,7 @@ We support
 
 - MySQL engine version 8 or higher
 - PostgreSQL engine version 12 or higher
-- ElasticSearch version 8.X (upto 8.10.2) or OpenSearch Version 2.X (upto 2.7)
+- ElasticSearch version 8.X (upto 8.11.4) or OpenSearch Version 2.X (upto 2.7)
 
 Once you have the External Database and Search Engine configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 

diff --git a/openmetadata-docs/content/v1.5.x/deployment/upgrade/versions/110-to-120.md b/openmetadata-docs/content/v1.5.x/deployment/upgrade/versions/110-to-120.md
@@ -161,7 +161,7 @@ For Kubernetes Deployment, `openmetadata.config.database.dbParams` is available
 ### Version Upgrades
 
 - The OpenMetadata Server is now based on **JDK 17**
-- OpenMetadata now **requires** **Elasticsearch** version **8.10.2** or **Opensearch** version **2.7**
+- OpenMetadata now **requires** **Elasticsearch** version **8.11.4** or **Opensearch** version **2.7**
 
 There is no direct migration to bump the indexes to the new supported versions. You might see errors like:
 

diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/bare-metal/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/bare-metal/index.md
@@ -54,7 +54,7 @@ You can refer a sample script [here](https://github.com/open-metadata/OpenMetada
 
 ## Elasticsearch (version 8.X)
 
-OpenMetadata supports ElasticSearch version up to 8.10.2. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
+OpenMetadata supports ElasticSearch version up to 8.11.4. To install or upgrade Elasticsearch to a supported version please see the instructions for your operating system at 
 [Installing ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
 
 Please follow the instructions here to [install ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/7.13/setup.html).
@@ -170,7 +170,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support 
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version between 12 or higher
 
 For Production Systems, we recommend Amazon RDS to be in Multiple Availability Zones. For Amazon OpenSearch (or ElasticSearch) Service, we recommend Multiple Availability Zones with minimum 3 Master Nodes.

diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/docker/index.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/docker/index.md
@@ -244,7 +244,7 @@ If you are running OpenMetadata in AWS, it is recommended to use [Amazon RDS](ht
 We support
 
 - Amazon RDS (MySQL) engine version 8 or higher
-- Amazon OpenSearch (ElasticSearch) engine version up to 8.10.2 or Amazon OpenSearch engine version up to 2.7
+- Amazon OpenSearch (ElasticSearch) engine version up to 8.11.4 or Amazon OpenSearch engine version up to 2.7
 - Amazon RDS (PostgreSQL) engine version 12 or higher
 
 Note:-

diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/aks.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/aks.md
@@ -16,7 +16,7 @@ We support
 
 - Azure SQL (MySQL) engine version 8 or higher
 - Azure SQL (PostgreSQL) engine version 12 or higher
-- Elastic Cloud (ElasticSearch version 8.10.2)
+- Elastic Cloud (ElasticSearch version 8.11.4)
 
 Once you have the Azure SQL and Elastic Cloud on Azure configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 

diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/on-prem.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/kubernetes/on-prem.md
@@ -23,7 +23,7 @@ We support
 
 - MySQL engine version 8 or higher
 - PostgreSQL engine version 12 or higher
-- ElasticSearch version 8.X (upto 8.10.2) or OpenSearch Version 2.X (upto 2.7)
+- ElasticSearch version 8.X (upto 8.11.4) or OpenSearch Version 2.X (upto 2.7)
 
 Once you have the External Database and Search Engine configured, you can update the environment variables below for OpenMetadata kubernetes deployments to connect with Database and ElasticSearch.
 

diff --git a/...metadata-docs/content/v1.6.x-SNAPSHOT/deployment/upgrade/versions/110-to-120.md b/...metadata-docs/content/v1.6.x-SNAPSHOT/deployment/upgrade/versions/110-to-120.md
@@ -161,7 +161,7 @@ For Kubernetes Deployment, `openmetadata.config.database.dbParams` is available
 ### Version Upgrades
 
 - The OpenMetadata Server is now based on **JDK 17**
-- OpenMetadata now **requires** **Elasticsearch** version **8.10.2** or **Opensearch** version **2.7**
+- OpenMetadata now **requires** **Elasticsearch** version **8.11.4** or **Opensearch** version **2.7**
 
 There is no direct migration to bump the indexes to the new supported versions. You might see errors like:
 

diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml
@@ -710,7 +710,7 @@
                 <systemPropertyVariables>
                   <jdbcContainerClassName>org.testcontainers.containers.PostgreSQLContainer</jdbcContainerClassName>
                   <jdbcContainerImage>postgres:15</jdbcContainerImage>
-                  <elasticSearchContainerClassName>docker.elastic.co/elasticsearch/elasticsearch:8.10.2</elasticSearchContainerClassName>
+                  <elasticSearchContainerClassName>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</elasticSearchContainerClassName>
                   <openSearchContainerClassName>opensearchproject/opensearch:2.7.0</openSearchContainerClassName>
                   <runESTestCases>false</runESTestCases>
                 </systemPropertyVariables>
@@ -750,7 +750,7 @@
                 <systemPropertyVariables>
                   <jdbcContainerClassName>org.testcontainers.containers.MySQLContainer</jdbcContainerClassName>
                   <jdbcContainerImage>mysql:8.3.0</jdbcContainerImage>
-                  <elasticSearchContainerClassName>docker.elastic.co/elasticsearch/elasticsearch:8.10.2</elasticSearchContainerClassName>
+                  <elasticSearchContainerClassName>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</elasticSearchContainerClassName>
                   <openSearchContainerClassName>opensearchproject/opensearch:2.7.0</openSearchContainerClassName>
                   <runESTestCases>false</runESTestCases>
                 </systemPropertyVariables>