feat: BigQuery common and provisioner classes

DEPENDENCIES

@@ -1,14 +1,17 @@
 maven/mavencentral/com.apicatalog/carbon-did/0.0.2, Apache-2.0, approved, #9239
 maven/mavencentral/com.apicatalog/iron-verifiable-credentials/0.8.1, Apache-2.0, approved, #9234
 maven/mavencentral/com.apicatalog/titanium-json-ld/1.0.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.apicatalog/titanium-json-ld/1.3.1, Apache-2.0, approved, #8912
 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.10.3, Apache-2.0, approved, CQ21280
 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.15.3, Apache-2.0, approved, #7947
+maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.16.1, Apache-2.0, approved, #11606
 maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.16.2, Apache-2.0, approved, #11606
 maven/mavencentral/com.fasterxml.jackson.core/jackson-core/2.16.1, Apache-2.0 AND MIT, approved, #11602
 maven/mavencentral/com.fasterxml.jackson.core/jackson-core/2.16.2, Apache-2.0 AND MIT, approved, #11602
 maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.15.3, Apache-2.0, approved, #7934
+maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.16.1, Apache-2.0, approved, #11605
 maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.16.2, Apache-2.0, approved, #11605
+maven/mavencentral/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.16.1, Apache-2.0, approved, #11853
 maven/mavencentral/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.16.2, Apache-2.0, approved, #11853
 maven/mavencentral/com.fasterxml.jackson.module/jackson-module-jakarta-xmlbind-annotations/2.15.3, Apache-2.0, approved, #9241
 maven/mavencentral/com.fasterxml.jackson.module/jackson-module-jakarta-xmlbind-annotations/2.16.2, Apache-2.0, approved, #11856
@@ -21,23 +24,43 @@
 maven/mavencentral/com.google.api-client/google-api-client/2.2.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.api-client/google-api-client/2.3.0, Apache-2.0 AND BSD-3-Clause, approved, #13411
 maven/mavencentral/com.google.api.grpc/gapic-google-cloud-storage-v2/2.34.0-alpha, Apache-2.0, approved, #13418
+maven/mavencentral/com.google.api.grpc/grpc-google-cloud-bigquerystorage-v1/3.1.0, , restricted, clearlydefined
+maven/mavencentral/com.google.api.grpc/grpc-google-cloud-bigquerystorage-v1/3.2.0, , restricted, clearlydefined
+maven/mavencentral/com.google.api.grpc/grpc-google-cloud-bigquerystorage-v1beta1/0.173.0, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.google.api.grpc/grpc-google-cloud-bigquerystorage-v1beta1/0.174.0, , restricted, clearlydefined
+maven/mavencentral/com.google.api.grpc/grpc-google-cloud-bigquerystorage-v1beta2/0.173.0, , restricted, clearlydefined
+maven/mavencentral/com.google.api.grpc/grpc-google-cloud-bigquerystorage-v1beta2/0.174.0, , restricted, clearlydefined
 maven/mavencentral/com.google.api.grpc/grpc-google-cloud-storage-v2/2.34.0-alpha, Apache-2.0, approved, #13434
+maven/mavencentral/com.google.api.grpc/proto-google-cloud-bigquerystorage-v1/3.1.0, , restricted, clearlydefined
+maven/mavencentral/com.google.api.grpc/proto-google-cloud-bigquerystorage-v1/3.2.0, , restricted, clearlydefined
+maven/mavencentral/com.google.api.grpc/proto-google-cloud-bigquerystorage-v1beta1/0.173.0, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.google.api.grpc/proto-google-cloud-bigquerystorage-v1beta1/0.174.0, , restricted, clearlydefined
+maven/mavencentral/com.google.api.grpc/proto-google-cloud-bigquerystorage-v1beta2/0.173.0, , restricted, clearlydefined
+maven/mavencentral/com.google.api.grpc/proto-google-cloud-bigquerystorage-v1beta2/0.174.0, , restricted, clearlydefined
 maven/mavencentral/com.google.api.grpc/proto-google-cloud-iamcredentials-v1/2.36.0, Apache-2.0, approved, #13435
 maven/mavencentral/com.google.api.grpc/proto-google-cloud-secretmanager-v1/2.36.0, Apache-2.0, approved, #13395
 maven/mavencentral/com.google.api.grpc/proto-google-cloud-secretmanager-v1beta1/2.36.0, Apache-2.0, approved, #13409
 maven/mavencentral/com.google.api.grpc/proto-google-cloud-storage-v2/2.34.0-alpha, Apache-2.0, approved, #13426
 maven/mavencentral/com.google.api.grpc/proto-google-common-protos/2.34.0, Apache-2.0, approved, #13421
+maven/mavencentral/com.google.api.grpc/proto-google-common-protos/2.36.0, , restricted, clearlydefined
 maven/mavencentral/com.google.api.grpc/proto-google-iam-admin-v1/3.31.0, Apache-2.0, approved, #13440
 maven/mavencentral/com.google.api.grpc/proto-google-iam-v1/1.29.0, Apache-2.0, approved, #13417
 maven/mavencentral/com.google.api/api-common/2.26.0, BSD-3-Clause, approved, #13419
+maven/mavencentral/com.google.api/api-common/2.28.0, , restricted, clearlydefined
 maven/mavencentral/com.google.api/gax-grpc/2.43.0, BSD-3-Clause, approved, #13436
 maven/mavencentral/com.google.api/gax-httpjson/2.43.0, BSD-3-Clause, approved, #13396
 maven/mavencentral/com.google.api/gax/2.43.0, BSD-3-Clause, approved, #13405
+maven/mavencentral/com.google.api/gax/2.45.0, , restricted, clearlydefined
+maven/mavencentral/com.google.apis/google-api-services-bigquery/v2-rev20240203-2.0.0, , restricted, clearlydefined
 maven/mavencentral/com.google.apis/google-api-services-iam/v2-rev20240108-2.0.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.google.apis/google-api-services-storage/v1-rev20240209-2.0.0, Apache-2.0, approved, #13416
 maven/mavencentral/com.google.auth/google-auth-library-credentials/1.23.0, BSD-3-Clause, approved, #13420
 maven/mavencentral/com.google.auth/google-auth-library-oauth2-http/1.23.0, BSD-3-Clause, approved, #13437
 maven/mavencentral/com.google.auto.value/auto-value-annotations/1.10.4, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.google.auto.value/auto-value/1.10.4, Apache-2.0, approved, #8470
+maven/mavencentral/com.google.cloud/google-cloud-bigquery/2.37.2, , restricted, clearlydefined
+maven/mavencentral/com.google.cloud/google-cloud-bigquerystorage/3.1.0, , restricted, clearlydefined
+maven/mavencentral/com.google.cloud/google-cloud-bigquerystorage/3.2.0, , restricted, clearlydefined
 maven/mavencentral/com.google.cloud/google-cloud-core-grpc/2.33.0, Apache-2.0, approved, #13408
 maven/mavencentral/com.google.cloud/google-cloud-core-http/2.33.0, Apache-2.0, approved, #13439
 maven/mavencentral/com.google.cloud/google-cloud-core/2.33.0, Apache-2.0, approved, #13406
@@ -56,6 +79,7 @@
 maven/mavencentral/com.google.errorprone/error_prone_annotations/2.23.0, Apache-2.0, approved, #11083
 maven/mavencentral/com.google.errorprone/error_prone_annotations/2.24.1, Apache-2.0, approved, #12448
 maven/mavencentral/com.google.errorprone/error_prone_annotations/2.26.1, Apache-2.0, approved, #13657
+maven/mavencentral/com.google.flatbuffers/flatbuffers-java/23.5.26, Apache-2.0, approved, #12857
 maven/mavencentral/com.google.guava/failureaccess/1.0.1, Apache-2.0, approved, CQ22654
 maven/mavencentral/com.google.guava/failureaccess/1.0.2, Apache-2.0, approved, CQ22654
 maven/mavencentral/com.google.guava/guava/29.0-android, Apache-2.0, approved, clearlydefined
@@ -116,6 +140,8 @@
 maven/mavencentral/io.grpc/grpc-stub/1.61.1, Apache-2.0, approved, #13398
 maven/mavencentral/io.grpc/grpc-util/1.61.1, Apache-2.0, approved, #13428
 maven/mavencentral/io.grpc/grpc-xds/1.61.1, Apache-2.0, approved, #13424
+maven/mavencentral/io.netty/netty-buffer/4.1.104.Final, Apache-2.0, approved, CQ21842
+maven/mavencentral/io.netty/netty-common/4.1.104.Final, Apache-2.0 AND MIT AND CC0-1.0, approved, CQ21843
 maven/mavencentral/io.opencensus/opencensus-api/0.31.1, Apache-2.0, approved, clearlydefined
 maven/mavencentral/io.opencensus/opencensus-contrib-http-util/0.31.1, Apache-2.0, approved, clearlydefined
 maven/mavencentral/io.opencensus/opencensus-proto/0.2.0, Apache-2.0, approved, clearlydefined
@@ -139,6 +165,10 @@
 maven/mavencentral/net.java.dev.jna/jna/5.13.0, Apache-2.0 AND LGPL-2.1-or-later, approved, #6709
 maven/mavencentral/net.sf.saxon/Saxon-HE/12.4, MPL-2.0 AND (MPL-2.0 AND Apache-2.0) AND (MPL-2.0 AND LicenseRef-X11-style) AND MPL-1.0 AND W3C, approved, #12716
 maven/mavencentral/org.antlr/antlr4-runtime/4.13.1, BSD-3-Clause, approved, #10767
+maven/mavencentral/org.apache.arrow/arrow-format/15.0.0, Apache-2.0, approved, #12854
+maven/mavencentral/org.apache.arrow/arrow-memory-core/15.0.0, Apache-2.0, approved, #12855
+maven/mavencentral/org.apache.arrow/arrow-memory-netty/15.0.0, Apache-2.0, approved, #12853
+maven/mavencentral/org.apache.arrow/arrow-vector/15.0.0, Apache-2.0 AND (Apache-2.0 AND FTL), approved, #12856
 maven/mavencentral/org.apache.commons/commons-compress/1.24.0, Apache-2.0 AND BSD-3-Clause AND bzip2-1.0.6 AND LicenseRef-Public-Domain, approved, #10368
 maven/mavencentral/org.apache.commons/commons-lang3/3.7, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.apache.commons/commons-lang3/3.8.1, Apache-2.0, approved, #815
@@ -161,6 +191,7 @@
 maven/mavencentral/org.bouncycastle/bcpkix-jdk18on/1.77, MIT, approved, #11593
 maven/mavencentral/org.bouncycastle/bcprov-jdk18on/1.77, MIT AND CC0-1.0, approved, #11595
 maven/mavencentral/org.bouncycastle/bcutil-jdk18on/1.77, MIT, approved, #11596
+maven/mavencentral/org.checkerframework/checker-compat-qual/2.5.6, GPL-2.0-only with Classpath-Exception-2.0, approved, #11598
 maven/mavencentral/org.checkerframework/checker-qual/3.37.0, MIT, approved, clearlydefined
 maven/mavencentral/org.checkerframework/checker-qual/3.42.0, MIT, approved, clearlydefined
 maven/mavencentral/org.codehaus.mojo/animal-sniffer-annotations/1.23, MIT, approved, clearlydefined
@@ -170,6 +201,8 @@
 maven/mavencentral/org.codehaus.plexus/plexus-utils/3.1.1, , approved, CQ16492
 maven/mavencentral/org.codehaus.plexus/plexus-utils/3.3.0, , approved, CQ21066
 maven/mavencentral/org.conscrypt/conscrypt-openjdk-uber/2.5.2, Apache-2.0, approved, clearlydefined
+maven/mavencentral/org.eclipse.collections/eclipse-collections-api/11.1.0, EPL-1.0 OR BSD-3-Clause, approved, technology.collections
+maven/mavencentral/org.eclipse.collections/eclipse-collections/11.1.0, EPL-1.0 OR BSD-3-Clause, approved, technology.collections
 maven/mavencentral/org.eclipse.edc/autodoc-processor/0.5.2-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/boot/0.5.2-SNAPSHOT, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/catalog-spi/0.5.2-SNAPSHOT, Apache-2.0, approved, technology.edc
@@ -259,6 +292,7 @@
 maven/mavencentral/org.jetbrains/annotations/13.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.jetbrains/annotations/17.0.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.jetbrains/annotations/24.1.0, Apache-2.0, approved, clearlydefined
+maven/mavencentral/org.json/json/20231013, OTHER, restricted, clearlydefined
 maven/mavencentral/org.junit-pioneer/junit-pioneer/2.2.0, EPL-2.0, approved, #11857
 maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.1, EPL-2.0, approved, #9714
 maven/mavencentral/org.junit.jupiter/junit-jupiter-api/5.10.2, EPL-2.0, approved, #9714
@@ -293,5 +327,6 @@
 maven/mavencentral/org.slf4j/slf4j-api/2.0.9, MIT, approved, #5915
 maven/mavencentral/org.testcontainers/junit-jupiter/1.19.7, MIT, approved, #10344
 maven/mavencentral/org.testcontainers/testcontainers/1.19.7, Apache-2.0 AND MIT, approved, #10347
+maven/mavencentral/org.threeten/threeten-extra/1.7.2, BSD-3-Clause, approved, #6493
 maven/mavencentral/org.threeten/threetenbp/1.6.8, BSD-3-Clause, approved, #6750
 maven/mavencentral/org.xmlresolver/xmlresolver/5.2.2, Apache-2.0, approved, clearlydefined

extensions/common/gcp/gcp-core/build.gradle.kts

@@ -8,7 +8,7 @@
  *  SPDX-License-Identifier: Apache-2.0
  *
  *  Contributors:
- *       Google LCC - Initial implementation
+ *       Google LLC - Initial implementation
  *
  */
 
@@ -18,17 +18,22 @@ plugins {
 
 dependencies {
     api(libs.edc.spi.core)
+    api(libs.edc.spi.dataplane)
     api(libs.edc.spi.transfer)
     api(libs.edc.util)
+    implementation(libs.edc.spi.validator)
+
 
     // GCP dependencies.
     implementation(platform(libs.googlecloud.bom))
+    implementation(libs.googlecloud.bigquery)
     implementation(libs.googlecloud.iam.admin)
     implementation(libs.googlecloud.storage)
     implementation(libs.googlecloud.iam.credentials)
     implementation(libs.googleapis.iam)
 
     testImplementation(libs.edc.junit)
+    testImplementation(libs.googlecloud.gax)
 }
 
 

extensions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/bigquery/BigQueryDataAddress.java

@@ -0,0 +1,99 @@
+/*
+ *  Copyright (c) 2024 Google LLC
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Google LLC
+ *
+ */
+
+package org.eclipse.edc.gcp.bigquery;
+
+import org.eclipse.edc.gcp.bigquery.service.BigQueryService;
+import org.eclipse.edc.spi.types.domain.DataAddress;
+
+import java.util.Optional;
+
+import static java.util.Collections.emptyMap;
+
+/**
+ * This is a wrapper class for the {@link DataAddress} object, which has typed accessors for
+ * properties specific to a BigQuery endpoint.
+ */
+public class BigQueryDataAddress extends DataAddress {
+    private BigQueryDataAddress() {
+        setType(BigQueryService.BIGQUERY_DATA);
+    }
+
+    public String getProject() {
+        return getStringProperty(BigQueryService.PROJECT);
+    }
+
+    public String getDataset() {
+        return getStringProperty(BigQueryService.DATASET);
+    }
+
+    public String getTable() {
+        return getStringProperty(BigQueryService.TABLE);
+    }
+
+    public String getQuery() {
+        return getStringProperty(BigQueryService.QUERY);
+    }
+
+    public String getServiceAccountName() {
+        return getStringProperty(BigQueryService.SERVICE_ACCOUNT_NAME);
+    }
+
+    public static final class Builder extends DataAddress.Builder<BigQueryDataAddress, Builder> {
+        private Builder() {
+            super(new BigQueryDataAddress());
+        }
+
+        public static Builder newInstance() {
+            return new Builder();
+        }
+
+        public Builder project(String project) {
+            property(BigQueryService.PROJECT, project);
+            return this;
+        }
+
+        public Builder dataset(String dataset) {
+            property(BigQueryService.DATASET, dataset);
+            return this;
+        }
+
+        public Builder table(String table) {
+            property(BigQueryService.TABLE, table);
+            return this;
+        }
+
+        public Builder query(String query) {
+            property(BigQueryService.QUERY, query);
+            return this;
+        }
+
+        public Builder serviceAccountName(String serviceAccountName) {
+            property(BigQueryService.SERVICE_ACCOUNT_NAME, serviceAccountName);
+            return this;
+        }
+
+        @Override
+        public BigQueryDataAddress build() {
+            return address;
+        }
+
+        public Builder copyFrom(DataAddress other) {
+            Optional.ofNullable(other)
+                .map(DataAddress::getProperties)
+                .orElse(emptyMap()).forEach(this::property);
+            return this;
+        }
+    }
+}

extensions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/bigquery/BigQueryPart.java

@@ -0,0 +1,54 @@
+/*
+ *  Copyright (c) 2024 Google LLC
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Google LLC
+ *
+ */
+
+package org.eclipse.edc.gcp.bigquery;
+
+import org.eclipse.edc.connector.dataplane.spi.pipeline.DataSource;
+
+import java.io.InputStream;
+
+public class BigQueryPart implements DataSource.Part {
+    private final String name;
+    private final InputStream inputStream;
+    private Exception sourceException;
+
+
+    public BigQueryPart(String name, InputStream inputStream) {
+        this.name = name;
+        this.inputStream = inputStream;
+    }
+
+    @Override
+    public String name() {
+        return name;
+    }
+
+    @Override
+    public InputStream openStream() {
+        return inputStream;
+    }
+
+    @Override
+    public long size() {
+        return SIZE_UNKNOWN;
+    }
+
+    public void setException(Exception exception) {
+        sourceException = exception;
+    }
+
+    public Exception getException() {
+        return sourceException;
+    }
+}

extensions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/bigquery/BigQueryTarget.java

@@ -0,0 +1,28 @@
+/*
+ *  Copyright (c) 2024 Google LLC
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Google LLC
+ *
+ */
+
+package org.eclipse.edc.gcp.bigquery;
+
+import com.google.cloud.bigquery.TableId;
+import com.google.cloud.bigquery.storage.v1.TableName;
+
+public record BigQueryTarget(String project, String dataset, String table) {
+    public TableId getTableId() {
+        return TableId.of(project, dataset, table);
+    }
+
+    public TableName getTableName() {
+        return TableName.of(project, dataset, table);
+    }
+}

extensions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/bigquery/service/BigQueryProvisionService.java

@@ -0,0 +1,29 @@
+/*
+ *  Copyright (c) 2024 Google LLC
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Google LLC
+ *
+ */
+
+package org.eclipse.edc.gcp.bigquery.service;
+
+import org.eclipse.edc.gcp.bigquery.BigQueryTarget;
+
+/**
+ * Interface used by BigQuery Provisioner.
+ */
+public interface BigQueryProvisionService {
+    /**
+     * Checks whether the table defined exists.
+     *
+     * @return true if the table targeted exists.
+     */
+    boolean tableExists(BigQueryTarget target);
+}

extensions/common/gcp/gcp-core/src/main/java/org/eclipse/edc/gcp/bigquery/service/BigQueryProvisionServiceFactory.java

@@ -0,0 +1,28 @@
+/*
+ *  Copyright (c) 2024 Google LLC
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Google LLC
+ *
+ */
+
+package org.eclipse.edc.gcp.bigquery.service;
+
+/**
+ * Interface used to provide a specific factory for the provision service.
+ * Can be used by test to provide mocks.
+ */
+public interface BigQueryProvisionServiceFactory {
+    /**
+     * Provides instances of the provision service.
+     *
+     * @return the instance of the service.
+     */
+    BigQueryProvisionService get();
+}