diff --git a/.github/workflows/deploy-connector.yaml b/.github/workflows/deploy-connector.yaml new file mode 100644 index 0000000..b6a422f --- /dev/null +++ b/.github/workflows/deploy-connector.yaml @@ -0,0 +1,183 @@ +name: Deploy Connector + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'deployment/**' + pull_request: + paths: + - 'deployment/**' + +permissions: + contents: read + +jobs: + + Deploy-Connector: + defaults: + run: + working-directory: ./deployment/connector + + env: + CLUSTER_NAME: eonax-cluster + DID_WEB: did:web:localhost:ih:did + permissions: + checks: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CICD_TOKEN }} + + - name: 'Setup Terraform' + uses: hashicorp/setup-terraform@v3.1.1 + with: + terraform_version: 1.6.0 + terraform_wrapper: false + + - name: 'Create Kubernetes cluster' + uses: helm/kind-action@v1.10.0 + with: + cluster_name: ${{ env.CLUSTER_NAME }} + config: ./deployment/kind.config.yaml + + - name: 'Create Ingress Controller' + shell: bash + run: | + kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml + kubectl wait \ + --namespace ingress-nginx \ + --for=condition=ready pod \ + --selector=app.kubernetes.io/component=controller \ + --timeout=90s + + - name: 'Install Vault and DB' + working-directory: ./deployment/storage + shell: bash + run: | + terraform init + terraform apply -auto-approve + + - name: 'Login to Docker registry' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_TOKEN }} + + - name: 'Pull Docker images and Helm charts' + shell: bash + run: | + for i in control-plane data-plane identity-hub; do \ + image=eonax-$i-postgresql-hashicorpvault; \ + + ## pull the Docker image + docker pull ${{ vars.DOCKER_REPO }}/$image:${{ vars.EONAX_VERSION }}; \ + ## tag image with version latest + docker tag ${{ vars.DOCKER_REPO }}/$image:${{ vars.EONAX_VERSION }} $image:latest; \ + ## load image to the cluster + kind load docker-image $image:latest --name ${{ env.CLUSTER_NAME }}; \ + + ## pull Helm charts + chart=${i//-/}; \ + helm pull ${{ vars.HELM_REPO }}/$chart --version ${{ vars.EONAX_VERSION }}; \ + mv $chart-${{ vars.EONAX_VERSION }}.tgz $chart.tgz; \ + done + + - name: 'Download SQL files' + shell: bash + run: | + jq -r --arg version "${{ vars.EDC_VERSION }}" '.files[] | "https://raw.githubusercontent.com/eclipse-edc/\(.repo)/\($version)/\(.path)/src/main/resources/\(.file_name)"' sql.json | \ + tr -d '\r' | \ + while read -r url; do curl -o "./connector/sql/$(basename "$url")" "$url"; done + + - name: 'Install connector' + shell: bash + run: | + terraform init + terraform apply -auto-approve + + - name: 'Generate keys' + shell: bash + run: | + openssl genpkey -algorithm RSA -out private-key.pem -pkeyopt rsa_keygen_bits:2048 && \ + openssl rsa -pubout -in private-key.pem -out public-key.pem && \ + for k in public-key private-key; do VAULT_TOKEN=root VAULT_ADDR=http://localhost/vault vault kv put secret/$k content=@$k.pem; done + + - name: 'Create participant context' + shell: bash + env: + IH_RESOLUTION_URL: http://localhost/ih/resolution + CP_DSP_URL: http://localhost/cp/dsp + run: | + didBase64Url=$(echo -n "$DID_WEB" | base64 | tr '+/' '-_' | tr -d '=') + + curl -X POST -H "Content-Type: application/json" -d "$(cat < serialized as JSON'; COMMENT ON COLUMN edc_policydefinitions.prohibitions IS 'Java List serialized as JSON'; COMMENT ON COLUMN edc_policydefinitions.duties IS 'Java List serialized as JSON'; +COMMENT ON COLUMN edc_policydefinitions.profiles IS 'Java List serialized as JSON'; COMMENT ON COLUMN edc_policydefinitions.extensible_properties IS 'Java Map serialized as JSON'; COMMENT ON COLUMN edc_policydefinitions.policy_type IS 'Java PolicyType serialized as JSON'; CREATE UNIQUE INDEX IF NOT EXISTS edc_policydefinitions_id_uindex - ON edc_policydefinitions (policy_id); \ No newline at end of file + ON edc_policydefinitions (policy_id); diff --git a/deployment/connector/sql/policy-monitor-schema.sql b/deployment/connector/sql/policy-monitor-schema.sql new file mode 100644 index 0000000..571e69e --- /dev/null +++ b/deployment/connector/sql/policy-monitor-schema.sql @@ -0,0 +1,36 @@ +-- Statements are designed for and tested with Postgres only! + +CREATE TABLE IF NOT EXISTS edc_lease +( + leased_by VARCHAR NOT NULL, + leased_at BIGINT, + lease_duration INTEGER NOT NULL, + lease_id VARCHAR NOT NULL + CONSTRAINT lease_pk + PRIMARY KEY +); + +COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; +COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; + +CREATE TABLE IF NOT EXISTS edc_policy_monitor +( + entry_id VARCHAR NOT NULL PRIMARY KEY, + state INTEGER NOT NULL , + created_at BIGINT NOT NULL , + updated_at BIGINT NOT NULL , + state_count INTEGER DEFAULT 0 NOT NULL, + state_time_stamp BIGINT, + trace_context JSON, + error_detail VARCHAR, + lease_id VARCHAR + CONSTRAINT policy_monitor_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL, + properties JSON, + contract_id VARCHAR +); + + +-- This will help to identify states that need to be transitioned without a table scan when the entries grow +CREATE INDEX IF NOT EXISTS policy_monitor_state ON edc_policy_monitor (state,state_time_stamp); \ No newline at end of file diff --git a/deployment/connector/sql/target-node-directory-schema.sql b/deployment/connector/sql/target-node-directory-schema.sql new file mode 100644 index 0000000..d4fab3a --- /dev/null +++ b/deployment/connector/sql/target-node-directory-schema.sql @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2024 Amadeus IT Group + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Amadeus IT Group - initial API and implementation + * + */ + +-- only intended for and tested with Postgres! +CREATE TABLE IF NOT EXISTS edc_target_node_directory +( + id VARCHAR PRIMARY KEY NOT NULL, + name VARCHAR NOT NULL, + target_url VARCHAR NOT NULL, + supported_protocols JSON +); + +COMMENT ON COLUMN edc_target_node_directory.supported_protocols IS 'List serialized as JSON'; \ No newline at end of file diff --git a/deployment/modules/db/sql/transfer-process.sql b/deployment/connector/sql/transfer-process-schema.sql similarity index 69% rename from deployment/modules/db/sql/transfer-process.sql rename to deployment/connector/sql/transfer-process-schema.sql index 44baf17..ab16436 100644 --- a/deployment/modules/db/sql/transfer-process.sql +++ b/deployment/connector/sql/transfer-process-schema.sql @@ -36,10 +36,17 @@ CREATE TABLE IF NOT EXISTS edc_transfer_process pending BOOLEAN DEFAULT FALSE, transfer_type VARCHAR, protocol_messages JSON, + data_plane_id VARCHAR, + correlation_id VARCHAR, + counter_party_address VARCHAR, + protocol VARCHAR, + asset_id VARCHAR, + contract_id VARCHAR, + data_destination JSON, lease_id VARCHAR - CONSTRAINT transfer_process_lease_lease_id_fk - REFERENCES edc_lease - ON DELETE SET NULL + CONSTRAINT transfer_process_lease_lease_id_fk + REFERENCES edc_lease + ON DELETE SET NULL ); COMMENT ON COLUMN edc_transfer_process.trace_context IS 'Java Map serialized as JSON'; @@ -56,27 +63,8 @@ COMMENT ON COLUMN edc_transfer_process.deprovisioned_resources IS 'List of depro CREATE UNIQUE INDEX IF NOT EXISTS transfer_process_id_uindex ON edc_transfer_process (transferprocess_id); -CREATE TABLE IF NOT EXISTS edc_data_request -( - datarequest_id VARCHAR NOT NULL - CONSTRAINT data_request_pk - PRIMARY KEY, - process_id VARCHAR NOT NULL, - connector_address VARCHAR NOT NULL, - protocol VARCHAR NOT NULL, - asset_id VARCHAR NOT NULL, - contract_id VARCHAR NOT NULL, - data_destination JSON NOT NULL, - transfer_process_id VARCHAR NOT NULL - CONSTRAINT data_request_transfer_process_id_fk - REFERENCES edc_transfer_process - ON UPDATE RESTRICT ON DELETE CASCADE -); - -COMMENT ON COLUMN edc_data_request.data_destination IS 'DataAddress serialized as JSON'; - -CREATE UNIQUE INDEX IF NOT EXISTS data_request_id_uindex - ON edc_data_request (datarequest_id); - CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex - ON edc_lease (lease_id); \ No newline at end of file + ON edc_lease (lease_id); + +-- This will help to identify states that need to be transitioned without a table scan when the entries grow +CREATE INDEX IF NOT EXISTS transfer_process_state ON edc_transfer_process (state,state_time_stamp); \ No newline at end of file diff --git a/deployment/connector/variables.tf b/deployment/connector/variables.tf new file mode 100644 index 0000000..dcd39b0 --- /dev/null +++ b/deployment/connector/variables.tf @@ -0,0 +1,84 @@ +variable "kube_context" { + description = "(Optional) Kubernetes cluster context" + default = "kind-eonax-cluster" +} + +variable "postgres_admin_credentials_secret_name" { + description = "(Optional) Secret containing the DB Admin credentials" + default = "postgresql" +} + +variable "control_plane_dsp_url" { + description = "(Optional) Internet facing URL of the Control Plane DSP api" + default = "http://localhost/cp/dsp" +} + +variable "data_plane_public_url" { + description = "(Optional) Internet facing URL of the Data Plane public api" + default = "http://localhost/dp/public" +} + +variable "identity_hub_did_web_url" { + description = "(Optional) did:web url that should resolve to the internet facing url serving the DID document" + default = "did:web:localhost:ih:did" +} + +variable "vault_url" { + description = "(Optional) Hashicorp Vault url" + default = "http://vault:8200" +} + +variable "db_server_fqdn" { + description = "(Optional) Fqdn of the DB server" + default = "postgres" +} + +variable "db_name" { + description = "(Optional) Name of the connector DB" + default = "connectordb" +} + +variable "db_username" { + description = "(Optional) DB username" + default = "connector" +} + +variable "db_password" { + description = "(Optional) DB password" + default = "connectorpwd" +} + +variable "vault_token_secret_name" { + description = "(Optional) Name of the Secret containing the Vault token" + default = "vault" +} + +variable "public_key_alias" { + description = "(Optional) Alias of the public key in the Vault" + default = "public-key" +} + +variable "private_key_alias" { + description = "(Optional) Alias of the private key in the Vault" + default = "private-key" +} + +variable "negotiation_state_machine_wait_millis" { + description = "(Optional) Wait time of the contract state machines in milliseconds" + default = 2000 +} + +variable "transfer_state_machine_wait_millis" { + description = "(Optional) Wait time of the transfer state machines in milliseconds" + default = 2000 +} + +variable "policy_monitor_state_machine_wait_millis" { + description = "(Optional) Wait time of the policy_monitor state machines in milliseconds" + default = 5000 +} + +variable "data_plane_state_machine_wait_millis" { + description = "(Optional) Wait time of the data plane state machines in milliseconds" + default = 5000 +} \ No newline at end of file diff --git a/deployment/data/Seeding.postman_collection.json b/deployment/data/Seeding.postman_collection.json deleted file mode 100644 index 2cf8fd3..0000000 --- a/deployment/data/Seeding.postman_collection.json +++ /dev/null @@ -1,650 +0,0 @@ -{ - "info": { - "_postman_id": "29e34005-d2be-4e9d-a11c-4db521a16896", - "name": "EonaX-Shared-Resource", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" - }, - "item": [ - { - "name": "Authority", - "item": [ - { - "name": "Public key", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "auth": { - "type": "apikey", - "apikey": [ - { - "key": "value", - "value": "{{vaultToken}}", - "type": "string" - }, - { - "key": "key", - "value": "X-Vault-Token", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"data\": {\r\n \"content\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEcDS3f35yBqbi9q35KaJ4CixIFxvt\\n6afy2Btfi8v+OZjT0RBS98r8ztPujN4o1ECGEgpak2+HpKYOkd5d8kQpjg==\\n-----END PUBLIC KEY-----\"\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{authorityName}}/vault/v1/secret/data/{{authorityName}}-pub", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{authorityName}}", - "vault", - "v1", - "secret", - "data", - "{{authorityName}}-pub" - ] - } - }, - "response": [] - }, - { - "name": "Private key", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "auth": { - "type": "apikey", - "apikey": [ - { - "key": "value", - "value": "{{vaultToken}}", - "type": "string" - }, - { - "key": "key", - "value": "X-Vault-Token", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"data\": {\r\n \"content\": \"-----BEGIN EC PRIVATE KEY-----\\nMHcCAQEEIPDpR1AbQwqpuFozaM3+2Jk1gKF3EpreBBPVYgS39aeAoAoGCCqGSM49\\nAwEHoUQDQgAEcDS3f35yBqbi9q35KaJ4CixIFxvt6afy2Btfi8v+OZjT0RBS98r8\\nztPujN4o1ECGEgpak2+HpKYOkd5d8kQpjg==\\n-----END EC PRIVATE KEY-----\"\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{authorityName}}/vault/v1/secret/data/{{authorityName}}", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{authorityName}}", - "vault", - "v1", - "secret", - "data", - "{{authorityName}}" - ] - } - }, - "response": [] - }, - { - "name": "Create Participant Context (IdentityHub)", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"participantId\": \"did:web:{{authorityName}}-identityhub%3A8383:api:did\",\r\n \"did\": \"did:web:{{authorityName}}-identityhub%3A8383:api:did\",\r\n \"active\": true,\r\n \"key\": {\r\n \"keyId\": \"my-key\",\r\n \"privateKeyAlias\": \"{{authorityName}}\",\r\n \"publicKeyPem\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEcDS3f35yBqbi9q35KaJ4CixIFxvt\\n6afy2Btfi8v+OZjT0RBS98r8ztPujN4o1ECGEgpak2+HpKYOkd5d8kQpjg==\\n-----END PUBLIC KEY-----\"\r\n },\r\n \"serviceEndpoints\": [\r\n {\r\n \"id\": \"credential-service-url\",\r\n \"type\": \"CredentialService\",\r\n \"serviceEndpoint\": \"http://{{authorityName}}-identityhub:8282/api/resolution/v1/participants/ZGlkOndlYjphdXRob3JpdHktaWRlbnRpdHlodWIlM0E4MzgzOmFwaTpkaWQ\"\r\n }\r\n ]\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{authorityName}}/ih/management/v1/participants", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{authorityName}}", - "ih", - "management", - "v1", - "participants" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Consumer", - "item": [ - { - "name": "Public key", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "auth": { - "type": "apikey", - "apikey": [ - { - "key": "value", - "value": "{{vaultToken}}", - "type": "string" - }, - { - "key": "key", - "value": "X-Vault-Token", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"data\": {\r\n \"content\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEOc48T9d92Rr5bXOyUqDs4HFvkRXF\\nDA6wGzrk3+zp8o9Unc/mutDK+ntdDycTDkNXeeJwMVQJWfKE5I9Wgb682w==\\n-----END PUBLIC KEY-----\"\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{consumerName}}/vault/v1/secret/data/{{consumerName}}-pub", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{consumerName}}", - "vault", - "v1", - "secret", - "data", - "{{consumerName}}-pub" - ] - } - }, - "response": [] - }, - { - "name": "Private key", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "auth": { - "type": "apikey", - "apikey": [ - { - "key": "value", - "value": "{{vaultToken}}", - "type": "string" - }, - { - "key": "key", - "value": "X-Vault-Token", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"data\": {\r\n \"content\": \"-----BEGIN EC PRIVATE KEY-----\\nMHcCAQEEICNv5iWlHoubyqa0xzjiU5q4ZXO7ZZG6iSqrjUWyv1QDoAoGCCqGSM49\\nAwEHoUQDQgAEOc48T9d92Rr5bXOyUqDs4HFvkRXFDA6wGzrk3+zp8o9Unc/mutDK\\n+ntdDycTDkNXeeJwMVQJWfKE5I9Wgb682w==\\n-----END EC PRIVATE KEY-----\"\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{consumerName}}/vault/v1/secret/data/{{consumerName}}", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{consumerName}}", - "vault", - "v1", - "secret", - "data", - "{{consumerName}}" - ] - } - }, - "response": [] - }, - { - "name": "Create Participant Context (IdentityHub)", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"participantId\": \"did:web:{{consumerName}}-identityhub%3A8383:api:did\",\r\n \"did\": \"did:web:{{consumerName}}-identityhub%3A8383:api:did\",\r\n \"active\": true,\r\n \"key\": {\r\n \"keyId\": \"my-key\",\r\n \"privateKeyAlias\": \"{{consumerName}}\",\r\n \"publicKeyPem\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEOc48T9d92Rr5bXOyUqDs4HFvkRXF\\nDA6wGzrk3+zp8o9Unc/mutDK+ntdDycTDkNXeeJwMVQJWfKE5I9Wgb682w==\\n-----END PUBLIC KEY-----\"\r\n },\r\n \"serviceEndpoints\": [\r\n {\r\n \"id\": \"credential-service-url\",\r\n \"type\": \"CredentialService\",\r\n \"serviceEndpoint\": \"http://{{consumerName}}-identityhub:8282/api/resolution/v1/participants/ZGlkOndlYjpjb25zdW1lci1pZGVudGl0eWh1YiUzQTgzODM6YXBpOmRpZA\"\r\n },\r\n {\r\n \"id\": \"dsp-url\",\r\n \"type\": \"DSPMessaging\",\r\n \"serviceEndpoint\": \"http://{{consumerName}}-connector:8282/api/dsp\"\r\n }\r\n ]\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{consumerName}}/ih/management/v1/participants", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{consumerName}}", - "ih", - "management", - "v1", - "participants" - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - }, - { - "name": "Provider", - "item": [ - { - "name": "Public key", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "auth": { - "type": "apikey", - "apikey": [ - { - "key": "value", - "value": "{{vaultToken}}", - "type": "string" - }, - { - "key": "key", - "value": "X-Vault-Token", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"data\": {\r\n \"content\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAENqKFUb7sxqJOb8TDX2PmRVRaW7R8\\n48S4RlUncssxMWHJXsrRbkv3naFOh9uB4s6++7hezPop50EdidCiK2Nfyw==\\n-----END PUBLIC KEY-----\"\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{providerName}}/vault/v1/secret/data/{{providerName}}-pub", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{providerName}}", - "vault", - "v1", - "secret", - "data", - "{{providerName}}-pub" - ] - } - }, - "response": [] - }, - { - "name": "Private key", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "auth": { - "type": "apikey", - "apikey": [ - { - "key": "value", - "value": "{{vaultToken}}", - "type": "string" - }, - { - "key": "key", - "value": "X-Vault-Token", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"data\": {\r\n \"content\": \"-----BEGIN EC PRIVATE KEY-----\\nMHcCAQEEIB2n2AQoTo+pHUBOakicb3UaQf+VeyQTX0RH0axZdEDtoAoGCCqGSM49\\nAwEHoUQDQgAENqKFUb7sxqJOb8TDX2PmRVRaW7R848S4RlUncssxMWHJXsrRbkv3\\nnaFOh9uB4s6++7hezPop50EdidCiK2Nfyw==\\n-----END EC PRIVATE KEY-----\"\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{providerName}}/vault/v1/secret/data/{{providerName}}", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{providerName}}", - "vault", - "v1", - "secret", - "data", - "{{providerName}}" - ] - } - }, - "response": [] - }, - { - "name": "Create asset \"Hello World!\"", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"@context\": {\r\n \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"\r\n },\r\n \"@id\": \"hello-world\",\r\n \"properties\": {\r\n \"name\": \"Hello World asset\",\r\n \"contenttype\": \"application/json\",\r\n \"version\": \"1.0\",\r\n \"description\": \"An asset that says Hello World!\"\r\n },\r\n \"dataAddress\": {\r\n \"type\": \"HttpData\",\r\n \"baseUrl\": \"http://provider-backend:8080/api/provider/data\"\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{providerName}}/cp/management/v3/assets", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{providerName}}", - "cp", - "management", - "v3", - "assets" - ] - } - }, - "response": [] - }, - { - "name": "Create policy Eona-X Membership", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"@context\": {\r\n \"@vocab\": \"https://w3id.org/edc/v0.0.1/ns/\"\r\n },\r\n \"@type\": \"PolicyDefinitionDto\",\r\n \"@id\": \"eonax-member-only\",\r\n \"policy\": {\r\n \"@context\": \"http://www.w3.org/ns/odrl.jsonld\",\r\n \"@type\": \"http://www.w3.org/ns/odrl/2/Set\",\r\n \"permission\": [\r\n {\r\n \"action\": \"use\",\r\n \"constraint\": {\r\n \"@type\": \"Constraint\",\r\n \"leftOperand\": \"MembershipCredential\",\r\n \"operator\": \"odrl:eq\",\r\n \"rightOperand\": \"active\"\r\n }\r\n }\r\n ]\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{providerName}}/cp/management/v2/policydefinitions", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{providerName}}", - "cp", - "management", - "v2", - "policydefinitions" - ] - } - }, - "response": [] - }, - { - "name": "Create contract def \"Hello World\"", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"@id\": \"6b8f1075-e48b-4c02-9c0b-14f2e3b9bde9\",\r\n \"@type\": \"https://w3id.org/edc/v0.0.1/ns/ContractDefinition\",\r\n \"https://w3id.org/edc/v0.0.1/ns/accessPolicyId\": \"eonax-member-only\",\r\n \"https://w3id.org/edc/v0.0.1/ns/contractPolicyId\": \"eonax-member-only\",\r\n \"https://w3id.org/edc/v0.0.1/ns/assetsSelector\": [\r\n {\r\n \"@type\": \"https://w3id.org/edc/v0.0.1/ns/Criterion\",\r\n \"https://w3id.org/edc/v0.0.1/ns/operandLeft\": \"https://w3id.org/edc/v0.0.1/ns/id\",\r\n \"https://w3id.org/edc/v0.0.1/ns/operator\": \"=\",\r\n \"https://w3id.org/edc/v0.0.1/ns/operandRight\": \"hello-world\"\r\n }\r\n ]\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{providerName}}/cp/management/v2/contractdefinitions", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{providerName}}", - "cp", - "management", - "v2", - "contractdefinitions" - ] - } - }, - "response": [] - }, - { - "name": "Create Participant Context (IdentityHub)", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"participantId\": \"did:web:{{providerName}}-identityhub%3A8383:api:did\",\r\n \"did\": \"did:web:{{providerName}}-identityhub%3A8383:api:did\",\r\n \"active\": true,\r\n \"key\": {\r\n \"keyId\": \"my-key\",\r\n \"privateKeyAlias\": \"{{providerName}}\",\r\n \"publicKeyPem\": \"-----BEGIN PUBLIC KEY-----\\nMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAENqKFUb7sxqJOb8TDX2PmRVRaW7R8\\n48S4RlUncssxMWHJXsrRbkv3naFOh9uB4s6++7hezPop50EdidCiK2Nfyw==\\n-----END PUBLIC KEY-----\"\r\n },\r\n \"serviceEndpoints\": [\r\n {\r\n \"id\": \"credential-service-url\",\r\n \"type\": \"CredentialService\",\r\n \"serviceEndpoint\": \"http://{{providerName}}-identityhub:8282/api/resolution/v1/participants/ZGlkOndlYjpwcm92aWRlci1pZGVudGl0eWh1YiUzQTgzODM6YXBpOmRpZA\"\r\n },\r\n {\r\n \"id\": \"dsp-url\",\r\n \"type\": \"DSPMessaging\",\r\n \"serviceEndpoint\": \"http://{{providerName}}-connector:8282/api/dsp\"\r\n }\r\n ]\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "http://localhost:80/{{providerName}}/ih/management/v1/participants", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "80", - "path": [ - "{{providerName}}", - "ih", - "management", - "v1", - "participants" - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Status code is 200 or 204\", function () {", - " pm.expect(pm.response.code).to.be.oneOf([204, 200])", - "});" - ] - } - } - ], - "variable": [ - { - "key": "vaultToken", - "value": "root" - }, - { - "key": "authorityName", - "value": "authority" - }, - { - "key": "providerName", - "value": "provider" - }, - { - "key": "consumerName", - "value": "consumer" - } - ] -} \ No newline at end of file diff --git a/deployment/main.tf b/deployment/main.tf deleted file mode 100644 index c93423f..0000000 --- a/deployment/main.tf +++ /dev/null @@ -1,116 +0,0 @@ -locals { - postgres_credentials_secret_name = "postgres-db" - - docker_image_pull_secret_name = "dockerconfigjson-github-com" - base64Token = base64encode("${var.container_registry_username}:${var.container_registry_token}") - secretJson = "{\"auths\":{\"ghcr.io\":{\"auth\":\"${local.base64Token}\"}}}" - - participants = [var.data_provider, var.data_consumer] -} - -################### -## POSTGRESQL DB ## -################### - -module "db" { - source = "./modules/db" - - authority_name = var.authority.name - participant_names = [for p in local.participants : p.name] -} - -#################################### -## K8S SECRET WITH DB CREDENTIALS ## -#################################### - -resource "kubernetes_secret" "postgresql-db-secret" { - - metadata { - name = local.postgres_credentials_secret_name - } - - data = { - "username" = module.db.postgres_username - "password" = module.db.postgres_password - } -} - -############################## -## DOCKER IMAGE PULL SECRET ## -############################## - -resource "kubernetes_secret_v1" "docker-image-pull-secret" { - - metadata { - name = local.docker_image_pull_secret_name - } - - data = { - ".dockerconfigjson" = local.secretJson - } - - type = "kubernetes.io/dockerconfigjson" -} - -################## -## PARTICIPANTS ## -################## - -module "participant" { - source = "./modules/participant" - - for_each = { for p in local.participants : p.name => p } - participant = each.value - - # POSTGRES - postgres_host = module.db.postgres_host - postgres_credentials_secret_name = kubernetes_secret.postgresql-db-secret.metadata.0.name - - # DOCKER - docker_image_pull_secret_name = kubernetes_secret_v1.docker-image-pull-secret.metadata.0.name - helm_chart_repo = var.helm_chart_repo - - # CONNECTOR - connector_repo = var.connector_repo - connector_chart_name = var.connector_chart_name - connector_version = var.connector_version - - # IDENTITY HUB - identityhub_repo = var.identityhub_repo - identityhub_chart_name = var.identityhub_chart_name - identityhub_version = var.identityhub_version -} - -######################### -## DATASPACE AUTHORITY ## -######################### - -module "authority" { - source = "./modules/authority" - - authority = var.authority - - participants = [ - for p in local.participants : merge(p, { - did : module.participant[p.name].did_url - }) - ] - - # POSTGRES - postgres_host = module.db.postgres_host - postgres_credentials_secret_name = kubernetes_secret.postgresql-db-secret.metadata.0.name - - # DOCKER - docker_image_pull_secret_name = kubernetes_secret_v1.docker-image-pull-secret.metadata.0.name - helm_chart_repo = var.helm_chart_repo - - # FEDERATED CATALOG - federatedcatalog_chart_name = var.federatedcatalog_chart_name - federatedcatalog_repo = var.federatedcatalog_repo - federatedcatalog_version = var.federatedcatalog_version - - # IDENTITY HUB - identityhub_chart_name = var.identityhub_chart_name - identityhub_version = var.identityhub_version - identityhub_repo = var.identityhub_repo -} \ No newline at end of file diff --git a/deployment/modules/authority/catalog.tf b/deployment/modules/authority/catalog.tf deleted file mode 100644 index 8c1efc3..0000000 --- a/deployment/modules/authority/catalog.tf +++ /dev/null @@ -1,83 +0,0 @@ -locals { - catalog_release_name = "${var.authority.name}-federatedcatalog" - - crawler_initial_delay = 10 - crawler_execution_period = 10 -} - -resource "helm_release" "federated-catalog" { - name = local.catalog_release_name - cleanup_on_fail = true - dependency_update = true - recreate_pods = true - repository = var.helm_chart_repo - chart = var.federatedcatalog_chart_name - version = var.federatedcatalog_version - - values = [ - yamlencode({ - - "imagePullSecrets" : [ - { - "name" : var.docker_image_pull_secret_name - } - ], - - "federatedcatalog" : { - "image" : { - "repository" : var.federatedcatalog_repo - "tag" : var.federatedcatalog_version - }, - "did" : { - "web" : { - "url" : local.did_url, - "useHttps" : false - } - }, - "crawler" : { - "participantsRegistry" : { - "url" : "http://${local.participants_registry_name}:8080/participants.json" - }, - "cache" : { - "executionPeriodSeconds" : local.crawler_execution_period - "executionDelaySeconds" : local.crawler_initial_delay - } - }, - "trustedIssuers" : { - "authority" : { - "did" : local.did_url - } - }, - "keys" : { - "sts" : { - "privateKeyVaultAlias" : local.privatekey_alias, - "publicKeyDid" : local.did_url - } - }, - "ingress" : { - "enabled" : true - "className" : "nginx" - "annotations" : { - "nginx.ingress.kubernetes.io/ssl-redirect" : "false" - "nginx.ingress.kubernetes.io/use-regex" : "true" - "nginx.ingress.kubernetes.io/rewrite-target" : "/api/$1$2" - }, - "endpoints" : [ - { - "port" : 8181, - "path" : "/${var.authority.name}/catalog/(management)(.*)" - } - ] - }, - "vault" : { - "hashicorp" : { - "url" : module.vault.vault_url - "token" : module.vault.vault_token - } - } - } - }) - ] - - depends_on = [kubernetes_service.registry-service, module.vault] -} \ No newline at end of file diff --git a/deployment/modules/authority/identityhub.tf b/deployment/modules/authority/identityhub.tf deleted file mode 100644 index e111e01..0000000 --- a/deployment/modules/authority/identityhub.tf +++ /dev/null @@ -1,97 +0,0 @@ -locals { - identityhub_release_name = "${var.authority.name}-identityhub" - - credential_service_url = "http://${local.identityhub_release_name}:8282/api/resolution" - did_url = "did:web:${local.identityhub_release_name}%3A8383:api:did" -} - -############################ -## VERIFIABLE CREDENTIALS ## -############################ - -resource "kubernetes_config_map" "verifiable-credentials" { - - metadata { - name = "${local.identityhub_release_name}-credentials" - } - - data = { - "credentials.json" = jsonencode(var.authority.vc) - } -} - -################## -## IDENTITY HUB ## -################## - -resource "helm_release" "identity-hub" { - name = local.identityhub_release_name - cleanup_on_fail = true - dependency_update = true - recreate_pods = true - repository = var.helm_chart_repo - chart = var.identityhub_chart_name - version = var.identityhub_version - - values = [ - yamlencode({ - - "imagePullSecrets" : [ - { - "name" : var.docker_image_pull_secret_name - } - ], - - "identityhub" : { - "image" : { - "repository" : var.identityhub_repo - "tag" : var.identityhub_version - }, - "keys" : { - "sts" : { - "publicKeyVaultAlias" : local.publickey_alias - } - }, - "did" : { - "web" : { - "url" : local.did_url, - "useHttps" : false - } - }, - "postgresql" : { - "jdbcUrl" : "jdbc:postgresql://${var.postgres_host}/${var.authority.name}", - "secret" : { - "name" : var.postgres_credentials_secret_name - } - }, - "ingress" : { - "enabled" : true - "className" : "nginx" - "annotations" : { - "nginx.ingress.kubernetes.io/ssl-redirect" : "false" - "nginx.ingress.kubernetes.io/use-regex" : "true" - "nginx.ingress.kubernetes.io/rewrite-target" : "/api/$1$2" - }, - "endpoints" : [ - { - "port" : 8181, - "path" : "/${var.authority.name}/ih/(management)(.*)" - }, - { - "port" : 8282, - "path" : "/${var.authority.name}/ih/(resolution)(.*)" - } - ] - }, - "vault" : { - "hashicorp" : { - "url" : module.vault.vault_url - "token" : module.vault.vault_token - } - } - } - }) - ] - - depends_on = [module.vault] -} \ No newline at end of file diff --git a/deployment/modules/authority/main.tf b/deployment/modules/authority/main.tf deleted file mode 100644 index a28f18f..0000000 --- a/deployment/modules/authority/main.tf +++ /dev/null @@ -1,10 +0,0 @@ -locals { - privatekey_alias = var.authority.name - publickey_alias = "${local.privatekey_alias}-pub" -} - -module "vault" { - source = "../vault" - - participant_name = var.authority.name -} \ No newline at end of file diff --git a/deployment/modules/authority/registry.tf b/deployment/modules/authority/registry.tf deleted file mode 100644 index 6890311..0000000 --- a/deployment/modules/authority/registry.tf +++ /dev/null @@ -1,111 +0,0 @@ -locals { - server_image = "nginxinc/nginx-unprivileged:1.25.3" - participants_registry_name = "registry" - server_port = 8080 -} - -resource "kubernetes_deployment" "registry" { - metadata { - name = local.participants_registry_name - labels = { - app = local.participants_registry_name - } - } - - spec { - replicas = 1 - selector { - match_labels = { - app = local.participants_registry_name - } - } - template { - metadata { - labels = { - app = local.participants_registry_name - } - } - spec { - container { - image = local.server_image - name = local.participants_registry_name - - env_from { - config_map_ref { - name = kubernetes_config_map.registry-config.metadata[0].name - } - } - port { - container_port = local.server_port - name = "registry-port" - } - - volume_mount { - mount_path = "/usr/share/nginx/html" - name = "registry-config" - } - } - volume { - name = "registry-config" - config_map { - name = kubernetes_config_map.registry-config.metadata.0.name - } - } - } - } - } -} - -resource "kubernetes_config_map" "registry-config" { - metadata { - name = "participants-registry-config" - } - - data = { - "participants.json" : jsonencode(var.participants) - } -} - -resource "kubernetes_service" "registry-service" { - metadata { - name = local.participants_registry_name - } - spec { - selector = { - app = kubernetes_deployment.registry.spec.0.template.0.metadata[0].labels.app - } - port { - name = "registry-port" - port = local.server_port - target_port = local.server_port - } - } -} - -resource "kubernetes_ingress_v1" "registry-ingress" { - metadata { - name = "participants-registry-ingress" - annotations = { - "nginx.ingress.kubernetes.io/rewrite-target" = "/$2" - "nginx.ingress.kubernetes.io/use-regex" = "true" - } - } - spec { - ingress_class_name = "nginx" - rule { - http { - path { - path = "/${var.authority.name}/${local.participants_registry_name}(/|$)(.*)" - backend { - service { - name = kubernetes_service.registry-service.metadata.0.name - port { - number = local.server_port - } - } - } - } - } - } - } -} diff --git a/deployment/modules/authority/variables.tf b/deployment/modules/authority/variables.tf deleted file mode 100644 index bb5dc58..0000000 --- a/deployment/modules/authority/variables.tf +++ /dev/null @@ -1,34 +0,0 @@ -variable "authority" { - type = object({ - name = string, - vc = list(any) - }) -} - -variable "participants" { - type = list(object({ - name = string - did = string - })) -} - -# POSTGRES -variable "postgres_host" {} -variable "postgres_credentials_secret_name" {} - -# FEDERATED CATALOG -variable "federatedcatalog_repo" {} -variable "federatedcatalog_chart_name" {} -variable "federatedcatalog_version" {} - -# IDENTITY HUB -variable "identityhub_repo" {} -variable "identityhub_chart_name" {} -variable "identityhub_version" {} - -# DOCKER PULL -variable "helm_chart_repo" {} -variable "docker_image_pull_secret_name" {} - - - diff --git a/deployment/modules/db/init.sh b/deployment/modules/db/init.sh deleted file mode 100644 index 4a9f17e..0000000 --- a/deployment/modules/db/init.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -e - -## Participants DB initialization -for db in ${participants} -do - psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -c "CREATE DATABASE $db" - psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$db" -a -f /docker-entrypoint-initdb.d/db_bootstrap_script.sql -done \ No newline at end of file diff --git a/deployment/modules/db/main.tf b/deployment/modules/db/main.tf deleted file mode 100644 index 8495ecd..0000000 --- a/deployment/modules/db/main.tf +++ /dev/null @@ -1,116 +0,0 @@ -locals { - pg_image = "postgres:15.3-alpine3.18" - pg_username = "postgres" - pg_password = "postgres" - - sql_files_path = fileset(path.module, "sql/*.sql") - sql_files_full_path = formatlist("${path.module}/%s", local.sql_files_path) - sql_files = [for p in local.sql_files_full_path : file(p)] - db_bootstrap_script = join("\n", local.sql_files) -} - -resource "kubernetes_deployment" "postgres" { - metadata { - name = "postgres" - labels = { - app = "postgres" - } - } - - spec { - replicas = 1 - selector { - match_labels = { - app = "postgres" - } - } - template { - metadata { - labels = { - app = "postgres" - } - } - spec { - container { - image = local.pg_image - name = "postgres" - - env { - name = "POSTGRES_USER" - value = local.pg_username - } - - env { - name = "POSTGRES_PASSWORD" - value = local.pg_password - } - - port { - container_port = var.postgres_port - name = "postgres-port" - } - - volume_mount { - mount_path = "/docker-entrypoint-initdb.d/" - name = "pg-initdb" - } - - # Uncomment this to assign (more) resources - # resources { - # limits = { - # cpu = "2" - # memory = "512Mi" - # } - # requests = { - # cpu = "250m" - # memory = "50Mi" - # } - # } - liveness_probe { - exec { - command = ["pg_isready", "-U", "postgres"] - } - failure_threshold = 10 - period_seconds = 5 - timeout_seconds = 30 - } - } - volume { - name = "pg-initdb" - config_map { - name = kubernetes_config_map.postgres-config.metadata.0.name - } - } - } - } - } -} - -resource "kubernetes_config_map" "postgres-config" { - metadata { - name = "pg-initdb-config" - } - - data = { - "db_bootstrap_script.sql" = local.db_bootstrap_script - "init.sh" = templatefile("${path.module}/init.sh", { - participants : join(" ", concat([var.authority_name], var.participant_names)) - }) - } -} - -resource "kubernetes_service" "pg-service" { - metadata { - name = "postgres" - } - spec { - selector = { - app = kubernetes_deployment.postgres.spec.0.template.0.metadata[0].labels.app - } - port { - name = "pg-port" - port = var.postgres_port - target_port = var.postgres_port - } - } -} diff --git a/deployment/modules/db/output.tf b/deployment/modules/db/output.tf deleted file mode 100644 index 402f105..0000000 --- a/deployment/modules/db/output.tf +++ /dev/null @@ -1,11 +0,0 @@ -output "postgres_host" { - value = "${kubernetes_service.pg-service.metadata.0.name}:${var.postgres_port}" -} - -output "postgres_username" { - value = local.pg_username -} - -output "postgres_password" { - value = local.pg_password -} \ No newline at end of file diff --git a/deployment/modules/db/sql/edr.sql b/deployment/modules/db/sql/edr.sql deleted file mode 100644 index ac91434..0000000 --- a/deployment/modules/db/sql/edr.sql +++ /dev/null @@ -1,56 +0,0 @@ --- --- Copyright (c) 2023 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) --- --- This program and the accompanying materials are made available under the --- terms of the Apache License, Version 2.0 which is available at --- https://www.apache.org/licenses/LICENSE-2.0 --- --- SPDX-License-Identifier: Apache-2.0 --- --- Contributors: --- Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation --- - - -CREATE TABLE IF NOT EXISTS edc_lease -( - leased_by VARCHAR NOT NULL, - leased_at BIGINT, - lease_duration INTEGER DEFAULT 60000 NOT NULL, - lease_id VARCHAR NOT NULL - CONSTRAINT lease_pk - PRIMARY KEY -); - -COMMENT ON COLUMN edc_lease.leased_at IS 'posix timestamp of lease'; - -COMMENT ON COLUMN edc_lease.lease_duration IS 'duration of lease in milliseconds'; - - -CREATE UNIQUE INDEX IF NOT EXISTS lease_lease_id_uindex - ON edc_lease (lease_id); - -CREATE TABLE IF NOT EXISTS edc_edr_cache -( - transfer_process_id VARCHAR NOT NULL PRIMARY KEY, - agreement_id VARCHAR NOT NULL, - asset_id VARCHAR NOT NULL, - edr_id VARCHAR NOT NULL, - contract_negotiation_id VARCHAR, - provider_id VARCHAR, - expiration_timestamp BIGINT, - state INTEGER DEFAULT 0 NOT NULL, - state_count INTEGER DEFAULT 0, - state_timestamp BIGINT, - error_detail VARCHAR, - lease_id VARCHAR CONSTRAINT edc_edr_cache_lease_lease_id_fk REFERENCES edc_lease ON DELETE SET NULL, - created_at BIGINT NOT NULL, - updated_at BIGINT NOT NULL -); - -CREATE INDEX IF NOT EXISTS edc_edr_asset_id_index - ON edc_edr_cache (asset_id); - - -CREATE INDEX IF NOT EXISTS edc_edr_agreement_id_index - ON edc_edr_cache (agreement_id); \ No newline at end of file diff --git a/deployment/modules/db/variables.tf b/deployment/modules/db/variables.tf deleted file mode 100644 index 4513cc3..0000000 --- a/deployment/modules/db/variables.tf +++ /dev/null @@ -1,9 +0,0 @@ -variable "postgres_port" { - default = 5432 -} - -variable "authority_name" {} - -variable "participant_names" { - type = list(string) -} \ No newline at end of file diff --git a/deployment/modules/participant/backend-api.tf b/deployment/modules/participant/backend-api.tf deleted file mode 100644 index f6adbd1..0000000 --- a/deployment/modules/participant/backend-api.tf +++ /dev/null @@ -1,111 +0,0 @@ -locals { - server_image = "nginxinc/nginx-unprivileged:1.25.3" - backend-api-name = "${var.participant.name}-api" - server_port = 8080 -} - -resource "kubernetes_deployment" "backend-api" { - metadata { - name = local.backend-api-name - labels = { - app = local.backend-api-name - } - } - - spec { - replicas = 1 - selector { - match_labels = { - app = local.backend-api-name - } - } - template { - metadata { - labels = { - app = local.backend-api-name - } - } - spec { - container { - image = local.server_image - name = local.backend-api-name - - env_from { - config_map_ref { - name = kubernetes_config_map.backend-api-config.metadata[0].name - } - } - port { - container_port = local.server_port - name = "api-port" - } - - volume_mount { - mount_path = "/usr/share/nginx/html" - name = "api-config" - } - } - volume { - name = "api-config" - config_map { - name = kubernetes_config_map.backend-api-config.metadata.0.name - } - } - } - } - } -} - -resource "kubernetes_config_map" "backend-api-config" { - metadata { - name = local.backend-api-name - } - - data = { - "data.json" : jsonencode({ message = "Hello World!" }) - } -} - -resource "kubernetes_service" "backend-api-service" { - metadata { - name = local.backend-api-name - } - spec { - selector = { - app = kubernetes_deployment.backend-api.spec.0.template.0.metadata[0].labels.app - } - port { - name = "api-port" - port = local.server_port - target_port = local.server_port - } - } -} - -resource "kubernetes_ingress_v1" "backend-api-ingress" { - metadata { - name = local.backend-api-name - annotations = { - "nginx.ingress.kubernetes.io/rewrite-target" = "/$2" - "nginx.ingress.kubernetes.io/use-regex" = "true" - } - } - spec { - ingress_class_name = "nginx" - rule { - http { - path { - path = "/${var.participant.name}/${local.backend-api-name}(/|$)(.*)" - backend { - service { - name = kubernetes_service.backend-api-service.metadata.0.name - port { - number = local.server_port - } - } - } - } - } - } - } -} diff --git a/deployment/modules/participant/connector.tf b/deployment/modules/participant/connector.tf deleted file mode 100644 index e7655b1..0000000 --- a/deployment/modules/participant/connector.tf +++ /dev/null @@ -1,109 +0,0 @@ -locals { - connector_release_name = "${var.participant.name}-connector" - - ################## - ## IMPORT NOTE! ## - ############################################################################################ - # These URLs must be the external routes exposed by the participant over the public internet - # which, are typically exposed through an API gateway, an external Load Balancer... - # In the case of this MVD we use internal routes for simplicity, but this should not - # reproduce in prod-grade deployment as all connectors of a dataspace will not be deployed - # in the same Kubernetes cluster in the real life - ############################################################################################ - protocol_url = "http://${local.connector_release_name}:8282/api/dsp" - public_url = "http://${local.connector_release_name}:8484/api/public" -} - -resource "helm_release" "connector" { - name = local.connector_release_name - cleanup_on_fail = true - dependency_update = true - recreate_pods = true - repository = var.helm_chart_repo - chart = var.connector_chart_name - version = var.connector_version - - values = [ - yamlencode({ - - "imagePullSecrets" : [ - { - "name" : var.docker_image_pull_secret_name - } - ], - - "connector" : { - "image" : { - "repository" : var.connector_repo - "tag" : var.connector_version - }, - "keys" : { - // use the same key pair for simplicity - "dataplane" : { - "privateKeyVaultAlias" : local.privatekey_alias, - "publicKeyVaultAlias" : local.publickey_alias - }, - "sts" : { - "privateKeyVaultAlias" : local.privatekey_alias, - "publicKeyDid" : local.did_url - } - }, - "did" : { - "web" : { - "url" : local.did_url - "useHttps" : false - } - }, - "trustedIssuers" : { - "authority" : { - "did" : "did:web:authority-identityhub%3A8383:api:did" - } - }, - "ingress" : { - "enabled" : true - "className" : "nginx" - "annotations" : { - "nginx.ingress.kubernetes.io/ssl-redirect" : "false" - "nginx.ingress.kubernetes.io/use-regex" : "true" - "nginx.ingress.kubernetes.io/rewrite-target" : "/api/$1$2" - }, - "endpoints" : [ - { - "port" : 8181, - "path" : "/${var.participant.name}/cp/(management)(.*)" - }, - { - "port" : 8282, - "path" : "/${var.participant.name}/cp/(dsp)(.*)" - }, - { - "port" : 8484, - "path" : "/${var.participant.name}/dp/(public)(.*)" - }, - { - "port" : 8585, - "path" : "/${var.participant.name}/dp/(data)(.*)" - } - ] - }, - "url" : { - "public" : local.public_url - }, - "postgresql" : { - "jdbcUrl" : "jdbc:postgresql://${var.postgres_host}/${var.participant.name}", - "secret" : { - "name" : var.postgres_credentials_secret_name - } - }, - "vault" : { - "hashicorp" : { - "url" : module.vault.vault_url - "token" : module.vault.vault_token - } - } - } - }) - ] - - depends_on = [module.vault] -} \ No newline at end of file diff --git a/deployment/modules/participant/identityhub.tf b/deployment/modules/participant/identityhub.tf deleted file mode 100644 index 79f559d..0000000 --- a/deployment/modules/participant/identityhub.tf +++ /dev/null @@ -1,103 +0,0 @@ -locals { - identityhub_release_name = "${var.participant.name}-identityhub" - - ################## - ## IMPORT NOTE! ## - ############################################################################################ - # These URLs must be the external routes exposed by the participant over the public internet - # which, are typically exposed through an API gateway, an external Load Balancer... - # In the case of this MVD we use internal routes for simplicity, but this should not - # reproduce in prod-grade deployment as all connectors of a dataspace will not be deployed - # in the same Kubernetes cluster in the real life - ############################################################################################ - did_url = "did:web:${local.identityhub_release_name}%3A8383:api:did" -} - -############################ -## VERIFIABLE CREDENTIALS ## -############################ - -resource "kubernetes_config_map" "verifiable-credentials" { - - metadata { - name = "${local.identityhub_release_name}-credentials" - } - - data = { - "credentials.json" = jsonencode(var.participant.vc) - - } -} - -resource "helm_release" "identity-hub" { - name = local.identityhub_release_name - cleanup_on_fail = true - dependency_update = true - recreate_pods = true - repository = var.helm_chart_repo - chart = var.identityhub_chart_name - version = var.identityhub_version - - values = [ - yamlencode({ - - "imagePullSecrets" : [ - { - "name" : var.docker_image_pull_secret_name - } - ], - - "identityhub" : { - "image" : { - "repository" : var.identityhub_repo - "tag" : var.identityhub_version - }, - "keys" : { - "sts" : { - "publicKeyVaultAlias" : local.publickey_alias - } - }, - "did" : { - "web" : { - "url" : local.did_url, - "useHttps" : false - } - }, - "postgresql" : { - "jdbcUrl" : "jdbc:postgresql://${var.postgres_host}/${var.participant.name}", - "secret" : { - "name" : var.postgres_credentials_secret_name - } - }, - "ingress" : { - "enabled" : true - "className" : "nginx" - "annotations" : { - "nginx.ingress.kubernetes.io/ssl-redirect" : "false" - "nginx.ingress.kubernetes.io/use-regex" : "true" - "nginx.ingress.kubernetes.io/rewrite-target" : "/api/$1$2" - }, - "endpoints" : [ - { - "port" : 8181, - "path" : "/${var.participant.name}/ih/(management)(.*)" - }, - { - "port" : 8282, - "path" : "/${var.participant.name}/ih/(resolution)(.*)" - } - ] - }, - "vault" : { - "hashicorp" : { - "url" : module.vault.vault_url - "token" : module.vault.vault_token - } - } - } - - }) - ] - - depends_on = [module.vault] -} \ No newline at end of file diff --git a/deployment/modules/participant/main.tf b/deployment/modules/participant/main.tf deleted file mode 100644 index fd9037d..0000000 --- a/deployment/modules/participant/main.tf +++ /dev/null @@ -1,10 +0,0 @@ -locals { - privatekey_alias = var.participant.name - publickey_alias = "${local.privatekey_alias}-pub" -} - -module "vault" { - source = "../vault" - - participant_name = var.participant.name -} \ No newline at end of file diff --git a/deployment/modules/participant/output.tf b/deployment/modules/participant/output.tf deleted file mode 100644 index bc2fb87..0000000 --- a/deployment/modules/participant/output.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "did_url" { - value = local.did_url -} \ No newline at end of file diff --git a/deployment/modules/participant/variables.tf b/deployment/modules/participant/variables.tf deleted file mode 100644 index 62e2366..0000000 --- a/deployment/modules/participant/variables.tf +++ /dev/null @@ -1,24 +0,0 @@ -variable "participant" { - type = object({ - name = string, - vc = list(any) - }) -} - -# POSTGRES -variable "postgres_host" {} -variable "postgres_credentials_secret_name" {} - -# FEDERATED CATALOG -variable "connector_repo" {} -variable "connector_chart_name" {} -variable "connector_version" {} - -# IDENTITY HUB -variable "identityhub_repo" {} -variable "identityhub_chart_name" {} -variable "identityhub_version" {} - -# DOCKER PULL -variable "helm_chart_repo" {} -variable "docker_image_pull_secret_name" {} \ No newline at end of file diff --git a/deployment/modules/vault/output.tf b/deployment/modules/vault/output.tf deleted file mode 100644 index f54a71f..0000000 --- a/deployment/modules/vault/output.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "vault_url" { - value = "http://${var.participant_name}-vault:${local.vault_port}" -} - -output "vault_token" { - value = local.vault_token -} \ No newline at end of file diff --git a/deployment/modules/vault/providers.tf b/deployment/modules/vault/providers.tf deleted file mode 100644 index 154e7de..0000000 --- a/deployment/modules/vault/providers.tf +++ /dev/null @@ -1,4 +0,0 @@ -terraform { - required_providers { - } -} \ No newline at end of file diff --git a/deployment/modules/vault/variables.tf b/deployment/modules/vault/variables.tf deleted file mode 100644 index 7ddefa3..0000000 --- a/deployment/modules/vault/variables.tf +++ /dev/null @@ -1,9 +0,0 @@ -variable "participant_name" {} - -#variable "publickey_alias" {} -# -#variable "publickey" {} -# -#variable "privatekey_alias" {} -# -#variable "privatekey" {} \ No newline at end of file diff --git a/deployment/output.tf b/deployment/output.tf deleted file mode 100644 index 03ed8c8..0000000 --- a/deployment/output.tf +++ /dev/null @@ -1,3 +0,0 @@ -#output "db_bootstrap_script" { -# value = module.db.db_bootstrap_script -#} \ No newline at end of file diff --git a/deployment/readme.md b/deployment/readme.md index e77c6f9..62d3211 100644 --- a/deployment/readme.md +++ b/deployment/readme.md @@ -4,10 +4,11 @@ - Terraform - Kind +- Docker desktop - cURL or Postman -- Vault CLI +- Hashicorp Vault CLI -## Create and prepare a local Kubernetes cluster +## Create a local Kubernetes cluster ```bash kind create cluster -n eonax-cluster --config kind.config.yaml @@ -18,6 +19,7 @@ kind create cluster -n eonax-cluster --config kind.config.yaml We install an Ingress Controller in order to interact with the microservice running in the cluster from the host. Below command will wait until the ingress controller is ready before returning. + ```bash kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml kubectl wait --namespace ingress-nginx \ @@ -26,44 +28,197 @@ kubectl wait --namespace ingress-nginx \ --timeout=90s ``` -### Configure secret for pulling Eona-X Docker images +## Deploy the Vault and DB (optional) + +```bash +cd storage +terraform init +terraform apply -auto-approve +``` + +## Deploy the connector -You need a token to pull the Docker image of the Eona-X connector. This token is provided by Amadeus. +```bash +cd connector +``` -Once you obtained your token, you'll have _inject_ in the Terraform script. Simply run the following command: +### Specify the Eona-X/EDC version ```bash -echo 'container_registry_token = ""' > terraform.tfvars +EDC_VERSION=v0.9.0 +EONAX_VERSION=0.2.0 ``` -## Deploy the dataspace +### Login to the Docker registry -Once you have configured the participants you want to deploy using the `participants` field of -the [variables.tf](variables.tf) file, -simply run the following Terraform command to deploy the dataspace: +Use the token provided by Amadeus in order to log to the Docker registry. + +```bash +GITHUB_TOKEN="" +echo $GITHUB_TOKEN | docker login ghcr.io -u amadeusitgroup --password-stdin +``` + +### Pull Helm chart and Docker images + +```bash +CLUSTER=eonax-cluster +DOCKER_IMAGE_REPO=ghcr.io/amadeusitgroup/dataspace_ecosystem +HELM_CHART_REPO=oci://ghcr.io/amadeusitgroup/dataspace_ecosystem/helm + +for i in control-plane data-plane identity-hub; do \ + image=eonax-$i-postgresql-hashicorpvault; \ + + ## pull the Docker image + docker pull $DOCKER_IMAGE_REPO/$image:$EONAX_VERSION; \ + ## tag image with version latest + docker tag $DOCKER_IMAGE_REPO/$image:$EONAX_VERSION $image:latest; \ + ## load image to the cluster + kind load docker-image $image:latest --name $CLUSTER; \ + + ## pull Helm chart + chart=${i//-/}; \ + helm pull $HELM_CHART_REPO/$chart --version $EONAX_VERSION; \ + mv $chart-$EONAX_VERSION.tgz $chart.tgz; \ +done +``` + +### Download SQL files + +```bash +jq -r --arg version "$EDC_VERSION" '.files[] | "https://raw.githubusercontent.com/eclipse-edc/\(.repo)/\($version)/\(.path)/src/main/resources/\(.file_name)"' sql.json | \ +tr -d '\r' | \ +while read -r url; do curl -o "./connector/sql/$(basename "$url")" "$url"; done +``` + +### Deploy the connector + +If you are using your own Vault and/or DB, please take care of updating the vault url and token secret name, and the DB +url, username and password in the `variables.tf` file. ```bash terraform init terraform apply -auto-approve ``` -### Configure the connector(s) +### Initialize connector -Once the deployment is complete, you can now use the standard APIs of -the [EDC connector](https://app.swaggerhub.com/apis/eclipse-edc-bot/management-api/) to configure -the connectors. For example, to get the federated catalog of a participant called `company1`: +#### Generate key pair ```bash -curl -X POST -d "{\"criteria\":[]}" -H "content-type: application/json" http://localhost/company1/management/federatedcatalog +openssl genpkey -algorithm RSA -out private-key.pem -pkeyopt rsa_keygen_bits:2048 && \ +openssl rsa -pubout -in private-key.pem -out public-key.pem && \ +for k in public-key private-key; do VAULT_TOKEN=root VAULT_ADDR=http://localhost/vault vault kv put secret/$k content=@$k.pem; done ``` -It is also possible to add/remove new secrets within the vault of each participant using the Hashicorp Vault Secret -Engine API. -For example, the following adds a new secret called `foo` with value `bar` within the Vault of the -participant `company1`: +#### Create participant context ```bash -VAULT_TOKEN=root VAULT_ADDR=http://localhost/company1/vault vault kv put secret/foo content=bar +DID_WEB="did:web:localhost:ih:did" +DID_WEB_BASE64_URL=$(echo -n "$DID_WEB" | base64 | tr '+/' '-_' | tr -d '=') +IH_RESOLUTION_URL=http://localhost/ih/resolution +CP_DSP_URL=http://localhost/cp/dsp + +curl -X POST -H "Content-Type: application/json" -d "$(cat <