From c94f04e40f70948cac80cf1b6b21e7aa832796fe Mon Sep 17 00:00:00 2001 From: Will Liu Date: Tue, 23 Jan 2024 12:50:26 -0500 Subject: [PATCH 1/8] run e2e test --- .github/workflows/e2e-test-coverage.yml | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/e2e-test-coverage.yml diff --git a/.github/workflows/e2e-test-coverage.yml b/.github/workflows/e2e-test-coverage.yml new file mode 100644 index 0000000000..be1955cae9 --- /dev/null +++ b/.github/workflows/e2e-test-coverage.yml @@ -0,0 +1,71 @@ +name: e2e test Build, Test & Coverage +on: # yamllint disable-line rule:truthy + pull_request: + paths: + - 'indexer/**' + - 'e2e-testing/**' + - 'protocol/**' + push: + branches: + - main + - 'wl/**' # e.g. wl/test + paths: + - 'indexer/**' + - 'e2e-testing/**' + - 'protocol/**' + +# Ensure only a single instance of this workflow is running, and cancel any that are in-progress +# before this workflow instance starts +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + + +jobs: + setup: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./e2e-testing + + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + + - name: Install Node.js + uses: actions/setup-node@v3 + with: + node-version: 16 + + - name: Install pnpm + run: + npm install -g pnpm@6.34.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build and install Indexer + run: | + pnpm install --loglevel warn --frozen-lockfile + pnpm run build:all + working-directory: ./indexer + + - name: Build and Start Docker Compose + run: | + cd ../protocol + make e2etest-build-image + cd ../e2e-testing + docker compose -f docker-compose-e2e-test.yml up -d + + build_and_test: + needs: setup + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./e2e-testing + + steps: + - name: Build and Test + run: | + pnpm build + pnpm test From 1efc5848bdb68a875acddc265a8ca8decfa8e6ab Mon Sep 17 00:00:00 2001 From: Will Liu Date: Tue, 23 Jan 2024 12:53:15 -0500 Subject: [PATCH 2/8] update --- .github/workflows/e2e-test-coverage.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test-coverage.yml b/.github/workflows/e2e-test-coverage.yml index be1955cae9..8a1fcc6e8a 100644 --- a/.github/workflows/e2e-test-coverage.yml +++ b/.github/workflows/e2e-test-coverage.yml @@ -2,17 +2,13 @@ name: e2e test Build, Test & Coverage on: # yamllint disable-line rule:truthy pull_request: paths: - - 'indexer/**' - - 'e2e-testing/**' - - 'protocol/**' + - '**' push: branches: - main - 'wl/**' # e.g. wl/test paths: - - 'indexer/**' - - 'e2e-testing/**' - - 'protocol/**' + - '**' # Ensure only a single instance of this workflow is running, and cancel any that are in-progress # before this workflow instance starts From c572194e71c7075c9da436c24a3a6070ba53c0ae Mon Sep 17 00:00:00 2001 From: Will Liu Date: Tue, 23 Jan 2024 12:56:00 -0500 Subject: [PATCH 3/8] fix --- .github/workflows/e2e-test-coverage.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test-coverage.yml b/.github/workflows/e2e-test-coverage.yml index 8a1fcc6e8a..acbda4e770 100644 --- a/.github/workflows/e2e-test-coverage.yml +++ b/.github/workflows/e2e-test-coverage.yml @@ -34,17 +34,17 @@ jobs: node-version: 16 - name: Install pnpm - run: - npm install -g pnpm@6.34.0 + run: + npm install -g pnpm@6.34.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - name: Build and install Indexer - run: | - pnpm install --loglevel warn --frozen-lockfile - pnpm run build:all - working-directory: ./indexer + run: | + pnpm install --loglevel warn --frozen-lockfile + pnpm run build:all + working-directory: ./indexer - name: Build and Start Docker Compose run: | From 30bbdace37b472fc7e6c59148c0cacf73d789868 Mon Sep 17 00:00:00 2001 From: Will Liu Date: Tue, 23 Jan 2024 13:43:18 -0500 Subject: [PATCH 4/8] fix --- .github/workflows/e2e-test-coverage.yml | 29 ++- .../docker-compose-e2e-test-remote.yml | 232 ++++++++++++++++++ 2 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 e2e-testing/docker-compose-e2e-test-remote.yml diff --git a/.github/workflows/e2e-test-coverage.yml b/.github/workflows/e2e-test-coverage.yml index acbda4e770..25c187c606 100644 --- a/.github/workflows/e2e-test-coverage.yml +++ b/.github/workflows/e2e-test-coverage.yml @@ -51,7 +51,7 @@ jobs: cd ../protocol make e2etest-build-image cd ../e2e-testing - docker compose -f docker-compose-e2e-test.yml up -d + docker compose -f docker-compose-e2e-test-remote up -d build_and_test: needs: setup @@ -60,6 +60,33 @@ jobs: run: working-directory: ./e2e-testing + services: + postgres: + image: postgres:12-alpine + env: + POSTGRES_PASSWORD: dydxserver123 + POSTGRES_USER: dydx_test + ports: + - 5432:5432 + redis: + image: redis:5.0.6-alpine + ports: + - 6379:6379 + kafka: + image: blacktop/kafka:2.6 + env: + KAFKA_ADVERTISED_HOST_NAME: localhost + KAFKA_CREATE_TOPICS: + "to-ender:1:1,\ + to-vulcan:1:1,\ + to-websockets-orderbooks:1:1,\ + to-websockets-subaccounts:1:1,\ + to-websockets-trades:1:1,\ + to-websockets-markets:1:1,\ + to-websockets-candles:1:1" + ports: + - 9092:9092 + steps: - name: Build and Test run: | diff --git a/e2e-testing/docker-compose-e2e-test-remote.yml b/e2e-testing/docker-compose-e2e-test-remote.yml new file mode 100644 index 0000000000..d3e8791380 --- /dev/null +++ b/e2e-testing/docker-compose-e2e-test-remote.yml @@ -0,0 +1,232 @@ +version: '3' +services: + kafka: + image: blacktop/kafka:2.6 + ports: + - 9092:9092 + environment: + KAFKA_ADVERTISED_HOST_NAME: kafka + KAFKA_CREATE_TOPICS: + "to-ender:1:1,\ + to-vulcan:1:1,\ + to-websockets-orderbooks:1:1,\ + to-websockets-subaccounts:1:1,\ + to-websockets-trades:1:1,\ + to-websockets-markets:1:1,\ + to-websockets-candles:1:1" + KAFKA_LISTENERS: INTERNAL://:9092,EXTERNAL_SAME_HOST://:29092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL_SAME_HOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL_SAME_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + DD_AGENT_HOST: datadog-agent + healthcheck: + test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic to-websockets-candles --describe"] + interval: 5s + timeout: 20s + retries: 50 + postgres: + build: + context: ../indexer + dockerfile: ../indexer/Dockerfile.postgres.local + ports: + - 5435:5432 + environment: + POSTGRES_PASSWORD: dydxserver123 + POSTGRES_USER: dydx_dev + DATADOG_POSTGRES_PASSWORD: dydxserver123 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dydx_dev"] + interval: 5s + timeout: 20s + retries: 10 + redis: + image: redis:5.0.6-alpine + ports: + - 6382:6379 + dydxprotocold0: + image: local:e2etest-dydxprotocol + platform: linux/amd64 + entrypoint: + - cosmovisor + - run + - start + - --log_level + # Note that only this validator has a log-level of `info`; other validators use `error` by default. + # Change to `debug` for more verbose log-level. + - info + - --home + - /dydxprotocol/chain/.alice + - --p2p.persistent_peers + - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656" + - --bridge-daemon-eth-rpc-endpoint + - "${ETH_RPC_ENDPOINT}" + environment: + - DAEMON_HOME=/dydxprotocol/chain/.alice + volumes: + - ../protocol/localnet/dydxprotocol0:/dydxprotocol/chain/.alice/data + ports: + - "26657:26657" + - "9090:9090" + - "1317:1317" + + # This is the Indexer connected node. + # TODO: remove stake and make this a full node. + dydxprotocold1: + image: local:e2etest-dydxprotocol + platform: linux/amd64 + entrypoint: + - cosmovisor + - run + - start + - --log_level + - error + - --home + - /dydxprotocol/chain/.bob + - --p2p.persistent_peers + - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,b69182310be02559483e42c77b7b104352713166@dydxprotocold1:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656" + - --non-validating-full-node=true + - --bridge-daemon-eth-rpc-endpoint + - "${ETH_RPC_ENDPOINT}" + - --indexer-kafka-conn-str + - "kafka:9092" + environment: + - DAEMON_HOME=/dydxprotocol/chain/.bob + volumes: + - ../protocol/localnet/dydxprotocol1:/dydxprotocol/chain/.bob/data + ports: + - "26658:26657" + depends_on: + kafka: + condition: service_healthy + + dydxprotocold2: + image: local:e2etest-dydxprotocol + platform: linux/amd64 + entrypoint: + - cosmovisor + - run + - start + - --log_level + - error + - --home + - /dydxprotocol/chain/.carl + - --p2p.persistent_peers + - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656" + - --bridge-daemon-eth-rpc-endpoint + - "${ETH_RPC_ENDPOINT}" + environment: + - DAEMON_HOME=/dydxprotocol/chain/.carl + volumes: + - ../protocol/localnet/dydxprotocol2:/dydxprotocol/chain/.carl/data + + dydxprotocold3: + image: local:e2etest-dydxprotocol + platform: linux/amd64 + entrypoint: + - cosmovisor + - run + - start + - --log_level + - error + - --home + - /dydxprotocol/chain/.dave + - --p2p.persistent_peers + - "17e5e45691f0d01449c84fd4ae87279578cdd7ec@dydxprotocold0:26656,47539956aaa8e624e0f1d926040e54908ad0eb44@dydxprotocold2:26656,5882428984d83b03d0c907c1f0af343534987052@dydxprotocold3:26656" + - --bridge-daemon-eth-rpc-endpoint + - "${ETH_RPC_ENDPOINT}" + environment: + - DAEMON_HOME=/dydxprotocol/chain/.dave + volumes: + - ../protocol/localnet/dydxprotocol3:/dydxprotocol/chain/.dave/data + + postgres-package: + build: + context: ../indexer + dockerfile: ../indexer/Dockerfile.postgres-package.local + links: + - postgres + depends_on: + postgres: + condition: service_healthy + ender: + build: + context: ../indexer + dockerfile: ../indexer/Dockerfile.service.remote + args: + service: ender + ports: + - 3001:3001 + links: + - postgres + environment: + - REDIS_URL=redis://redis:6379 + depends_on: + kafka: + condition: service_healthy + postgres-package: + condition: service_completed_successfully + comlink: + build: + context: ../indexer + dockerfile: ../indexer/Dockerfile.service.remote + args: + service: comlink + environment: + - PORT=3002 + - REDIS_URL=redis://redis:6379 + - RATE_LIMIT_REDIS_URL=redis://redis:6379 + - RATE_LIMIT_ENABLED=false + - COMPLIANCE_DATA_CLIENT=PLACEHOLDER + ports: + - 3002:3002 + links: + - postgres + depends_on: + postgres-package: + condition: service_completed_successfully + socks: + build: + context: ../indexer + dockerfile: ../indexer/Dockerfile.service.remote + args: + service: socks + ports: + - 3003:3003 + links: + - postgres + environment: + - WS_PORT=3003 + - COMLINK_URL=host.docker.internal:3002 + depends_on: + kafka: + condition: service_healthy + postgres-package: + condition: service_completed_successfully + roundtable: + build: + context: ../indexer + dockerfile: ../indexer/Dockerfile.service.remote + args: + service: roundtable + ports: + - 3004:3004 + links: + - postgres + depends_on: + kafka: + condition: service_healthy + postgres-package: + condition: service_completed_successfully + vulcan: + build: + context: ../indexer + dockerfile: ../indexer/Dockerfile.service.remote + args: + service: vulcan + environment: + - REDIS_URL=redis://redis:6379 + ports: + - 3005:3005 + depends_on: + kafka: + condition: service_healthy From 3d304d3431bf50c46bb6e2dbd8d8bba9fbec5e11 Mon Sep 17 00:00:00 2001 From: Will Liu Date: Tue, 23 Jan 2024 13:47:15 -0500 Subject: [PATCH 5/8] fix? --- .github/workflows/e2e-test-coverage.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-test-coverage.yml b/.github/workflows/e2e-test-coverage.yml index 25c187c606..d3c6423862 100644 --- a/.github/workflows/e2e-test-coverage.yml +++ b/.github/workflows/e2e-test-coverage.yml @@ -65,18 +65,19 @@ jobs: image: postgres:12-alpine env: POSTGRES_PASSWORD: dydxserver123 - POSTGRES_USER: dydx_test + POSTGRES_USER: dydx_dev + DATADOG_POSTGRES_PASSWORD: dydxserver123 ports: - - 5432:5432 + - 5435:5432 redis: image: redis:5.0.6-alpine ports: - - 6379:6379 + - 6382:6379 kafka: image: blacktop/kafka:2.6 env: - KAFKA_ADVERTISED_HOST_NAME: localhost - KAFKA_CREATE_TOPICS: + KAFKA_ADVERTISED_HOST_NAME: kafka + KAFKA_CREATE_TOPICS: "to-ender:1:1,\ to-vulcan:1:1,\ to-websockets-orderbooks:1:1,\ @@ -84,10 +85,22 @@ jobs: to-websockets-trades:1:1,\ to-websockets-markets:1:1,\ to-websockets-candles:1:1" + KAFKA_LISTENERS: INTERNAL://:9092,EXTERNAL_SAME_HOST://:29092 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL_SAME_HOST://localhost:29092 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL_SAME_HOST:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL ports: - 9092:9092 steps: + - name: Setup database + id: setup-database + env: + DB_PORT: 5435 + NODE_ENV: development + run: | + cd packages/postgres && pnpm run migrate + - name: Build and Test run: | pnpm build From 78eb8add4d65428fa418ef632e7a6c88654cc26c Mon Sep 17 00:00:00 2001 From: Will Liu Date: Tue, 23 Jan 2024 13:59:09 -0500 Subject: [PATCH 6/8] update --- .../docker-compose-e2e-test-remote.yml | 84 ------------------- 1 file changed, 84 deletions(-) diff --git a/e2e-testing/docker-compose-e2e-test-remote.yml b/e2e-testing/docker-compose-e2e-test-remote.yml index d3e8791380..0cab13055c 100644 --- a/e2e-testing/docker-compose-e2e-test-remote.yml +++ b/e2e-testing/docker-compose-e2e-test-remote.yml @@ -1,48 +1,5 @@ version: '3' services: - kafka: - image: blacktop/kafka:2.6 - ports: - - 9092:9092 - environment: - KAFKA_ADVERTISED_HOST_NAME: kafka - KAFKA_CREATE_TOPICS: - "to-ender:1:1,\ - to-vulcan:1:1,\ - to-websockets-orderbooks:1:1,\ - to-websockets-subaccounts:1:1,\ - to-websockets-trades:1:1,\ - to-websockets-markets:1:1,\ - to-websockets-candles:1:1" - KAFKA_LISTENERS: INTERNAL://:9092,EXTERNAL_SAME_HOST://:29092 - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,EXTERNAL_SAME_HOST://localhost:29092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL_SAME_HOST:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - DD_AGENT_HOST: datadog-agent - healthcheck: - test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic to-websockets-candles --describe"] - interval: 5s - timeout: 20s - retries: 50 - postgres: - build: - context: ../indexer - dockerfile: ../indexer/Dockerfile.postgres.local - ports: - - 5435:5432 - environment: - POSTGRES_PASSWORD: dydxserver123 - POSTGRES_USER: dydx_dev - DATADOG_POSTGRES_PASSWORD: dydxserver123 - healthcheck: - test: ["CMD-SHELL", "pg_isready -U dydx_dev"] - interval: 5s - timeout: 20s - retries: 10 - redis: - image: redis:5.0.6-alpine - ports: - - 6382:6379 dydxprotocold0: image: local:e2etest-dydxprotocol platform: linux/amd64 @@ -95,9 +52,6 @@ services: - ../protocol/localnet/dydxprotocol1:/dydxprotocol/chain/.bob/data ports: - "26658:26657" - depends_on: - kafka: - condition: service_healthy dydxprotocold2: image: local:e2etest-dydxprotocol @@ -139,15 +93,6 @@ services: volumes: - ../protocol/localnet/dydxprotocol3:/dydxprotocol/chain/.dave/data - postgres-package: - build: - context: ../indexer - dockerfile: ../indexer/Dockerfile.postgres-package.local - links: - - postgres - depends_on: - postgres: - condition: service_healthy ender: build: context: ../indexer @@ -156,15 +101,8 @@ services: service: ender ports: - 3001:3001 - links: - - postgres environment: - REDIS_URL=redis://redis:6379 - depends_on: - kafka: - condition: service_healthy - postgres-package: - condition: service_completed_successfully comlink: build: context: ../indexer @@ -179,11 +117,6 @@ services: - COMPLIANCE_DATA_CLIENT=PLACEHOLDER ports: - 3002:3002 - links: - - postgres - depends_on: - postgres-package: - condition: service_completed_successfully socks: build: context: ../indexer @@ -192,16 +125,9 @@ services: service: socks ports: - 3003:3003 - links: - - postgres environment: - WS_PORT=3003 - COMLINK_URL=host.docker.internal:3002 - depends_on: - kafka: - condition: service_healthy - postgres-package: - condition: service_completed_successfully roundtable: build: context: ../indexer @@ -210,13 +136,6 @@ services: service: roundtable ports: - 3004:3004 - links: - - postgres - depends_on: - kafka: - condition: service_healthy - postgres-package: - condition: service_completed_successfully vulcan: build: context: ../indexer @@ -227,6 +146,3 @@ services: - REDIS_URL=redis://redis:6379 ports: - 3005:3005 - depends_on: - kafka: - condition: service_healthy From 4e9869eb7c0c56c0c2bbf662b4efe426b7c0574c Mon Sep 17 00:00:00 2001 From: Will Liu Date: Tue, 23 Jan 2024 14:14:14 -0500 Subject: [PATCH 7/8] update --- .github/workflows/e2e-test-coverage.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test-coverage.yml b/.github/workflows/e2e-test-coverage.yml index d3c6423862..e812a25080 100644 --- a/.github/workflows/e2e-test-coverage.yml +++ b/.github/workflows/e2e-test-coverage.yml @@ -46,12 +46,9 @@ jobs: pnpm run build:all working-directory: ./indexer - - name: Build and Start Docker Compose - run: | - cd ../protocol - make e2etest-build-image - cd ../e2e-testing - docker compose -f docker-compose-e2e-test-remote up -d + - name: Build + run: make e2etest-build-image + working-directory: ./protocol build_and_test: needs: setup @@ -101,6 +98,10 @@ jobs: run: | cd packages/postgres && pnpm run migrate + - name: Docker Compose + run: docker compose -f docker-compose-e2e-test-remote.yml up -d + working-directory: ./e2e-testing + - name: Build and Test run: | pnpm build From b2fb8b07e1572e6eae2cd443d1707c2bf333360d Mon Sep 17 00:00:00 2001 From: Will Liu Date: Tue, 23 Jan 2024 14:23:34 -0500 Subject: [PATCH 8/8] add working dir --- .github/workflows/e2e-test-coverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-test-coverage.yml b/.github/workflows/e2e-test-coverage.yml index e812a25080..117024313e 100644 --- a/.github/workflows/e2e-test-coverage.yml +++ b/.github/workflows/e2e-test-coverage.yml @@ -95,8 +95,8 @@ jobs: env: DB_PORT: 5435 NODE_ENV: development - run: | - cd packages/postgres && pnpm run migrate + run: cd packages/postgres && pnpm run migrate + working-directory: ./indexer - name: Docker Compose run: docker compose -f docker-compose-e2e-test-remote.yml up -d