From 9449f24f359519828c452a62fc2b7d7018f3a5c1 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 28 Nov 2023 11:06:47 -0600 Subject: [PATCH] MAINT: GitHub action for e2e secrets (#3704) * ADD: initial AWS testing resources CDK Signed-off-by: George Chen * MNT: renaming Signed-off-by: George Chen * E2E: basicLogWithAwsSecretsEndToEndTest Signed-off-by: George Chen * MAINT: refactoring on build.gradle Signed-off-by: George Chen * MAINT: use AWS_PROFILE Signed-off-by: George Chen * MAINT: check aws credentials Signed-off-by: George Chen * maint: check absolute path of .aws Signed-off-by: George Chen * MAINT: check output Signed-off-by: George Chen * MAINT: check output Signed-off-by: George Chen * MAINT: check step output Signed-off-by: George Chen * MAINT: aws configure Signed-off-by: George Chen * MAINT: set files Signed-off-by: George Chen * MNT: format Signed-off-by: George Chen * MAINT: check files Signed-off-by: George Chen * testing Signed-off-by: George Chen * MAINT: run actual tests Signed-off-by: George Chen * TST: assume new role Signed-off-by: George Chen * MAINT: add test Signed-off-by: George Chen * MAINT: title Signed-off-by: George Chen * MAINT: back test on branch Signed-off-by: George Chen * MAINT: revert Signed-off-by: George Chen * MAINT: job name Signed-off-by: George Chen * MAINT: file name Signed-off-by: George Chen * MAINT: us-east-2 Signed-off-by: George Chen * MAINT: manual workflow trigger Signed-off-by: George Chen * MAINT: add back variable Signed-off-by: George Chen * MAINT: use repo variables Signed-off-by: George Chen * MAINT: target change Signed-off-by: George Chen * TST: any push Signed-off-by: George Chen * MAINT: revert change Signed-off-by: George Chen --------- Signed-off-by: George Chen --- .../data-prepper-aws-secrets-e2e-tests.yml | 53 +++++++++++++++++++ ...sic-grok-e2e-pipeline-with-aws-secrets.yml | 1 + 2 files changed, 54 insertions(+) create mode 100644 .github/workflows/data-prepper-aws-secrets-e2e-tests.yml diff --git a/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml new file mode 100644 index 0000000000..26247886f8 --- /dev/null +++ b/.github/workflows/data-prepper-aws-secrets-e2e-tests.yml @@ -0,0 +1,53 @@ +# This workflow will build a Java project with Gradle +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle + +name: Data Prepper AWS secrets End-to-end test with Gradle +on: + push: + branches: [ main ] + pull_request_target: + types: [ opened, synchronize, reopened ] + workflow_dispatch: +# permission can be added at job level or workflow level +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +jobs: + build: + strategy: + matrix: + java: [ 11, 17, 21, docker ] + test: [ 'basicLogWithAwsSecretsEndToEndTest' ] + fail-fast: false + + runs-on: ubuntu-latest + steps: + - name: Git clone the repository + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: configure aws credentials + id: creds + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.TEST_IAM_ROLE_ARN }} + aws-region: ${{ secrets.TEST_REGION }} + output-credentials: true + - name: get caller identity 1 + run: | + aws sts get-caller-identity + - name: Configure AWS Credentials file + run: | + aws configure set default.region ${{ secrets.TEST_REGION }} + aws configure set default.aws_access_key_id ${{ steps.creds.outputs.aws-access-key-id }} + aws configure set default.aws_secret_access_key ${{ steps.creds.outputs.aws-secret-access-key }} + aws configure set default.aws_session_token ${{ steps.creds.outputs.aws-session-token }} + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Checkout Data Prepper + uses: actions/checkout@v2 + - name: Run basic grok with AWS secrets end-to-end tests with Gradle + run: AWS_PROFILE=default ./gradlew -PendToEndJavaVersion=${{ matrix.java }} :e2e-test:log:${{ matrix.test }} diff --git a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml index 34be38f12b..b05b774386 100644 --- a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml +++ b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml @@ -3,6 +3,7 @@ pipeline_configurations: secrets: opensearch-sink: secret_id: "opensearch-sink-basic-credentials" + region: "us-east-2" grok-pipeline: source: http: