From d1e3610a01182d1f52a00c0f522cfa13bb855c1d Mon Sep 17 00:00:00 2001 From: neel-astro Date: Tue, 17 Sep 2024 15:00:23 +0530 Subject: [PATCH] code review changes: allow inputs for worflow_dispatch --- .../e2e/get_astro_env_info/action.yaml | 74 ++++++ .../e2e/validate_deployment/action.yaml | 2 +- .github/workflows/tests.yaml | 216 ++++++++++++++---- e2e-setup/mocks/git.sh | 11 +- 4 files changed, 251 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/e2e/get_astro_env_info/action.yaml diff --git a/.github/workflows/e2e/get_astro_env_info/action.yaml b/.github/workflows/e2e/get_astro_env_info/action.yaml new file mode 100644 index 0000000..2032782 --- /dev/null +++ b/.github/workflows/e2e/get_astro_env_info/action.yaml @@ -0,0 +1,74 @@ +name: Get Astro Environment Info +description: Get information about the Astro environment +inputs: + input_workspace_id: + description: The workspace ID via input + required: true + secret_workspace_id: + description: The workspace ID via secret + required: true + input_organization_id: + description: The organization ID via input + required: true + secret_organization_id: + description: The organization ID via secret + required: true + input_astro_api_token: + description: The Astronomer API token via input + required: true + secret_astro_api_token: + description: The Astronomer API token via secret + required: true + input_astronomer_host: + description: The Astronomer host via input + required: true + secret_astronomer_host: + description: The Astronomer host via secret + required: true + +outputs: + organization_id: + value: ${{ steps.get-info.outputs.ORGANIZATION_ID }} + workspace_id: + value: ${{ steps.get-info.outputs.WORKSPACE_ID }} + astronomer_host: + value: ${{ steps.get-info.outputs.ASTRONOMER_HOST }} + astro_api_token: + value: ${{ steps.get-info.outputs.ASTRO_API_TOKEN }} + +runs: + using: "composite" + steps: + - name: Get info from inputs or secrets + shell: bash + id: get-info + run: | + if [ "${{ inputs.input_workspace_id }}" != "" ]; then + echo "Using provided workspace_id" + echo "WORKSPACE_ID=${{ inputs.input_workspace_id }}" >> $GITHUB_OUTPUT + else + echo "WORKSPACE_ID=${{ inputs.secret_workspace_id }}" >> $GITHUB_OUTPUT + fi + + if [ "${{ inputs.input_organization_id }}" != "" ]; then + echo "Using provided org_id" + echo "ORGANIZATION_ID=${{ inputs.input_organization_id }}" >> $GITHUB_OUTPUT + else + echo "ORGANIZATION_ID=${{ inputs.secret_organization_id }}" >> $GITHUB_OUTPUT + fi + + if [ "${{ inputs.input_astronomer_host }}" != "" ]; then + echo "Using provided astronomer_host" + echo "ASTRONOMER_HOST=${{ inputs.input_astronomer_host }}" >> $GITHUB_OUTPUT + else + echo "ASTRONOMER_HOST=${{ inputs.secret_astronomer_host }}" >> $GITHUB_OUTPUT + fi + + if [ "${{ inputs.input_astro_api_token }}" != "" ]; then + echo "Using provided token" + echo "ASTRO_API_TOKEN=${{ inputs.input_astro_api_token }}" >> $GITHUB_OUTPUT + echo "ASTRO_API_TOKEN=${{ inputs.input_astro_api_token }}" >> $GITHUB_ENV + else + echo "ASTRO_API_TOKEN=${{ inputs.secret_astro_api_token }}" >> $GITHUB_OUTPUT + echo "ASTRO_API_TOKEN=${{ inputs.secret_astro_api_token }}" >> $GITHUB_ENV + fi diff --git a/.github/workflows/e2e/validate_deployment/action.yaml b/.github/workflows/e2e/validate_deployment/action.yaml index fe1ed92..650c622 100644 --- a/.github/workflows/e2e/validate_deployment/action.yaml +++ b/.github/workflows/e2e/validate_deployment/action.yaml @@ -1,4 +1,4 @@ -name: Validate Deployment +name: Validate Deployment Deploy Versions description: Validate deployment info from Astronomer API inputs: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9df2560..037aed2 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -6,14 +6,52 @@ on: - main - e2e-tests workflow_dispatch: + inputs: + workspace_id: + description: "Workspace ID" + required: false + default: "" + org_id: + description: "Organization ID" + required: false + default: "" + astronomer_host: + description: "Astronomer Host" + required: false + default: "" + token: + description: "API Token" + required: false + default: "" env: ASTRO_API_TOKEN: ${{ secrets.ASTRO_API_TOKEN }} jobs: + redact-inputs: + name: Redact Inputs + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Redact Inputs + run: | + + # add-mask will redact the value of the input, but will still log the value, hence using a known workaround: https://github.com/actions/runner/issues/643 + WORKSPACE_ID=$(jq -r '.inputs.workspace_id' $GITHUB_EVENT_PATH) + echo ::add-mask::$WORKSPACE_ID + ORG_ID=$(jq -r '.inputs.org_id' $GITHUB_EVENT_PATH) + echo ::add-mask::$ORG_ID + ASTRONOMER_HOST=$(jq -r '.inputs.astronomer_host' $GITHUB_EVENT_PATH) + echo ::add-mask::$ASTRONOMER_HOST + TOKEN=$(jq -r '.inputs.token' $GITHUB_EVENT_PATH) + echo ::add-mask::$TOKEN + # Create test deployments would use the template files and generate deployments with unique names create-test-deployments: name: Create Deployment + needs: redact-inputs runs-on: ubuntu-latest strategy: matrix: @@ -25,6 +63,19 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + - name: Install dependencies run: | @@ -33,8 +84,8 @@ jobs: sudo apt install yq -y curl -sSL https://install.astronomer.io | sudo bash -s - astro context switch ${{ secrets.ASTRONOMER_HOST }} - astro workspace switch ${{ secrets.WORKSPACE_ID }} + astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }} # Generate a random test_uid and set it as an environment variable, so we can append it to the deployment name test_uid=$(uuidgen) @@ -58,7 +109,7 @@ jobs: dag-deploy-test: name: DAG Deploy Test runs-on: ubuntu-latest - needs: create-test-deployments + needs: [create-test-deployments] strategy: matrix: deployment_id: @@ -70,6 +121,19 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + - name: Mock git commands run: | mv e2e-setup/mocks/git.sh /usr/local/bin/git @@ -83,22 +147,22 @@ jobs: curl -sSL https://install.astronomer.io | sudo bash -s - name: Set CLI context - run: astro context switch ${{ secrets.ASTRONOMER_HOST }} + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Get Deployment Info Before Test id: get-deployment-before uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ matrix.deployment_id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Deploy to Astro uses: ./ with: deployment-id: ${{ matrix.deployment_id }} - workspace: ${{ secrets.WORKSPACE_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} parse: true root-folder: e2e-setup/astro-project @@ -107,9 +171,9 @@ jobs: uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ matrix.deployment_id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Validate Deploy Action uses: ./.github/workflows/e2e/validate_deployment @@ -135,6 +199,19 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + - name: Install dependencies run: | @@ -144,22 +221,22 @@ jobs: curl -sSL https://install.astronomer.io | sudo bash -s - name: Set CLI context - run: astro context switch ${{ secrets.ASTRONOMER_HOST }} + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Get Deployment Info Before Test id: get-deployment-before uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ matrix.deployment_id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Run Pytest uses: ./ with: deployment-id: ${{ matrix.deployment_id }} - workspace: ${{ secrets.WORKSPACE_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} pytest: true root-folder: e2e-setup/astro-project @@ -168,9 +245,9 @@ jobs: uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ matrix.deployment_id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Validate Deploy Action uses: ./.github/workflows/e2e/validate_deployment @@ -195,6 +272,19 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + - name: Install dependencies run: | @@ -207,22 +297,22 @@ jobs: run: cd e2e-setup/astro-project && docker build -t custom-image:latest . - name: Set CLI context - run: astro context switch ${{ secrets.ASTRONOMER_HOST }} + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Get Deployment Info Before Test id: get-deployment-before uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ matrix.deployment_id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Deploy to Astro uses: ./ with: deployment-id: ${{ matrix.deployment_id }} - workspace: ${{ secrets.WORKSPACE_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} image-name: custom-image:latest root-folder: e2e-setup/astro-project @@ -231,9 +321,9 @@ jobs: uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ matrix.deployment_id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Validate Deploy Action uses: ./.github/workflows/e2e/validate_deployment @@ -247,7 +337,7 @@ jobs: deployment-preview-test: name: Deployment Preview Test runs-on: ubuntu-latest - needs: create-test-deployments + needs: [create-test-deployments] strategy: matrix: deployment_id: @@ -259,6 +349,19 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + - name: Install dependencies id: install-pre-reqs run: | @@ -269,14 +372,14 @@ jobs: curl -sSL https://install.astronomer.io | sudo bash -s - name: Set CLI context - run: astro context switch ${{ secrets.ASTRONOMER_HOST }} + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Create Deployment Preview id: create-deployment-preview uses: ./ with: deployment-id: ${{ matrix.deployment_id }} - workspace: ${{ secrets.WORKSPACE_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} action: create-deployment-preview preview-name: test-preview-${{ matrix.deployment_id }} root-folder: e2e-setup/astro-project @@ -290,9 +393,9 @@ jobs: uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Validate Deploy Action id: validate-deployment-preview-create @@ -308,9 +411,9 @@ jobs: uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Deploy to Deployment Preview id: deployment-preview-deploy @@ -327,9 +430,9 @@ jobs: uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ steps.deployment-preview-deploy.outputs.preview-id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Validate Deploy Action id: validate-deployment-preview-deploy @@ -350,9 +453,9 @@ jobs: uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: DAG Deploy to Astro id: deployment-preview-dag-deploy @@ -369,9 +472,9 @@ jobs: uses: ./.github/workflows/e2e/get_deployment_info with: deployment_id: ${{ steps.deployment-preview-dag-deploy.outputs.preview-id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} - name: Validate DAG Deploy Action id: validate-dag-deploy-action @@ -402,9 +505,9 @@ jobs: with: expected_status_code: 404 deployment_id: ${{ steps.create-deployment-preview.outputs.preview-id }} - organization_id: ${{ secrets.ORGANIZATION_ID }} - astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} - astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} delete-test-deployments: name: Delete Deployment @@ -423,11 +526,24 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + - name: Install Astro CLI and set context run: | curl -sSL https://install.astronomer.io | sudo bash -s - astro context switch ${{ secrets.ASTRONOMER_HOST }} - astro workspace switch ${{ secrets.WORKSPACE_ID }} + astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }} - name: Delete Deployment run: | diff --git a/e2e-setup/mocks/git.sh b/e2e-setup/mocks/git.sh index ffd3371..8365d3e 100755 --- a/e2e-setup/mocks/git.sh +++ b/e2e-setup/mocks/git.sh @@ -1,4 +1,13 @@ #!/bin/bash # hack to mock git commands as part of action.yaml so that we could simulate dags only deploy scenario without making any additional commits -echo "e2e-setup/astro-project/dags/exampledag.py" + +# Check if the script was invoked with "git diff" +if [[ "$1" == "diff" ]]; then + echo "e2e-setup/astro-project/dags/exampledag.py" +elif [[ "$1" == "fetch" ]]; then + echo "Handling git fetch, doing nothing" +else + echo "Error: git mock script isn't configured to handle $1" >&2 + exit 1 +fi \ No newline at end of file