diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b5654d7..e17291b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -100,11 +100,99 @@ jobs: DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -) echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT + default-deploy-tests: + name: Default Deploy Test + runs-on: ubuntu-latest + needs: [create-test-deployments] + strategy: + max-parallel: 1 + matrix: + deployment_id: + ["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"] + deploy_type: [dags, image-and-dags] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + + - name: Mock git commands + run: | + + # if deploy_type is dags, then use dags-deploy-git.sh as mock otherwise use image-deploy-git.sh as mock + if [[ "${{ matrix.deploy_type }}" == "dags" ]]; then + mv e2e-setup/mocks/default-dag-deploy-git.sh /usr/local/bin/git + else + mv e2e-setup/mocks/default-image-deploy-git.sh /usr/local/bin/git + fi + chmod +x /usr/local/bin/git + + - name: Install dependencies + run: | + + sudo apt-get install jq + + # we need to pre-install the CLI to set the context + curl -sSL https://install.astronomer.io | sudo bash -s + + - name: Setup astro project in root folder + run: | + + # copy the astro project to the root folder + cp -r e2e-setup/astro-project/ . + + - name: Set CLI context + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Get Deployment Info Before Test + id: get-deployment-before + uses: ./.github/workflows/e2e/get_deployment_info + with: + deployment_id: ${{ matrix.deployment_id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Run Deploy Action + uses: ./ + with: + deployment-id: ${{ matrix.deployment_id }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + + - name: Get Deployment Info After Test + id: get-deployment-after + uses: ./.github/workflows/e2e/get_deployment_info + with: + deployment_id: ${{ matrix.deployment_id }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + - name: Validate Deploy Action + uses: ./.github/workflows/e2e/validate_deployment + with: + is_dag_only_deploy: ${{ matrix.deploy_type == 'dags' }} + dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }} + image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }} + dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }} + image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }} + # DAG Deploy test would test the DAG only deploy functionality in deploy action dag-deploy-test: name: DAG Deploy Test runs-on: ubuntu-latest - needs: [create-test-deployments] + needs: [default-deploy-tests, create-test-deployments] strategy: matrix: deployment_id: diff --git a/action.yaml b/action.yaml index ca0e2bd..bcd0b0c 100644 --- a/action.yaml +++ b/action.yaml @@ -7,7 +7,7 @@ branding: inputs: root-folder: required: false - default: "./" + default: "" description: "Path to the Astro project, or dbt project for dbt deploys." parse: required: false @@ -300,7 +300,11 @@ runs: run: | echo ::group::Get DBT Deploy Options - cd ${{ inputs.root-folder }} + + if [[ ${{ inputs.root-folder }} != "" ]]; then + cd ${{ inputs.root-folder }} + fi + branch=$(echo "${GITHUB_REF#refs/heads/}") echo "Branch pushed to: $branch" git fetch origin $branch @@ -308,13 +312,8 @@ runs: echo "Files changed: $files" DBT_DEPLOY=false - root_folder="${{ inputs.root-folder }}" - if [[ $root_folder == "./" ]]; then - root_folder="" - fi - for file in $files; do - if [[ $file =~ ^"$root_folder".* ]]; then + if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then echo $file is part of configured root folder, so would be triggering a dbt deploy DBT_DEPLOY=true fi @@ -344,10 +343,13 @@ runs: # infer based on files changed to deploy only dags or image and dags echo ::group::Get Deploy Type - cd ${{ inputs.root-folder }} + + if [[ ${{ inputs.root-folder }} != "" ]]; then + cd ${{ inputs.root-folder }} + fi + branch=$(echo "${GITHUB_REF#refs/heads/}") echo "Branch pushed to: $branch" - pwd git fetch origin $branch files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }}) echo "files changed: $files" @@ -359,17 +361,12 @@ runs: SKIP_IMAGE_OR_DAGS_DEPLOY=true - root_folder="${{ inputs.root-folder }}" - if [[ $root_folder == "./" ]]; then - root_folder="" - fi - # This for loop checks for following cases: # 1. If no file is part of the input root folder, then it skips deploy # 2. If any file is not part of the dags folder, then it triggers a full image build # 3. If all files are part of the dags folder and input root folder, then it triggers a DAG-only deploy for file in $files; do - if [[ $file =~ ^"$root_folder".* ]]; then + if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then echo $file is part of the input root folder SKIP_IMAGE_OR_DAGS_DEPLOY=false if [[ $file != *"dags/"* ]]; then @@ -454,7 +451,11 @@ runs: echo ::group::DAG Deploy to Astro # Deploy only dags - cd ${{ inputs.root-folder }} + + if [[ ${{ inputs.root-folder }} != "" ]]; then + cd ${{ inputs.root-folder }} + fi + astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --dags ${{steps.deploy-options.outputs.OPTIONS}} echo ::endgroup:: shell: bash @@ -464,7 +465,11 @@ runs: run: | echo ::group::Image and DAG Deploy to Astro # Deploy image and DAGs - cd ${{ inputs.root-folder }} + + if [[ ${{ inputs.root-folder }} != "" ]]; then + cd ${{ inputs.root-folder }} + fi + astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.deploy-options.outputs.OPTIONS}} echo ::endgroup:: shell: bash @@ -473,7 +478,11 @@ runs: run: | echo ::group::DBT Deploy to Astro - cd ${{ inputs.root-folder }} + + if [[ ${{ inputs.root-folder }} != "" ]]; then + cd ${{ inputs.root-folder }} + fi + astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.dbt-deploy-options.outputs.DBT_OPTIONS}} echo ::endgroup:: shell: bash diff --git a/e2e-setup/mocks/default-dag-deploy-git.sh b/e2e-setup/mocks/default-dag-deploy-git.sh new file mode 100644 index 0000000..5c54749 --- /dev/null +++ b/e2e-setup/mocks/default-dag-deploy-git.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# hack to mock git commands as part of action.yaml so that we could simulate dags only deploy scenario without making any additional commits + +# Check if the script was invoked with "git diff" +if [[ "$1" == "diff" ]]; then + echo "dags/exampledag.py" +elif [[ "$1" == "fetch" ]]; then + echo "Handling git fetch, doing nothing" +else + echo "Error: git mock script isn't configured to handle $1" >&2 + exit 1 +fi diff --git a/e2e-setup/mocks/default-image-deploy-git.sh b/e2e-setup/mocks/default-image-deploy-git.sh new file mode 100644 index 0000000..873387e --- /dev/null +++ b/e2e-setup/mocks/default-image-deploy-git.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# hack to mock git commands as part of action.yaml so that we could simulate image deploy scenario without making any additional commits + +# Check if the script was invoked with "git diff" +if [[ "$1" == "diff" ]]; then + echo "Dockerfile" +elif [[ "$1" == "fetch" ]]; then + echo "Handling git fetch, doing nothing" +else + echo "Error: git mock script isn't configured to handle $1" >&2 + exit 1 +fi