Skip to content

Commit

Permalink
add tests for default case and address code review comments
Browse files Browse the repository at this point in the history
  • Loading branch information
neel-astro committed Sep 27, 2024
1 parent 97d3b2b commit e49d5e6
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 20 deletions.
90 changes: 89 additions & 1 deletion .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,99 @@ jobs:
DEPLOYMENT_ID=$(astro deployment create --deployment-file e2e-setup/deployment-templates/${{ matrix.deployment }} | yq e '.deployment.metadata.deployment_id' -)
echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT
default-deploy-tests:
name: Default Deploy Test
runs-on: ubuntu-latest
needs: [create-test-deployments]
strategy:
max-parallel: 1
matrix:
deployment_id:
["${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}"]
deploy_type: [dags, image-and-dags]
steps:
- name: Checkout code
uses: actions/checkout@v4

- name: Get Astro Environment Info
id: get-astro-env-info
uses: ./.github/workflows/e2e/get_astro_env_info
with:
input_workspace_id: ${{ github.event.inputs.workspace_id }}
input_organization_id: ${{ github.event.inputs.org_id }}
input_astronomer_host: ${{ github.event.inputs.astronomer_host }}
input_astro_api_token: ${{ github.event.inputs.token }}
secret_workspace_id: ${{ secrets.WORKSPACE_ID }}
secret_organization_id: ${{ secrets.ORGANIZATION_ID }}
secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}

- name: Mock git commands
run: |
# if deploy_type is dags, then use dags-deploy-git.sh as mock otherwise use image-deploy-git.sh as mock
if [[ "${{ matrix.deploy_type }}" == "dags" ]]; then
mv e2e-setup/mocks/default-dag-deploy-git.sh /usr/local/bin/git
else
mv e2e-setup/mocks/default-image-deploy-git.sh /usr/local/bin/git
fi
chmod +x /usr/local/bin/git
- name: Install dependencies
run: |
sudo apt-get install jq
# we need to pre-install the CLI to set the context
curl -sSL https://install.astronomer.io | sudo bash -s
- name: Setup astro project in root folder
run: |
# copy the astro project to the root folder
cp -r e2e-setup/astro-project/ .
- name: Set CLI context
run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }}

- name: Get Deployment Info Before Test
id: get-deployment-before
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}

- name: Run Deploy Action
uses: ./
with:
deployment-id: ${{ matrix.deployment_id }}
workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }}

- name: Get Deployment Info After Test
id: get-deployment-after
uses: ./.github/workflows/e2e/get_deployment_info
with:
deployment_id: ${{ matrix.deployment_id }}
organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}

- name: Validate Deploy Action
uses: ./.github/workflows/e2e/validate_deployment
with:
is_dag_only_deploy: ${{ matrix.deploy_type == 'dags' }}
dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}

# DAG Deploy test would test the DAG only deploy functionality in deploy action
dag-deploy-test:
name: DAG Deploy Test
runs-on: ubuntu-latest
needs: [create-test-deployments]
needs: [default-deploy-tests, create-test-deployments]
strategy:
matrix:
deployment_id:
Expand Down
47 changes: 28 additions & 19 deletions action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ branding:
inputs:
root-folder:
required: false
default: "./"
default: ""
description: "Path to the Astro project, or dbt project for dbt deploys."
parse:
required: false
Expand Down Expand Up @@ -300,21 +300,20 @@ runs:
run: |
echo ::group::Get DBT Deploy Options
cd ${{ inputs.root-folder }}
if [[ ${{ inputs.root-folder }} != "" ]]; then
cd ${{ inputs.root-folder }}
fi
branch=$(echo "${GITHUB_REF#refs/heads/}")
echo "Branch pushed to: $branch"
git fetch origin $branch
files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }})
echo "Files changed: $files"
DBT_DEPLOY=false
root_folder="${{ inputs.root-folder }}"
if [[ $root_folder == "./" ]]; then
root_folder=""
fi
for file in $files; do
if [[ $file =~ ^"$root_folder".* ]]; then
if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then
echo $file is part of configured root folder, so would be triggering a dbt deploy
DBT_DEPLOY=true
fi
Expand Down Expand Up @@ -344,10 +343,13 @@ runs:
# infer based on files changed to deploy only dags or image and dags
echo ::group::Get Deploy Type
cd ${{ inputs.root-folder }}
if [[ ${{ inputs.root-folder }} != "" ]]; then
cd ${{ inputs.root-folder }}
fi
branch=$(echo "${GITHUB_REF#refs/heads/}")
echo "Branch pushed to: $branch"
pwd
git fetch origin $branch
files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }})
echo "files changed: $files"
Expand All @@ -359,17 +361,12 @@ runs:
SKIP_IMAGE_OR_DAGS_DEPLOY=true
root_folder="${{ inputs.root-folder }}"
if [[ $root_folder == "./" ]]; then
root_folder=""
fi
# This for loop checks for following cases:
# 1. If no file is part of the input root folder, then it skips deploy
# 2. If any file is not part of the dags folder, then it triggers a full image build
# 3. If all files are part of the dags folder and input root folder, then it triggers a DAG-only deploy
for file in $files; do
if [[ $file =~ ^"$root_folder".* ]]; then
if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then
echo $file is part of the input root folder
SKIP_IMAGE_OR_DAGS_DEPLOY=false
if [[ $file != *"dags/"* ]]; then
Expand Down Expand Up @@ -454,7 +451,11 @@ runs:
echo ::group::DAG Deploy to Astro
# Deploy only dags
cd ${{ inputs.root-folder }}
if [[ ${{ inputs.root-folder }} != "" ]]; then
cd ${{ inputs.root-folder }}
fi
astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} --dags ${{steps.deploy-options.outputs.OPTIONS}}
echo ::endgroup::
shell: bash
Expand All @@ -464,7 +465,11 @@ runs:
run: |
echo ::group::Image and DAG Deploy to Astro
# Deploy image and DAGs
cd ${{ inputs.root-folder }}
if [[ ${{ inputs.root-folder }} != "" ]]; then
cd ${{ inputs.root-folder }}
fi
astro deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.deploy-options.outputs.OPTIONS}}
echo ::endgroup::
shell: bash
Expand All @@ -473,7 +478,11 @@ runs:
run: |
echo ::group::DBT Deploy to Astro
cd ${{ inputs.root-folder }}
if [[ ${{ inputs.root-folder }} != "" ]]; then
cd ${{ inputs.root-folder }}
fi
astro dbt deploy ${{steps.deployment-preview.outputs.FINAL_DEPLOYMENT_ID}} ${{steps.dbt-deploy-options.outputs.DBT_OPTIONS}}
echo ::endgroup::
shell: bash
Expand Down
13 changes: 13 additions & 0 deletions e2e-setup/mocks/default-dag-deploy-git.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/bin/bash

# hack to mock git commands as part of action.yaml so that we could simulate dags only deploy scenario without making any additional commits

# Check if the script was invoked with "git diff"
if [[ "$1" == "diff" ]]; then
echo "dags/exampledag.py"
elif [[ "$1" == "fetch" ]]; then
echo "Handling git fetch, doing nothing"
else
echo "Error: git mock script isn't configured to handle $1" >&2
exit 1
fi
13 changes: 13 additions & 0 deletions e2e-setup/mocks/default-image-deploy-git.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
#!/bin/bash

# hack to mock git commands as part of action.yaml so that we could simulate image deploy scenario without making any additional commits

# Check if the script was invoked with "git diff"
if [[ "$1" == "diff" ]]; then
echo "Dockerfile"
elif [[ "$1" == "fetch" ]]; then
echo "Handling git fetch, doing nothing"
else
echo "Error: git mock script isn't configured to handle $1" >&2
exit 1
fi

0 comments on commit e49d5e6

Please sign in to comment.