update README and add test for infer case

.github/workflows/tests.yaml

@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - main
-      - e2e-tests
+      - dbt-deploy
   workflow_dispatch:
     inputs:
       workspace_id:
@@ -260,10 +260,89 @@ jobs:
           dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
           image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
 
+  infer-image-deploy:
+    name: Infer Image Deploy Test
+    runs-on: ubuntu-latest
+    needs: [pytests-test, create-test-deployments]
+    strategy:
+      matrix:
+        deployment_id:
+          [
+            "${{ needs.create-test-deployments.outputs.DEPLOYMENT_ID }}",
+            "${{ needs.create-test-deployments.outputs.HIBERNATE_DEPLOYMENT_ID }}",
+          ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get Astro Environment Info
+        id: get-astro-env-info
+        uses: ./.github/workflows/e2e/get_astro_env_info
+        with:
+          input_workspace_id: ${{ github.event.inputs.workspace_id }}
+          input_organization_id: ${{ github.event.inputs.org_id }}
+          input_astronomer_host: ${{ github.event.inputs.astronomer_host }}
+          input_astro_api_token: ${{ github.event.inputs.token }}
+          secret_workspace_id: ${{ secrets.WORKSPACE_ID }}
+          secret_organization_id: ${{ secrets.ORGANIZATION_ID }}
+          secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }}
+          secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }}
+
+      - name: Mock git commands
+        run: |
+          mv e2e-setup/mocks/image-deploy-git.sh /usr/local/bin/git
+          chmod +x /usr/local/bin/git
+
+      - name: Install dependencies
+        run: |
+
+          sudo apt-get install jq
+
+          # we need to pre-install the CLI to set the context
+          curl -sSL https://install.astronomer.io | sudo bash -s
+
+      - name: Set CLI context
+        run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Get Deployment Info Before Test
+        id: get-deployment-before
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Run Deploy Action
+        uses: ./
+        with:
+          deployment-id: ${{ matrix.deployment_id }}
+          workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }}
+          parse: true
+          action: deploy
+          root-folder: e2e-setup/astro-project
+
+      - name: Get Deployment Info After Test
+        id: get-deployment-after
+        uses: ./.github/workflows/e2e/get_deployment_info
+        with:
+          deployment_id: ${{ matrix.deployment_id }}
+          organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }}
+          astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }}
+          astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }}
+
+      - name: Validate Deploy Action
+        uses: ./.github/workflows/e2e/validate_deployment
+        with:
+          dag_tarball_version_before: ${{ steps.get-deployment-before.outputs.desired_dag_tarball_version }}
+          image_version_before: ${{ steps.get-deployment-before.outputs.desired_image_version }}
+          dag_tarball_version_after: ${{ steps.get-deployment-after.outputs.desired_dag_tarball_version }}
+          image_version_after: ${{ steps.get-deployment-after.outputs.desired_image_version }}
+
   custom-docker-image-test:
     name: Custom Docker Image Test
     runs-on: ubuntu-latest
-    needs: [pytests-test, create-test-deployments]
+    needs: [infer-image-deploy, create-test-deployments]
     strategy:
       matrix:
         deployment_id:

README.md

@@ -48,7 +48,8 @@ The following table lists the configuration options for the Deploy to Astro acti
 
 | Name | Default | Description |
 | ---|---|--- |
-| `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `create-deployment-preview`, `delete-deployment-preview`, `deploy-deployment-preview` or `dbt-deploy`. |
+| `action` | `deploy` | Specify what action you would like to take. Use this option to create or delete deployment previews. Specify either `deploy`, `create-deployment-preview`, `delete-deployment-preview` or `deploy-deployment-preview`. If using `deploy` or `deploy-deployment-preview` one should also specify `deploy-type`. |
+| `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs or DBT project. Possible options are `infer`, `dags-only`, `image-and-dags` or `dbt`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. |
 | `deployment-id` | `false` | Specifies the id of the deployment you to make a preview from or are deploying too. |
 | `deployment-name` | `false` | Specifies The name of the deployment you want to make preview from or are deploying too. Cannot be used with `deployment-id` |
 | `description` |  | Configure a description for a deploy to Astro. Description will be visible in the Deploy History tab. |
@@ -57,11 +58,11 @@ The following table lists the configuration options for the Deploy to Astro acti
 | `pytest` | `false` | When set to `true`, all pytests in the `tests` directory of your Astro project are run before deploying to Astro. See [Run tests with pytest](https://docs.astronomer.io/astro/cli/test-your-astro-project-locally#run-tests-with-pytest) |
 | `pytest-file` | (all tests run) | Specifies a custom pytest file to run with the pytest command. For example, you could specify `/tests/test-tags.py`.|
 | `force` | `false` | When set to `true`, your code is deployed and skips any pytest or parsing errors. |
-| `image-name` | | Specifies a custom, locally built image to deploy. |
+| `image-name` | | Specifies a custom, locally built image to deploy. To be used with `deploy-type` set to `image-and-dags` or `infer` |
 | `workspace` | | Workspace id to select. Only required when `ASTRO_API_TOKEN` is given an organization token. |
 | `preview-name` | `false` | Specifies custom preview name. By default this is branch name “_” deployment name. |
 | `checkout` | `true` | Whether to checkout the repo as the first step. Set this to false if you want to modify repo contents before invoking the action. Your custom checkout step needs to have `fetch-depth` of `0` and `ref` equal to `${{ github.event.after }}` so all the commits in the PR are checked out. Look at the checkout step that runs within this action for reference. |
-| `deploy-image` | `false` | If true image and DAGs will deploy for any action that deploys code. |
+| `deploy-image` | `false` | If true image and DAGs will deploy for any action that deploys code. NOTE: This option is deprecated and will be removed in a future release. Use `deploy-type: image-and-dags` instead. |
 | `build-secrets` | `` | Mimics docker build --secret flag. See https://docs.docker.com/build/building/secrets/ for more information. Example input 'id=mysecret,src=secrets.txt'. |
 | `mount-path` | `` | Path to mount dbt project in Airflow, for reference by DAGs. Default /usr/local/airflow/dbt/{dbt project name} |
 | `checkout-submodules` | `false` | Whether to checkout submodules when cloning the repository: `false` to disable (default), `true` to checkout submodules or `recursive` to recursively checkout submodules. Works only when `checkout` is set to `true`. Works only when `checkout` is set to `true`. |
@@ -186,6 +187,7 @@ jobs:
       uses: astronomer/[email protected]
       with:
         deployment-id: <deployment id>
+        deploy-type: image-and-dags
         image-name: ${{ steps.image_tag.outputs.image_tag }}
 
 ```
@@ -200,7 +202,7 @@ steps:
   uses: astronomer/[email protected]
   with:
     deployment-id: <deployment id>
-    action: dbt-deploy
+    deploy-type: dbt
     root-folder: dbt
 ```
 
@@ -214,12 +216,12 @@ steps:
   uses: astronomer/[email protected]
   with:
     deployment-id: <deployment id>
+    deploy-type: dbt
     root-folder: dbt
 - name: DAGs/Image Deploy to Astro
   uses: astronomer/[email protected]
   with:
     deployment-id: <deployment id>
-    action: deploy
     root-folder: astro-project/
     parse: true
 ```
@@ -304,7 +306,8 @@ jobs:
     - name: Deploy to Deployment Preview
       uses: astronomer/[email protected]
       with:
-        action: dbt-deploy-deployment-preview
+        action: deploy-deployment-preview
+        deploy-type: dbt
         deployment-id: <orginal deployment id>
         root-folder: dbt
 ```

action.yaml

@@ -28,7 +28,7 @@ inputs:
   image-name:
     required: false
     default: no-custom-image
-    description: "Specify a custom built image to deploy to an Asto Deployment."
+    description: "Specify a custom built image to deploy to an Asto Deployment.  To be used with 'deploy-type' set to 'image-and-dags' or 'infer'"
   action:
     required: false
     default: deploy
@@ -82,7 +82,7 @@ inputs:
   deploy-type:
     required: false
     default: "infer"
-    description: "Specify the type of deploy you would like to do. Use this option to deploy only DAGs or the image or DBT project. Specify either 'infer', 'dags-only', 'image-and-dags', or 'dbt'."
+    description: "Specify the type of deploy you would like to do. Use this option to deploy only DAGs or the image or DBT project. Specify either 'infer', 'dags-only', 'image-and-dags', or 'dbt'. 'infer' option would infer between DAG only deploy and image and DAG deploy based on updated files."
   build-secrets:
     required: false
     description: "Mimics docker build --secret flag. See https://docs.docker.com/build/building/secrets/ for more information. Example input 'id=mysecret,src=secrets.txt'"
@@ -129,7 +129,7 @@ runs:
         # Check if the Astro CLI version is less than 1.28.1 for dbt-deploy
         if [[ "${{ inputs.deploy-type }}" == "dbt" && $CLI_VERSION != "" ]]; then
           REQUIRED_VERSION="1.28.1"
-          CURRENT_VERSION=$(astro --version | awk '{print $3}')
+          CURRENT_VERSION=$(astro version | awk '{print $4}')
           if [[ $CURRENT_VERSION =~ ([0-9]+)\.([0-9]+)\.([0-9]+) ]]; then
             CURRENT_MAJOR=${BASH_REMATCH[1]}
             CURRENT_MINOR=${BASH_REMATCH[2]}
@@ -348,8 +348,13 @@ runs:
         echo "Files changed: $files"
         dbt_deploy=0
 
+        root_folder="${{ inputs.root-folder }}"
+        if [[ $root_folder == "./" ]]; then
+          root_folder=""
+        fi
+
         for file in $files; do
-          if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then
+          if [[ $file =~ ^"$root_folder".* ]]; then
             echo $file is part of configured root folder, so would be triggering a dbt deploy
             dbt_deploy=1
           fi
@@ -374,7 +379,7 @@ runs:
       shell: bash
       id: dbt-deploy-options
     - name: Get Deploy Type
-      if: ${{ inputs.deploy-type == 'infer' }}
+      if: ${{ inputs.deploy-type == 'infer' || inputs.deploy-type == 'dags-only' }}
       run: |
 
         # infer based on files changed to deploy only dags or image and dags
@@ -389,12 +394,19 @@ runs:
 
         skip_deploy=1
 
+        # This is to ensure that the we are backward compatible for the case when
+        # the git repo only contains astro project and root-folder is not set in the GHA
+        root_folder="${{ inputs.root-folder }}"
+        if [[ $root_folder == "./" ]]; then
+          root_folder=""
+        fi
+
         # This for loop checks for following cases:
         # 1. If no file is part of the input root folder, then it skips deploy
         # 2. If any file is not part of the dags folder, then it triggers a full image build
         # 3. If all files are part of the dags folder and input root folder, then it triggers a DAG-only deploy
         for file in $files; do
-          if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then
+          if [[ $file =~ ^"$root_folder".* ]]; then
             echo $file is part of the input root folder
             skip_deploy=0
             if [[ $file != *"dags/"* ]]; then
@@ -447,7 +459,7 @@ runs:
         fi
 
         # add custom image option
-        if [[ ${{ inputs.image-name }} != no-custom-image ]]; then
+        if [[ ${{ inputs.image-name }} != no-custom-image && inputs.deploy-type != "dags-only" ]]; then
           options="$options --image-name ${{ inputs.image-name }}"
         fi
 
@@ -504,7 +516,7 @@ runs:
         echo ::endgroup::
       shell: bash
     - name: DAG Deploy to Astro
-      if: ${{ inputs.deploy-type == 'dags-only' || (inputs.deploy-type == 'infer' && steps.deployment-type.outputs.DAGS_ONLY == '1') }}
+      if: ${{ (inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'infer') && steps.deployment-type.outputs.DAGS_ONLY == '1' }}
       run: |
 
         echo ::group::DAG Deploy to Astro

e2e-setup/README.md

@@ -6,18 +6,27 @@ The E2E tests could be triggered automatically when a new commit is pushed to ma
 
 ## Folder Structure
 
+```
 e2e-setup/
 ├── astro-project/
 ├── deployment-templates/
-│ ├── deployment-hibernate.yaml
-│ └── deployment.yaml
+│   ├── deployment-hibernate.yaml
+│   └── deployment.yaml
 ├── mocks/
-│ └── git.sh
+│   ├── dag-deploy-git.sh
+│   └── dbt-deploy-git.sh
+├── dbt/
+│   └── dbt_project.yml
+```
 
 ### astro-project
 
 Astro project folder contains a basic airflow project initialized via Astro CLI, which will deployed as part of tests via deploy action
 
+### dbt
+
+The dbt folder contains a basic sample dbt_project.yml file, which is used to define the configuration of a dbt project. This file includes settings such as the project name, version, and other configurations necessary for running dbt commands.
+
 ### deployment-templates
 
 Deployment templates contains the basic templates used by e2e tests to create required deployments against which tests would be executed

e2e-setup/mocks/dbt-deploy-git.sh

@@ -1,4 +1,13 @@
 #!/bin/bash
 
 # hack to mock git commands as part of action.yaml so that we could simulate dbt deploy scenario without making any additional commits
-echo "e2e-setup/dbt/dbt_project.yml"
+
+# Check if the script was invoked with "git diff"
+if [[ "$1" == "diff" ]]; then
+  echo "e2e-setup/dbt/dbt_project.yml"
+elif [[ "$1" == "fetch" ]]; then
+  echo "Handling git fetch, doing nothing"
+else
+  echo "Error: git mock script isn't configured to handle $1" >&2
+  exit 1
+fi

e2e-setup/mocks/git.sh → e2e-setup/mocks/image-deploy-git.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 
-# hack to mock git commands as part of action.yaml so that we could simulate dags only deploy scenario without making any additional commits
+# hack to mock git commands as part of action.yaml so that we could simulate image deploy scenario without making any additional commits
 
 # Check if the script was invoked with "git diff"
 if [[ "$1" == "diff" ]]; then
-  echo "e2e-setup/astro-project/dags/exampledag.py"
+  echo "e2e-setup/astro-project/Dockerfile"
 elif [[ "$1" == "fetch" ]]; then
   echo "Handling git fetch, doing nothing"
 else
   echo "Error: git mock script isn't configured to handle $1" >&2
   exit 1
-fi
+fi