aws_datalake.yaml

name: Build image and deploy Prefect flow - S3 Datalake

env:
  PROJECT_NAME: S3 Datalake
  PROJECT_DIRECTORY: flows/aws/datalake
  PROD_WORKPOOL: Demo-ECS
  DEV_WORKPOOL: Dev-ECS
  CLOUD_ENV: AWS # AWS, GCP, AZURE

on:
  push:
    branches:
      - main
      - Dev
    paths:
      - "$PROJECT_DIRECTORY/**"
  workflow_dispatch:

jobs:
  deploy:
      name: Deploy $PROJECT_NAME flows
      runs-on: ubuntu-latest

      steps:
        - name: Checkout
          uses: actions/checkout@v4

        - name: Log in to image registry
          uses: docker/login-action@v3
          if: env.CLOUD_ENV == 'AWS'
          with:
            registry: ${{ secrets.ECR_REPO }}
            username: ${{ secrets.AWS_ACCESS_KEY_ID }}
            password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

        - name: Get commit hash
          id: get-commit-hash
          run: echo "COMMIT_HASH=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"

        - name: Setup Python
          uses: actions/setup-python@v5
          with:
            python-version: "3.11"
            cache: "pip"

        - name: Prefect Deploy
          env:
            BRANCH: ${{ github.ref_name }}
            GITHUB_SHA: ${{ steps.get-commit-hash.outputs.COMMIT_HASH }}
            PREFECT_API_KEY: ${{ secrets.PREFECT_API_KEY }}
            IMG_REPO: ${{ secrets.ECR_REPO }}
            WORKSPACE: ${{ github.ref == 'refs/heads/main' && 'se-demos' || 'se-demos-dev' }}
            WORK_POOL_NAME: ${{ github.ref == 'refs/heads/main' && '$PROD_WORKPOOL' || '$DEV_WORKPOOL' }}
            SCHEDULES_ACTIVE: ${{ github.ref == 'refs/heads/main' && True || False }}
          run: |
            cd $PROJECT_DIRECTORY
            pip install -r requirements-ci.txt
            prefect cloud workspace set -w sales-engineering/$WORKSPACE
            python deploy.py