Merge pull request #22 from jaceksan/jacek/gitlab2github

GHA: scheduled jobs

.github/workflows/elta_dev.yml

@@ -15,6 +15,8 @@ on:
       - data_pipeline/meltano-plugins.yml
       - data_pipeline/plugins/**/*.lock
       - Dockerfile_meltano
+      - Dockerfile_dbt
+      - Dockerfile_gooddata
       # Extract load
       - data_pipeline/meltano.yml
       - data_pipeline/meltano_conf/**/*

.github/workflows/elta_schedule_prod.yml

@@ -0,0 +1,15 @@
+name: Extract, Load, Transform, and Analytics (Prod Schedule)
+
+on:
+  schedule:
+    - cron: "0 5 * * *"
+
+jobs:
+  elta:
+    name: ELTA (staging)
+    uses: ./.github/workflows/reusable_elta.yml
+    with:
+      ENVIRONMENT: "prod"
+      BRANCH_NAME: "prod"
+      RUN_ANALYTICS: "false"
+    secrets: inherit

.github/workflows/elta_schedule_staging.yml

@@ -0,0 +1,15 @@
+name: Extract, Load, Transform, and Analytics (Staging Schedule)
+
+on:
+  schedule:
+    - cron: "50 15 * * *"
+
+jobs:
+  elta:
+    name: ELTA (staging)
+    uses: ./.github/workflows/reusable_elta.yml
+    with:
+      ENVIRONMENT: "staging"
+      BRANCH_NAME: "main"
+      RUN_ANALYTICS: "false"
+    secrets: inherit

.github/workflows/elta_staging_prod.yml

@@ -1,9 +1,6 @@
 name: Extract, Load, Transform, and Analytics (Staging/Production)
 
 on:
-  # TODO - schedules
-  #  schedule:
-  #    - cron: "0 4 * * *"
   push:
     branches:
       - main

.github/workflows/reusable_elta.yml

@@ -6,6 +6,12 @@ on:
       ENVIRONMENT:
         required: true
         type: string
+      BRANCH_NAME:
+        type: string
+      RUN_ANALYTICS:
+        required: false
+        type: string
+        default: "true"
 
 jobs:
   ####################################
@@ -16,6 +22,7 @@ jobs:
       ENVIRONMENT: "${{ inputs.ENVIRONMENT }}"
       FULL_REFRESH: "false" # TODO: define workflow_dispatch with this parameter and set it here
       MELTANO_CUSTOM_IMAGE: "ghcr.io/${{ github.repository }}/${{ vars.MELTANO_CUSTOM_IMAGE_BASE }}:${{ vars.MELTANO_VERSION }}"
+      BRANCH_NAME: "${{ inputs.BRANCH_NAME }}"
     secrets: inherit
   ####################################
   # Transform
@@ -27,14 +34,17 @@ jobs:
       ENVIRONMENT: "${{ inputs.ENVIRONMENT }}"
       FULL_REFRESH: "false" # TODO: define workflow_dispatch with this parameter and set it here
       DBT_CUSTOM_IMAGE: "ghcr.io/${{ github.repository }}/${{ vars.DBT_CUSTOM_IMAGE_BASE }}:${{ vars.DBT_VERSION }}"
+      BRANCH_NAME: "${{ inputs.BRANCH_NAME }}"
     secrets: inherit
   ####################################
   # Analytics
   analytics:
+    if: ${{ inputs.RUN_ANALYTICS == 'true' }}
     needs:
       - transform
     uses: ./.github/workflows/reusable_analytics.yml
     with:
       ENVIRONMENT: "${{ inputs.ENVIRONMENT }}"
       GOODDATA_CUSTOM_IMAGE: "ghcr.io/${{ github.repository }}/${{ vars.GOODDATA_IMAGE_BASE }}:${{ vars.GOODDATA_VERSION }}"
+      BRANCH_NAME: "${{ inputs.BRANCH_NAME }}"
     secrets: inherit