Merge branch 'prod' into dev

kelvinou01 · Aug 16, 2023 · 5c60405 · 5c60405
2 parents ce805a0 + c9ddf2d
commit 5c60405
Show file tree

Hide file tree

Showing 9 changed files with 103 additions and 45 deletions.
diff --git a/.github/workflows/build_images.yml b/.github/workflows/build_images.yml
@@ -1,14 +1,14 @@
-name: 'Build Docker Images'
+name: "Build Docker Images"
 
 on:
   workflow_run:
     workflows: ["Run Tests"]
-    types: 
+    types:
       - completed
 
 jobs:
   build:
-    name: 'Build Docker Images'
+    name: "Build Docker Images"
     runs-on: ubuntu-latest
     environment: ${GITHUB_REF##*/}
 
@@ -18,19 +18,23 @@ jobs:
         working-directory: ./
 
     steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-
-    - name: 'Authenticate to Google Cloud'
-      uses: 'google-github-actions/auth@v1'
-      with:
-        credentials_json: '${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}'
-
-    - name: 'Build Extract'
-      run: gcloud builds submit --config=cloudbuild.yml --substitutions=IMAGE_NAME='extract' .
-
-    - name: 'Build Transform'
-      run: gcloud builds submit --config=cloudbuild.yml --substitutions=IMAGE_NAME='transform' .
-
-    - name: 'Build Load'
-      run: gcloud builds submit --config=cloudbuild.yml --substitutions=IMAGE_NAME='load' .
+      - name: "Terminate if tests failed"
+        if: ${{ github.event.workflow_run.conclusion != 'success' }}
+        run: exit 1
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: "Authenticate to Google Cloud"
+        uses: "google-github-actions/auth@v1"
+        with:
+          credentials_json: "${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}"
+
+      - name: "Build Extract"
+        run: gcloud builds submit --config=dockerfiles/cloudbuild.yml --substitutions=_IMAGE_NAME="extract" .
+
+      - name: "Build Transform"
+        run: gcloud builds submit --config=dockerfiles/cloudbuild.yml --substitutions=_IMAGE_NAME="transform" .
+
+      - name: "Build Load"
+        run: gcloud builds submit --config=dockerfiles/cloudbuild.yml --substitutions=_IMAGE_NAME="load" .
diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml
@@ -3,7 +3,7 @@ name: 'Terraform Apply'
 on:
   workflow_run:
     workflows: ["Build Docker Images"]
-    types: 
+    types:
       - completed
 
 jobs:
@@ -18,6 +18,10 @@ jobs:
         working-directory: ./terraform
 
     steps:
+    - name: "Terminate if building images failed"
+      if: ${{ github.event.workflow_run.conclusion != 'success' }}
+      run: exit 1
+
     - name: Checkout
       uses: actions/checkout@v3
 
@@ -28,13 +32,13 @@ jobs:
       uses: 'google-github-actions/auth@v1'
       with:
         credentials_json: '${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}'
-    
+
     - name: Get Image Digests
       id: get_image_digests
       run: |
-        echo 'extract_image_digest=$(gcloud artifacts docker images describe asia-southeast1-docker.pkg.dev/university-subreddits/etl/extract:latest --format="value(image_summary.digest)")' >> $GITHUB_OUTPUT
-        echo 'transform_image_digest=$(gcloud artifacts docker images describe asia-southeast1-docker.pkg.dev/university-subreddits/etl/transform:latest --format="value(image_summary.digest)")' >> $GITHUB_OUTPUT
-        echo 'load_image_digest=$(gcloud artifacts docker images describe asia-southeast1-docker.pkg.dev/university-subreddits/etl/load:latest --format="value(image_summary.digest)")' >> $GITHUB_OUTPUT
+        echo "extract_image_digest=$(gcloud artifacts docker images describe asia-southeast1-docker.pkg.dev/university-subreddits/etl-images/extract:latest --format='value(image_summary.digest)')" >> $GITHUB_OUTPUT
+        echo "transform_image_digest=$(gcloud artifacts docker images describe asia-southeast1-docker.pkg.dev/university-subreddits/etl-images/transform:latest --format='value(image_summary.digest)')" >> $GITHUB_OUTPUT
+        echo "load_image_digest=$(gcloud artifacts docker images describe asia-southeast1-docker.pkg.dev/university-subreddits/etl-images/load:latest --format='value(image_summary.digest)')" >> $GITHUB_OUTPUT
 
     - name: Terraform Init
       run: terraform init
@@ -49,9 +53,9 @@ jobs:
       run: |
         terraform plan  \
           -input=false  \
-          -var extract_image_digest=${{steps.get_image_digest.outputs.extract_image_digest}}  \
-          -var transform_image_digest=${{steps.get_image_digest.outputs.transform_image_digest}}  \
-          -var load_image_digest=${{steps.get_image_digest.outputs.load_image_digest}}  \
+          -var extract_image_digest=${{steps.get_image_digests.outputs.extract_image_digest}}  \
+          -var transform_image_digest=${{steps.get_image_digests.outputs.transform_image_digest}}  \
+          -var load_image_digest=${{steps.get_image_digests.outputs.load_image_digest}}  \
           -var credentials=$GOOGLE_APPLICATION_CREDENTIALS  \
           -var reddit_client_id=${{secrets.REDDIT_CLIENT_ID}}  \
           -var reddit_client_secret=${{secrets.REDDIT_CLIENT_SECRET}}  \
@@ -67,16 +71,16 @@ jobs:
           #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
           #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
           #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
- 
+
           <details><summary>Show Plan</summary>
- 
+
           \`\`\`\n
           ${process.env.PLAN}
           \`\`\`
- 
+
           </details>
           *Pushed by: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`;
- 
+
           github.rest.issues.createComment({
             issue_number: context.issue.number,
             owner: context.repo.owner,
@@ -93,9 +97,9 @@ jobs:
         terraform apply  \
           -auto-approve  \
           -input=false \
-          -var extract_image_digest=${{steps.get_image_digest.outputs.extract_image_digest}}  \
-          -var transform_image_digest=${{steps.get_image_digest.outputs.transform_image_digest}}  \
-          -var load_image_digest=${{steps.get_image_digest.outputs.load_image_digest}}  \
+          -var extract_image_digest=${{steps.get_image_digests.outputs.extract_image_digest}}  \
+          -var transform_image_digest=${{steps.get_image_digests.outputs.transform_image_digest}}  \
+          -var load_image_digest=${{steps.get_image_digests.outputs.load_image_digest}}  \
           -var credentials=$GOOGLE_APPLICATION_CREDENTIALS  \
           -var reddit_client_id=${{secrets.REDDIT_CLIENT_ID}}  \
           -var reddit_client_secret=${{secrets.REDDIT_CLIENT_SECRET}}  \

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -17,12 +17,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Checkout
+      uses: actions/checkout@v3
+
     - name: Set up Python 3.11
       uses: actions/setup-python@v3
       with:
         python-version: "3.11"
+
+    - name: Install test dependencies
+      run: pip3 install .[test]
+
     - name: Run pre-commit
-      run: Make pre-commit
+      run: make pre-commit
+
     - name: Run tests
-      run: Make test
+      run: make test
diff --git a/.gitignore b/.gitignore
@@ -10,3 +10,5 @@ src/scripts/
 .vscode
 .pytest_cache
 .historical-data
+build/
+*university_subreddits.egg-info
diff --git a/Makefile b/Makefile
@@ -5,7 +5,6 @@ pre-commit:
 test: pre-commit
 	cd tests && export REDDIT_CLIENT_ID="" REDDIT_CLIENT_SECRET="" SUBREDDITS="" HUGGINGFACE_TOKEN="" GCS_RAW_BUCKET_NAME="" GCS_TRANSFORMED_BUCKET_NAME="" && python3 -m pytest -v
 
-
 first-time-setup:
 	gcloud artifacts repositories create etl-images --location=asia-southeast1 --repository-format=docker
 	cd terraform && terraform init
diff --git a/dockerfiles/cloudbuild.yml b/dockerfiles/cloudbuild.yml
@@ -0,0 +1,42 @@
+steps:
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - run
+      - "--privileged"
+      - "linuxkit/binfmt:v0.7"
+    id: initialize-qemu
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - buildx
+      - create
+      - "--name"
+      - mybuilder
+    id: create-builder
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - buildx
+      - use
+      - mybuilder
+    id: select-builder
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - buildx
+      - inspect
+      - "--bootstrap"
+    id: show-target-build-platforms
+  - name: gcr.io/cloud-builders/docker
+    args:
+      - buildx
+      - build
+      - "--platform"
+      - "linux/amd64,linux/arm64"
+      - "-t"
+      - "asia-southeast1-docker.pkg.dev/university-subreddits/etl-images/$_IMAGE_NAME:latest"
+      - "-f"
+      - "dockerfiles/Dockerfile.$_IMAGE_NAME"
+      - "--push"
+      - .
+    id: build-multiarch-image
+options:
+  env:
+    - DOCKER_CLI_EXPERIMENTAL=enabled
diff --git a/pyproject.toml b/pyproject.toml
@@ -30,6 +30,9 @@ test = [
     "pre-commit~=3.3.3",
     "pytest~=7.4.0",
     "pytest-cov~=4.1.0",
+    "university-subreddits[extract]",
+    "university-subreddits[transform]",
+    "university-subreddits[load]",
 ]
 
 [tool.black]

diff --git a/terraform/backend.tf b/terraform/backend.tf
@@ -9,4 +9,4 @@ terraform {
     bucket = "university-subreddits-terraform"
     prefix = "state/"
   }
-}
+}
diff --git a/terraform/variables.tf b/terraform/variables.tf
@@ -35,18 +35,15 @@ variable "load_image_digest" {
 }
 
 variable "reddit_client_id" {
-  type    = string
-  default = "w4YVv8aziZ7DgUQDjAooWQ"
+  type = string
 }
 
 variable "reddit_client_secret" {
-  type    = string
-  default = "4r5ZL-Z2N08taBxS5E8MCs3LXkjphQ"
+  type = string
 }
 
 variable "huggingface_token" {
-  type    = string
-  default = "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"
+  type = string
 }
 
 variable "subreddits" {