Merge pull request #7 from pdcp1/master

Refactor logic to a separate Github actions (public and private)

.github/workflows/run_private_backup.yml

@@ -0,0 +1,73 @@
+name: Private backup to the cloud
+
+on:
+  push:
+    branches:
+      - master
+  schedule:
+    - cron: "0 10 * * 1" # At 10:00 on Monday.
+  workflow_dispatch:
+    inputs:
+      home:
+        description: 'This can be triggered from the GH page'
+        required: false
+        default: 'This is not used'
+
+env:
+  AWS_REGION: us-east-1                   # set this to your preferred AWS region, e.g. us-west-1
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build-and-run-backups:
+    name: Backup
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile
+          load: true
+          tags: aiid-docker-backup:latest
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_S3_BACKUP_USER_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_S3_BACKUP_USER_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Generate private backup
+        run: |
+          docker run --rm \
+            -e IS_PUBLIC_BACKUP="false" \
+            -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_S3_BACKUP_USER_ACCESS_KEY_ID }}" \
+            -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_S3_BACKUP_USER_SECRET_ACCESS_KEY }}" \
+            -e TARGET_PRIVATE_BUCKET_URL="${{ vars.AWS_S3_PRIVATE_BUCKET }}" \
+            -e TARGET_PUBLIC_BUCKET_URL="${{ vars.AWS_S3_PUBLIC_BUCKET }}" \
+            -e CLOUDFLARE_ACCOUNT_ID="${{ vars.CLOUDFLARE_ACCOUNT_ID }}" \
+            -e CLOUDFLARE_R2_ACCESS_KEY="${{ secrets.CLOUDFLARE_R2_ACCESS_KEY }}" \
+            -e CLOUDFLARE_R2_SECRET_KEY="${{ secrets.CLOUDFLARE_R2_SECRET_KEY }}" \
+            -e CLOUDFLARE_R2_BUCKET="${{ vars.CLOUDFLARE_R2_PRIVATE_BUCKET }}" \
+            -e BACKUPFILE_PREFIX="backup" \
+            -e MONGODB_HOST="mongo" \
+            -e MONGODB_URI="mongodb+srv://${{ secrets.MONGO_BACKUP_USER }}:${{ secrets.MONGO_BACKUP_USER_PASSWORD }}@aiiddev-aqdmh.gcp.mongodb.net/aiidprod" \
+            -e MONGODB_DBNAME="aiidprod" \
+            -e MONGODB_URI_TRANSLATIONS="mongodb+srv://${{ secrets.MONGO_BACKUP_USER }}:${{ secrets.MONGO_BACKUP_USER_PASSWORD }}@aiiddev-aqdmh.gcp.mongodb.net/translations" \
+            -e MONGODB_DBNAME_TRANSLATIONS="translations" \
+            -e MONGODB_USERNAME="${{ secrets.MONGO_BACKUP_USER }}" \
+            -e MONGODB_PASSWORD="${{ secrets.MONGO_BACKUP_USER_PASSWORD }}" \
+            -e MONGODB_AUTHDB="admin" \
+            -v ~:/mab \
+            aiid-docker-backup:latest
+

.github/workflows/run_backup.yml → .github/workflows/run_public_backup.yml

@@ -1,11 +1,9 @@
-name: Backup to the cloud
+name: Public backup to the cloud
 
 on:
   push:
     branches:
       - master
-      - staging
-      - classifications-csv
   schedule:
     - cron: "0 10 * * 1" # At 10:00 on Monday.
   workflow_dispatch:
@@ -17,41 +15,50 @@ on:
 
 env:
   AWS_REGION: us-east-1                   # set this to your preferred AWS region, e.g. us-west-1
-  ECR_REPOSITORY: mongodb-awesome-backup  # set this to your Amazon ECR repository name
 
 defaults:
   run:
     shell: bash
 
 jobs:
-  backup-to-s3:
+  build-and-run-backups:
     name: Backup
     runs-on: ubuntu-latest
 
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile
+          load: true
+          tags: aiid-docker-backup:latest
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_S3_BACKUP_USER_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_S3_BACKUP_USER_SECRET_ACCESS_KEY }}
           aws-region: ${{ env.AWS_REGION }}
 
-      - name: Generate public and private backups
-        id: build-image
+      - name: Generate public backup
         run: |
-          ls
-          pwd
-          ls -a
           docker run --rm \
+            -e IS_PUBLIC_BACKUP="true" \
             -e AWS_ACCESS_KEY_ID="${{ secrets.AWS_S3_BACKUP_USER_ACCESS_KEY_ID }}" \
             -e AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_S3_BACKUP_USER_SECRET_ACCESS_KEY }}" \
             -e TARGET_PRIVATE_BUCKET_URL="${{ vars.AWS_S3_PRIVATE_BUCKET }}" \
             -e TARGET_PUBLIC_BUCKET_URL="${{ vars.AWS_S3_PUBLIC_BUCKET }}" \
             -e CLOUDFLARE_ACCOUNT_ID="${{ vars.CLOUDFLARE_ACCOUNT_ID }}" \
             -e CLOUDFLARE_R2_ACCESS_KEY="${{ secrets.CLOUDFLARE_R2_ACCESS_KEY }}" \
             -e CLOUDFLARE_R2_SECRET_KEY="${{ secrets.CLOUDFLARE_R2_SECRET_KEY }}" \
-            -e CLOUDFLARE_R2_PUBLIC_BUCKET="${{ vars.CLOUDFLARE_R2_PUBLIC_BUCKET }}" \
-            -e CLOUDFLARE_R2_PRIVATE_BUCKET="${{ vars.CLOUDFLARE_R2_PRIVATE_BUCKET }}" \
+            -e CLOUDFLARE_R2_BUCKET="${{ vars.CLOUDFLARE_R2_PUBLIC_BUCKET }}" \
             -e BACKUPFILE_PREFIX="backup" \
             -e MONGODB_HOST="mongo" \
             -e MONGODB_URI="mongodb+srv://${{ secrets.MONGO_BACKUP_USER }}:${{ secrets.MONGO_BACKUP_USER_PASSWORD }}@aiiddev-aqdmh.gcp.mongodb.net/aiidprod" \
@@ -61,7 +68,6 @@ jobs:
             -e MONGODB_USERNAME="${{ secrets.MONGO_BACKUP_USER }}" \
             -e MONGODB_PASSWORD="${{ secrets.MONGO_BACKUP_USER_PASSWORD }}" \
             -e MONGODB_AUTHDB="admin" \
-            -e AWSCLI_ENDPOINT_OPT="" \
             -v ~:/mab \
-            public.ecr.aws/${{ vars.AWS_ECR_ID || 'd4w2c1x5'}}/mongodb-awesome-backup:latest
+            aiid-docker-backup:latest
 

Dockerfile

@@ -22,4 +22,4 @@ ENV AWS_DEFAULT_REGION=ap-northeast-1
 COPY bin /opt/bin
 WORKDIR /opt/bin
 ENTRYPOINT ["/opt/bin/entrypoint.sh"]
-CMD ["backup_full_snapshot", "backup_filtered_data", "prune", "list"]
+CMD ["backup", "prune", "list"]

README.md

@@ -1,6 +1,6 @@
 This is a quick port of the forked project to support JSON and CSV backups of the [AIID](https://incidentdatabase.ai/).
 
-The complete state of the database will be backed up on a weekly basis in both JSON and CSV form. The backups can be downloaded from here: todo
+The complete state of the database will be backed up on a weekly basis in both JSON and CSV form. The backups can be downloaded from [here](https://incidentdatabase.ai/research/snapshots/).
 
 What is mongodb-awesome-backup?
 -------------------------------
@@ -13,30 +13,29 @@ Requirements
 ------------
 
 Amazon IAM Access Key ID/Secret Access Key, which must have the access rights of the target Amazon S3 bucket.
+Cloudflare R2 Access Key ID/Secret Access Key, which must have the access rights of the target Cloudflare R2 bucket.
+MongoDB credentials with read access to the target database.
 
 Usage
 -----
-Note that either AWS_ or GCP_ vars are required not both.
-
 ```bash
 docker run --rm \
   -e AWS_ACCESS_KEY_ID=<Your IAM Access Key ID> \
   -e AWS_SECRET_ACCESS_KEY=<Your IAM Secret Access Key> \
-  -e TARGET_PRIVATE_BUCKET_URL=<Target public Bucket URL ([s3://...|gs://...])> \
-  -e TARGET_PUBLIC_BUCKET_URL=<Target private Bucket URL ([s3://...|gs://...])> \
+  -e TARGET_PRIVATE_BUCKET_URL=<Target public Bucket URL (s3://...)> \
+  -e TARGET_PUBLIC_BUCKET_URL=<Target private Bucket URL (s3://...)> \
   -e CLOUDFLARE_ACCOUNT_ID=<Cloudflare R2 account ID> \
   -e CLOUDFLARE_R2_ACCESS_KEY=<Cloudflare R2 Access ID Key> \
   -e CLOUDFLARE_R2_SECRET_KEY=<Cloudflare R2 Access Secret Key> \
   -e CLOUDFLARE_R2_PUBLIC_BUCKET=<Cloudflare R2 public bucket name (ie: "aiid-public")> \
-  -e CLOUDFLARE_R2_PRIVATE_BUCKET=<Cloudflare R2 private bucket name (ie: "aiid-private")> \
+  -e CLOUDFLARE_R2_BUCKET=<Cloudflare R2 bucket name (ie: "aiid-public")> \
   [ -e BACKUPFILE_PREFIX=<Prefix of Backup Filename (default: "backup") \ ]
   [ -e MONGODB_URI=<Target MongoDB URI> \ ]
   [ -e MONGODB_HOST=<Target MongoDB Host (default: "mongo")> \ ]
   [ -e MONGODB_DBNAME=<Target DB name> \ ]
   [ -e MONGODB_USERNAME=<DB login username> \ ]
   [ -e MONGODB_PASSWORD=<DB login password> \ ]
   [ -e MONGODB_AUTHDB=<Authentication DB name> \ ]
-  [ -e AWSCLI_ENDPOINT_OPT=<S3 endpoint URL (ex. https://fra1.digitaloceanspaces.com)> \ ]
   [ -v ~:/mab \ ]
   weseek/mongodb-awesome-backup
 ```
@@ -55,8 +54,8 @@ Environment variables
 | --------------------- | ------------------------------------------------------------------------------ | ------- |
 | AWS_ACCESS_KEY_ID     | Your IAM Access Key ID                                                         | -       |
 | AWS_SECRET_ACCESS_KEY | Your IAM Secret Access Key                                                     | -       |
-| TARGET_PRIVATE_BUCKET_URL     | Target private Bucket URL ([s3://...\|gs://...]). **URL is needed to be end with '/'** | -       |
-| TARGET_PUBLIC_BUCKET_URL     | Target public Bucket URL ([s3://...\|gs://...]). **URL is needed to be end with '/'** | -       |
+| TARGET_PRIVATE_BUCKET_URL     | Target private Bucket URL (s3://...). **URL is needed to be end with '/'** | -       |
+| TARGET_PUBLIC_BUCKET_URL     | Target public Bucket URL (s3://...). **URL is needed to be end with '/'** | -       |
 | CLOUDFLARE_ACCOUNT_ID     | Cloudflare R2 account ID | -       |
 | CLOUDFLARE_R2_ACCESS_KEY     | Cloudflare R2 Access Key ID | -       |
 | CLOUDFLARE_R2_SECRET_KEY     | Cloudflare R2 Access Secret ID | -       |
@@ -67,10 +66,6 @@ Environment variables
 
 | Variable                          | Description                                                                                                                                                                                               | Default  |
 | --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- |
-| GCP_SERVICE_ACCOUNT_KEY_JSON_PATH | JSON file path to your GCP Service Account Key                                                                                                                                                            | -        |
-| GCP_ACCESS_KEY_ID                 | Your GCP Access Key                                                                                                                                                                                       | -        |
-| GCP_SECRET_ACCESS_KEY             | Your GCP Secret                                                                                                                                                                                           | -        |
-| GCP_PROJECT_ID                    | Your GCP Project ID                                                                                                                                                                                       | -        |
 | BACKUPFILE_PREFIX                 | Prefix of Backup Filename                                                                                                                                                                                 | "backup" |
 | MONGODB_URI                       | Target MongoDB URI (ex. `mongodb://mongodb?replicaSet=rs0`). If set, the other `MONGODB_*` variables will be ignored.                                                                                     | -        |
 | MONGODB_HOST                      | Target MongoDB Host                                                                                                                                                                                       | "mongo"  |
@@ -80,7 +75,3 @@ Environment variables
 | MONGODB_AUTHDB                    | Authentication DB name                                                                                                                                                                                    | -        |
 | CRONMODE                          | If set "true", this container is executed in cron mode.  In cron mode, the script will be executed with the specified arguments and at the time specified by CRON_EXPRESSION.                             | "false"  |
 | CRON_EXPRESSION                   | Cron expression (ex. "CRON_EXPRESSION=0 4 * * *" if you want to run at 4:00 every day)                                                                                                                    | -        |
-| AWSCLI_ENDPOINT_OPT               | Set a custom S3 endpoint if you use a S3 based service like DigitalOcean Spaces. (ex. AWSCLI_ENDPOINT_OPT="https://fra1.digitaloceanspaces.com") If not set the Amazon S3 standard endpoint will be used. | -        |
-| AWSCLIOPT                         | Other options you want to pass to `aws` command                                                                                                                                                           | -        |
-| GCSCLIOPT                         | Other options you want to pass to `gsutil` command                                                                                                                                                        | -        |
-| HEALTHCHECKS_URL                  | URL that gets called after a successful backup (eg. https://healthchecks.io)                                                                                                                              | -        |