Merge pull request #9 from City-Bureau/batch-update-workflow

🔧 Clean up: Github workflows

.github/workflows/archive.yml

@@ -7,6 +7,7 @@ on:
 
 env:
   CI: true
+  PYTHON_VERSION: 3.9
   PIPENV_VENV_IN_PROJECT: true
   SCRAPY_SETTINGS_MODULE: city_scrapers.settings.archive
   AUTOTHROTTLE_MAX_DELAY: 30.0
@@ -17,29 +18,29 @@ jobs:
   crawl:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v1
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: pip install --user pipenv
 
       - name: Cache Python dependencies
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: .venv
-          key: pip-3.8-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}
           restore-keys: |
-            pip-3.8-
+            ${{ env.PYTHON_VERSION }}-
             pip-
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: 3.8
+          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
 
       - name: Run scrapers
         run: |

.github/workflows/ci.yml

@@ -18,18 +18,20 @@ jobs:
         python-version: [3.9]
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: pip install --user pipenv
 
       - name: Cache Python dependencies
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: .venv
           key: pip-${{ matrix.python-version }}-${{ hashFiles('**/Pipfile.lock') }}
@@ -55,8 +57,9 @@ jobs:
           pipenv run flake8 .
 
       - name: Test with pytest
+        # Ignores exit code 5 (no tests collected)
         run: |
-          pipenv run pytest
+          pipenv run pytest || [ $? -eq 5 ]
 
       - name: Validate output with scrapy
         if: github.event_name == 'pull_request'

.github/workflows/cron.yml

@@ -2,58 +2,51 @@ name: Cron
 
 on:
   schedule:
-    # Set any time that you'd like scrapers to run (in UTC)
-    - cron: "27 6 * * *"
+    - cron: "1 6 * * *"
   workflow_dispatch:
 
 env:
   CI: true
+  PYTHON_VERSION: 3.9
   PIPENV_VENV_IN_PROJECT: true
   SCRAPY_SETTINGS_MODULE: city_scrapers.settings.prod
   WAYBACK_ENABLED: true
   AUTOTHROTTLE_MAX_DELAY: 30.0
   AUTOTHROTTLE_START_DELAY: 1.5
   AUTOTHROTTLE_TARGET_CONCURRENCY: 3.0
-  # Add secrets for the platform you're using and uncomment here
-  # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  # S3_BUCKET: ${{ secrets.S3_BUCKET }}
   AZURE_ACCOUNT_KEY: ${{ secrets.AZURE_ACCOUNT_KEY }}
   AZURE_ACCOUNT_NAME: ${{ secrets.AZURE_ACCOUNT_NAME }}
   AZURE_CONTAINER: ${{ secrets.AZURE_CONTAINER }}
   AZURE_STATUS_CONTAINER: ${{ secrets.AZURE_STATUS_CONTAINER }}
-  # GOOGLE_APPLICATION_CREDENTIALS = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
-  # GCS_BUCKET = os.getenv("GCS_BUCKET")
-  # Setup Sentry, add the DSN to secrets and uncomment here
-  # SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
 
 jobs:
   crawl:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v1
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Pipenv
-        uses: dschep/install-pipenv-action@v1
+        run: pip install --user pipenv
 
       - name: Cache Python dependencies
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: .venv
-          key: pip-3.9-${{ hashFiles('**/Pipfile.lock') }}
+          key: ${{ env.PYTHON_VERSION }}-${{ hashFiles('**/Pipfile.lock') }}
           restore-keys: |
-            pip-3.9-
+            ${{ env.PYTHON_VERSION }}-
             pip-
 
       - name: Install dependencies
         run: pipenv sync
         env:
-          PIPENV_DEFAULT_PYTHON_VERSION: 3.9
+          PIPENV_DEFAULT_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
 
       - name: Run scrapers
         run: |
@@ -63,4 +56,10 @@ jobs:
       - name: Combine output feeds
         run: |
           export PYTHONPATH=$(pwd):$PYTHONPATH
-          pipenv run scrapy combinefeeds -s LOG_ENABLED=True
+          pipenv run scrapy combinefeeds -s LOG_ENABLED=False
+
+      - name: Prevent workflow deactivation
+        uses: gautamkrishnar/keepalive-workflow@v1
+        with:
+          committer_username: "citybureau-bot"
+          committer_email: "[email protected]"