Merge branch 'develop' of github.com:cisagov/XFD into dependabot/pip/…

…backend/worker/pipreqs-0.4.12

.bandit.yml

@@ -6,8 +6,6 @@
 # If `tests` is empty, all tests are considered included.
 
 tests:
-# - B101
-# - B102
 
 skips:
-# - B101 # skip "assert used" check since assertions are required in pytests
+  - B101  # skip "assert used" check since assertions are required in pytests

.dockerignore

@@ -11,4 +11,4 @@ nvd-dump
 minio-data
 **/node_modules
 **/.cache
-./docs/node_modules
+./docs/node_modules

.github/dependabot.yml

@@ -1,66 +1,60 @@
 ---
 version: 2
 updates:
-  - directory: /
-    # ignore:
-    #   # Managed by cisagov/ASM-Dashboard
-    #   - dependency-name: actions/cache
-    #   - dependency-name: actions/checkout
-    #   - dependency-name: actions/setup-go
-    #   - dependency-name: actions/setup-python
-    #   - dependency-name: crazy-max/ghaction-dump-context
-    #   - dependency-name: crazy-max/ghaction-github-labeler
-    #   - dependency-name: crazy-max/ghaction-github-status
-    #   - dependency-name: hashicorp/setup-terraform
-    #   - dependency-name: mxschmitt/action-tmate
-    #   - dependency-name: step-security/harden-runner
-    package-ecosystem: github-actions
+  - package-ecosystem: github-actions
+    directory: /
     schedule:
       interval: weekly
-  - directory: /
-    package-ecosystem: terraform
+    ignore:
+      - dependency-name: '*'
+        update-types: [version-update:semver-patch, version-update:semver-minor]
+      # Managed by cisagov/skeleton-generic
+      - dependency-name: actions/cache
+      - dependency-name: actions/checkout
+      - dependency-name: actions/setup-go
+      - dependency-name: actions/setup-python
+      - dependency-name: crazy-max/ghaction-dump-context
+      - dependency-name: crazy-max/ghaction-github-labeler
+      - dependency-name: crazy-max/ghaction-github-status
+      - dependency-name: hashicorp/setup-terraform
+      - dependency-name: mxschmitt/action-tmate
+      - dependency-name: step-security/harden-runner
+  - package-ecosystem: terraform
+    directory: /infrastructure
     schedule:
       interval: weekly
-  - package-ecosystem: 'npm'
-    directory: '/'
-    schedule:
-      interval: 'weekly'
-    ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-patch","version-update:semver-minor"]
-  - package-ecosystem: "npm"
-    directory: "/frontend"
+  - package-ecosystem: npm
+    directory: /
     schedule:
-      interval: "weekly"
+      interval: weekly
     ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-patch","version-update:semver-minor"]
-  - package-ecosystem: "npm"
-    directory: "/backend"
+      - dependency-name: '*'
+        update-types: [version-update:semver-patch, version-update:semver-minor]
+  - package-ecosystem: npm
+    directory: /frontend
     schedule:
-      interval: "weekly"
+      interval: weekly
     ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-patch","version-update:semver-minor"]
-  - package-ecosystem: "pip"
-    directory: "/backend/worker"
+      - dependency-name: '*'
+        update-types: [version-update:semver-patch, version-update:semver-minor]
+  - package-ecosystem: npm
+    directory: /backend
     schedule:
-      interval: "weekly"
+      interval: weekly
     ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-patch","version-update:semver-minor"]
-  - package-ecosystem: 'docker'
-    directory: '/'
+      - dependency-name: '*'
+        update-types: [version-update:semver-patch, version-update:semver-minor]
+  - package-ecosystem: pip
+    directory: /backend/worker
     schedule:
-      interval: 'weekly'
+      interval: weekly
     ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-patch","version-update:semver-minor"]
-  - package-ecosystem: 'github-actions'
-    directory: '/'
+      - dependency-name: '*'
+        update-types: [version-update:semver-patch, version-update:semver-minor]
+  - package-ecosystem: docker
+    directory: /
     schedule:
-      interval: 'weekly'
+      interval: weekly
     ignore:
-      - dependency-name: "*"
-        update-types: ["version-update:semver-patch","version-update:semver-minor"]
-
+      - dependency-name: '*'
+        update-types: [version-update:semver-patch, version-update:semver-minor]

.github/workflows/backend.yml

@@ -7,15 +7,15 @@ on:
       - develop
       - production
     paths:
-      - 'backend/**'
-      - '.github/workflows/backend.yml'
+      - backend/**
+      - .github/workflows/backend.yml
   pull_request:
     branches:
       - develop
       - production
     paths:
-      - 'backend/**'
-      - '.github/workflows/backend.yml'
+      - backend/**
+      - .github/workflows/backend.yml
 
 defaults:
   run:
@@ -33,9 +33,8 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+          restore-keys: ${{ runner.os }}-node-
       - name: Install dependencies
         run: npm ci
       - name: Lint
@@ -53,8 +52,7 @@ jobs:
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
+          restore-keys: ${{ runner.os }}-node-
       - name: Install dependencies
         run: npm ci
       - name: Run site locally
@@ -86,8 +84,7 @@ jobs:
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
+          restore-keys: ${{ runner.os }}-node-
       - name: Install dependencies
         run: npm ci
       - name: Build
@@ -114,12 +111,13 @@ jobs:
         uses: actions/[email protected]
         with:
           python-version: '3.10'
+      - name: Copy .env file
+        run: cp ../dev.env.example .env
       - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
           key: pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            pip-
+          restore-keys: pip-
       - run: pip install -r worker/requirements.txt
       - run: pytest
   build_worker:
@@ -134,9 +132,8 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+          restore-keys: ${{ runner.os }}-node-
       - name: Install dependencies
         run: npm ci
       - name: Build worker container
@@ -157,9 +154,8 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+          restore-keys: ${{ runner.os }}-node-
       - name: Install dependencies
         run: npm ci
 
@@ -185,7 +181,18 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Run syncdb
-        run: aws lambda invoke --function-name crossfeed-staging-syncdb --region us-east-1 /dev/stdout
+        run: |
+          aws lambda invoke --function-name crossfeed-staging-syncdb \
+          --region us-east-1 /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Run syncmdl
+        run: |
+          aws lambda invoke --function-name crossfeed-staging-syncmdl \
+          --region us-east-1 /dev/stdout
         working-directory: backend
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -206,9 +213,8 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
+          key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+          restore-keys: ${{ runner.os }}-node-
       - name: Install dependencies
         run: npm ci
 
@@ -234,7 +240,18 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
       - name: Run syncdb
-        run: aws lambda invoke --function-name crossfeed-prod-syncdb --region us-east-1 /dev/stdout
+        run: |
+          aws lambda invoke --function-name crossfeed-prod-syncdb --region us-east-1 \
+          /dev/stdout
+        working-directory: backend
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Run syncmdl
+        run: |
+          aws lambda invoke --function-name crossfeed-prod-syncmdl --region us-east-1 \
+          /dev/stdout
         working-directory: backend
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/build.yml

@@ -47,7 +47,7 @@ jobs:
       - id: setup-python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: '3.11'
       # We need the Go version and Go cache location for the actions/cache step,
       # so the Go installation must happen before that.
       - id: setup-go
@@ -56,17 +56,15 @@ jobs:
           # There is no expectation for actual Go code so we disable caching as
           # it relies on the existence of a go.sum file.
           cache: false
-          go-version: "1.20"
+          go-version: '1.20'
       - name: Lookup Go cache directory
         id: go-cache
-        run: |
-          echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT
+        run: echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT
       - uses: actions/cache@v3
         env:
           BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\
             py${{ steps.setup-python.outputs.python-version }}-\
             go${{ steps.setup-go.outputs.go-version }}-\
-            packer${{ steps.setup-env.outputs.packer-version }}-\
             tf${{ steps.setup-env.outputs.terraform-version }}-"
         with:
           # Note that the .terraform directory IS NOT included in the
@@ -78,52 +76,20 @@ jobs:
           path: |
             ${{ env.PIP_CACHE_DIR }}
             ${{ env.PRE_COMMIT_CACHE_DIR }}
-            ${{ env.CURL_CACHE_DIR }}
             ${{ steps.go-cache.outputs.dir }}
           key: "${{ env.BASE_CACHE_KEY }}\
             ${{ hashFiles('**/requirements-test.txt') }}-\
             ${{ hashFiles('**/requirements.txt') }}-\
             ${{ hashFiles('**/.pre-commit-config.yaml') }}"
-          restore-keys: |
-            ${{ env.BASE_CACHE_KEY }}
-      - name: Setup curl cache
-        run: mkdir -p ${{ env.CURL_CACHE_DIR }}
-      - name: Install Packer
-        env:
-          PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }}
-        run: |
-          PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip"
-          curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \
-            --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \
-            --location \
-            "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}"
-          sudo unzip -d /opt/packer \
-            ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}"
-          sudo mv /usr/local/bin/packer /usr/local/bin/packer-default
-          sudo ln -s /opt/packer/packer /usr/local/bin/packer
+          restore-keys: ${{ env.BASE_CACHE_KEY }}
       - uses: hashicorp/setup-terraform@v2
         with:
           terraform_version: ${{ steps.setup-env.outputs.terraform-version }}
-      - name: Install go-critic
-        env:
-          PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic
-          PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }}
-        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
-      - name: Install gosec
-        env:
-          PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec
-          PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }}
-        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
       - name: Install shfmt
         env:
           PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt
           PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }}
         run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
-      - name: Install staticcheck
-        env:
-          PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck
-          PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }}
-        run: go install ${PACKAGE_URL}@${PACKAGE_VERSION}
       - name: Install Terraform-docs
         env:
           PACKAGE_URL: github.com/terraform-docs/terraform-docs
@@ -135,6 +101,8 @@ jobs:
           pip install --upgrade --requirement requirements-test.txt
       - name: Set up pre-commit hook environments
         run: pre-commit install-hooks
+      - name: Create .env file needed by docker-compose-check pre-commit hook
+        run: cp dev.env.example .env
       - name: Run pre-commit on all files
         run: pre-commit run --all-files
       - name: Setup tmate debug session