breaking: Added GitHub selfhosted runners configuration (#47)

* github runner container apps env * init identity * fix terraform storage account for state * added github iac pipelines * added github iac pipelines for diego_common domain * pre-commit fixs * updated static analysis * added main ci for static analysis * added main ci for static analysis * minor fixs * fix container app env * added custom image for container app runner * minor fix * added pipeline diego-app * upgrated azurerm to 3.43 to support container apps * assigned service principal to group, to avoid directory reader association * diego-app: added association to github service principal * service principal: created password for kubelogin * created pipelines for aks plan * created pipelines for aks plan * minor fix * upgrated terraform to 1.3.8 * upgrated terraform to 1.3.8 * added Azure Kubernetes Service Cluster User Role to SP * upgrated providers to last version * added kv access policy for domain * removed system namespace * terraform lock updated * added Azure Kubernetes Service RBAC Admin to SP * increased premissions to Azure Kubernetes Service RBAC Cluster Admin * Created custom role * removed permissions not needed * removed client secret * removed password in client secret * removed old folders * github action fix custome role for aks * removed old files * minor fix * test aks resource group auto search * pre-commit fixs
pagopa · Mar 2, 2023 · b18bbd7 · b18bbd7
1 parent d4a162d
commit b18bbd7
Show file tree

Hide file tree

Showing 45 changed files with 1,589 additions and 390 deletions.
diff --git a/.github/workflows/dev_cd_diego_common.yml b/.github/workflows/dev_cd_diego_common.yml
@@ -0,0 +1,102 @@
+name: Continuous Delivery on dev core
+
+on:
+  workflow_dispatch:
+  # Trigger the workflow on push on the main branch
+  push:
+    branches:
+      - main
+    paths:
+      - 'src/domains/diego-common/**'
+      - '.github/workflows/dev**'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  DIR: "src/domains/diego-common"
+  AZURE_ENVIRONMENT: dev
+
+jobs:
+  create_runner:
+    name: Create Runner
+    runs-on: ubuntu-22.04
+    environment: dev-runner
+    outputs:
+      runner_name: ${{ steps.create_github_runner.outputs.runner_name }}
+    steps:
+      - name: Create GitHub Runner
+        id: create_github_runner
+        # from https://github.com/pagopa/github-self-hosted-runner-azure-create-action/commits/main
+        uses: pagopa/github-self-hosted-runner-azure-create-action@b4590e069e753daee6bc9809d484523cc7026035
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          container_app_environment_name: ${{ secrets.AZURE_CONTAINER_APP_ENVIRONMENT_NAME }}
+          resource_group_name: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
+          pat_token: ${{ secrets.BOT_TOKEN }}
+
+  terraform_preapply_job:
+    name: Terraform Pre Apply
+    runs-on: [self-hosted, "${{ needs.create_runner.outputs.runner_name }}"]
+    environment: dev-ci
+    needs: create_runner
+    steps:
+      - name: Checkout
+        id: checkout
+        # from https://github.com/actions/checkout/commits/main
+        uses: actions/checkout@1f9a0c22da41e6ebfa534300ef656657ea2c6707
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup terraform
+        id: setup-version
+        # https://github.com/pagopa/terraform-install-action/commits/main
+        uses: pagopa/terraform-install-action@1f76f593176e58c423b88d72273a612ba7ba430b
+
+      - name: Terraform pre apply
+        # from https://github.com/pagopa/terraform-preapply-azure-action/commits/main
+        uses: pagopa/terraform-preapply-azure-action@2b2413f53494fc0296dde8292aeec51c36c31a5c
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          dir: ${{ env.DIR }}
+          azure_environment: ${{ env.AZURE_ENVIRONMENT }}
+
+  terraform_apply_job:
+    name: Terraform Apply
+    runs-on: [self-hosted, "${{ needs.create_runner.outputs.runner_name }}"]
+    environment: dev-cd
+    needs: [create_runner, terraform_preapply_job]
+    steps:
+      - name: Terraform apply
+        # from https://github.com/pagopa/terraform-apply-azure-action/commits/main
+        uses: pagopa/terraform-apply-azure-action@5d84f089c3a0dd451ee2898280c2cd80362d113c
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          dir: ${{ env.DIR }}
+
+  cleanup_runner:
+    name: Cleanup Runner
+    if: always()
+    runs-on: ubuntu-22.04
+    environment: dev-runner
+    needs: [create_runner, terraform_preapply_job, terraform_apply_job]
+    steps:
+      - name: Cleanup GitHub Runner
+        id: cleanup_github_runner
+        # from https://github.com/pagopa/github-self-hosted-runner-azure-cleanup-action/commits/main
+        uses: pagopa/github-self-hosted-runner-azure-cleanup-action@97731a35e6ffc79b66c4dfd2aae5e4fd04e3ebb5
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resource_group_name: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
+          runner_name: ${{ needs.create_runner.outputs.runner_name }}
+          pat_token: ${{ secrets.BOT_TOKEN }}
diff --git a/.github/workflows/dev_ci_diego_app.yml b/.github/workflows/dev_ci_diego_app.yml
@@ -0,0 +1,92 @@
+name: Continuous Integration on DEV@src/domains/diego-app
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+    paths:
+      - 'src/domains/diego-app/**'
+      - '.github/workflows/dev**'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  DIR: "src/domains/diego-app"
+  AZURE_ENVIRONMENT: dev
+
+jobs:
+  create_runner:
+    name: Create Runner
+    runs-on: ubuntu-22.04
+    environment: dev-runner
+    outputs:
+      runner_name: ${{ steps.create_github_runner.outputs.runner_name }}
+    steps:
+      - name: Create GitHub Runner
+        id: create_github_runner
+        # from https://github.com/pagopa/github-self-hosted-runner-azure-create-action/commits/main
+        uses: pagopa/github-self-hosted-runner-azure-create-action@b4590e069e753daee6bc9809d484523cc7026035
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          container_app_environment_name: ${{ secrets.AZURE_CONTAINER_APP_ENVIRONMENT_NAME }}
+          resource_group_name: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
+          pat_token: ${{ secrets.BOT_TOKEN }}
+          self_hosted_runner_image_tag: "beta-kubelogin-setup"
+
+  terraform_plan_job:
+    name: Terraform Plan
+    runs-on: [self-hosted, "${{ needs.create_runner.outputs.runner_name }}"]
+    environment: dev-ci
+    needs: create_runner
+    steps:
+      - name: Checkout
+        id: checkout
+        # from https://github.com/actions/checkout/commits/main
+        uses: actions/checkout@1f9a0c22da41e6ebfa534300ef656657ea2c6707
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup terraform
+        id: setup-version
+        # https://github.com/pagopa/terraform-install-action/commits/main
+        uses: pagopa/terraform-install-action@1f76f593176e58c423b88d72273a612ba7ba430b
+
+      - name: Terraform plan
+        # from https://github.com/pagopa/terraform-aks-azure-action/blob/main/action.yml
+        uses: pagopa/terraform-aks-azure-action@beta-version
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          dir: ${{ env.DIR }}
+          azure_environment: ${{ env.AZURE_ENVIRONMENT }}
+          terraform_action: "plan"
+          aks_name: "dvopla-d-neu-dev01-aks"
+
+  cleanup_runner:
+    name: Cleanup Runner
+    if: always()
+    runs-on: ubuntu-22.04
+    environment: dev-runner
+    needs: [create_runner, terraform_plan_job]
+    steps:
+      - name: Cleanup GitHub Runner
+        id: cleanup_github_runner
+        # from https://github.com/pagopa/github-self-hosted-runner-azure-cleanup-action/commits/main
+        uses: pagopa/github-self-hosted-runner-azure-cleanup-action@97731a35e6ffc79b66c4dfd2aae5e4fd04e3ebb5
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resource_group_name: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
+          runner_name: ${{ needs.create_runner.outputs.runner_name }}
+          pat_token: ${{ secrets.BOT_TOKEN }}
diff --git a/.github/workflows/dev_ci_diego_common.yml b/.github/workflows/dev_ci_diego_common.yml
@@ -0,0 +1,90 @@
+name: Continuous Integration on dev core
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+    paths:
+      - 'src/domains/diego-common/**'
+      - '.github/workflows/dev**'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  DIR: "src/domains/diego-common"
+  AZURE_ENVIRONMENT: dev
+
+jobs:
+  create_runner:
+    name: Create Runner
+    runs-on: ubuntu-22.04
+    environment: dev-runner
+    outputs:
+      runner_name: ${{ steps.create_github_runner.outputs.runner_name }}
+    steps:
+      - name: Create GitHub Runner
+        id: create_github_runner
+        # from https://github.com/pagopa/github-self-hosted-runner-azure-create-action/commits/main
+        uses: pagopa/github-self-hosted-runner-azure-create-action@b4590e069e753daee6bc9809d484523cc7026035
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          container_app_environment_name: ${{ secrets.AZURE_CONTAINER_APP_ENVIRONMENT_NAME }}
+          resource_group_name: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
+          pat_token: ${{ secrets.BOT_TOKEN }}
+          self_hosted_runner_image_tag: "beta-kubelogin-setup"
+
+  terraform_plan_job:
+    name: Terraform Plan
+    runs-on: [self-hosted, "${{ needs.create_runner.outputs.runner_name }}"]
+    environment: dev-ci
+    needs: create_runner
+    steps:
+      - name: Checkout
+        id: checkout
+        # from https://github.com/actions/checkout/commits/main
+        uses: actions/checkout@1f9a0c22da41e6ebfa534300ef656657ea2c6707
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup terraform
+        id: setup-version
+        # https://github.com/pagopa/terraform-install-action/commits/main
+        uses: pagopa/terraform-install-action@1f76f593176e58c423b88d72273a612ba7ba430b
+
+      - name: Terraform plan
+        # from https://github.com/pagopa/terraform-plan-azure-action/commits/main
+        uses: pagopa/terraform-plan-azure-action@392aca28cbb33f5dc28215dfb72385e136fd813b
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          dir: ${{ env.DIR }}
+          azure_environment: ${{ env.AZURE_ENVIRONMENT }}
+
+  cleanup_runner:
+    name: Cleanup Runner
+    if: always()
+    runs-on: ubuntu-22.04
+    environment: dev-runner
+    needs: [create_runner, terraform_plan_job]
+    steps:
+      - name: Cleanup GitHub Runner
+        id: cleanup_github_runner
+        # from https://github.com/pagopa/github-self-hosted-runner-azure-cleanup-action/commits/main
+        uses: pagopa/github-self-hosted-runner-azure-cleanup-action@97731a35e6ffc79b66c4dfd2aae5e4fd04e3ebb5
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resource_group_name: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
+          runner_name: ${{ needs.create_runner.outputs.runner_name }}
+          pat_token: ${{ secrets.BOT_TOKEN }}
diff --git a/.github/workflows/dev_drift_diego_common.yml b/.github/workflows/dev_drift_diego_common.yml
@@ -0,0 +1,82 @@
+name: Drift Detection on dev core
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '08 00 * * *'
+
+permissions:
+  id-token: write
+  contents: read
+
+env:
+  DIR: "src/domains/diego-common"
+  AZURE_ENVIRONMENT: dev
+
+jobs:
+  create_runner:
+    name: Create Runner
+    runs-on: ubuntu-22.04
+    environment: dev-runner
+    outputs:
+      runner_name: ${{ steps.create_github_runner.outputs.runner_name }}
+    steps:
+      - name: Create GitHub Runner
+        id: create_github_runner
+        # from https://github.com/pagopa/github-self-hosted-runner-azure-create-action/commits/main
+        uses: pagopa/github-self-hosted-runner-azure-create-action@b4590e069e753daee6bc9809d484523cc7026035
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          container_app_environment_name: ${{ secrets.AZURE_CONTAINER_APP_ENVIRONMENT_NAME }}
+          resource_group_name: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
+          pat_token: ${{ secrets.BOT_TOKEN }}
+
+  terraform_driftdetection_job:
+    name: Terraform Drift Detection
+    runs-on: [self-hosted, "${{ needs.create_runner.outputs.runner_name }}"]
+    environment: dev-ci
+    needs: create_runner
+    steps:
+      - name: Checkout
+        id: checkout
+        # from https://github.com/actions/checkout/commits/main
+        uses: actions/checkout@1f9a0c22da41e6ebfa534300ef656657ea2c6707
+        with:
+          persist-credentials: false
+          fetch-depth: 0
+
+      - name: Setup terraform
+        id: setup-version
+        # https://github.com/pagopa/terraform-install-action/commits/main
+        uses: pagopa/terraform-install-action@1f76f593176e58c423b88d72273a612ba7ba430b
+
+      - name: Terraform drift detection
+        # from https://github.com/pagopa/terraform-driftdetection-azure-action/commits/main
+        uses: pagopa/terraform-driftdetection-azure-action@71bd771b3a071c78b36e5e0ecbd666ac39b1113c
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          dir: ${{ env.DIR }}
+          azure_environment: ${{ env.AZURE_ENVIRONMENT }}
+
+  cleanup_runner:
+    name: Cleanup Runner
+    if: always()
+    runs-on: ubuntu-22.04
+    environment: dev-runner
+    needs: [create_runner, terraform_driftdetection_job]
+    steps:
+      - name: Cleanup GitHub Runner
+        id: cleanup_github_runner
+        # from https://github.com/pagopa/github-self-hosted-runner-azure-cleanup-action/commits/main
+        uses: pagopa/github-self-hosted-runner-azure-cleanup-action@97731a35e6ffc79b66c4dfd2aae5e4fd04e3ebb5
+        with:
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resource_group_name: ${{ secrets.AZURE_RESOURCE_GROUP_NAME }}
+          runner_name: ${{ needs.create_runner.outputs.runner_name }}
+          pat_token: ${{ secrets.BOT_TOKEN }}
diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml
@@ -21,17 +21,18 @@ jobs:
             src/domains/diego-app
             src/domains/diego-common
             src/matteo
+            .identity
         run: |
           pids=()
 
           TAG=$(cat .terraform-version)
           for f in $FOLDERS; do
             pushd "$(pwd)/${f}"
-              sed -i -e 's/  backend "azurerm" {}//g' 99_main.tf # use local backend
+              rm -rf 99_main.tf
+              mv 99_main.tf.ci 99_main.tf
               docker run -v $(pwd):/tmp -w /tmp hashicorp/terraform:$TAG init &
               pids+=($!)
             popd
-
           done
 
           # Wait for each specific process to terminate.