InfraCI - ByoVnet cluster #2687
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: InfraCI - ByoVnet cluster | |
| # Credential prerequisites | |
| # 1. IAM Owner on the Resource Group you're deploying into (we're making role assignments) | |
| # 2. IAM Owner on the Vnet (we're making role assignments) | |
| on: | |
| # Triggering on push causes a problem with the Azure Login Federated Identity Subject Claim. | |
| # Disabling this trigger as having Push + PR is OTT | |
| # push: | |
| # paths: | |
| # #- ".github/workflows/ByoVnetCI.yml" | |
| # - ".github/workflows_dep/AksDeploy-ByoVnet.parameters.json" | |
| # - "bicep/*" | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - "bicep/*" | |
| - ".github/workflows/ByoVnetCI.yml" | |
| types: #Type filtering should stop CI/CD running on draft status workflows | |
| - opened | |
| - reopened | |
| - synchronize | |
| - ready_for_review | |
| #Running on a schedule helps trap issues like Kubernetes versions not being available in the region, or Well Architected recommendations changing | |
| schedule: | |
| # At 11:00pm, every Wednesday week | |
| - cron: "0 23 * * 3" | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'Which environment to deploy to' | |
| required: true | |
| default: "csu" | |
| type: environment | |
| ResourceGroup: | |
| description: 'Which Resource Group to deploy to' | |
| default: 'AksBicepAcc-Ci-ByoDeploy' | |
| type: string | |
| required: false | |
| region: | |
| description: 'Region (needs to be same as byo vnet location)' | |
| default: 'southcentralus' | |
| type: string | |
| required: false | |
| doWellArchitected: | |
| description: 'Perform the Well Architected Framework assesment' | |
| default: false | |
| type: boolean | |
| required: false | |
| doCostEstimate: | |
| description: 'Perform a template Cost Estimate' | |
| default: false | |
| type: boolean | |
| required: false | |
| doDebugSteps: | |
| description: 'Run informational steps' | |
| default: true | |
| type: boolean | |
| required: false | |
| doVerifySteps: | |
| description: 'Run optional verify steps' | |
| default: true | |
| type: boolean | |
| required: false | |
| doSmokeTestUninstall: | |
| description: 'Uninstall smoke test apps after tests' | |
| default: true | |
| type: boolean | |
| required: false | |
| defaultdeny: | |
| description: 'Default Deny NetworkPolicy?' | |
| default: false | |
| type: boolean | |
| required: false | |
| permissions: | |
| id-token: write | |
| contents: read | |
| concurrency: ci-${{ github.ref }} | |
| env: | |
| AZCLIVERSION: 2.53.0 #2.43.0 #2.34.1 #2.29.2 #2.26.0 #latest | |
| ParamFilePath: ".github/workflows_dep/AksDeploy-ByoVnet.parameters.json" | |
| RESNAME: "Byov" | |
| DEPNAME: "Dep${{ github.run_number }}" | |
| KVACCESS: "IAM" | |
| jobs: | |
| Well_Architected: | |
| runs-on: ubuntu-latest | |
| needs: [Validation] | |
| if: github.event_name != 'pull_request' || github.event.inputs.doWellArchitected == 'true' || contains( github.event.pull_request.labels.*.name, 'test-deploy-byoconfig') | |
| steps: | |
| - uses: actions/[email protected] | |
| # PSRule does this cool thing where it traverse the parameter file through to the arm template | |
| # PSRule performs IaC recommendations of the template. | |
| # https://azure.github.io/PSRule.Rules.Azure/ | |
| - name: PSRule for Azure - Well Architected | |
| uses: microsoft/[email protected] | |
| #uses: microsoft/ps-rule@main | |
| continue-on-error: true #Setting this whilst PSRule gets bedded in, in this project | |
| with: | |
| modules: 'PSRule.Rules.Azure' | |
| inputPath: "${{ env.ParamFilePath }}" | |
| prerelease: true | |
| # In addition to the Well Architected analysis, we can also leverage the CAF rules | |
| # Run analysis against PSRule for Cloud Adoption Framework | |
| - name: PSRule for CAF | |
| uses: microsoft/[email protected] | |
| continue-on-error: true #Setting this whilst PSRule gets bedded in, in this project | |
| with: | |
| modules: 'PSRule.Rules.CAF' | |
| inputPath: "${{ env.ParamFilePath }}" | |
| prerelease: false | |
| CostEstimate: | |
| needs: [ReusableWF, Validation] | |
| if: github.event.inputs.doCostEstimate == 'true' | |
| uses: TheCloudTheory/arm-estimator/.github/workflows/estimateFromUrl.yml@main | |
| #uses: Gordonby/arm-estimator/.github/workflows/estimateFromUrl.yml@gb-workflowpolish | |
| with: | |
| rg: ${{ needs.ReusableWF.outputs.RG }} | |
| environment: ${{needs.ReusableWF.outputs.Environment}} | |
| aceVersion: 1.1-beta2 | |
| templateFileURL: https://github.com/Azure/AKS-Construction/releases/download/${{ needs.ReusableWF.outputs.LatestAkscVersionTag }}/main.json | |
| templateParamFileURL: https://raw.githubusercontent.com/Azure/AKS-Construction/${{ github.event_name == 'pull_request' && github.head_ref || github.ref_name }}/.github/workflows_dep/AksDeploy-ByoVnet.parameters.json | |
| templateParams: ${{ needs.Validation.outputs.PARAMOVERRIDESNOSECRETS}} #${{ needs.Validation.outputs.PARAMOVERRIDES}} | |
| secrets: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| #Pre deployment validation of the parameters + bicep code | |
| Validation: | |
| runs-on: ubuntu-latest | |
| environment: ${{ github.event.inputs.environment }} | |
| if: ${{ !github.event.pull_request.head.repo.fork && !github.event.pull_request.draft }} | |
| outputs: | |
| RESOURCEGROUP: ${{ steps.params.outputs.RESOURCEGROUP}} | |
| REGION: ${{ steps.params.outputs.REGION}} | |
| RESNAME: ${{ steps.params.outputs.NEWRESNAME}} | |
| PARAMOVERRIDES: ${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}} #This won't work if the string includes any SECRETS!!! | |
| PARAMOVERRIDESNOSECRETS: ${{ steps.imperitiveparams.outputs.PARAMOVERRIDESNOSECRETS}} | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Job parameter check | |
| run: | | |
| echo "Param file path is: ${{ env.ParamFilePath }}" | |
| echo "Resource name is ${{ env.RESNAME }}" | |
| echo "Deployment name is ${{ env.DEPNAME }}" | |
| echo "Ref is ${{ github.ref }}" | |
| echo "Ref name is ${{GITHUB.REF_NAME}}" | |
| echo "EventTrigger name is ${{github.event_name}}" | |
| echo "PR contains bug : ${{contains(github.event.pull_request.labels.*.name, 'bug')}}" | |
| echo "PR labels : ${{github.event.pull_request.labels.*.name}}" | |
| echo "PR draft : ${{github.event.pull_request.draft}}" | |
| echo "AZCLIVERSION is ${{ env.AZCLIVERSION }}" | |
| echo "doDebugSteps is ${{ github.event.inputs.doDebugSteps }}" | |
| echo "doVerifySteps is ${{ github.event.inputs.doVerifySteps }}" | |
| echo "doSmokeTestUninstall is ${{ github.event.inputs.doSmokeTestUninstall }}" | |
| - name: Arm Parameter file check | |
| if: github.event.inputs.doVerifySteps == 'true' | |
| shell: pwsh | |
| run: | | |
| Write-Output "Checking parameter file existence/contents" | |
| $paramFilePath="${{ env.ParamFilePath }}" | |
| Test-Path $paramFilePath | |
| if (Test-Path $paramFilePath) { | |
| $paramFileContent=Get-Content $paramFilePath | |
| Write-Output $paramFileContent | |
| Write-Output "Test Pulling a param (ingressApplicationGateway)" | |
| $params=$paramFileContent|ConvertFrom-Json | |
| Write-Output $params.parameters.ingressApplicationGateway.value | |
| } | |
| - name: Parameter Value Augmentation | |
| id: params | |
| env: | |
| DEFAULTRGNAME: AksBicepAcc-Ci-ByoDeploy | |
| run: | | |
| if [ -z "${{ github.event.inputs.region }}" ] | |
| then | |
| echo "Region parameter not available through GitHub event data, setting default" | |
| REGION="southcentralus" | |
| else | |
| echo "Region parameter found in GitHub event (${{ github.event.inputs.region }})" | |
| REGION="${{ github.event.inputs.region }}" | |
| fi | |
| echo $REGION | |
| echo "REGION=$REGION" >> $GITHUB_OUTPUT | |
| if [ -z "${{ github.event.inputs.ResourceGroup }}" ] | |
| then | |
| echo "ResourceGroup parameter not available through GitHub event data, setting to default" | |
| echo $DEFAULTRGNAME | |
| echo "RESOURCEGROUP=$DEFAULTRGNAME" >> $GITHUB_OUTPUT | |
| else | |
| echo "Resource Group parameter found in GitHub event (${{ github.event.inputs.ResourceGroup }})" | |
| echo "RESOURCEGROUP=${{ github.event.inputs.ResourceGroup }}" >> $GITHUB_OUTPUT | |
| fi | |
| NEWRESNAME="$RESNAME${REGION:0:3}" | |
| echo "Setting new Resource Name $NEWRESNAME" | |
| echo "NEWRESNAME=$NEWRESNAME" >> $GITHUB_OUTPUT | |
| - name: Azure Login | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| enable-AzPSSession: true | |
| environment: azurecloud | |
| allow-no-subscriptions: false | |
| - name: Install Pwsh modules | |
| shell: pwsh | |
| run: | | |
| Set-PSRepository -Name PSGallery -InstallationPolicy Trusted | |
| Install-Module -Name Az.KeyVault -Force | |
| - name: Verify any active Azure Resource Group Deployments #These can mess up our deployment | |
| id: activedeps | |
| if: github.event.inputs.doVerifySteps == 'true' | |
| env: | |
| RG: ${{ steps.params.outputs.RESOURCEGROUP }} | |
| uses: Azure/cli@v2 | |
| with: | |
| azcliversion: ${{ env.AZCLIVERSION }} | |
| inlineScript: | | |
| echo "AZ CLI version" | |
| az version | |
| RUNCOUNT=$(az deployment group list -g $RG --query "[?properties.provisioningState=='Running'].[properties.provisioningState, name] | length(@)" -o tsv) | |
| echo "Active deployments : $RUNCOUNT" | |
| echo 'Active deployment list' | |
| az deployment group list -g $RG --query "[?properties.provisioningState=='Running'].[properties.provisioningState, name]" | |
| #echo 'Verbose deployment list' | |
| #az deployment group list -g $RG --query "[].[properties.provisioningState, name]" | |
| echo "RUNCOUNT=$RUNCOUNT" >> $GITHUB_OUTPUT | |
| - name: Verify AKS Preview Features are available in target Subscription | |
| if: github.event.inputs.doVerifySteps == 'true' | |
| shell: pwsh | |
| run: | | |
| write-output 'Full list of features of AKS' | |
| az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService')].{Name:name,State:properties.state}" | |
| write-output 'Features that are still registering' | |
| az feature list -o table --query "[?contains(name, 'Microsoft.ContainerService') && properties.state=='Registering'].{Name:name,State:properties.state}" | |
| write-output 'Checking to ensure no features are still registering' | |
| #Not going to use JMES as i'm getting a weird error that i don't see locally in powershell | |
| $aksfeatures = az feature list --query "[?contains(name, 'Microsoft.ContainerService')]" | ConvertFrom-Json | |
| #$aksfeatures = $allfeatures | Where-Object {$_.Name -like 'Microsoft.ContainerService*'} | |
| $registeringfeatures = $aksfeatures | Where-Object {$_.properties.state -eq 'Registering'} | |
| if ($registeringfeatures.count -gt 0) { | |
| Write-Error "There are still features registering" | |
| } else { Write-Output "-- All good, no features in the process of registering" } | |
| write-output 'Check specific features.' | |
| $paramFilePath="${{ env.ParamFilePath }}" | |
| $paramFileContent=Get-Content $paramFilePath | |
| $params=$paramFileContent|ConvertFrom-Json | |
| if($params.parameters.keyVaultAksCSI.value -eq $true) { | |
| $feature='AKS-AzureKeyVaultSecretsProvider' | |
| write-output "-- $feature" | |
| $featureCsi = $aksfeatures | Where-Object {$_.name -like "*$feature"} | |
| $featureCsi.properties.state | |
| if ($featureCsi.properties.state -ne 'Registered') { | |
| Write-Output $featureCsi | |
| Write-Error "$feature NOT registered" | |
| } else { Write-Output "-- Looks like $feature is registered properly" } | |
| } | |
| - name: Create Parameter file imperative override string | |
| uses: Azure/cli@v2 | |
| id: imperitiveparams | |
| env: | |
| RG: ${{ steps.params.outputs.RESOURCEGROUP }} | |
| with: | |
| azcliversion: ${{ env.AZCLIVERSION }} | |
| inlineScript: | | |
| PARAMOVERRIDES="resourceName=$RESNAME location=${{ steps.params.outputs.REGION}} byoAKSSubnetId=${{ secrets.ByoAksSubnetId }} byoAGWSubnetId=${{ secrets.ByoAgwSubnetId }} dnsZoneId=${{ secrets.BYODNSZONEID }} enableTelemetry=${{ VARS.ENABLETELEMETRY == 'true' }}" | |
| echo $PARAMOVERRIDES | |
| echo "PARAMOVERRIDES=$PARAMOVERRIDES" >> $GITHUB_OUTPUT | |
| PARAMOVERRIDESNOSECRETS="resourceName=$RESNAME location=${{ steps.params.outputs.REGION}} byoAKSSubnetId='' byoAGWSubnetId='' dnsZoneId=''" | |
| echo $PARAMOVERRIDESNOSECRETS | |
| echo "PARAMOVERRIDESNOSECRETS=$PARAMOVERRIDESNOSECRETS" >> $GITHUB_OUTPUT | |
| - name: Validate Infrastructure deployment | |
| uses: Azure/cli@v2 | |
| env: | |
| RG: ${{ steps.params.outputs.RESOURCEGROUP }} | |
| with: | |
| azcliversion: ${{ env.AZCLIVERSION }} | |
| inlineScript: | | |
| RESNAME='${{ env.RESNAME }}' | |
| DEPNAME='Dep${{ github.run_number }}' | |
| PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}}' | |
| echo $PARAMS | |
| az deployment group validate -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS --verbose | |
| - name: What If (GA edge Az cli version) (has issues) | |
| uses: Azure/cli@v2 | |
| id: edgeWhatIf | |
| env: | |
| RG: ${{ steps.params.outputs.RESOURCEGROUP }} | |
| continue-on-error: true #Setting to true due to bug in the AzureCLI https://github.com/Azure/azure-cli/issues/19850 | |
| with: | |
| azcliversion: ${{ env.AZCLIVERSION }} #latest | |
| inlineScript: | | |
| RESNAME='${{ env.RESNAME }}' | |
| DEPNAME='${{ env.DEPNAME }}' | |
| WHATIFPATH='whatif.json' | |
| PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}}' | |
| #az deployment group what-if --debug -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p resourceName=$RESNAME byoAKSSubnetId=${{ secrets.ByoAksSubnetId }} byoAGWSubnetId=${{ secrets.ByoAgwSubnetId }} | |
| az deployment group what-if --no-pretty-print -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS > $WHATIFPATH | |
| if [[ -f $WHATIFPATH ]] | |
| then | |
| echo "The WhatIf json file was created" | |
| fi | |
| cat $WHATIFPATH | |
| echo "edgeSuccess=true" >> $GITHUB_OUTPUT | |
| - name: What If (No Secrets param string, used for Cost Estimate) | |
| uses: Azure/cli@v2 | |
| env: | |
| RG: ${{ steps.params.outputs.RESOURCEGROUP }} | |
| continue-on-error: true | |
| with: | |
| azcliversion: ${{ env.AZCLIVERSION }} #latest | |
| inlineScript: | | |
| DEPNAME='${{ env.DEPNAME }}' | |
| WHATIFPATH='whatif.json' | |
| PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDESNOSECRETS}}' | |
| az deployment group what-if --no-pretty-print -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS > $WHATIFPATH | |
| if [[ -f $WHATIFPATH ]] | |
| then | |
| echo "The WhatIf json file was created" | |
| fi | |
| cat $WHATIFPATH | |
| - name: What If (workaround task) | |
| uses: Azure/cli@v2 | |
| continue-on-error: true #Setting to true due to bug in the AzureCLI https://github.com/Azure/azure-cli/issues/19850 | |
| if: steps.edgeWhatIf.outputs.edgeSuccess != 'true' | |
| env: | |
| RG: ${{ steps.params.outputs.RESOURCEGROUP }} | |
| with: | |
| azcliversion: 2.26.0 | |
| inlineScript: | | |
| RESNAME='${{ env.RESNAME }}' | |
| DEPNAME='${{ env.DEPNAME }}' | |
| WHATIFPATH='whatif.json' | |
| PARAMS='${{ steps.imperitiveparams.outputs.PARAMOVERRIDES}}' | |
| az deployment group what-if --no-pretty-print -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS > $WHATIFPATH | |
| if [[ -f $WHATIFPATH ]] | |
| then | |
| echo "The WhatIf json file was created" | |
| fi | |
| cat $WHATIFPATH | |
| - name: What If Analysis Output - Parse output | |
| if: github.event.inputs.doVerifySteps == 'true' | |
| shell: pwsh | |
| run: | | |
| $whatifpath='whatif.json' | |
| Write-Output "Checking for JSON What-If" | |
| $whatifexists=Test-Path -path $whatifpath | |
| Write-Output $whatifexists | |
| if ($whatifexists) { | |
| $jsonFileRaw=Get-Content $whatifpath | |
| Write-Output $jsonFileRaw | |
| $whatIf=$jsonFileRaw | ConvertFrom-Json | |
| if ($null -eq $whatIf) { | |
| Write-Output "What If results are null" | |
| #We might want to throw an error here, after the whatif bug gets fixed https://github.com/Azure/azure-cli/issues/19850 | |
| } else { | |
| Write-Output $whatif.changes[0].after.type | |
| } | |
| } | |
| - name: What If Analysis Output - Pester Testing | |
| shell: pwsh | |
| run: | | |
| Write-Output "https://pester.dev/" | |
| Write-Output "TODO" | |
| - name: What If Analysis Gaps - Check for Soft deleted KeyVaults | |
| shell: pwsh | |
| run: | | |
| #This is a current gap in the What-If analysis. | |
| $whatifpath='whatif.json' | |
| $whatifexists=Test-Path -path $whatifpath | |
| if ($whatifexists) { | |
| $whatIf = Get-Content $whatifpath | ConvertFrom-Json | |
| if ($null -eq $whatIf) { | |
| Write-Output "What If results are null" | |
| #We might want to throw an error here, after the whatif bug gets fixed https://github.com/Azure/azure-cli/issues/19850 | |
| } else { | |
| $paramFilePath="${{ env.ParamFilePath }}" | |
| $paramFileContent=Get-Content $paramFilePath | |
| $params=$paramFileContent|ConvertFrom-Json | |
| if($params.parameters.keyVaultCreate.value -eq $true) { | |
| Write-Output "Checking for already existing soft deleted KV" | |
| #lets do this properly and interrogate the whatif file for the KVNAME | |
| $kvafter = $whatif.changes | where-object {$_.after.type -eq 'Microsoft.KeyVault/vaults'} | where {$_.changeType -ne 'Ignore'} | select after | |
| $vaultname = $kvafter.after.name | |
| write-output "Looking for KeyVault $vaultname" | |
| Get-AzKeyVault -InRemovedState | ft | |
| $deletedKv = Get-AzKeyVault -InRemovedState | where-object {$_.vaultname -eq $vaultname} | |
| if($deletedKv.length -eq 0) { | |
| Write-Output "All is good, soft deleted KV not found in soft deleted list." | |
| } | |
| else { | |
| Write-Output "Soft deleted KV output" | |
| Write-Output $deletedKv.length | |
| Write-Output $deletedKv | |
| Write-Error "Existing soft deleted Key Vault ($vaultName) Found - Cannot proceed, as deployment will fail" | |
| } | |
| } | |
| } | |
| } | |
| Deploy: | |
| runs-on: ubuntu-latest | |
| needs: [Validation] | |
| environment: ${{ github.event.inputs.environment }} | |
| env: | |
| RG: ${{ needs.Validation.outputs.RESOURCEGROUP }} | |
| REGION: ${{ needs.Validation.outputs.REGION }} | |
| RESNAME: ${{ needs.Validation.outputs.RESNAME }} | |
| outputs: | |
| AKSNAME: ${{ steps.deployAks.outputs.AKSNAME}} | |
| LANAME : ${{ steps.deployAks.outputs.LANAME}} | |
| LAWGUID : ${{ steps.deployAks.outputs.LAWGUID}} | |
| AKVNAME : ${{ steps.deployAks.outputs.AKVNAME}} | |
| ACRNAME : ${{ steps.deployAks.outputs.ACRNAME}} | |
| AGNAME : ${{ steps.deployAks.outputs.AGNAME}} | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || contains( github.event.pull_request.labels.*.name, 'test-deploy-byoconfig') | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Job parameter check | |
| if: github.event.inputs.doDebugSteps == 'true' | |
| run: | | |
| echo "RG is: ${{ env.RG }}" | |
| echo "Param file path is: ${{ env.ParamFilePath }}" | |
| echo "Resource name is ${{ env.RESNAME }} | |
| echo "Deployment name is ${{ env.DEPNAME }} | |
| - name: Parameter file dependency check | |
| if: github.event.inputs.doVerifySteps == 'true' | |
| shell: pwsh | |
| run: | | |
| Write-Output "Checking parameter file existence/contents" | |
| $paramFilePath="${{ env.ParamFilePath }}" | |
| Test-Path $paramFilePath | |
| if (Test-Path $paramFilePath) { | |
| $paramFileContent=Get-Content $paramFilePath | |
| Write-Output $paramFileContent | |
| } | |
| - name: Azure Login | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| enable-AzPSSession: true | |
| environment: azurecloud | |
| allow-no-subscriptions: false | |
| - name: Deploy Infrastructure | |
| id: deployAks | |
| uses: Azure/cli@v2 | |
| with: | |
| azcliversion: ${{ env.AZCLIVERSION }} | |
| inlineScript: | | |
| RESNAME='${{ env.RESNAME }}' | |
| DEPNAME='Dep${{ github.run_number }}' | |
| PARAMS="resourceName=$RESNAME location=$REGION byoAKSSubnetId=${{ secrets.ByoAksSubnetId }} byoAGWSubnetId=${{ secrets.ByoAgwSubnetId }} dnsZoneId=${{ secrets.BYODNSZONEID }}" | |
| az deployment group create -f bicep/main.bicep -g $RG -p ${{ env.ParamFilePath }} -p $PARAMS --name $DEPNAME --verbose | |
| DEPSTATUS=$(az deployment operation group list --resource-group $RG --name $DEPNAME) #--query "[?properties.provisioningState=='Failed']" | |
| echo $DEPSTATUS | |
| #outputs | |
| AKSNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.aksClusterName.value" -o tsv) | |
| echo "AKSNAME=$AKSNAME" >> $GITHUB_OUTPUT | |
| AGNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.applicationGatewayName.value" -o tsv) | |
| echo "AGNAME=$AGNAME" >> $GITHUB_OUTPUT | |
| AKVNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.keyVaultName.value" -o tsv) | |
| echo "AKVNAME=$AKVNAME" >> $GITHUB_OUTPUT | |
| LANAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.logAnalyticsName.value" -o tsv) | |
| echo "LANAME=$LANAME" >> $GITHUB_OUTPUT | |
| LAWGUID=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.logAnalyticsGuid.value" -o tsv) | |
| echo "LAWGUID=$LAWGUID" >> $GITHUB_OUTPUT | |
| ACRNAME=$(az deployment group show -n $DEPNAME -g $RG --query "properties.outputs.containerRegistryName.value" -o tsv) | |
| echo "ACRNAME=$ACRNAME" >> $GITHUB_OUTPUT | |
| - name: Enable AGIC Addon | |
| uses: Azure/cli@v2 | |
| if: ${{ needs.Deploy.outputs.AGNAME}} != '' | |
| with: | |
| azcliversion: ${{ env.AZCLIVERSION }} | |
| inlineScript: | | |
| AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}' | |
| AGNAME='${{ steps.deployAks.outputs.AGNAME}}' | |
| echo "AKS $AKSNAME" | |
| echo "AG $AGNAME" | |
| echo "Checking if ingressApplicationGateway is enabled" | |
| AGICEnabled=$(az aks show -n $AKSNAME -g $RG --query "addonProfiles.ingressApplicationGateway.enabled") | |
| echo "AGICEnabled = $AGICEnabled" | |
| if [ "$AGICEnabled" != "true" ]; | |
| then | |
| az aks enable-addons -n $AKSNAME -g $RG -a ingress-appgw --appgw-id $(az network application-gateway show -g $RG -n $AGNAME --query id -o tsv) | |
| fi | |
| - name: Create AGIC role assignments (if not exist) | |
| if: ${{ needs.Deploy.outputs.AGNAME}} != '' | |
| continue-on-error: true | |
| shell: pwsh | |
| run: | | |
| $AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}' | |
| $AGNAME='${{ steps.deployAks.outputs.AGNAME}}' | |
| write-output "AKS $AKSNAME" | |
| write-output "AG $AGNAME" | |
| write-output "Checking if ingressApplicationGateway is enabled" | |
| $AGICEnabled=az aks show -n $AKSNAME -g $RG --query "addonProfiles.ingressApplicationGateway.enabled" | |
| write-output "AGICEnabled = $AGICEnabled" | |
| if ($AGICEnabled -eq $true) { | |
| $agicobjid=az aks show -g $RG -n $AKSNAME --query "addonProfiles.ingressApplicationGateway.identity.objectId" -o tsv | |
| az role assignment list --assignee $agicobjid | |
| $rgScope=az group show -n $RG --query id | |
| az role assignment create --role Reader --assignee-object-id $agicobjid --assignee-principal-type ServicePrincipal --scope $rgScope | |
| $agwScope=az network application-gateway show -g $RG -n $AGNAME --query id -o tsv | |
| az role assignment create --role Contributor --assignee-object-id $agicobjid --assignee-principal-type ServicePrincipal --scope $agwScope | |
| az role assignment list --assignee $agicobjid | |
| } | |
| - name: Create Default Deny NetworkPolicy | |
| if: github.event.inputs.defaultdeny == 'true' | |
| run: | |
| AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}' | |
| netpolicycmd="kubectl apply -f https://raw.githubusercontent.com/Azure/Aks-Construction/0.4.3/postdeploy/k8smanifests/networkpolicy-deny-all.yml"; | |
| echo "Sending command $netpolicycmd to $AKSNAME in $RG"; | |
| cmdOut=$(az aks command invoke -g $RG -n $AKSNAME -o json --command "${netpolicycmd}"); | |
| echo $cmdOut; | |
| - name: Verify Default Deny NetworkPolicy | |
| if: github.event.inputs.defaultdeny == 'true' && github.event.inputs.doVerifySteps == 'true' | |
| run: | |
| AKSNAME='${{ steps.deployAks.outputs.AKSNAME}}' | |
| netpolicycmd="kubectl get networkpolicy -A"; | |
| echo "Sending command $netpolicycmd to $AKSNAME in $RG"; | |
| cmdOut=$(az aks command invoke -g $RG -n $AKSNAME -o json --command "${netpolicycmd}"); | |
| echo $cmdOut; | |
| ReusableWF: | |
| runs-on: ubuntu-latest | |
| needs: [Validation] | |
| env: | |
| RG: ${{ needs.Validation.outputs.RESOURCEGROUP }} | |
| outputs: | |
| RG: ${{ needs.Validation.outputs.RESOURCEGROUP }} | |
| ENVIRONMENT: ${{ github.event.inputs.Environment != '' && github.event.inputs.Environment || 'csu' }} | |
| Uninstall: ${{ github.event.inputs.doSmokeTestUninstall }} | |
| ExistingDnsDomainName: aksc.msftcsu.net | |
| ExistingDnsDomainRg: aksbicepacc-ci-deployvnet | |
| LatestAkscVersionTag: ${{ steps.AkscTags.outputs.LATEST}} | |
| steps: | |
| - name: Dummy step | |
| run: echo "Resuable workflows can't be directly passed ENV/INPUTS (yet)" | |
| - name: Explain more | |
| run: echo "see https://github.community/t/reusable-workflow-env-context-not-available-in-jobs-job-id-with/206111" | |
| - name: Job parameter inspection | |
| run: | | |
| echo "PARAMOVERRIDES: ${{ needs.Validation.outputs.PARAMOVERRIDES}}" | |
| echo "doSmokeTestUninstall: ${{ github.event.inputs.doSmokeTestUninstall }}" | |
| echo "RG: $RG" | |
| - name: Get latest AKSC version | |
| id: AkscTags | |
| run: | | |
| LATEST=$(curl https://api.github.com/repos/Azure/AKS-Construction/releases/latest | jq '.tag_name' -r) | |
| echo "LATEST=$LATEST" >> $GITHUB_OUTPUT | |
| Post-Deploy: | |
| needs: [Validation, Deploy, ReusableWF] | |
| uses: ./.github/workflows/PostDeploy.yml | |
| with: | |
| RG: ${{needs.ReusableWF.outputs.RG}} | |
| AKSNAME: ${{needs.Deploy.outputs.AKSNAME}} | |
| DNSDOMAIN: ${{needs.ReusableWF.outputs.ExistingDnsDomainName}} | |
| DNSRG: ${{needs.ReusableWF.outputs.ExistingDnsDomainRg}} | |
| CERTMANAGEREMAIL: "[email protected]" | |
| secrets: | |
| AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} | |
| SmokeTest_SimpleApp: | |
| needs: [Deploy, ReusableWF] | |
| uses: ./.github/workflows/AppDeploy_AzureVote.yml | |
| with: | |
| ENVIRONMENT: ${{ needs.ReusableWF.outputs.ENVIRONMENT }} | |
| RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #$RG | |
| AKSNAME: ${{needs.Deploy.outputs.AKSNAME}} | |
| AGNAME: ${{ needs.Deploy.outputs.AGNAME}} | |
| APPNAME: avote-public | |
| INGRESSTYPE: "AGIC-Public-Ingress" | |
| FORCEHELMCLEANINSTALL: true | |
| UNINSTALLAFTERVERIFY: ${{ needs.ReusableWF.outputs.Uninstall == 'true' }} | |
| secrets: | |
| AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} | |
| SmokeTest_JavaApp-certmgr: | |
| needs: [Deploy, ReusableWF, Post-Deploy] | |
| #uses: azure-samples/java-aks-keyvault-tls/.github/workflows/deployapp.yml@gb-workflow-bumps | |
| uses: azure-samples/java-aks-keyvault-tls/.github/workflows/[email protected] | |
| with: | |
| #ENVIRONMENT: ${{ needs.ReusableWF.outputs.ENVIRONMENT }} | |
| REPOREF: "0.9.5" | |
| RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows | |
| AKSNAME: ${{needs.Deploy.outputs.AKSNAME}} | |
| DNSDOMAIN: ${{needs.ReusableWF.outputs.ExistingDnsDomainName}} | |
| DNSRG: ${{needs.ReusableWF.outputs.ExistingDnsDomainRg}} | |
| DNSRECORDNAME: openjdk-demo | |
| AKVNAME: ${{ needs.Deploy.outputs.AKVNAME}} | |
| AGNAME: ${{ needs.Deploy.outputs.AGNAME}} | |
| APPNAME: openjdk-demo | |
| FRONTENDCERTTYPE: certmanager-staging | |
| FORCEHELMCLEANINSTALL: true | |
| UNINSTALLAFTERVERIFY: ${{ needs.ReusableWF.outputs.Uninstall == 'true' }} | |
| secrets: | |
| AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} | |
| # AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| # AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| # AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| SmokeTest_JavaApp-appgw: | |
| needs: [Deploy, ReusableWF, Post-Deploy, SmokeTest_JavaApp-certmgr] | |
| uses: azure-samples/java-aks-keyvault-tls/.github/workflows/[email protected] | |
| with: | |
| #ENVIRONMENT: ${{ needs.ReusableWF.outputs.ENVIRONMENT }} | |
| REPOREF: "0.9.5" | |
| RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows | |
| AKSNAME: ${{needs.Deploy.outputs.AKSNAME}} | |
| DNSDOMAIN: ${{needs.ReusableWF.outputs.ExistingDnsDomainName}} | |
| DNSRG: ${{needs.ReusableWF.outputs.ExistingDnsDomainRg}} | |
| DNSRECORDNAME: openjdk-keyvssl | |
| AKVNAME: ${{ needs.Deploy.outputs.AKVNAME}} | |
| AGNAME: ${{ needs.Deploy.outputs.AGNAME}} | |
| APPNAME: openjdk-keyvssl | |
| FRONTENDCERTTYPE: appgw-selfsigned | |
| FORCEHELMCLEANINSTALL: true | |
| UNINSTALLAFTERVERIFY: ${{ needs.ReusableWF.outputs.Uninstall == true }} | |
| secrets: | |
| AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} | |
| # AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| # AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| # AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| DeployVerify: | |
| runs-on: ubuntu-latest | |
| needs: [Validation, Deploy] | |
| env: | |
| RG: ${{ needs.Validation.outputs.RESOURCEGROUP }} | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Param check | |
| if: github.event.inputs.doDebugSteps == 'true' | |
| run: | | |
| echo "RG is $RG" | |
| echo "Param file path is: ${{ env.ParamFilePath }}" | |
| echo "Resource name is ${{ env.RESNAME }}" | |
| echo "Deployment name is ${{ env.DEPNAME }}" | |
| echo "AKS name is ${{ needs.Deploy.outputs.AKSNAME }}" | |
| - name: Azure Login | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| enable-AzPSSession: true | |
| - name: Test Control Plane Config | |
| shell: pwsh | |
| run: | | |
| $RG='${{ env.RG }}' | |
| $AKSNAME='${{ needs.Deploy.outputs.AKSNAME }}' | |
| $paramFilePath="${{ env.ParamFilePath }}" | |
| $params = Get-Content $paramFilePath | ConvertFrom-Json | |
| Write-Output "Checking for User Assigned Identity" | |
| if($params.parameters.custom_vnet.value -eq $true) { | |
| $aksId=az aks show -n $AKSNAME -g $RG --query "identity.type" -o tsv | |
| write-output $aksId | |
| if($aksId -eq "UserAssigned"){ | |
| Write-Output "UAI set correctly" | |
| } else { | |
| Write-Error "UAI not set properly"} | |
| } | |
| Write-Output "Checking for CSI Secrets driver" | |
| if($params.parameters.keyVaultAksCSI.value -eq $true) { | |
| $csiEnabled=az aks show -n $AKSNAME -g $RG --query "addonProfiles.omsagent.enabled" -o tsv | |
| write-output $csiEnabled | |
| if($csiEnabled -eq "true"){ | |
| Write-Output "CSI Secrets driver set correctly" | |
| } else { | |
| Write-Error "CSI Secrets driver not set properly"} | |
| } | |
| #grep KeyvaultSecretsProvider | |
| #Sometimes cluster config works for deployment, but misconfig can | |
| #Prevent the cluster from scaling... So lets make sure it can. | |
| - name: Scale the cluster up by 1 node | |
| shell: pwsh | |
| run: | | |
| $RG='${{ env.RG }}' | |
| $AKSNAME='${{ needs.Deploy.outputs.AKSNAME }}' | |
| Write-Output "Scaling $AKSNAME in $RG" | |
| $manualScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount==null].{name:name, count:count}" -o json | ConvertFrom-Json | |
| $manualScalePools | ForEach-Object { Write-Output "scaling [m] pool $($_.name)"; az aks scale -g $RG -n $AKSNAME --node-count $($_.pool + 1) --nodepool-name $_.name } | |
| $autoScalePools = az aks show -n $AKSNAME -g $RG --query "agentPoolProfiles[?maxCount!=null].{name:name, minCount:minCount, maxCount:maxCount}" -o json | ConvertFrom-Json | |
| $autoScalePools | ForEach-Object { Write-Output "scaling [a] pool $($_.name)"; az aks nodepool update --update-cluster-autoscaler -g $RG --cluster-name $AKSNAME --name $_.name --min-count $($_.minCount + 1) --max-count $($_.maxCount + 1) } | |
| Troubleshoot: | |
| needs: [Deploy, ReusableWF, Post-Deploy, SmokeTest_JavaApp-certmgr, SmokeTest_JavaApp-appgw] | |
| uses: ./.github/workflows/AksTroubleshooting.yml | |
| if: always() | |
| with: | |
| Environment: '' | |
| RG: ${{ needs.ReusableWF.outputs.RG }} #Automation-Actions-AksDeployCI #'${{ env.RG }}' There seems to be an issue passing Env variables in reusable workflows | |
| AKSNAME: ${{needs.Deploy.outputs.AKSNAME}} | |
| AGNAME: ${{ needs.Deploy.outputs.AGNAME}} | |
| LANAME: ${{ needs.Deploy.outputs.LANAME}} | |
| LAWGUID: ${{ needs.Deploy.outputs.LAWGUID}} | |
| USERUNCMD: false | |
| secrets: | |
| AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }} | |
| Infra_Destroy: | |
| uses: ./.github/workflows/cleanupRg.yml | |
| if: github.event_name == 'schedule' | |
| needs: [ReusableWF, Validation, Deploy, Troubleshoot, DeployVerify, SmokeTest_SimpleApp, Well_Architected] | |
| with: | |
| environment: ${{ needs.ReusableWF.outputs.ENVIRONMENT }} | |
| rg: ${{ needs.Validation.outputs.RESOURCEGROUP }} | |
| secrets: | |
| AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} | |
| AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} | |
| AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} |