Use managed identity for blob upload of release assets (#4680)

- Change release tool to use manage identity for uploads
- Use WIF for authentication for release tool
- Ensure log publishing doesn't collide on retry
- Disable SBOM for logs and test binary assets
- Fix token names and add appropriate groups

diagnostics.yml

@@ -302,10 +302,12 @@ extends:
                 displayName: 'Publish Bundled Tools'
                 condition: succeeded()
               - output: pipelineArtifact
-                artifact: Logs_Packaging_Signing
+                artifact: Logs_Packaging_Signing_Attempt$(System.JobAttempt)
                 path: '$(Build.SourcesDirectory)/artifacts/log'
                 displayName: 'Publish Signing and Packaging Logs'
                 condition: always()
+                continueOnError: true
+                sbomEnabled: false # we don't need SBOM for logs
             steps:
             - task: DownloadPipelineArtifact@2
               displayName: 'Download release builds'

eng/pipelines/build.yml

@@ -260,6 +260,7 @@ jobs:
           inputs:
             targetPath: $(Build.ArtifactStagingDirectory)/artifacts_on_failure
             artifactName: Artifacts_On_Failure_$(_PhaseName)_$(System.JobAttempt)
+            sbomEnabled: false # we don't need SBOM for non-shipping diagnostics assets
           continueOnError: true
           condition: failed()
 
@@ -278,6 +279,7 @@ jobs:
           inputs:
             targetPath: '$(Build.StagingDirectory)/BuildLogs'
             artifactName: Logs_$(_PhaseName)_$(System.JobAttempt)
+            sbomEnabled: false # we don't need SBOM for logs
           continueOnError: true
           condition: always()
 

eng/pipelines/prepare-release.yml

@@ -3,7 +3,7 @@ stages:
   displayName: Release Preparation
   jobs:
   - job: PrepareReleaseJob
-    displayName: Prepare release with Darc
+    displayName: Prepare Release
     ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}:
       templateContext:
         outputs:
@@ -15,14 +15,14 @@ stages:
     variables:
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}:
       - group: DotNet-Diagnostics-Storage
-      - group: DotNet-DotNetStage-Storage
+      - group: DotNetBuilds storage account read tokens
       - group: Release-Pipeline
     steps:
     - ${{ if in(variables['Build.Reason'], 'PullRequest') }}:
       - script: '$(Build.Repository.LocalPath)\dotnet.cmd build $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release /bl'
         workingDirectory: '$(System.ArtifactsDirectory)'
         displayName: 'Build Manifest generation and asset publishing tool'
-    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}:
+    - ${{ elseif and(ne(variables['System.TeamProject'], 'public'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) }}:
       - task: UseDotNet@2
         displayName: 'Use .NET Core runtime 6.x'
         inputs:
@@ -37,13 +37,25 @@ stages:
           filePath: '$(Build.Repository.LocalPath)/eng/release/Scripts/AcquireBuild.ps1'
           arguments: >-
             -BarBuildId "$(BARBuildId)"
-            -AzdoToken "$(dn-bot-dotnet-all-scopes)"
-            -MaestroToken "$(MaestroAccessToken)"
-            -GitHubToken "$(BotAccount-dotnet-bot-repo-PAT)"
-            -DownloadTargetPath "$(System.ArtifactsDirectory)\ReleaseTarget"
-            -SasSuffixes "$(dotnetclichecksumsmsrc-dotnet-read-list-sas-token),$(dotnetclimsrc-read-sas-token)"
             -ReleaseVersion "$(Build.BuildNumber)"
+            -DownloadTargetPath "$(System.ArtifactsDirectory)\ReleaseTarget"
+            -AzdoToken "$(dn-bot-all-drop-rw-code-rw-release-all)"
+            -MaestroToken "$(MaestroAccessToken)"
+            -SasSuffixes "$(dotnetbuilds-internal-checksums-container-read-token),$(dotnetbuilds-internal-container-read-token)"
           workingDirectory: '$(Build.Repository.LocalPath)'
+      - task: AzureCLI@2
+        displayName: 'Use WIF to obtain credentials for Azure CLI'
+        inputs:
+          azureSubscription: 'dotnetstage-diagnostics-tools-rw'
+          scriptType: pscore
+          scriptLocation: inlineScript
+          addSpnToEnvironment: true
+          inlineScript: |
+              echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId"
+              echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken"
+              echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId"
+      - script: az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN)
+        displayName: 'Use az to authenticate using managed identity'
       - script: >-
           $(Build.Repository.LocalPath)\dotnet.cmd run --project $(Build.Repository.LocalPath)\eng\release\DiagnosticsReleaseTool\DiagnosticsReleaseTool.csproj -c Release
           --
@@ -53,9 +65,8 @@ stages:
           --staging-directory "$(System.ArtifactsDirectory)\ReleaseStaging"
           --release-name "$(Build.BuildNumber)"
           --account-name "$(dotnet-diagnostics-storage-accountname)"
-          --account-key "$(dotnetstage-storage-key)"
+          --client-id $(ARM_CLIENT_ID)
           --container-name "$(dotnet-diagnostics-container-name)"
-          --sas-valid-days "$(dotnet-diagnostics-storage-retentiondays)"
           -v True
         workingDirectory: '$(Build.Repository.LocalPath)\'
         displayName: 'Manifest generation and asset publishing'

eng/pipelines/publish-pipeline-artifact-shim.yml

@@ -3,6 +3,7 @@ parameters:
   displayName: 'Publish Pipeline Artifact'
   condition: succeeded()
   continueOnError: true
+  enableSbom: true
 
 steps:
 - ${{ if ne(variables['System.TeamProject'], 'public') }}:
@@ -11,6 +12,7 @@ steps:
     inputs:
       targetPath: ${{ parameters.inputs.targetPath }}
       artifactName: ${{ parameters.inputs.artifactName }}
+      enableSbom: ${{ parameters.enableSbom }}
     condition: ${{ parameters.condition }}
     displayName: ${{ parameters.displayName }}
     continueOnError: ${{ parameters.continueOnError }}

eng/release/DiagnosticsReleaseTool/Common/AzureBlobPublisher.cs

@@ -7,6 +7,8 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using Azure.Core;
+using Azure.Identity;
 using Azure.Storage;
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
@@ -17,17 +19,14 @@ namespace ReleaseTool.Core
 {
     public class AzureBlobBublisher : IPublisher
     {
-        private const int ClockSkewSec = 15 * 60;
         private const int MaxRetries = 15;
         private const int MaxFullLoopRetries = 5;
         private readonly TimeSpan FullLoopRetryDelay = TimeSpan.FromSeconds(1);
-        private const string AccessPolicyDownloadId = "DownloadDrop";
 
         private readonly string _accountName;
-        private readonly string _accountKey;
+        private readonly string _clientId;
         private readonly string _containerName;
         private readonly string _releaseName;
-        private readonly int _sasValidDays;
         private readonly ILogger _logger;
 
         private BlobContainerClient _client;
@@ -40,12 +39,17 @@ private Uri AccountBlobUri
             }
         }
 
-        private StorageSharedKeyCredential AccountCredential
+        private TokenCredential Credentials
         {
             get
             {
-                StorageSharedKeyCredential credential = new(_accountName, _accountKey);
-                return credential;
+                if (_clientId == null)
+                {
+                    // Local development scenario. Use the default credential.
+                    return new DefaultAzureCredential();
+                }
+
+                return new DefaultAzureCredential(new DefaultAzureCredentialOptions { ManagedIdentityClientId = _clientId });
             }
         }
 
@@ -68,13 +72,12 @@ private static BlobClientOptions BlobOptions
             }
         }
 
-        public AzureBlobBublisher(string accountName, string accountKey, string containerName, string releaseName, int sasValidDays, ILogger logger)
+        public AzureBlobBublisher(string accountName, string clientId, string containerName, string releaseName, ILogger logger)
         {
             _accountName = accountName;
-            _accountKey = accountKey;
+            _clientId = clientId;
             _containerName = containerName;
             _releaseName = releaseName;
-            _sasValidDays = sasValidDays;
             _logger = logger;
         }
 
@@ -107,20 +110,11 @@ public async Task<string> PublishFileAsync(FileMapping fileMap, CancellationToke
 
                     await blobClient.UploadAsync(srcStream, overwrite: true, ct);
 
-                    BlobSasBuilder sasBuilder = new()
-                    {
-                        BlobContainerName = client.Name,
-                        BlobName = blobClient.Name,
-                        Identifier = AccessPolicyDownloadId,
-                        Protocol = SasProtocol.Https
-                    };
-                    Uri accessUri = blobClient.GenerateSasUri(sasBuilder);
-
                     using BlobDownloadStreamingResult blobStream = (await blobClient.DownloadStreamingAsync(cancellationToken: ct)).Value;
                     srcStream.Position = 0;
                     completed = await VerifyFileStreamsMatchAsync(srcStream, blobStream, ct);
 
-                    result = accessUri;
+                    result = blobClient.Uri;
                 }
                 catch (IOException ioEx) when (ioEx is not PathTooLongException)
                 {
@@ -155,7 +149,7 @@ private async Task<BlobContainerClient> GetClient(CancellationToken ct)
         {
             if (_client == null)
             {
-                BlobServiceClient serviceClient = new(AccountBlobUri, AccountCredential, BlobOptions);
+                BlobServiceClient serviceClient = new(AccountBlobUri, Credentials, BlobOptions);
                 _logger.LogInformation($"Attempting to connect to {serviceClient.Uri} to store blobs.");
 
                 BlobContainerClient newClient;
@@ -165,9 +159,9 @@ private async Task<BlobContainerClient> GetClient(CancellationToken ct)
                     try
                     {
                         newClient = serviceClient.GetBlobContainerClient(_containerName);
-                        if (!(await newClient.ExistsAsync(ct)).Value)
+                        if (!await newClient.ExistsAsync(ct))
                         {
-                            newClient = (await serviceClient.CreateBlobContainerAsync(_containerName, PublicAccessType.None, metadata: null, ct));
+                            newClient = await serviceClient.CreateBlobContainerAsync(_containerName, PublicAccessType.None, metadata: null, ct);
                         }
                     }
                     catch (Exception ex)
@@ -176,31 +170,6 @@ private async Task<BlobContainerClient> GetClient(CancellationToken ct)
                         continue;
                     }
 
-                    try
-                    {
-                        DateTime baseTime = DateTime.UtcNow;
-                        // Add the new (or update existing) "download" policy to the container
-                        // This is used to mint the SAS tokens without an expiration policy
-                        // Expiration can be added later by modifying this policy
-                        BlobSignedIdentifier downloadPolicyIdentifier = new()
-                        {
-                            Id = AccessPolicyDownloadId,
-                            AccessPolicy = new BlobAccessPolicy()
-                            {
-                                Permissions = "r",
-                                PolicyStartsOn = new DateTimeOffset(baseTime.AddSeconds(-ClockSkewSec)),
-                                PolicyExpiresOn = new DateTimeOffset(DateTime.UtcNow.AddDays(_sasValidDays).AddSeconds(ClockSkewSec)),
-                            }
-                        };
-                        _logger.LogInformation($"Writing download access policy: {AccessPolicyDownloadId} to {_containerName}.");
-                        await newClient.SetAccessPolicyAsync(PublicAccessType.None, new BlobSignedIdentifier[] { downloadPolicyIdentifier }, cancellationToken: ct);
-                    }
-                    catch (Exception ex)
-                    {
-                        _logger.LogWarning(ex, $"Failed to write access policy for {_containerName}, retrying.");
-                        continue;
-                    }
-
                     _logger.LogInformation($"Container {_containerName} is ready.");
                     _client = newClient;
                     break;

eng/release/DiagnosticsReleaseTool/Config.cs

@@ -13,9 +13,8 @@ internal sealed class Config
         public DirectoryInfo StagingDirectory { get; }
         public string ReleaseName { get; }
         public string AccountName { get; }
-        public string AccountKey { get; }
+        public string ClientId { get; }
         public string ContainerName { get; }
-        public int SasValidDays { get; }
 
         public Config(
             FileInfo toolManifest,
@@ -24,19 +23,17 @@ public Config(
             DirectoryInfo stagingDirectory,
             string releaseName,
             string accountName,
-            string accountKey,
-            string containerName,
-            int sasValidDays)
+            string clientId,
+            string containerName)
         {
             ToolManifest = toolManifest;
             ShouldVerifyManifest = verifyToolManifest;
             DropPath = inputDropPath;
             StagingDirectory = stagingDirectory;
             ReleaseName = releaseName;
             AccountName = accountName;
-            AccountKey = accountKey;
+            ClientId = clientId;
             ContainerName = containerName;
-            SasValidDays = sasValidDays;
         }
     }
 }

eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseCommandLine.cs

@@ -38,19 +38,19 @@ public static Command PrepareRelease() =>
                 ToolManifestVerificationOption(), DiagnosticLoggingOption(),
                 // Outputs
                 StagingPathOption(),
-                AzureStorageAccountNameOption(), AzureStorageAccountKeyOption(), AzureStorageContainerNameOption(), AzureStorageSasExpirationOption()
+                AzureStorageAccountNameOption(), AzureStorageAccountKeyOption(), AzureStorageContainerNameOption()
             };
 
 
         private static Option<bool> DiagnosticLoggingOption() =>
             new(
-                aliases: new[] { "-v", "--verbose" },
+                aliases: ["-v", "--verbose"],
                 description: "Enables diagnostic logging",
                 getDefaultValue: () => false);
 
         private static Option ToolManifestPathOption() =>
             new Option<FileInfo>(
-                aliases: new[] { "--tool-manifest", "-t" },
+                aliases: ["--tool-manifest", "-t"],
                 description: "Full path to the manifest of tools and packages to publish.")
             {
                 IsRequired = true
@@ -64,56 +64,51 @@ private static Option<bool> ToolManifestVerificationOption() =>
 
         private static Option<DirectoryInfo> InputDropPathOption() =>
             new Option<DirectoryInfo>(
-                aliases: new[] { "-i", "--input-drop-path" },
+                aliases: ["-i", "--input-drop-path"],
                 description: "Path to drop generated by `darc gather-drop`")
             {
                 IsRequired = true
             }.ExistingOnly();
 
         private static Option<string> ReleaseNameOption() =>
             new(
-                aliases: new[] { "-r", "--release-name" },
+                aliases: ["-r", "--release-name"],
                 description: "Name of this release.")
             {
                 IsRequired = true,
             };
 
         private static Option StagingPathOption() =>
             new Option<DirectoryInfo>(
-                aliases: new[] { "--staging-directory", "-s" },
+                aliases: ["--staging-directory", "-s"],
                 description: "Full path to the staging path.",
                 getDefaultValue: () => new DirectoryInfo(
                     Path.Join(Path.GetTempPath(), Path.GetRandomFileName())))
             .LegalFilePathsOnly();
 
         private static Option<string> AzureStorageAccountNameOption() =>
             new(
-                aliases: new[] { "-n", "--account-name" },
+                aliases: ["-n", "--account-name"],
                 description: "Storage account name, must be in public azure cloud.")
             {
                 IsRequired = true,
             };
 
         private static Option<string> AzureStorageAccountKeyOption() =>
             new(
-                aliases: new[] { "-k", "--account-key" },
-                description: "Storage account key, in base 64 format.")
+                aliases: ["-k", "--client-id"],
+                description: "Identity Client ID. If left blank, ambient identity will be used.",
+                getDefaultValue: () => null)
             {
                 IsRequired = true,
             };
 
         private static Option<string> AzureStorageContainerNameOption() =>
             new(
-                aliases: new[] { "-c", "--container-name" },
+                aliases: ["-c", "--container-name"],
                 description: "Storage account container name where the files will be uploaded.")
             {
                 IsRequired = true,
             };
-
-        private static Option<int> AzureStorageSasExpirationOption() =>
-            new(
-                aliases: new[] { "--sas-valid-days" },
-                description: "Number of days to allow access to the blobs via the provided SAS URIs.",
-                getDefaultValue: () => 1);
     }
 }

eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseRunner.cs

@@ -51,7 +51,7 @@ internal static async Task<int> PrepareRelease(Config releaseConfig, bool verbos
             DirectoryInfo basePublishDirectory = darcLayoutHelper.GetShippingDirectoryForSingleProjectVariants(DiagnosticsRepoHelpers.ProductNames);
             string publishManifestPath = Path.Combine(releaseConfig.StagingDirectory.FullName, ManifestName);
 
-            IPublisher releasePublisher = new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.AccountKey, releaseConfig.ContainerName, releaseConfig.ReleaseName, releaseConfig.SasValidDays, logger);
+            IPublisher releasePublisher = new AzureBlobBublisher(releaseConfig.AccountName, releaseConfig.ClientId, releaseConfig.ContainerName, releaseConfig.ReleaseName, logger);
             IManifestGenerator manifestGenerator = new DiagnosticsManifestGenerator(releaseMetadata, releaseConfig.ToolManifest, logger);
 
             using Release diagnosticsRelease = new(

eng/release/DiagnosticsReleaseTool/DiagnosticsReleaseTool.csproj

@@ -21,7 +21,8 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
 
-    <PackageReference Include="Azure.Storage.Blobs" Version="[12.13.0]" />
+    <PackageReference Include="Azure.Identity" Version="[1.11.3]" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="[12.20.0]" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20468.1" />
   </ItemGroup>