From 2af16fd6a22de0a4ea6837de4caf44550972c5a1 Mon Sep 17 00:00:00 2001 From: Michael Pysson Date: Tue, 29 Aug 2023 23:55:29 +0000 Subject: [PATCH] Merged PR 735951: Update SBOM related packages Updates various SBOM related packages to work with the new Component Detector. These changes are necessary in order for CloudBuild Repo to update to the new version of Component Detector and SBOM tooling --- Public/Src/Tools/DropDaemon/DropDaemon.cs | 18 +++++++++--------- .../Src/Tools/DropDaemon/Tool.DropDaemon.dsc | 14 +++++++------- .../DropDaemon/SbomGenerationTests.cs | 8 ++++---- .../DropDaemon/Test.Tool.DropDaemon.dsc | 1 + cg/nuget/cgmanifest.json | 18 +++++++++--------- config.dsc | 2 +- config.microsoftInternal.dsc | 14 +++++++------- config.nuget.dotnetcore.dsc | 4 ++-- 8 files changed, 40 insertions(+), 39 deletions(-) diff --git a/Public/Src/Tools/DropDaemon/DropDaemon.cs b/Public/Src/Tools/DropDaemon/DropDaemon.cs index 2c1cec3f81..8855fef7c0 100644 --- a/Public/Src/Tools/DropDaemon/DropDaemon.cs +++ b/Public/Src/Tools/DropDaemon/DropDaemon.cs @@ -885,7 +885,7 @@ private async Task GenerateAndUploadBuildManifestFileWithSignedCatal hasMoreData = bxlResult.Result.HasMoreData; } while (hasMoreData); - IEnumerable manifestFileList = fileList.Select(ToSbomFile); + IEnumerable manifestFileList = fileList.Select(ToSbomFile); string sbomGenerationRootDirectory = null; var logger = GetDropSpecificLogger(dropConfig); @@ -905,13 +905,13 @@ private async Task GenerateAndUploadBuildManifestFileWithSignedCatal FileUtilities.CreateDirectory(sbomGenerationRootDirectory); // Always generate SPDX, but exclude CloudBuild manifest if configured to do so - var specs = new List() { new("SPDX", "2.2") }; + var specs = new List() { new("SPDX", "2.2") }; if (!m_disableCloudBuildManifest) { specs.Add(new("CloudBuildManifest", "1.0.0")); } - Possible> maybePackages; + Possible> maybePackages; using (m_counters.StartStopwatch(DropDaemonCounter.BuildManifestComponentConversionDuration)) { maybePackages = await GetSbomPackagesAsync(logger); @@ -924,7 +924,7 @@ private async Task GenerateAndUploadBuildManifestFileWithSignedCatal var packages = maybePackages.Result; logger.Verbose("Starting SBOM Generation"); - var result = await m_sbomGenerator.GenerateSBOMAsync(sbomGenerationRootDirectory, manifestFileList, packages, metadata, specs); + var result = await m_sbomGenerator.GenerateSbomAsync(sbomGenerationRootDirectory, manifestFileList, packages, metadata, specs); logger.Verbose("Finished SBOM Generation"); if (!result.IsSuccessful) @@ -991,7 +991,7 @@ private static string GetSbomGenerationErrorDetails(IList errors) return sb.ToString(); } - private SBOMFile ToSbomFile(BuildXL.Ipc.ExternalApi.Commands.BuildManifestFileInfo fileInfo) + private SbomFile ToSbomFile(BuildXL.Ipc.ExternalApi.Commands.BuildManifestFileInfo fileInfo) { // Include artifacts hash only when computing CloudBuildV1 Manifest var maybeArtifactsHash = m_disableCloudBuildManifest ? Array.Empty() : new[] { fileInfo.AzureArtifactsHash }; @@ -1021,14 +1021,14 @@ static AlgorithmName mapHashType(HashType hashType) } /// - /// Tries to convert output from component detection to a list of . + /// Tries to convert output from component detection to a list of . /// /// - /// A converted list of if successful. + /// A converted list of if successful. /// If not successful, errors messages will be logged and also returned as a Failure{string}. /// Any warnings raised by the tooling are also logged via the APIServer. /// - private async Task>> GetSbomPackagesAsync(IIpcLogger logger) + private async Task>> GetSbomPackagesAsync(IIpcLogger logger) { // Read Path for bcde output from environment, this should already be set by Cloudbuild var bcdeOutputJsonPath = Environment.GetEnvironmentVariable(Constants.ComponentGovernanceBCDEOutputFilePath); @@ -1039,7 +1039,7 @@ private async Task>> GetSbomPackagesAsync(IIpc // and the SBOM creation here can still happen without a set of packages. // Log a message on the ApiServer it and return an empty set. Analysis.IgnoreResult(await ApiClient.LogMessage($"[GetSbomPackages] The '{Constants.ComponentGovernanceBCDEOutputFilePath}' environment variable was not found. This happens when component governance on the build runner is disabled. Component detection data will not be included in build manifest.", isWarning: false)); - return new List(); + return new List(); } else if (!System.IO.File.Exists(bcdeOutputJsonPath)) { diff --git a/Public/Src/Tools/DropDaemon/Tool.DropDaemon.dsc b/Public/Src/Tools/DropDaemon/Tool.DropDaemon.dsc index 0f820b2f0d..fd26262f7d 100644 --- a/Public/Src/Tools/DropDaemon/Tool.DropDaemon.dsc +++ b/Public/Src/Tools/DropDaemon/Tool.DropDaemon.dsc @@ -16,7 +16,6 @@ export namespace DropDaemon { appConfig: f`DropDaemon.exe.config`, assemblyBindingRedirects: dropDaemonBindingRedirects(), sources: globR(d`.`, "*.cs"), - references: [ importFrom("BuildXL.Cache.ContentStore").Hashing.dll, importFrom("BuildXL.Utilities.Instrumentation").Tracing.dll, @@ -56,6 +55,7 @@ export namespace DropDaemon { internalsVisibleTo: [ "Test.Tool.DropDaemon", ], + deploymentOptions: { ignoredSelfContainedRuntimeFilenames: [a`System.Text.Encodings.Web.dll`, a`System.Text.Json.dll`] }, runtimeContentToSkip: dropDaemonRuntimeContentToSkip() }); @@ -136,15 +136,15 @@ export namespace DropDaemon { name: "System.Text.Json", publicKeyToken: "cc7b13ffcd2ddd51", culture: "neutral", - oldVersion: "0.0.0.0-5.0.0.0", - newVersion: "5.0.0.0", + oldVersion: "0.0.0.0-7.0.0.0", + newVersion: "7.0.0.0" }, { name: "System.Text.Encodings.Web", publicKeyToken: "cc7b13ffcd2ddd51", culture: "neutral", - oldVersion: "0.0.0.0-5.0.0.1", - newVersion: "5.0.0.1", // Corresponds to { id: "System.Text.Encodings.Web", version: "4.7.2" }, + oldVersion: "0.0.0.0-7.0.0.0", + newVersion: "7.0.0.0" } ]; } @@ -172,9 +172,9 @@ export namespace DropDaemon { importFrom("Microsoft.ComponentDetection.Contracts").pkg, importFrom("Microsoft.Sbom.Adapters").pkg, importFrom("packageurl-dotnet").pkg, - importFrom("System.Text.Json.v5.0.0").pkg, + importFrom("System.Text.Json.v7.0.0").pkg, importFrom("Newtonsoft.Json").pkg, - importFrom("System.Text.Encodings.Web.v5.0.1").pkg, + importFrom("System.Text.Encodings.Web.v7.0.0").pkg, ]; } diff --git a/Public/Src/Tools/UnitTests/DropDaemon/SbomGenerationTests.cs b/Public/Src/Tools/UnitTests/DropDaemon/SbomGenerationTests.cs index 2c3c9c7fc4..b21c451844 100644 --- a/Public/Src/Tools/UnitTests/DropDaemon/SbomGenerationTests.cs +++ b/Public/Src/Tools/UnitTests/DropDaemon/SbomGenerationTests.cs @@ -67,10 +67,10 @@ public async Task GenerateSbom() var sbomGenerationRootDirectory = Path.Combine(Path.GetTempPath(), "sbom"); - var specs = new List() { new("SPDX", "2.2"), new("CloudBuildManifest", "1.0.0") }; + var specs = new List() { new("SPDX", "2.2"), new("CloudBuildManifest", "1.0.0") }; // A file with VSO and SHA1 hashes to generate both SPDX and CBManifest - var myfile = new SBOMFile() + var myfile = new SbomFile() { Id = "MyFileId", Path = "Oh/What/A/Cool/Path.txt", @@ -93,7 +93,7 @@ public async Task GenerateSbom() }, } }; - IEnumerable files = new List() { myfile }; + IEnumerable files = new List() { myfile }; var (adapterReport, packages) = new ComponentDetectionToSBOMPackageAdapter().TryConvert(GenerateBcdeOutput(Path.GetTempFileName())); XAssert.IsNotNull(packages); @@ -105,7 +105,7 @@ public async Task GenerateSbom() } } - var result = await sbomGenerator.GenerateSBOMAsync(sbomGenerationRootDirectory, files, packages, metadata, specs); + var result = await sbomGenerator.GenerateSbomAsync(sbomGenerationRootDirectory, files, packages, metadata, specs); if (!result.IsSuccessful) { var errorDetails = GetSbomGenerationErrorDetails(result.Errors); diff --git a/Public/Src/Tools/UnitTests/DropDaemon/Test.Tool.DropDaemon.dsc b/Public/Src/Tools/UnitTests/DropDaemon/Test.Tool.DropDaemon.dsc index 8ecd0aa0c3..6773d011f9 100644 --- a/Public/Src/Tools/UnitTests/DropDaemon/Test.Tool.DropDaemon.dsc +++ b/Public/Src/Tools/UnitTests/DropDaemon/Test.Tool.DropDaemon.dsc @@ -39,6 +39,7 @@ namespace Test.Tool.DropDaemon { ...importFrom("BuildXL.Tools.DropDaemon").dropDaemonSbomPackages(), ], + deploymentOptions: { ignoredSelfContainedRuntimeFilenames: [a`System.Text.Encodings.Web.dll`, a`System.Text.Json.dll`] }, runtimeContentToSkip: importFrom("BuildXL.Tools.DropDaemon").dropDaemonRuntimeContentToSkip(), }); } diff --git a/cg/nuget/cgmanifest.json b/cg/nuget/cgmanifest.json index f12522ae75..e4f23713e2 100644 --- a/cg/nuget/cgmanifest.json +++ b/cg/nuget/cgmanifest.json @@ -1122,7 +1122,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.ComponentDetection.Contracts", - "Version": "2.0.2" + "Version": "3.6.4" } } }, @@ -1779,7 +1779,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.Parsers.ManifestGenerator", - "Version": "2.3.3" + "Version": "3.7.0" } } }, @@ -1815,7 +1815,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.SBOMCore", - "Version": "2.3.3" + "Version": "3.8.0" } } }, @@ -1824,7 +1824,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.Sbom.Adapters", - "Version": "0.2.6" + "Version": "1.5.2" } } }, @@ -1833,7 +1833,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.Sbom.Contracts", - "Version": "0.2.6" + "Version": "1.5.2" } } }, @@ -1842,7 +1842,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.Sbom.Extensions", - "Version": "0.2.6" + "Version": "1.5.2" } } }, @@ -1851,7 +1851,7 @@ "Type": "NuGet", "NuGet": { "Name": "Microsoft.Sbom.Parsers.Spdx22SbomParser", - "Version": "0.2.6" + "Version": "1.5.2" } } }, @@ -4029,7 +4029,7 @@ "Type": "NuGet", "NuGet": { "Name": "System.Text.Encodings.Web", - "Version": "5.0.1" + "Version": "7.0.0" } } }, @@ -4047,7 +4047,7 @@ "Type": "NuGet", "NuGet": { "Name": "System.Text.Json", - "Version": "5.0.0" + "Version": "7.0.0" } } }, diff --git a/config.dsc b/config.dsc index 906db3e0a0..a0d0d2b4de 100644 --- a/config.dsc +++ b/config.dsc @@ -396,7 +396,7 @@ config({ // Needed for SBOM Generation { id: "Microsoft.Extensions.Logging.Abstractions", version: "6.0.3", alias: "Microsoft.Extensions.Logging.Abstractions.v6.0.3", dependentPackageIdsToSkip: ["System.Buffers", "System.Memory"] }, - { id: "System.Text.Encodings.Web", version: "5.0.1", dependentPackageIdsToSkip: ["System.Buffers", "System.Memory"], alias: "System.Text.Encodings.Web.v5.0.1" }, + { id: "System.Text.Encodings.Web", version: "7.0.0", dependentPackageIdsToSkip: ["System.Buffers", "System.Memory"], alias: "System.Text.Encodings.Web.v7.0.0" }, { id: "packageurl-dotnet", version: "1.1.0" }, { id: "System.Reactive", version: "4.4.1" }, diff --git a/config.microsoftInternal.dsc b/config.microsoftInternal.dsc index f0eeb8ffe2..d485d83398 100644 --- a/config.microsoftInternal.dsc +++ b/config.microsoftInternal.dsc @@ -79,13 +79,13 @@ export const pkgs = isMicrosoftInternal ? [ { id: "VisualCppTools.Internal.VS2017Layout", version: "14.16.27034", osSkip: [ "macOS", "unix" ] }, // SBOM Generation - { id: "Microsoft.SBOMCore", version: "2.3.3" }, - { id: "Microsoft.Parsers.ManifestGenerator", version: "2.3.3", dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] }, - { id: "Microsoft.Sbom.Parsers.Spdx22SbomParser", version: "0.2.6" }, - { id: "Microsoft.Sbom.Adapters", version: "0.2.6" }, - { id: "Microsoft.ComponentDetection.Contracts", version: "2.0.2" }, - { id: "Microsoft.Sbom.Contracts", version: "0.2.6" }, - { id: "Microsoft.Sbom.Extensions", version: "0.2.6", dependentPackageIdsToSkip: ["System.Text.Json"] }, + { id: "Microsoft.SBOMCore", version: "3.8.0" }, + { id: "Microsoft.Parsers.ManifestGenerator", version: "3.7.0", dependentPackageIdsToIgnore: ["BuildXL.Cache.Hashing"] }, + { id: "Microsoft.Sbom.Parsers.Spdx22SbomParser", version: "1.5.2" }, + { id: "Microsoft.Sbom.Adapters", version: "1.5.2" }, + { id: "Microsoft.ComponentDetection.Contracts", version: "3.6.4" }, + { id: "Microsoft.Sbom.Contracts", version: "1.5.2" }, + { id: "Microsoft.Sbom.Extensions", version: "1.5.2", dependentPackageIdsToSkip: ["System.Text.Json"] }, { id: "Microsoft.Bcl.HashCode", version: "1.1.1" }, // Process remoting diff --git a/config.nuget.dotnetcore.dsc b/config.nuget.dotnetcore.dsc index 2284dd4041..ddd6ec6b8f 100644 --- a/config.nuget.dotnetcore.dsc +++ b/config.nuget.dotnetcore.dsc @@ -244,9 +244,9 @@ export const pkgs = [ { id: "System.Text.Json", version: "4.7.2", dependentPackageIdsToSkip: ["System.Memory", "System.ValueTuple", "System.Runtime.CompilerServices.Unsafe", "System.Numerics.Vectors", "System.Threading.Tasks.Extensions", "Microsoft.Bcl.AsyncInterfaces"], }, - { id: "System.Text.Json", version: pkgVersion5, + { id: "System.Text.Json", version: pkgVersion7, dependentPackageIdsToSkip: ["System.Memory", "System.Buffers", "System.ValueTuple", "System.Runtime.CompilerServices.Unsafe", "System.Numerics.Vectors", "System.Threading.Tasks.Extensions", "Microsoft.Bcl.AsyncInterfaces", "System.Text.Encodings.Web"], - alias: "System.Text.Json.v5.0.0" + alias: "System.Text.Json.v7.0.0" }, { id: "System.Threading.AccessControl", version: pkgVersionNext },