From 0270b04106d4adfafc28cca8ed9196b34d449a07 Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Mon, 23 Sep 2024 16:53:21 +0200 Subject: [PATCH 01/18] merge vertical manifests --- eng/pipelines/templates/jobs/vmr-build.yml | 34 +- .../templates/stages/vmr-final-join.yml | 52 ++++ .../templates/steps/vmr-join-verticals.yml | 37 +++ src/SourceBuild/content/build.proj | 3 + .../content/eng/join-verticals.proj | 46 +++ .../content/eng/join-verticals.ps1 | 33 ++ .../content/eng/merge-asset-manifests.proj | 3 +- src/SourceBuild/content/eng/pipelines/ci.yml | 3 + src/SourceBuild/content/eng/pipelines/pr.yml | 3 + .../AzureDevOpsClient.cs | 197 ++++++++++++ .../JoinVerticals.cs | 290 ++++++++++++++++++ .../MergeAssetManifests.cs | 8 + .../Models/AzureDevOpsArtifactInformation.cs | 10 + 13 files changed, 716 insertions(+), 3 deletions(-) create mode 100644 eng/pipelines/templates/stages/vmr-final-join.yml create mode 100644 eng/pipelines/templates/steps/vmr-join-verticals.yml create mode 100644 src/SourceBuild/content/eng/join-verticals.proj create mode 100644 src/SourceBuild/content/eng/join-verticals.ps1 create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs create mode 100644 src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 68de499d0c7d..c8526d806258 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -172,6 +172,12 @@ jobs: displayName: Publish Artifacts sbomEnabled: true + - output: buildArtifacts + PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml + ArtifactName: VerticalManifests + displayName: Publish Vertical Manifest + sbomEnabled: false + - ${{ if not(parameters.isBuiltFromVmr) }}: - output: pipelineArtifact displayName: Upload failed patches @@ -270,7 +276,7 @@ jobs: - script: | set extraBuildProperties= if not [${{ parameters.buildPass }}]==[] set extraBuildProperties=%extraBuildProperties% /p:DotNetBuildPass=${{ parameters.buildPass }} - call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} %extraBuildProperties% ${{ parameters.extraProperties }} + call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }} displayName: Build workingDirectory: ${{ variables.sourcesPath }} env: @@ -281,7 +287,7 @@ jobs: - ${{ if eq(parameters.runTests, 'True') }}: - script: | - call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }} + call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }} displayName: Run Tests workingDirectory: ${{ variables.sourcesPath }} timeoutInMinutes: ${{ variables.runTestsTimeout }} @@ -379,6 +385,8 @@ jobs: extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}" fi + extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)" + buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties" # Only use Docker when a container is specified @@ -454,6 +462,8 @@ jobs: customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}" fi + extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)" + if [[ -n "${{ parameters.extraProperties }}" ]]; then extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}" fi @@ -595,6 +605,19 @@ jobs: TargetFolder: $(Build.ArtifactStagingDirectory)/publishing displayName: Copy artifacts to Artifact Staging Directory + - ${{ if eq(parameters.targetOS, 'windows') }}: + - powershell: | + $sourcePath = "$(sourcesPath)/artifacts/VerticalManifest.xml" + $targetPath = "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml" + New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/manifests" -Force | Out-Null + Copy-Item $sourcePath -Destination $targetPath -Force + displayName: Copy vertical manifest to Artifact Staging Directory + - ${{ else }}: + - script: | + mkdir -p "$(Build.ArtifactStagingDirectory)/manifests" + cp "$(sourcesPath)/artifacts/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml" + displayName: Copy vertical manifest to Artifact Staging Directory + # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages - ${{ if ne(parameters.buildSourceOnly, 'true') }}: - task: CopyFiles@2 @@ -608,3 +631,10 @@ jobs: artifact: $(Agent.JobName)_Artifacts$(artifactsSuffix) displayName: Publish Artifacts continueOnError: true + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml + ArtifactName: VerticalManifests + displayName: Publish Vertical Manifest + condition: succeededOrFailed() diff --git a/eng/pipelines/templates/stages/vmr-final-join.yml b/eng/pipelines/templates/stages/vmr-final-join.yml new file mode 100644 index 000000000000..947e045c91ea --- /dev/null +++ b/eng/pipelines/templates/stages/vmr-final-join.yml @@ -0,0 +1,52 @@ +parameters: +# Branch of the VMR to use (to push to for internal builds) +- name: vmrBranch + type: string + default: $(Build.SourceBranch) + +- name: pool_Windows + type: object + default: + name: $(defaultPoolName) + image: $(poolImage_Windows) + demands: ImageOverride -equals $(poolImage_Windows) + os: windows + +stages: +- stage: VMR_Final_Join + displayName: VMR Final Join + dependsOn: VMR_Vertical_Build + condition: succeededOrFailed() + variables: + - template: ../variables/vmr-build.yml + parameters: + vmrBranch: ${{ parameters.vmrBranch }} + + jobs: + - job: FinalJoin + displayName: Final Build Pass + pool: ${{ parameters.pool_Windows }} + timeoutInMinutes: 240 + templateContext: + outputs: + - output: buildArtifacts + PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/MergedManifest.xml + ArtifactName: AssetManifests + displayName: Publish Merged Manifest + sbomEnabled: false + - output: buildArtifacts + PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/assets + ArtifactName: BlobArtifacts + displayName: Publish Blob Artifacts + sbomEnabled: false + - output: buildArtifacts + PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/packages + ArtifactName: PackageArtifacts + displayName: Publish Package Artifacts + sbomEnabled: false + steps: + - template: ../steps/vmr-join-verticals.yml + parameters: + dotNetBuildPass: final + primaryDependentJob: Windows_x64 + outputFolder: $(Build.ArtifactStagingDirectory)/artifacts \ No newline at end of file diff --git a/eng/pipelines/templates/steps/vmr-join-verticals.yml b/eng/pipelines/templates/steps/vmr-join-verticals.yml new file mode 100644 index 000000000000..2d4dfd0658a4 --- /dev/null +++ b/eng/pipelines/templates/steps/vmr-join-verticals.yml @@ -0,0 +1,37 @@ +parameters: +- name: dotNetBuildPass + type: string + default: final + +- name: primaryDependentJob + type: string + default: Windows_x64 + +- name: outputFolder + type: string + default: $(Build.ArtifactStagingDirectory)/artifacts + +steps: +- task: DownloadPipelineArtifact@2 + inputs: + artifactName: 'VerticalManifests' + targetPath: $(Build.ArtifactStagingDirectory)/VerticalManifests + checkDownloadedFiles: true + +- task: DownloadPipelineArtifact@2 + inputs: + artifactName: ${{ parameters.primaryDependentJob }}_Artifacts + targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts + checkDownloadedFiles: true + +- powershell: eng/join-verticals.ps1 + /p:VerticalManifestsPath=$(Build.ArtifactStagingDirectory)/VerticalManifests + /p:MainVertical=${{ parameters.primaryDependentJob }} + /p:DotNetBuildPass=${{ parameters.dotNetBuildPass }} + /p:BuildId=$(Build.BuildId) + /p:AzureDevOpsToken=$(System.AccessToken) + /p:AzureDevOpsBaseUri=$(System.CollectionUri) + /p:AzureDevOpsProject=$(System.TeamProject) + /p:MainVerticalArtifactsFolder=$(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts + /p:OutputFolder=${{ parameters.outputFolder }} + displayName: Join Verticals \ No newline at end of file diff --git a/src/SourceBuild/content/build.proj b/src/SourceBuild/content/build.proj index 2c8614b7619d..839ce4da7a9f 100644 --- a/src/SourceBuild/content/build.proj +++ b/src/SourceBuild/content/build.proj @@ -30,9 +30,12 @@ source-build non-source-build + 1 + $(DotNetBuildPass) + diff --git a/src/SourceBuild/content/eng/join-verticals.proj b/src/SourceBuild/content/eng/join-verticals.proj new file mode 100644 index 000000000000..caf981c0dffd --- /dev/null +++ b/src/SourceBuild/content/eng/join-verticals.proj @@ -0,0 +1,46 @@ + + + + $(NetCurrent) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/SourceBuild/content/eng/join-verticals.ps1 b/src/SourceBuild/content/eng/join-verticals.ps1 new file mode 100644 index 000000000000..bd19e2617cbe --- /dev/null +++ b/src/SourceBuild/content/eng/join-verticals.ps1 @@ -0,0 +1,33 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string][Alias('v')]$verbosity = "minimal", + [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties +) + +$useGlobalNuGetCache=$false +$ci = $true + +. $PSScriptRoot\common\tools.ps1 + +$project = Join-Path $EngRoot "join-verticals.proj" +$arguments = @() +$targets = "/t:JoinVerticals" + +try { + $bl = '/bl:' + (Join-Path $LogDir 'JoinVerticals.binlog') + + MSBuild -restore ` + $project ` + $bl ` + $targets ` + /p:Configuration=Release ` + @properties ` + @arguments +} +catch { + Write-Host $_.ScriptStackTrace + Write-PipelineTelemetryError -Category 'Build' -Message $_ + ExitWithExitCode 1 +} + +ExitWithExitCode 0 diff --git a/src/SourceBuild/content/eng/merge-asset-manifests.proj b/src/SourceBuild/content/eng/merge-asset-manifests.proj index fb05993aa871..4bfc9e7a9d6b 100644 --- a/src/SourceBuild/content/eng/merge-asset-manifests.proj +++ b/src/SourceBuild/content/eng/merge-asset-manifests.proj @@ -21,7 +21,8 @@ + VmrBuildNumber="$(BUILD_BUILDNUMBER)" + VerticalName="$(VerticalName)" /> diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml index 33b41f2c9163..97c1a32df276 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/ci.yml @@ -95,3 +95,6 @@ extends: scope: lite ${{ else }}: scope: full + + - ${{ if eq(variables['isSourceOnlyBuild'], 'false') }}: + - template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self \ No newline at end of file diff --git a/src/SourceBuild/content/eng/pipelines/pr.yml b/src/SourceBuild/content/eng/pipelines/pr.yml index 471f6a92b751..fd681be1a679 100644 --- a/src/SourceBuild/content/eng/pipelines/pr.yml +++ b/src/SourceBuild/content/eng/pipelines/pr.yml @@ -48,3 +48,6 @@ stages: scope: lite ${{ else }}: scope: full + +- ${{ if eq(variables['isSourceOnlyBuild'], 'false') }}: + - template: /src/sdk/eng/pipelines/templates/stages/vmr-post-build.yml@self \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs new file mode 100644 index 000000000000..3cb37c352be2 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs @@ -0,0 +1,197 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text; +using System.Threading.Tasks; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.UnifiedBuild.Tasks; + +public class AzureDevOpsClient : IDisposable +{ + private readonly HttpClient _httpClient; + private readonly TaskLoggingHelper _logger; + + private const string _azureDevOpsApiVersion = "7.1-preview.5"; + // download in 100 MB chunks + private const int _downloadBufferSize = 1024 * 1024 * 100; + private const int _httpTimeoutSeconds = 300; + + public AzureDevOpsClient( + string? azureDevOpsToken, + string azureDevOpsBaseUri, + string azureDevOpsProject, + TaskLoggingHelper logger) + { + + _logger = logger; + + _httpClient = new(new HttpClientHandler { CheckCertificateRevocationList = true }); + + _httpClient.BaseAddress = new Uri($"{azureDevOpsBaseUri}/{azureDevOpsProject}/_apis/"); + + _httpClient.Timeout = TimeSpan.FromSeconds(_httpTimeoutSeconds); + + if (!string.IsNullOrEmpty(azureDevOpsToken)) + { + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( + "Basic", + Convert.ToBase64String(Encoding.UTF8.GetBytes($":{azureDevOpsToken}"))); + } + } + + /// + /// Downloads a build artifact as zip file + /// + public async Task DownloadArtifactZip(string buildId, string artifactName, string downloadPath, int retryCount) + { + var artifactInformation = await GetArtifactInformation(buildId, artifactName, retryCount); + string downloadUrl = artifactInformation.Resource.DownloadUrl; + + _logger.LogMessage(MessageImportance.High, $"Downloading artifact zip from {downloadUrl}"); + + try + { + using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(downloadUrl, retryCount); + using Stream readStream = await httpResponse.Content.ReadAsStreamAsync(); + using FileStream writeStream = File.Create(downloadPath); + + await readStream.CopyToAsync(writeStream, _downloadBufferSize); + } + catch (Exception ex) + { + _logger.LogError($"Failed to download artifact zip: {ex.Message}"); + throw; + } + } + + public async Task DownloadSingleFileFromArtifact(string buildId, string artifactName, string itemId, string itemSubPath, string downloadPath, int retryCount) + { + try + { + var downloadFileUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&fileId={itemId}&fileName={itemSubPath}&api-version={_azureDevOpsApiVersion}"; + + _logger.LogMessage(MessageImportance.High, $"Downloading file {itemSubPath} from {downloadFileUrl}"); + + using HttpResponseMessage fileDownloadResponse = await ExecuteApiCallWithRetry(downloadFileUrl, retryCount); + using Stream readStream = await fileDownloadResponse.Content.ReadAsStreamAsync(); + using FileStream writeStream = File.Create(downloadPath); + + await readStream.CopyToAsync(writeStream, _downloadBufferSize); + } + catch (Exception ex) + { + _logger.LogError($"Failed to download file: {ex.Message}"); + throw; + } + } + + public async Task GetArtifactFilesInformation(string buildId, string artifactName, int retryCount) + { + var artifactInformation = await GetArtifactInformation(buildId, artifactName, retryCount); + string artifactId = artifactInformation.Resource.Data; + + var getManifestUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&fileId={artifactId}&fileName={artifactName}&api-version={_azureDevOpsApiVersion}"; + + try + { + using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(getManifestUrl, retryCount); + + ArtifactFiles filesInformation = await httpResponse.Content.ReadFromJsonAsync() + ?? throw new ArgumentException($"Couldn't parse AzDo response {httpResponse.Content} to {nameof(ArtifactFiles)}"); + + return filesInformation; + } + catch (Exception ex) + { + _logger.LogError($"Failed to download file: {ex.Message}"); + throw; + } + } + + public async Task GetArtifactInformation(string buildId, string artifactName, int retryCount) + { + string relativeUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&api-version={_azureDevOpsApiVersion}"; + + _logger.LogMessage(MessageImportance.High, $"Getting download link from {relativeUrl}"); + + try + { + using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(relativeUrl, retryCount); + + AzureDevOpsArtifactInformation artifactInformation = await httpResponse.Content.ReadFromJsonAsync() + ?? throw new ArgumentException($"Couldn't parse AzDo response {httpResponse.Content} to {nameof(AzureDevOpsArtifactInformation)}"); + + return artifactInformation; + } + catch(Exception ex) + { + _logger.LogError($"Failed to get artifact download URL: {ex.Message}"); + throw; + } + } + + private async Task ExecuteApiCallWithRetry(string relativeUrl, int retryCount) + { + int retriesRemaining = retryCount; + + while (true) + { + try + { + HttpResponseMessage httpResponse = await _httpClient.GetAsync(relativeUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); + + httpResponse.EnsureSuccessStatusCode(); + + return httpResponse; + } + catch (Exception ex) when (ex is HttpRequestException || ex is TaskCanceledException) + { + if (ex is HttpRequestException && ex.Message.Contains(((int)HttpStatusCode.NotFound).ToString())) + { + _logger.LogError($"Resource not found at {relativeUrl}: {ex.Message}"); + throw; + } + + if (ex is HttpRequestException && ex.Message.Contains(((int)HttpStatusCode.Unauthorized).ToString())) + { + _logger.LogError($"Failure to authenticate: {ex.Message}"); + throw; + } + + if (retriesRemaining <= 0) + { + _logger.LogError($"There was an error calling AzureDevOps API against URI '{relativeUrl}' " + + $"after {retryCount} attempts. Exception: {ex}"); + throw; + } + + _logger.LogWarning($"There was an error calling AzureDevOps API against URI against URI '{relativeUrl}'. " + + $"{retriesRemaining} attempts remaining. Exception: {ex.ToString()}"); + } + + --retriesRemaining; + await Task.Delay(5000); + } + } + + public void Dispose() + { + _httpClient.Dispose(); + } + + public record Blob(string Id, int Size); + public record ArtifactItem(string Path, Blob Blob); + public record ArtifactFiles(string ManifestFormat, ArtifactItem[] Items, string[] ManifestReferences); +} \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs new file mode 100644 index 000000000000..4f827076b0bb --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs @@ -0,0 +1,290 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using Microsoft.Build.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using static Microsoft.DotNet.UnifiedBuild.Tasks.AzureDevOpsClient; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.UnifiedBuild.Tasks; + +public class JoinVerticals : Microsoft.Build.Utilities.Task +{ + /// + /// Paths to Verticals Manifests + /// + [Required] + public required ITaskItem[] VerticalManifest { get; init; } + + /// + /// Optional subset of the Verticals we want to join, separated by ; + /// + /// Windows_x64;Windows_x86 + public string? VerticalSubSet { get; set; } + + /// + /// Name of the main vertical that we'll take all artifacts from + /// + [Required] + public required string MainVertical { get; init; } + + /// + /// Azure DevOps build id + /// + [Required] + public required string BuildId { get; init; } + + /// + /// Azure DevOps token, required scopes: "Build (read)", allowed to be empty when running in a public project + /// + public string? AzureDevOpsToken { get; set; } + + /// + /// Azure DevOps organization + /// + [Required] + public required string AzureDevOpsBaseUri { get; init; } + + /// + /// Azure DevOps project + /// + [Required] + public required string AzureDevOpsProject { get; init; } + + /// + /// Location of sownloaded artifacts from the main vertical + /// + [Required] + public required string MainVerticalArtifactsFolder { get; init; } + + /// + /// Folder where packages and assets will be stored + /// + [Required] + public required string OutputFolder { get; init; } + + private const string _packageElementName = "Package"; + private const string _blobElementName = "Blob"; + private const string _idAttribute = "Id"; + private const string _verticalNameAttribute = "VerticalName"; + private const string _artifactNameSuffix = "_Artifacts"; + private const string _assetsFolderName = "assets"; + private const string _packagesFolderName = "packages"; + private const int _retryCount = 10; + + public override bool Execute() + { + ExecuteAsync().Wait(); + return !Log.HasLoggedErrors; + } + + private async Task ExecuteAsync() + { + List verticalManifests = VerticalManifest.Select(xmlPath => XDocument.Load(xmlPath.ItemSpec)).ToList(); + + if (!string.IsNullOrEmpty(VerticalSubSet)) + { + var verticalSubSet = VerticalSubSet.Split(';'); + verticalManifests = verticalManifests.Where(manifest => verticalSubSet.Contains(GetRequiredRootAttribute(manifest, _verticalNameAttribute))).ToList(); + } + + // Find the main manifest, and remove it from the list + XDocument mainVerticalManifest = verticalManifests.FirstOrDefault(manifest => GetRequiredRootAttribute(manifest, _verticalNameAttribute) == MainVertical) + ?? throw new ArgumentException($"Couldn't find main vertical manifest {MainVertical} in vertical manifest list"); + + int removed = verticalManifests.RemoveAll(manifest => GetRequiredRootAttribute(manifest, _verticalNameAttribute) == MainVertical); + + if (removed > 1) + { + throw new ArgumentException($"Found more than one main vertical manifest {MainVertical} in vertical manifest list"); + } + + if (!Directory.Exists(MainVerticalArtifactsFolder)) + { + throw new ArgumentException($"Main vertical artifacts directory {MainVerticalArtifactsFolder} not found."); + } + + string mainVerticalName = GetRequiredRootAttribute(mainVerticalManifest, _verticalNameAttribute); + + Dictionary packageElements = []; + Dictionary blobElements = []; + + List addedPackageIds = AddMissingElements(packageElements, mainVerticalManifest, _packageElementName); + List addedBlobIds = AddMissingElements(blobElements, mainVerticalManifest, _blobElementName); + + CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _packagesFolderName), Path.Combine(OutputFolder, _packagesFolderName)); + CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _assetsFolderName), Path.Combine(OutputFolder, _assetsFolderName)); + + foreach (XDocument verticalManifest in verticalManifests) + { + string verticalName = GetRequiredRootAttribute(verticalManifest, _verticalNameAttribute); + + addedPackageIds = AddMissingElements(packageElements, verticalManifest, _packageElementName); + addedBlobIds = AddMissingElements(blobElements, verticalManifest, _blobElementName); + + if (addedPackageIds.Count > 0 || addedBlobIds.Count > 0) + { + await DownloadArtifactFiles( + BuildId, + $"{verticalName}{_artifactNameSuffix}", + addedPackageIds, + addedBlobIds); + } + } + + // Create MergedManifest.xml + // taking the attributes from the main manifest + XElement mainManifestRoot = verticalManifests.First().Root + ?? throw new ArgumentException("The root element of the vertical manifest is null."); + mainManifestRoot.Attribute(_verticalNameAttribute)!.Remove(); + + string manifestOutputPath = Path.Combine(OutputFolder, "MergedManifest.xml"); + XDocument mergedManifest = new(new XElement( + mainManifestRoot.Name, + mainManifestRoot.Attributes(), + packageElements.Values.OrderBy(elem => elem.Attribute(_idAttribute)?.Value), + blobElements.Values.OrderBy(elem => elem.Attribute(_idAttribute)?.Value))); + + File.WriteAllText(manifestOutputPath, mergedManifest.ToString()); + } + + /// + /// Downloads specified packages and symbols from a specific build artifact and stores them in an output folder, + /// either flat or with the same relative path as in the artifact. + /// + private async Task DownloadArtifactFiles( + string buildId, + string artifactName, + List packageFileNames, + List assetFileNames) + { + string packagesOutputPath = Path.Combine(OutputFolder, _packagesFolderName); + string assetsOutputPath = Path.Combine(OutputFolder, _assetsFolderName); + + using AzureDevOpsClient azureDevOpsClient = new(AzureDevOpsToken, AzureDevOpsBaseUri, AzureDevOpsProject, Log); + + ArtifactFiles filesInformation = await azureDevOpsClient.GetArtifactFilesInformation(buildId, artifactName, _retryCount); + + foreach (var package in packageFileNames) + { + await DownloadFileFromArtifact(filesInformation, artifactName, azureDevOpsClient, buildId, package, packagesOutputPath); + } + + foreach (var asset in assetFileNames) + { + await DownloadFileFromArtifact(filesInformation, artifactName, azureDevOpsClient, buildId, asset, assetsOutputPath); + } + } + + private async Task DownloadFileFromArtifact(ArtifactFiles artifactFilesMetadata, string azureDevOpsArtifact, AzureDevOpsClient azureDevOpsClient, string buildId, string manifestFile, string destinationDirectory) + { + ArtifactItem fileItem; + + var matchingFilePaths = artifactFilesMetadata.Items.Where(f => Path.GetFileName(f.Path) == Path.GetFileName(manifestFile)); + + if (!matchingFilePaths.Any()) + { + throw new ArgumentException($"File {manifestFile} not found in source files."); + } + + if (matchingFilePaths.Count() > 1) + { + if (manifestFile.Contains("productVersion.txt")) + { + fileItem = matchingFilePaths.First(); + } + else + { + fileItem = matchingFilePaths.SingleOrDefault(f => f.Path.Contains(manifestFile) || f.Path.Contains(manifestFile.Replace("/", @"\"))) + ?? throw new ArgumentException($"File {manifestFile} not found in source files."); + } + } + else + { + fileItem = matchingFilePaths.Single(); + } + + string itemId = fileItem.Blob.Id; + string artifactSubPath = fileItem.Path; + + string destinationFilePath = Path.Combine(destinationDirectory, Path.GetFileName(manifestFile)); + + try + { + await azureDevOpsClient.DownloadSingleFileFromArtifact(buildId, azureDevOpsArtifact, itemId, artifactSubPath, destinationFilePath, _retryCount); + } + catch (Exception ex) + { + Log.LogError($"Failed to download file {manifestFile} from artifact {azureDevOpsArtifact}: {ex.Message}"); + throw; + } + } + + /// + /// Copy all files from the source directory to the destination directory, + /// in a flat layout + /// + private void CopyMainVerticalAssets(string sourceDirectory, string destinationDirectory) + { + var sourceFiles = Directory.EnumerateFiles(sourceDirectory, "*", SearchOption.AllDirectories); + + if (!Directory.Exists(destinationDirectory)) + { + Directory.CreateDirectory(destinationDirectory); + } + + foreach (var sourceFile in sourceFiles) + { + string destinationFilePath = Path.Combine(destinationDirectory, Path.GetFileName(sourceFile)); + + Log.LogMessage(MessageImportance.High, $"Copying {sourceFile} to {destinationFilePath}"); + File.Copy(sourceFile, destinationFilePath, true); + } + } + + /// + /// Find the artifacts from the vertical manifest that are not already in the dictionary and add them + /// Return a list of the added artifact ids + /// + private List AddMissingElements(Dictionary addedArtifacts, XDocument verticalManifest, string elementName) + { + List addedFiles = []; + + string verticalName = verticalManifest.Root!.Attribute(_verticalNameAttribute)!.Value; + + foreach (XElement artifactElement in verticalManifest.Descendants(elementName)) + { + string elementId = artifactElement.Attribute(_idAttribute)?.Value + ?? throw new ArgumentException($"Required attribute '{_idAttribute}' not found in {elementName} element."); + + if (addedArtifacts.TryAdd(elementId, artifactElement)) + { + if (elementName == _packageElementName) + { + string version = artifactElement.Attribute("Version")?.Value + ?? throw new ArgumentException($"Required attribute 'Version' not found in {elementName} element."); + + elementId += $".{version}.nupkg"; + } + + addedFiles.Add(elementId); + Log.LogMessage(MessageImportance.High, $"Taking {elementName} '{elementId}' from '{verticalName}'"); + } + } + + return addedFiles; + } + + private static string GetRequiredRootAttribute(XDocument document, string attributeName) + { + return document.Root?.Attribute(attributeName)?.Value + ?? throw new ArgumentException($"Required attribute '{attributeName}' not found in root element."); + } +} \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs index eff0413e88b8..c2b87171ad2c 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs @@ -33,7 +33,13 @@ public class MergeAssetManifests : Task /// public string VmrBuildNumber { get; set; } = string.Empty; + /// + /// Vmr Vertical Name, e.g. "Android_Shortstack_arm". Allowed to be empty for non official builds. + /// + public string VerticalName { get; set; } = string.Empty; + private static readonly string _buildIdAttribute = "BuildId"; + private static readonly string _verticalNameAttribute = "VerticalName"; private static readonly string _azureDevOpsBuildNumberAttribute = "AzureDevOpsBuildNumber"; private static readonly string[] _ignoredAttributes = [ _buildIdAttribute, @@ -54,6 +60,7 @@ public override bool Execute() // Set the BuildId and AzureDevOpsBuildNumber attributes to the value of VmrBuildNumber mergedManifestRoot.SetAttributeValue(_buildIdAttribute, VmrBuildNumber); mergedManifestRoot.SetAttributeValue(_azureDevOpsBuildNumberAttribute, VmrBuildNumber); + mergedManifestRoot.SetAttributeValue(_verticalNameAttribute, VerticalName); List packageElements = new(); List blobElements = new(); @@ -70,6 +77,7 @@ public override bool Execute() XDocument verticalManifest = new(new XElement(mergedManifestRoot.Name, mergedManifestRoot.Attributes(), packageElements, blobElements)); File.WriteAllText(MergedAssetManifestOutputPath, verticalManifest.ToString()); + Log.LogMessage(MessageImportance.High, $"Merged asset manifest written to {MergedAssetManifestOutputPath}"); return !Log.HasLoggedErrors; } diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs new file mode 100644 index 000000000000..d7d2ba96bd78 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +#nullable enable + +namespace Microsoft.DotNet.UnifiedBuild.Tasks; + +public record AzureDevOpsArtifactInformation(int Id, string Name, string Source, AzdoArtifactResources Resource); +public record AzdoArtifactResources(string Type, string Data, AzdoArtifactProperties Properties, string Url, string DownloadUrl); +public record AzdoArtifactProperties(string RootId, string Artifactsize, string HashType, string DomainId); \ No newline at end of file From 260f35625058d19e917c7d163523578e9607cafa Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Wed, 25 Sep 2024 16:45:09 +0200 Subject: [PATCH 02/18] exclude verticals from join --- .../content/eng/join-verticals.proj | 18 ++++++++--- .../AzureDevOpsClient.cs | 4 ++- .../JoinVerticals.cs | 30 +++++++++---------- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/SourceBuild/content/eng/join-verticals.proj b/src/SourceBuild/content/eng/join-verticals.proj index caf981c0dffd..32151d0cc958 100644 --- a/src/SourceBuild/content/eng/join-verticals.proj +++ b/src/SourceBuild/content/eng/join-verticals.proj @@ -25,11 +25,21 @@ + - - - + ExcludedVerticals="@(ExcludedVerticals)" /> \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs index 3cb37c352be2..d28e7ca24e00 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs @@ -104,6 +104,8 @@ public async Task GetArtifactFilesInformation(string buildId, str var getManifestUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&fileId={artifactId}&fileName={artifactName}&api-version={_azureDevOpsApiVersion}"; + _logger.LogMessage(MessageImportance.High, $"Getting {artifactName} artifact manifest"); + try { using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(getManifestUrl, retryCount); @@ -124,7 +126,7 @@ public async Task GetArtifactInformation(string { string relativeUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&api-version={_azureDevOpsApiVersion}"; - _logger.LogMessage(MessageImportance.High, $"Getting download link from {relativeUrl}"); + _logger.LogMessage(MessageImportance.High, $"Getting {artifactName} metadata from {relativeUrl}"); try { diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs index 4f827076b0bb..35489fc11c52 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs @@ -24,10 +24,9 @@ public class JoinVerticals : Microsoft.Build.Utilities.Task public required ITaskItem[] VerticalManifest { get; init; } /// - /// Optional subset of the Verticals we want to join, separated by ; + /// Optional subset of the Verticals to exclude from the join /// - /// Windows_x64;Windows_x86 - public string? VerticalSubSet { get; set; } + public required ITaskItem[] ExcludedVerticals { get; set; } /// /// Name of the main vertical that we'll take all artifacts from @@ -89,23 +88,10 @@ private async Task ExecuteAsync() { List verticalManifests = VerticalManifest.Select(xmlPath => XDocument.Load(xmlPath.ItemSpec)).ToList(); - if (!string.IsNullOrEmpty(VerticalSubSet)) - { - var verticalSubSet = VerticalSubSet.Split(';'); - verticalManifests = verticalManifests.Where(manifest => verticalSubSet.Contains(GetRequiredRootAttribute(manifest, _verticalNameAttribute))).ToList(); - } - // Find the main manifest, and remove it from the list XDocument mainVerticalManifest = verticalManifests.FirstOrDefault(manifest => GetRequiredRootAttribute(manifest, _verticalNameAttribute) == MainVertical) ?? throw new ArgumentException($"Couldn't find main vertical manifest {MainVertical} in vertical manifest list"); - int removed = verticalManifests.RemoveAll(manifest => GetRequiredRootAttribute(manifest, _verticalNameAttribute) == MainVertical); - - if (removed > 1) - { - throw new ArgumentException($"Found more than one main vertical manifest {MainVertical} in vertical manifest list"); - } - if (!Directory.Exists(MainVerticalArtifactsFolder)) { throw new ArgumentException($"Main vertical artifacts directory {MainVerticalArtifactsFolder} not found."); @@ -126,6 +112,18 @@ private async Task ExecuteAsync() { string verticalName = GetRequiredRootAttribute(verticalManifest, _verticalNameAttribute); + // We already processed the main vertical + if (verticalName == MainVertical) + { + continue; + } + + if (ExcludedVerticals.Any(v => v.ItemSpec == verticalName)) + { + Log.LogMessage(MessageImportance.High, $"Excluding {verticalName} from the join"); + continue; + } + addedPackageIds = AddMissingElements(packageElements, verticalManifest, _packageElementName); addedBlobIds = AddMissingElements(blobElements, verticalManifest, _blobElementName); From c5c14ac44b78baedfd06b23d994027f8c9e834cb Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Wed, 25 Sep 2024 17:02:19 +0200 Subject: [PATCH 03/18] add comment --- eng/pipelines/templates/jobs/vmr-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index c8526d806258..853f86423271 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -172,6 +172,7 @@ jobs: displayName: Publish Artifacts sbomEnabled: true + # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs - output: buildArtifacts PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml ArtifactName: VerticalManifests @@ -632,6 +633,7 @@ jobs: displayName: Publish Artifacts continueOnError: true + # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs - task: PublishBuildArtifacts@1 inputs: PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml From 0aa6762cec2d6c80e7ceecff08ff4a85d56c00fd Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Fri, 27 Sep 2024 11:37:43 +0200 Subject: [PATCH 04/18] list duplicate assets --- .../content/eng/join-verticals.proj | 22 ++------- .../JoinVerticals.cs | 46 +++++++++++-------- 2 files changed, 33 insertions(+), 35 deletions(-) diff --git a/src/SourceBuild/content/eng/join-verticals.proj b/src/SourceBuild/content/eng/join-verticals.proj index 32151d0cc958..2136f3e79c2f 100644 --- a/src/SourceBuild/content/eng/join-verticals.proj +++ b/src/SourceBuild/content/eng/join-verticals.proj @@ -14,7 +14,7 @@ - + @@ -24,20 +24,9 @@ - - + @@ -49,8 +38,7 @@ AzureDevOpsBaseUri="$(AzureDevOpsBaseUri)" AzureDevOpsProject="$(AzureDevOpsProject)" MainVerticalArtifactsFolder="$(MainVerticalArtifactsFolder)" - OutputFolder="$(OutputFolder)" - ExcludedVerticals="@(ExcludedVerticals)" /> + OutputFolder="$(OutputFolder)" /> \ No newline at end of file diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs index 35489fc11c52..f399ff5b5d2f 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs @@ -23,11 +23,6 @@ public class JoinVerticals : Microsoft.Build.Utilities.Task [Required] public required ITaskItem[] VerticalManifest { get; init; } - /// - /// Optional subset of the Verticals to exclude from the join - /// - public required ITaskItem[] ExcludedVerticals { get; set; } - /// /// Name of the main vertical that we'll take all artifacts from /// @@ -78,6 +73,8 @@ public class JoinVerticals : Microsoft.Build.Utilities.Task private const string _packagesFolderName = "packages"; private const int _retryCount = 10; + private Dictionary> duplicatedItems = new(); + public override bool Execute() { ExecuteAsync().Wait(); @@ -88,7 +85,6 @@ private async Task ExecuteAsync() { List verticalManifests = VerticalManifest.Select(xmlPath => XDocument.Load(xmlPath.ItemSpec)).ToList(); - // Find the main manifest, and remove it from the list XDocument mainVerticalManifest = verticalManifests.FirstOrDefault(manifest => GetRequiredRootAttribute(manifest, _verticalNameAttribute) == MainVertical) ?? throw new ArgumentException($"Couldn't find main vertical manifest {MainVertical} in vertical manifest list"); @@ -99,8 +95,8 @@ private async Task ExecuteAsync() string mainVerticalName = GetRequiredRootAttribute(mainVerticalManifest, _verticalNameAttribute); - Dictionary packageElements = []; - Dictionary blobElements = []; + Dictionary packageElements = []; + Dictionary blobElements = []; List addedPackageIds = AddMissingElements(packageElements, mainVerticalManifest, _packageElementName); List addedBlobIds = AddMissingElements(blobElements, mainVerticalManifest, _blobElementName); @@ -118,12 +114,6 @@ private async Task ExecuteAsync() continue; } - if (ExcludedVerticals.Any(v => v.ItemSpec == verticalName)) - { - Log.LogMessage(MessageImportance.High, $"Excluding {verticalName} from the join"); - continue; - } - addedPackageIds = AddMissingElements(packageElements, verticalManifest, _packageElementName); addedBlobIds = AddMissingElements(blobElements, verticalManifest, _blobElementName); @@ -147,10 +137,17 @@ await DownloadArtifactFiles( XDocument mergedManifest = new(new XElement( mainManifestRoot.Name, mainManifestRoot.Attributes(), - packageElements.Values.OrderBy(elem => elem.Attribute(_idAttribute)?.Value), - blobElements.Values.OrderBy(elem => elem.Attribute(_idAttribute)?.Value))); + packageElements.Values.Select(v => v.Element).OrderBy(elem => elem.Attribute(_idAttribute)?.Value), + blobElements.Values.Select(v => v.Element).OrderBy(elem => elem.Attribute(_idAttribute)?.Value))); File.WriteAllText(manifestOutputPath, mergedManifest.ToString()); + + Log.LogMessage(MessageImportance.High, $"### Duplicate items found in the following verticals: ###"); + + foreach (var item in duplicatedItems) + { + Log.LogMessage(MessageImportance.High, $"Item: {item.Key} -- Produced by: {string.Join(", ", item.Value)}"); + } } /// @@ -251,7 +248,7 @@ private void CopyMainVerticalAssets(string sourceDirectory, string destinationDi /// Find the artifacts from the vertical manifest that are not already in the dictionary and add them /// Return a list of the added artifact ids /// - private List AddMissingElements(Dictionary addedArtifacts, XDocument verticalManifest, string elementName) + private List AddMissingElements(Dictionary addedArtifacts, XDocument verticalManifest, string elementName) { List addedFiles = []; @@ -262,7 +259,7 @@ private List AddMissingElements(Dictionary addedArtifa string elementId = artifactElement.Attribute(_idAttribute)?.Value ?? throw new ArgumentException($"Required attribute '{_idAttribute}' not found in {elementName} element."); - if (addedArtifacts.TryAdd(elementId, artifactElement)) + if (addedArtifacts.TryAdd(elementId, new AddedElement(verticalName, artifactElement))) { if (elementName == _packageElementName) { @@ -275,6 +272,17 @@ private List AddMissingElements(Dictionary addedArtifa addedFiles.Add(elementId); Log.LogMessage(MessageImportance.High, $"Taking {elementName} '{elementId}' from '{verticalName}'"); } + else + { + AddedElement previouslyAddedArtifact = addedArtifacts[elementId]; + if (previouslyAddedArtifact.VerticalName != MainVertical) + { + if (!duplicatedItems.TryAdd(elementId, new List { verticalName, previouslyAddedArtifact.VerticalName })) + { + duplicatedItems[elementId].Add(verticalName); + } + } + } } return addedFiles; @@ -285,4 +293,6 @@ private static string GetRequiredRootAttribute(XDocument document, string attrib return document.Root?.Attribute(attributeName)?.Value ?? throw new ArgumentException($"Required attribute '{attributeName}' not found in root element."); } + + private record AddedElement(string VerticalName, XElement Element); } \ No newline at end of file From d1784b776a714f209b470e38811e6fd2d8ae8994 Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Fri, 27 Sep 2024 11:42:21 +0200 Subject: [PATCH 05/18] parallelism --- .../JoinVerticals.cs | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs index f399ff5b5d2f..f6424cb28baa 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs @@ -9,6 +9,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; using System.Xml.Linq; using static Microsoft.DotNet.UnifiedBuild.Tasks.AzureDevOpsClient; using Task = System.Threading.Tasks.Task; @@ -77,7 +78,7 @@ public class JoinVerticals : Microsoft.Build.Utilities.Task public override bool Execute() { - ExecuteAsync().Wait(); + ExecuteAsync().GetAwaiter().GetResult(); return !Log.HasLoggedErrors; } @@ -104,6 +105,9 @@ private async Task ExecuteAsync() CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _packagesFolderName), Path.Combine(OutputFolder, _packagesFolderName)); CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _assetsFolderName), Path.Combine(OutputFolder, _assetsFolderName)); + using var clientThrottle = new SemaphoreSlim(16, 16); + List downloadTasks = new(); + foreach (XDocument verticalManifest in verticalManifests) { string verticalName = GetRequiredRootAttribute(verticalManifest, _verticalNameAttribute); @@ -119,14 +123,18 @@ private async Task ExecuteAsync() if (addedPackageIds.Count > 0 || addedBlobIds.Count > 0) { - await DownloadArtifactFiles( - BuildId, - $"{verticalName}{_artifactNameSuffix}", - addedPackageIds, - addedBlobIds); + downloadTasks.Add( + DownloadArtifactFiles( + BuildId, + $"{verticalName}{_artifactNameSuffix}", + addedPackageIds, + addedBlobIds, + clientThrottle)); } } + await Task.WhenAll(downloadTasks); + // Create MergedManifest.xml // taking the attributes from the main manifest XElement mainManifestRoot = verticalManifests.First().Root @@ -158,23 +166,33 @@ private async Task DownloadArtifactFiles( string buildId, string artifactName, List packageFileNames, - List assetFileNames) + List assetFileNames, + SemaphoreSlim clientThrottle) { string packagesOutputPath = Path.Combine(OutputFolder, _packagesFolderName); string assetsOutputPath = Path.Combine(OutputFolder, _assetsFolderName); using AzureDevOpsClient azureDevOpsClient = new(AzureDevOpsToken, AzureDevOpsBaseUri, AzureDevOpsProject, Log); - ArtifactFiles filesInformation = await azureDevOpsClient.GetArtifactFilesInformation(buildId, artifactName, _retryCount); - - foreach (var package in packageFileNames) + try { - await DownloadFileFromArtifact(filesInformation, artifactName, azureDevOpsClient, buildId, package, packagesOutputPath); - } + await clientThrottle.WaitAsync(); + + ArtifactFiles filesInformation = await azureDevOpsClient.GetArtifactFilesInformation(buildId, artifactName, _retryCount); - foreach (var asset in assetFileNames) + foreach (var package in packageFileNames) + { + await DownloadFileFromArtifact(filesInformation, artifactName, azureDevOpsClient, buildId, package, packagesOutputPath); + } + + foreach (var asset in assetFileNames) + { + await DownloadFileFromArtifact(filesInformation, artifactName, azureDevOpsClient, buildId, asset, assetsOutputPath); + } + } + finally { - await DownloadFileFromArtifact(filesInformation, artifactName, azureDevOpsClient, buildId, asset, assetsOutputPath); + clientThrottle.Release(); } } From 745f5522a5255050a60897212c53bc5a78eeeb14 Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Fri, 27 Sep 2024 11:51:37 +0200 Subject: [PATCH 06/18] download build artifact --- eng/pipelines/templates/steps/vmr-join-verticals.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/vmr-join-verticals.yml b/eng/pipelines/templates/steps/vmr-join-verticals.yml index 2d4dfd0658a4..38ad876f25f3 100644 --- a/eng/pipelines/templates/steps/vmr-join-verticals.yml +++ b/eng/pipelines/templates/steps/vmr-join-verticals.yml @@ -12,10 +12,10 @@ parameters: default: $(Build.ArtifactStagingDirectory)/artifacts steps: -- task: DownloadPipelineArtifact@2 +- task: DownloadBuildArtifacts@1 inputs: artifactName: 'VerticalManifests' - targetPath: $(Build.ArtifactStagingDirectory)/VerticalManifests + downloadPath: $(Build.ArtifactStagingDirectory)/VerticalManifests checkDownloadedFiles: true - task: DownloadPipelineArtifact@2 From 6f70d33416371be043ab41b42f7289dfc70dad0c Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Fri, 27 Sep 2024 15:02:35 +0200 Subject: [PATCH 07/18] fix path --- eng/pipelines/templates/steps/vmr-join-verticals.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/vmr-join-verticals.yml b/eng/pipelines/templates/steps/vmr-join-verticals.yml index 38ad876f25f3..50b1866b098f 100644 --- a/eng/pipelines/templates/steps/vmr-join-verticals.yml +++ b/eng/pipelines/templates/steps/vmr-join-verticals.yml @@ -15,7 +15,7 @@ steps: - task: DownloadBuildArtifacts@1 inputs: artifactName: 'VerticalManifests' - downloadPath: $(Build.ArtifactStagingDirectory)/VerticalManifests + downloadPath: $(Build.ArtifactStagingDirectory) checkDownloadedFiles: true - task: DownloadPipelineArtifact@2 From 82c18d04336d56e52f0df8fe7f18e40999216437 Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Fri, 27 Sep 2024 15:07:25 +0200 Subject: [PATCH 08/18] add comment --- .../tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs index f6424cb28baa..6395a591db91 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs @@ -209,6 +209,7 @@ private async Task DownloadFileFromArtifact(ArtifactFiles artifactFilesMetadata, if (matchingFilePaths.Count() > 1) { + // Picking the first one until https://github.com/dotnet/source-build/issues/4596 is resolved if (manifestFile.Contains("productVersion.txt")) { fileItem = matchingFilePaths.First(); From ed711dcbc31261876a9fcb8fda50cd671628e16f Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Mon, 7 Oct 2024 16:18:40 +0200 Subject: [PATCH 09/18] add extra parallelism --- .../JoinVerticals.cs | 121 ++++++++++-------- 1 file changed, 68 insertions(+), 53 deletions(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs index 6395a591db91..0f7023e4577b 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs @@ -102,8 +102,11 @@ private async Task ExecuteAsync() List addedPackageIds = AddMissingElements(packageElements, mainVerticalManifest, _packageElementName); List addedBlobIds = AddMissingElements(blobElements, mainVerticalManifest, _blobElementName); - CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _packagesFolderName), Path.Combine(OutputFolder, _packagesFolderName)); - CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _assetsFolderName), Path.Combine(OutputFolder, _assetsFolderName)); + string packagesOutputDirectory = Path.Combine(OutputFolder, _packagesFolderName); + string blobsOutputDirectory = Path.Combine(OutputFolder, _assetsFolderName); + + CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _packagesFolderName), packagesOutputDirectory); + CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _assetsFolderName), blobsOutputDirectory); using var clientThrottle = new SemaphoreSlim(16, 16); List downloadTasks = new(); @@ -121,14 +124,25 @@ private async Task ExecuteAsync() addedPackageIds = AddMissingElements(packageElements, verticalManifest, _packageElementName); addedBlobIds = AddMissingElements(blobElements, verticalManifest, _blobElementName); - if (addedPackageIds.Count > 0 || addedBlobIds.Count > 0) + if (addedPackageIds.Count > 0) { downloadTasks.Add( DownloadArtifactFiles( BuildId, $"{verticalName}{_artifactNameSuffix}", addedPackageIds, + packagesOutputDirectory, + clientThrottle)); + } + + if (addedBlobIds.Count > 0) + { + downloadTasks.Add( + DownloadArtifactFiles( + BuildId, + $"{verticalName}{_artifactNameSuffix}", addedBlobIds, + blobsOutputDirectory, clientThrottle)); } } @@ -159,79 +173,76 @@ private async Task ExecuteAsync() } /// - /// Downloads specified packages and symbols from a specific build artifact and stores them in an output folder, - /// either flat or with the same relative path as in the artifact. + /// Downloads specified packages and symbols from a specific build artifact and stores them in an output folder /// private async Task DownloadArtifactFiles( string buildId, string artifactName, - List packageFileNames, - List assetFileNames, + List fileNamesToDownload, + string outputDirectory, SemaphoreSlim clientThrottle) { - string packagesOutputPath = Path.Combine(OutputFolder, _packagesFolderName); - string assetsOutputPath = Path.Combine(OutputFolder, _assetsFolderName); - using AzureDevOpsClient azureDevOpsClient = new(AzureDevOpsToken, AzureDevOpsBaseUri, AzureDevOpsProject, Log); + ArtifactFiles filesInformation = await azureDevOpsClient.GetArtifactFilesInformation(buildId, artifactName, _retryCount); + + await Task.WhenAll(fileNamesToDownload.Select(async fileName => + await DownloadFileFromArtifact( + filesInformation, + artifactName, + azureDevOpsClient, + buildId, + fileName, + outputDirectory, + clientThrottle))); + } + + private async Task DownloadFileFromArtifact( + ArtifactFiles artifactFilesMetadata, + string azureDevOpsArtifact, + AzureDevOpsClient azureDevOpsClient, + string buildId, + string manifestFile, + string destinationDirectory, + SemaphoreSlim clientThrottle) + { try { await clientThrottle.WaitAsync(); + + ArtifactItem fileItem; - ArtifactFiles filesInformation = await azureDevOpsClient.GetArtifactFilesInformation(buildId, artifactName, _retryCount); + var matchingFilePaths = artifactFilesMetadata.Items.Where(f => Path.GetFileName(f.Path) == Path.GetFileName(manifestFile)); - foreach (var package in packageFileNames) + if (!matchingFilePaths.Any()) { - await DownloadFileFromArtifact(filesInformation, artifactName, azureDevOpsClient, buildId, package, packagesOutputPath); + throw new ArgumentException($"File {manifestFile} not found in source files."); } - foreach (var asset in assetFileNames) + if (matchingFilePaths.Count() > 1) { - await DownloadFileFromArtifact(filesInformation, artifactName, azureDevOpsClient, buildId, asset, assetsOutputPath); - } - } - finally - { - clientThrottle.Release(); - } - } - - private async Task DownloadFileFromArtifact(ArtifactFiles artifactFilesMetadata, string azureDevOpsArtifact, AzureDevOpsClient azureDevOpsClient, string buildId, string manifestFile, string destinationDirectory) - { - ArtifactItem fileItem; - - var matchingFilePaths = artifactFilesMetadata.Items.Where(f => Path.GetFileName(f.Path) == Path.GetFileName(manifestFile)); - - if (!matchingFilePaths.Any()) - { - throw new ArgumentException($"File {manifestFile} not found in source files."); - } - - if (matchingFilePaths.Count() > 1) - { - // Picking the first one until https://github.com/dotnet/source-build/issues/4596 is resolved - if (manifestFile.Contains("productVersion.txt")) - { - fileItem = matchingFilePaths.First(); + // Picking the first one until https://github.com/dotnet/source-build/issues/4596 is resolved + if (manifestFile.Contains("productVersion.txt")) + { + fileItem = matchingFilePaths.First(); + } + else + { + fileItem = matchingFilePaths + .SingleOrDefault(f => f.Path.Contains(manifestFile) || f.Path.Contains(manifestFile.Replace("/", @"\"))) + ?? throw new ArgumentException($"File {manifestFile} not found in source files."); + } } else { - fileItem = matchingFilePaths.SingleOrDefault(f => f.Path.Contains(manifestFile) || f.Path.Contains(manifestFile.Replace("/", @"\"))) - ?? throw new ArgumentException($"File {manifestFile} not found in source files."); + fileItem = matchingFilePaths.Single(); } - } - else - { - fileItem = matchingFilePaths.Single(); - } - string itemId = fileItem.Blob.Id; - string artifactSubPath = fileItem.Path; + string itemId = fileItem.Blob.Id; + string artifactSubPath = fileItem.Path; - string destinationFilePath = Path.Combine(destinationDirectory, Path.GetFileName(manifestFile)); + string destinationFilePath = Path.Combine(destinationDirectory, Path.GetFileName(manifestFile)); - try - { await azureDevOpsClient.DownloadSingleFileFromArtifact(buildId, azureDevOpsArtifact, itemId, artifactSubPath, destinationFilePath, _retryCount); } catch (Exception ex) @@ -239,6 +250,10 @@ private async Task DownloadFileFromArtifact(ArtifactFiles artifactFilesMetadata, Log.LogError($"Failed to download file {manifestFile} from artifact {azureDevOpsArtifact}: {ex.Message}"); throw; } + finally + { + clientThrottle.Release(); + } } /// @@ -314,4 +329,4 @@ private static string GetRequiredRootAttribute(XDocument document, string attrib } private record AddedElement(string VerticalName, XElement Element); -} \ No newline at end of file +} From c34e878cacb9217caa2068b77340b45359ee0ab7 Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Mon, 7 Oct 2024 16:22:54 +0200 Subject: [PATCH 10/18] Update src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs Co-authored-by: Matt Mitchell --- .../tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs index 0f7023e4577b..e140ca186ee3 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs @@ -25,7 +25,7 @@ public class JoinVerticals : Microsoft.Build.Utilities.Task public required ITaskItem[] VerticalManifest { get; init; } /// - /// Name of the main vertical that we'll take all artifacts from + /// Name of the main vertical that we'll take all artifacts from if they exist in this vertical, and at least one other vertical. /// [Required] public required string MainVertical { get; init; } From a08dc294a1bf123b721784e9d40c659e408b3b05 Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Wed, 9 Oct 2024 11:27:05 +0200 Subject: [PATCH 11/18] add comment --- .../Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs index 0f7023e4577b..37f13cedc0e4 100644 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs @@ -228,8 +228,11 @@ private async Task DownloadFileFromArtifact( } else { + // For some files it's not enough to compare the filename because they have 2 copies in the artifact + // e.g. assets/Release/dotnet-sdk-*-win-x64.zip and assets/Release/Sdk/*/dotnet-sdk-*-win-x64.zip + // In this case take the one matching the full path from the manifest fileItem = matchingFilePaths - .SingleOrDefault(f => f.Path.Contains(manifestFile) || f.Path.Contains(manifestFile.Replace("/", @"\"))) + .SingleOrDefault(f => f.Path.EndsWith(manifestFile) || f.Path.EndsWith(manifestFile.Replace("/", @"\"))) ?? throw new ArgumentException($"File {manifestFile} not found in source files."); } } From ea8dd9539e73e11e0e857a51af521980847d052a Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Tue, 15 Oct 2024 18:05:14 +0200 Subject: [PATCH 12/18] Update src/SourceBuild/content/eng/join-verticals.proj Co-authored-by: Viktor Hofer --- src/SourceBuild/content/eng/join-verticals.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/join-verticals.proj b/src/SourceBuild/content/eng/join-verticals.proj index 2136f3e79c2f..173139fcfad2 100644 --- a/src/SourceBuild/content/eng/join-verticals.proj +++ b/src/SourceBuild/content/eng/join-verticals.proj @@ -14,7 +14,7 @@ - + From 90400d735014b4248a59c7d91966e4f763a3448b Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Tue, 15 Oct 2024 18:10:27 +0200 Subject: [PATCH 13/18] Update src/SourceBuild/content/eng/pipelines/ci.yml Co-authored-by: Viktor Hofer --- src/SourceBuild/content/eng/pipelines/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml index 97c1a32df276..e289903ee33a 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/ci.yml @@ -96,5 +96,6 @@ extends: ${{ else }}: scope: full - - ${{ if eq(variables['isSourceOnlyBuild'], 'false') }}: + - ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}: + - template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self \ No newline at end of file From 2e256badb4450972b6ebf854cf6cc557f67a2ad4 Mon Sep 17 00:00:00 2001 From: Milena Hristova Date: Tue, 15 Oct 2024 18:10:38 +0200 Subject: [PATCH 14/18] Update src/SourceBuild/content/eng/pipelines/pr.yml Co-authored-by: Viktor Hofer --- src/SourceBuild/content/eng/pipelines/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/pipelines/pr.yml b/src/SourceBuild/content/eng/pipelines/pr.yml index fd681be1a679..d1af7db2243f 100644 --- a/src/SourceBuild/content/eng/pipelines/pr.yml +++ b/src/SourceBuild/content/eng/pipelines/pr.yml @@ -49,5 +49,5 @@ stages: ${{ else }}: scope: full -- ${{ if eq(variables['isSourceOnlyBuild'], 'false') }}: +- ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}: - template: /src/sdk/eng/pipelines/templates/stages/vmr-post-build.yml@self \ No newline at end of file From 0d501d1fdb9d23bedca21b69309bb2797800ead6 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Wed, 16 Oct 2024 17:52:02 +0200 Subject: [PATCH 15/18] Remove the unused target --- src/SourceBuild/content/eng/join-verticals.proj | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/SourceBuild/content/eng/join-verticals.proj b/src/SourceBuild/content/eng/join-verticals.proj index 173139fcfad2..561464a10822 100644 --- a/src/SourceBuild/content/eng/join-verticals.proj +++ b/src/SourceBuild/content/eng/join-verticals.proj @@ -8,12 +8,7 @@ - - - - - From 5d36eba356e800f0b6836cea3c5f7c2bd7846552 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Wed, 16 Oct 2024 17:52:55 +0200 Subject: [PATCH 16/18] Update src/SourceBuild/content/eng/pipelines/ci.yml --- src/SourceBuild/content/eng/pipelines/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml index e289903ee33a..e00f5bc76fa5 100644 --- a/src/SourceBuild/content/eng/pipelines/ci.yml +++ b/src/SourceBuild/content/eng/pipelines/ci.yml @@ -97,5 +97,4 @@ extends: scope: full - ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}: - - template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self \ No newline at end of file From d33abcc9c6b9741d17793a38f368d4cbece6d139 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Wed, 16 Oct 2024 19:51:35 +0200 Subject: [PATCH 17/18] Update src/SourceBuild/content/eng/pipelines/pr.yml --- src/SourceBuild/content/eng/pipelines/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SourceBuild/content/eng/pipelines/pr.yml b/src/SourceBuild/content/eng/pipelines/pr.yml index d1af7db2243f..d1b51c7f322f 100644 --- a/src/SourceBuild/content/eng/pipelines/pr.yml +++ b/src/SourceBuild/content/eng/pipelines/pr.yml @@ -50,4 +50,4 @@ stages: scope: full - ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}: - - template: /src/sdk/eng/pipelines/templates/stages/vmr-post-build.yml@self \ No newline at end of file + - template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self \ No newline at end of file From 875c991a2f09372b303a7b57dd0aaf43c4db6858 Mon Sep 17 00:00:00 2001 From: Viktor Hofer Date: Thu, 17 Oct 2024 17:41:57 +0200 Subject: [PATCH 18/18] Update vmr-build.yml --- eng/pipelines/templates/jobs/vmr-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 1b707756e998..24f93dfcf380 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -598,7 +598,7 @@ jobs: - ${{ if eq(parameters.targetOS, 'windows') }}: - powershell: | - $sourcePath = "$(sourcesPath)/artifacts/VerticalManifest.xml" + $sourcePath = "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" $targetPath = "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml" New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/manifests" -Force | Out-Null Copy-Item $sourcePath -Destination $targetPath -Force @@ -606,7 +606,7 @@ jobs: - ${{ else }}: - script: | mkdir -p "$(Build.ArtifactStagingDirectory)/manifests" - cp "$(sourcesPath)/artifacts/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml" + cp "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml" displayName: Copy vertical manifest to Artifact Staging Directory # When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages