diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml
index d204c827fe2d..24f93dfcf380 100644
--- a/eng/pipelines/templates/jobs/vmr-build.yml
+++ b/eng/pipelines/templates/jobs/vmr-build.yml
@@ -168,6 +168,13 @@ jobs:
displayName: Publish Artifacts
sbomEnabled: true
+ # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
+ - output: buildArtifacts
+ PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
+ ArtifactName: VerticalManifests
+ displayName: Publish Vertical Manifest
+ sbomEnabled: false
+
- ${{ if not(parameters.isBuiltFromVmr) }}:
- output: pipelineArtifact
displayName: Upload failed patches
@@ -271,7 +278,7 @@ jobs:
- script: |
set extraBuildProperties=
if not [${{ parameters.buildPass }}]==[] set extraBuildProperties=%extraBuildProperties% /p:DotNetBuildPass=${{ parameters.buildPass }}
- call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} %extraBuildProperties% ${{ parameters.extraProperties }}
+ call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }}
displayName: Build
workingDirectory: ${{ variables.sourcesPath }}
env:
@@ -282,7 +289,7 @@ jobs:
- ${{ if eq(parameters.runTests, 'True') }}:
- script: |
- call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
+ call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }}
displayName: Run Tests
workingDirectory: ${{ variables.sourcesPath }}
timeoutInMinutes: ${{ variables.runTestsTimeout }}
@@ -380,6 +387,8 @@ jobs:
extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
fi
+ extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)"
+
buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
# Only use Docker when a container is specified
@@ -455,6 +464,8 @@ jobs:
customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}"
fi
+ extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)"
+
if [[ -n "${{ parameters.extraProperties }}" ]]; then
extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
fi
@@ -585,6 +596,19 @@ jobs:
TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
displayName: Copy artifacts to Artifact Staging Directory
+ - ${{ if eq(parameters.targetOS, 'windows') }}:
+ - powershell: |
+ $sourcePath = "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml"
+ $targetPath = "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
+ New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/manifests" -Force | Out-Null
+ Copy-Item $sourcePath -Destination $targetPath -Force
+ displayName: Copy vertical manifest to Artifact Staging Directory
+ - ${{ else }}:
+ - script: |
+ mkdir -p "$(Build.ArtifactStagingDirectory)/manifests"
+ cp "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
+ displayName: Copy vertical manifest to Artifact Staging Directory
+
# When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
- ${{ if ne(parameters.buildSourceOnly, 'true') }}:
- task: CopyFiles@2
@@ -598,3 +622,11 @@ jobs:
artifact: $(Agent.JobName)_Artifacts
displayName: Publish Artifacts
continueOnError: true
+
+ # Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
+ - task: PublishBuildArtifacts@1
+ inputs:
+ PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
+ ArtifactName: VerticalManifests
+ displayName: Publish Vertical Manifest
+ condition: succeededOrFailed()
diff --git a/eng/pipelines/templates/stages/vmr-final-join.yml b/eng/pipelines/templates/stages/vmr-final-join.yml
new file mode 100644
index 000000000000..947e045c91ea
--- /dev/null
+++ b/eng/pipelines/templates/stages/vmr-final-join.yml
@@ -0,0 +1,52 @@
+parameters:
+# Branch of the VMR to use (to push to for internal builds)
+- name: vmrBranch
+ type: string
+ default: $(Build.SourceBranch)
+
+- name: pool_Windows
+ type: object
+ default:
+ name: $(defaultPoolName)
+ image: $(poolImage_Windows)
+ demands: ImageOverride -equals $(poolImage_Windows)
+ os: windows
+
+stages:
+- stage: VMR_Final_Join
+ displayName: VMR Final Join
+ dependsOn: VMR_Vertical_Build
+ condition: succeededOrFailed()
+ variables:
+ - template: ../variables/vmr-build.yml
+ parameters:
+ vmrBranch: ${{ parameters.vmrBranch }}
+
+ jobs:
+ - job: FinalJoin
+ displayName: Final Build Pass
+ pool: ${{ parameters.pool_Windows }}
+ timeoutInMinutes: 240
+ templateContext:
+ outputs:
+ - output: buildArtifacts
+ PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/MergedManifest.xml
+ ArtifactName: AssetManifests
+ displayName: Publish Merged Manifest
+ sbomEnabled: false
+ - output: buildArtifacts
+ PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/assets
+ ArtifactName: BlobArtifacts
+ displayName: Publish Blob Artifacts
+ sbomEnabled: false
+ - output: buildArtifacts
+ PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/packages
+ ArtifactName: PackageArtifacts
+ displayName: Publish Package Artifacts
+ sbomEnabled: false
+ steps:
+ - template: ../steps/vmr-join-verticals.yml
+ parameters:
+ dotNetBuildPass: final
+ primaryDependentJob: Windows_x64
+ outputFolder: $(Build.ArtifactStagingDirectory)/artifacts
\ No newline at end of file
diff --git a/eng/pipelines/templates/steps/vmr-join-verticals.yml b/eng/pipelines/templates/steps/vmr-join-verticals.yml
new file mode 100644
index 000000000000..50b1866b098f
--- /dev/null
+++ b/eng/pipelines/templates/steps/vmr-join-verticals.yml
@@ -0,0 +1,37 @@
+parameters:
+- name: dotNetBuildPass
+ type: string
+ default: final
+
+- name: primaryDependentJob
+ type: string
+ default: Windows_x64
+
+- name: outputFolder
+ type: string
+ default: $(Build.ArtifactStagingDirectory)/artifacts
+
+steps:
+- task: DownloadBuildArtifacts@1
+ inputs:
+ artifactName: 'VerticalManifests'
+ downloadPath: $(Build.ArtifactStagingDirectory)
+ checkDownloadedFiles: true
+
+- task: DownloadPipelineArtifact@2
+ inputs:
+ artifactName: ${{ parameters.primaryDependentJob }}_Artifacts
+ targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts
+ checkDownloadedFiles: true
+
+- powershell: eng/join-verticals.ps1
+ /p:VerticalManifestsPath=$(Build.ArtifactStagingDirectory)/VerticalManifests
+ /p:MainVertical=${{ parameters.primaryDependentJob }}
+ /p:DotNetBuildPass=${{ parameters.dotNetBuildPass }}
+ /p:BuildId=$(Build.BuildId)
+ /p:AzureDevOpsToken=$(System.AccessToken)
+ /p:AzureDevOpsBaseUri=$(System.CollectionUri)
+ /p:AzureDevOpsProject=$(System.TeamProject)
+ /p:MainVerticalArtifactsFolder=$(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts
+ /p:OutputFolder=${{ parameters.outputFolder }}
+ displayName: Join Verticals
\ No newline at end of file
diff --git a/src/SourceBuild/content/eng/join-verticals.proj b/src/SourceBuild/content/eng/join-verticals.proj
new file mode 100644
index 000000000000..561464a10822
--- /dev/null
+++ b/src/SourceBuild/content/eng/join-verticals.proj
@@ -0,0 +1,39 @@
+
+
+
+ $(NetCurrent)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/src/SourceBuild/content/eng/join-verticals.ps1 b/src/SourceBuild/content/eng/join-verticals.ps1
new file mode 100644
index 000000000000..bd19e2617cbe
--- /dev/null
+++ b/src/SourceBuild/content/eng/join-verticals.ps1
@@ -0,0 +1,33 @@
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+ [string][Alias('v')]$verbosity = "minimal",
+ [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
+)
+
+$useGlobalNuGetCache=$false
+$ci = $true
+
+. $PSScriptRoot\common\tools.ps1
+
+$project = Join-Path $EngRoot "join-verticals.proj"
+$arguments = @()
+$targets = "/t:JoinVerticals"
+
+try {
+ $bl = '/bl:' + (Join-Path $LogDir 'JoinVerticals.binlog')
+
+ MSBuild -restore `
+ $project `
+ $bl `
+ $targets `
+ /p:Configuration=Release `
+ @properties `
+ @arguments
+}
+catch {
+ Write-Host $_.ScriptStackTrace
+ Write-PipelineTelemetryError -Category 'Build' -Message $_
+ ExitWithExitCode 1
+}
+
+ExitWithExitCode 0
diff --git a/src/SourceBuild/content/eng/merge-asset-manifests.proj b/src/SourceBuild/content/eng/merge-asset-manifests.proj
index e1b5b053b4ad..ae49332985b3 100644
--- a/src/SourceBuild/content/eng/merge-asset-manifests.proj
+++ b/src/SourceBuild/content/eng/merge-asset-manifests.proj
@@ -22,7 +22,8 @@
+ VmrBuildNumber="$(BUILD_BUILDNUMBER)"
+ VerticalName="$(VerticalName)" />
diff --git a/src/SourceBuild/content/eng/pipelines/ci.yml b/src/SourceBuild/content/eng/pipelines/ci.yml
index 33b41f2c9163..e00f5bc76fa5 100644
--- a/src/SourceBuild/content/eng/pipelines/ci.yml
+++ b/src/SourceBuild/content/eng/pipelines/ci.yml
@@ -95,3 +95,6 @@ extends:
scope: lite
${{ else }}:
scope: full
+
+ - ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}:
+ - template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self
\ No newline at end of file
diff --git a/src/SourceBuild/content/eng/pipelines/pr.yml b/src/SourceBuild/content/eng/pipelines/pr.yml
index 471f6a92b751..d1b51c7f322f 100644
--- a/src/SourceBuild/content/eng/pipelines/pr.yml
+++ b/src/SourceBuild/content/eng/pipelines/pr.yml
@@ -48,3 +48,6 @@ stages:
scope: lite
${{ else }}:
scope: full
+
+- ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}:
+ - template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self
\ No newline at end of file
diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs
new file mode 100644
index 000000000000..d28e7ca24e00
--- /dev/null
+++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/AzureDevOpsClient.cs
@@ -0,0 +1,199 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using System;
+using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using System.Text;
+using System.Threading.Tasks;
+using Task = System.Threading.Tasks.Task;
+
+namespace Microsoft.DotNet.UnifiedBuild.Tasks;
+
+public class AzureDevOpsClient : IDisposable
+{
+ private readonly HttpClient _httpClient;
+ private readonly TaskLoggingHelper _logger;
+
+ private const string _azureDevOpsApiVersion = "7.1-preview.5";
+ // download in 100 MB chunks
+ private const int _downloadBufferSize = 1024 * 1024 * 100;
+ private const int _httpTimeoutSeconds = 300;
+
+ public AzureDevOpsClient(
+ string? azureDevOpsToken,
+ string azureDevOpsBaseUri,
+ string azureDevOpsProject,
+ TaskLoggingHelper logger)
+ {
+
+ _logger = logger;
+
+ _httpClient = new(new HttpClientHandler { CheckCertificateRevocationList = true });
+
+ _httpClient.BaseAddress = new Uri($"{azureDevOpsBaseUri}/{azureDevOpsProject}/_apis/");
+
+ _httpClient.Timeout = TimeSpan.FromSeconds(_httpTimeoutSeconds);
+
+ if (!string.IsNullOrEmpty(azureDevOpsToken))
+ {
+ _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(
+ "Basic",
+ Convert.ToBase64String(Encoding.UTF8.GetBytes($":{azureDevOpsToken}")));
+ }
+ }
+
+ ///
+ /// Downloads a build artifact as zip file
+ ///
+ public async Task DownloadArtifactZip(string buildId, string artifactName, string downloadPath, int retryCount)
+ {
+ var artifactInformation = await GetArtifactInformation(buildId, artifactName, retryCount);
+ string downloadUrl = artifactInformation.Resource.DownloadUrl;
+
+ _logger.LogMessage(MessageImportance.High, $"Downloading artifact zip from {downloadUrl}");
+
+ try
+ {
+ using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(downloadUrl, retryCount);
+ using Stream readStream = await httpResponse.Content.ReadAsStreamAsync();
+ using FileStream writeStream = File.Create(downloadPath);
+
+ await readStream.CopyToAsync(writeStream, _downloadBufferSize);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError($"Failed to download artifact zip: {ex.Message}");
+ throw;
+ }
+ }
+
+ public async Task DownloadSingleFileFromArtifact(string buildId, string artifactName, string itemId, string itemSubPath, string downloadPath, int retryCount)
+ {
+ try
+ {
+ var downloadFileUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&fileId={itemId}&fileName={itemSubPath}&api-version={_azureDevOpsApiVersion}";
+
+ _logger.LogMessage(MessageImportance.High, $"Downloading file {itemSubPath} from {downloadFileUrl}");
+
+ using HttpResponseMessage fileDownloadResponse = await ExecuteApiCallWithRetry(downloadFileUrl, retryCount);
+ using Stream readStream = await fileDownloadResponse.Content.ReadAsStreamAsync();
+ using FileStream writeStream = File.Create(downloadPath);
+
+ await readStream.CopyToAsync(writeStream, _downloadBufferSize);
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError($"Failed to download file: {ex.Message}");
+ throw;
+ }
+ }
+
+ public async Task GetArtifactFilesInformation(string buildId, string artifactName, int retryCount)
+ {
+ var artifactInformation = await GetArtifactInformation(buildId, artifactName, retryCount);
+ string artifactId = artifactInformation.Resource.Data;
+
+ var getManifestUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&fileId={artifactId}&fileName={artifactName}&api-version={_azureDevOpsApiVersion}";
+
+ _logger.LogMessage(MessageImportance.High, $"Getting {artifactName} artifact manifest");
+
+ try
+ {
+ using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(getManifestUrl, retryCount);
+
+ ArtifactFiles filesInformation = await httpResponse.Content.ReadFromJsonAsync()
+ ?? throw new ArgumentException($"Couldn't parse AzDo response {httpResponse.Content} to {nameof(ArtifactFiles)}");
+
+ return filesInformation;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogError($"Failed to download file: {ex.Message}");
+ throw;
+ }
+ }
+
+ public async Task GetArtifactInformation(string buildId, string artifactName, int retryCount)
+ {
+ string relativeUrl = $"build/builds/{buildId}/artifacts?artifactName={artifactName}&api-version={_azureDevOpsApiVersion}";
+
+ _logger.LogMessage(MessageImportance.High, $"Getting {artifactName} metadata from {relativeUrl}");
+
+ try
+ {
+ using HttpResponseMessage httpResponse = await ExecuteApiCallWithRetry(relativeUrl, retryCount);
+
+ AzureDevOpsArtifactInformation artifactInformation = await httpResponse.Content.ReadFromJsonAsync()
+ ?? throw new ArgumentException($"Couldn't parse AzDo response {httpResponse.Content} to {nameof(AzureDevOpsArtifactInformation)}");
+
+ return artifactInformation;
+ }
+ catch(Exception ex)
+ {
+ _logger.LogError($"Failed to get artifact download URL: {ex.Message}");
+ throw;
+ }
+ }
+
+ private async Task ExecuteApiCallWithRetry(string relativeUrl, int retryCount)
+ {
+ int retriesRemaining = retryCount;
+
+ while (true)
+ {
+ try
+ {
+ HttpResponseMessage httpResponse = await _httpClient.GetAsync(relativeUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
+
+ httpResponse.EnsureSuccessStatusCode();
+
+ return httpResponse;
+ }
+ catch (Exception ex) when (ex is HttpRequestException || ex is TaskCanceledException)
+ {
+ if (ex is HttpRequestException && ex.Message.Contains(((int)HttpStatusCode.NotFound).ToString()))
+ {
+ _logger.LogError($"Resource not found at {relativeUrl}: {ex.Message}");
+ throw;
+ }
+
+ if (ex is HttpRequestException && ex.Message.Contains(((int)HttpStatusCode.Unauthorized).ToString()))
+ {
+ _logger.LogError($"Failure to authenticate: {ex.Message}");
+ throw;
+ }
+
+ if (retriesRemaining <= 0)
+ {
+ _logger.LogError($"There was an error calling AzureDevOps API against URI '{relativeUrl}' " +
+ $"after {retryCount} attempts. Exception: {ex}");
+ throw;
+ }
+
+ _logger.LogWarning($"There was an error calling AzureDevOps API against URI against URI '{relativeUrl}'. " +
+ $"{retriesRemaining} attempts remaining. Exception: {ex.ToString()}");
+ }
+
+ --retriesRemaining;
+ await Task.Delay(5000);
+ }
+ }
+
+ public void Dispose()
+ {
+ _httpClient.Dispose();
+ }
+
+ public record Blob(string Id, int Size);
+ public record ArtifactItem(string Path, Blob Blob);
+ public record ArtifactFiles(string ManifestFormat, ArtifactItem[] Items, string[] ManifestReferences);
+}
\ No newline at end of file
diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs
new file mode 100644
index 000000000000..8e2faef5c0b5
--- /dev/null
+++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/JoinVerticals.cs
@@ -0,0 +1,335 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using Microsoft.Build.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Xml.Linq;
+using static Microsoft.DotNet.UnifiedBuild.Tasks.AzureDevOpsClient;
+using Task = System.Threading.Tasks.Task;
+
+namespace Microsoft.DotNet.UnifiedBuild.Tasks;
+
+public class JoinVerticals : Microsoft.Build.Utilities.Task
+{
+ ///
+ /// Paths to Verticals Manifests
+ ///
+ [Required]
+ public required ITaskItem[] VerticalManifest { get; init; }
+
+ ///
+ /// Name of the main vertical that we'll take all artifacts from if they exist in this vertical, and at least one other vertical.
+ ///
+ [Required]
+ public required string MainVertical { get; init; }
+
+ ///
+ /// Azure DevOps build id
+ ///
+ [Required]
+ public required string BuildId { get; init; }
+
+ ///
+ /// Azure DevOps token, required scopes: "Build (read)", allowed to be empty when running in a public project
+ ///
+ public string? AzureDevOpsToken { get; set; }
+
+ ///
+ /// Azure DevOps organization
+ ///
+ [Required]
+ public required string AzureDevOpsBaseUri { get; init; }
+
+ ///
+ /// Azure DevOps project
+ ///
+ [Required]
+ public required string AzureDevOpsProject { get; init; }
+
+ ///
+ /// Location of sownloaded artifacts from the main vertical
+ ///
+ [Required]
+ public required string MainVerticalArtifactsFolder { get; init; }
+
+ ///
+ /// Folder where packages and assets will be stored
+ ///
+ [Required]
+ public required string OutputFolder { get; init; }
+
+ private const string _packageElementName = "Package";
+ private const string _blobElementName = "Blob";
+ private const string _idAttribute = "Id";
+ private const string _verticalNameAttribute = "VerticalName";
+ private const string _artifactNameSuffix = "_Artifacts";
+ private const string _assetsFolderName = "assets";
+ private const string _packagesFolderName = "packages";
+ private const int _retryCount = 10;
+
+ private Dictionary> duplicatedItems = new();
+
+ public override bool Execute()
+ {
+ ExecuteAsync().GetAwaiter().GetResult();
+ return !Log.HasLoggedErrors;
+ }
+
+ private async Task ExecuteAsync()
+ {
+ List verticalManifests = VerticalManifest.Select(xmlPath => XDocument.Load(xmlPath.ItemSpec)).ToList();
+
+ XDocument mainVerticalManifest = verticalManifests.FirstOrDefault(manifest => GetRequiredRootAttribute(manifest, _verticalNameAttribute) == MainVertical)
+ ?? throw new ArgumentException($"Couldn't find main vertical manifest {MainVertical} in vertical manifest list");
+
+ if (!Directory.Exists(MainVerticalArtifactsFolder))
+ {
+ throw new ArgumentException($"Main vertical artifacts directory {MainVerticalArtifactsFolder} not found.");
+ }
+
+ string mainVerticalName = GetRequiredRootAttribute(mainVerticalManifest, _verticalNameAttribute);
+
+ Dictionary packageElements = [];
+ Dictionary blobElements = [];
+
+ List addedPackageIds = AddMissingElements(packageElements, mainVerticalManifest, _packageElementName);
+ List addedBlobIds = AddMissingElements(blobElements, mainVerticalManifest, _blobElementName);
+
+ string packagesOutputDirectory = Path.Combine(OutputFolder, _packagesFolderName);
+ string blobsOutputDirectory = Path.Combine(OutputFolder, _assetsFolderName);
+
+ CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _packagesFolderName), packagesOutputDirectory);
+ CopyMainVerticalAssets(Path.Combine(MainVerticalArtifactsFolder, _assetsFolderName), blobsOutputDirectory);
+
+ using var clientThrottle = new SemaphoreSlim(16, 16);
+ List downloadTasks = new();
+
+ foreach (XDocument verticalManifest in verticalManifests)
+ {
+ string verticalName = GetRequiredRootAttribute(verticalManifest, _verticalNameAttribute);
+
+ // We already processed the main vertical
+ if (verticalName == MainVertical)
+ {
+ continue;
+ }
+
+ addedPackageIds = AddMissingElements(packageElements, verticalManifest, _packageElementName);
+ addedBlobIds = AddMissingElements(blobElements, verticalManifest, _blobElementName);
+
+ if (addedPackageIds.Count > 0)
+ {
+ downloadTasks.Add(
+ DownloadArtifactFiles(
+ BuildId,
+ $"{verticalName}{_artifactNameSuffix}",
+ addedPackageIds,
+ packagesOutputDirectory,
+ clientThrottle));
+ }
+
+ if (addedBlobIds.Count > 0)
+ {
+ downloadTasks.Add(
+ DownloadArtifactFiles(
+ BuildId,
+ $"{verticalName}{_artifactNameSuffix}",
+ addedBlobIds,
+ blobsOutputDirectory,
+ clientThrottle));
+ }
+ }
+
+ await Task.WhenAll(downloadTasks);
+
+ // Create MergedManifest.xml
+ // taking the attributes from the main manifest
+ XElement mainManifestRoot = verticalManifests.First().Root
+ ?? throw new ArgumentException("The root element of the vertical manifest is null.");
+ mainManifestRoot.Attribute(_verticalNameAttribute)!.Remove();
+
+ string manifestOutputPath = Path.Combine(OutputFolder, "MergedManifest.xml");
+ XDocument mergedManifest = new(new XElement(
+ mainManifestRoot.Name,
+ mainManifestRoot.Attributes(),
+ packageElements.Values.Select(v => v.Element).OrderBy(elem => elem.Attribute(_idAttribute)?.Value),
+ blobElements.Values.Select(v => v.Element).OrderBy(elem => elem.Attribute(_idAttribute)?.Value)));
+
+ File.WriteAllText(manifestOutputPath, mergedManifest.ToString());
+
+ Log.LogMessage(MessageImportance.High, $"### Duplicate items found in the following verticals: ###");
+
+ foreach (var item in duplicatedItems)
+ {
+ Log.LogMessage(MessageImportance.High, $"Item: {item.Key} -- Produced by: {string.Join(", ", item.Value)}");
+ }
+ }
+
+ ///
+ /// Downloads specified packages and symbols from a specific build artifact and stores them in an output folder
+ ///
+ private async Task DownloadArtifactFiles(
+ string buildId,
+ string artifactName,
+ List fileNamesToDownload,
+ string outputDirectory,
+ SemaphoreSlim clientThrottle)
+ {
+ using AzureDevOpsClient azureDevOpsClient = new(AzureDevOpsToken, AzureDevOpsBaseUri, AzureDevOpsProject, Log);
+
+ ArtifactFiles filesInformation = await azureDevOpsClient.GetArtifactFilesInformation(buildId, artifactName, _retryCount);
+
+ await Task.WhenAll(fileNamesToDownload.Select(async fileName =>
+ await DownloadFileFromArtifact(
+ filesInformation,
+ artifactName,
+ azureDevOpsClient,
+ buildId,
+ fileName,
+ outputDirectory,
+ clientThrottle)));
+ }
+
+ private async Task DownloadFileFromArtifact(
+ ArtifactFiles artifactFilesMetadata,
+ string azureDevOpsArtifact,
+ AzureDevOpsClient azureDevOpsClient,
+ string buildId,
+ string manifestFile,
+ string destinationDirectory,
+ SemaphoreSlim clientThrottle)
+ {
+ try
+ {
+ await clientThrottle.WaitAsync();
+
+ ArtifactItem fileItem;
+
+ var matchingFilePaths = artifactFilesMetadata.Items.Where(f => Path.GetFileName(f.Path) == Path.GetFileName(manifestFile));
+
+ if (!matchingFilePaths.Any())
+ {
+ throw new ArgumentException($"File {manifestFile} not found in source files.");
+ }
+
+ if (matchingFilePaths.Count() > 1)
+ {
+ // Picking the first one until https://github.com/dotnet/source-build/issues/4596 is resolved
+ if (manifestFile.Contains("productVersion.txt"))
+ {
+ fileItem = matchingFilePaths.First();
+ }
+ else
+ {
+ // For some files it's not enough to compare the filename because they have 2 copies in the artifact
+ // e.g. assets/Release/dotnet-sdk-*-win-x64.zip and assets/Release/Sdk/*/dotnet-sdk-*-win-x64.zip
+ // In this case take the one matching the full path from the manifest
+ fileItem = matchingFilePaths
+ .SingleOrDefault(f => f.Path.EndsWith(manifestFile) || f.Path.EndsWith(manifestFile.Replace("/", @"\")))
+ ?? throw new ArgumentException($"File {manifestFile} not found in source files.");
+ }
+ }
+ else
+ {
+ fileItem = matchingFilePaths.Single();
+ }
+
+ string itemId = fileItem.Blob.Id;
+ string artifactSubPath = fileItem.Path;
+
+ string destinationFilePath = Path.Combine(destinationDirectory, Path.GetFileName(manifestFile));
+
+ await azureDevOpsClient.DownloadSingleFileFromArtifact(buildId, azureDevOpsArtifact, itemId, artifactSubPath, destinationFilePath, _retryCount);
+ }
+ catch (Exception ex)
+ {
+ Log.LogError($"Failed to download file {manifestFile} from artifact {azureDevOpsArtifact}: {ex.Message}");
+ throw;
+ }
+ finally
+ {
+ clientThrottle.Release();
+ }
+ }
+
+ ///
+ /// Copy all files from the source directory to the destination directory,
+ /// in a flat layout
+ ///
+ private void CopyMainVerticalAssets(string sourceDirectory, string destinationDirectory)
+ {
+ var sourceFiles = Directory.EnumerateFiles(sourceDirectory, "*", SearchOption.AllDirectories);
+
+ if (!Directory.Exists(destinationDirectory))
+ {
+ Directory.CreateDirectory(destinationDirectory);
+ }
+
+ foreach (var sourceFile in sourceFiles)
+ {
+ string destinationFilePath = Path.Combine(destinationDirectory, Path.GetFileName(sourceFile));
+
+ Log.LogMessage(MessageImportance.High, $"Copying {sourceFile} to {destinationFilePath}");
+ File.Copy(sourceFile, destinationFilePath, true);
+ }
+ }
+
+ ///
+ /// Find the artifacts from the vertical manifest that are not already in the dictionary and add them
+ /// Return a list of the added artifact ids
+ ///
+ private List AddMissingElements(Dictionary addedArtifacts, XDocument verticalManifest, string elementName)
+ {
+ List addedFiles = [];
+
+ string verticalName = verticalManifest.Root!.Attribute(_verticalNameAttribute)!.Value;
+
+ foreach (XElement artifactElement in verticalManifest.Descendants(elementName))
+ {
+ string elementId = artifactElement.Attribute(_idAttribute)?.Value
+ ?? throw new ArgumentException($"Required attribute '{_idAttribute}' not found in {elementName} element.");
+
+ if (addedArtifacts.TryAdd(elementId, new AddedElement(verticalName, artifactElement)))
+ {
+ if (elementName == _packageElementName)
+ {
+ string version = artifactElement.Attribute("Version")?.Value
+ ?? throw new ArgumentException($"Required attribute 'Version' not found in {elementName} element.");
+
+ elementId += $".{version}.nupkg";
+ }
+
+ addedFiles.Add(elementId);
+ Log.LogMessage(MessageImportance.High, $"Taking {elementName} '{elementId}' from '{verticalName}'");
+ }
+ else
+ {
+ AddedElement previouslyAddedArtifact = addedArtifacts[elementId];
+ if (previouslyAddedArtifact.VerticalName != MainVertical)
+ {
+ if (!duplicatedItems.TryAdd(elementId, new List { verticalName, previouslyAddedArtifact.VerticalName }))
+ {
+ duplicatedItems[elementId].Add(verticalName);
+ }
+ }
+ }
+ }
+
+ return addedFiles;
+ }
+
+ private static string GetRequiredRootAttribute(XDocument document, string attributeName)
+ {
+ return document.Root?.Attribute(attributeName)?.Value
+ ?? throw new ArgumentException($"Required attribute '{attributeName}' not found in root element.");
+ }
+
+ private record AddedElement(string VerticalName, XElement Element);
+}
diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs
index eff0413e88b8..c2b87171ad2c 100644
--- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs
+++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/MergeAssetManifests.cs
@@ -33,7 +33,13 @@ public class MergeAssetManifests : Task
///
public string VmrBuildNumber { get; set; } = string.Empty;
+ ///
+ /// Vmr Vertical Name, e.g. "Android_Shortstack_arm". Allowed to be empty for non official builds.
+ ///
+ public string VerticalName { get; set; } = string.Empty;
+
private static readonly string _buildIdAttribute = "BuildId";
+ private static readonly string _verticalNameAttribute = "VerticalName";
private static readonly string _azureDevOpsBuildNumberAttribute = "AzureDevOpsBuildNumber";
private static readonly string[] _ignoredAttributes = [
_buildIdAttribute,
@@ -54,6 +60,7 @@ public override bool Execute()
// Set the BuildId and AzureDevOpsBuildNumber attributes to the value of VmrBuildNumber
mergedManifestRoot.SetAttributeValue(_buildIdAttribute, VmrBuildNumber);
mergedManifestRoot.SetAttributeValue(_azureDevOpsBuildNumberAttribute, VmrBuildNumber);
+ mergedManifestRoot.SetAttributeValue(_verticalNameAttribute, VerticalName);
List packageElements = new();
List blobElements = new();
@@ -70,6 +77,7 @@ public override bool Execute()
XDocument verticalManifest = new(new XElement(mergedManifestRoot.Name, mergedManifestRoot.Attributes(), packageElements, blobElements));
File.WriteAllText(MergedAssetManifestOutputPath, verticalManifest.ToString());
+ Log.LogMessage(MessageImportance.High, $"Merged asset manifest written to {MergedAssetManifestOutputPath}");
return !Log.HasLoggedErrors;
}
diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs
new file mode 100644
index 000000000000..d7d2ba96bd78
--- /dev/null
+++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/Models/AzureDevOpsArtifactInformation.cs
@@ -0,0 +1,10 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+#nullable enable
+
+namespace Microsoft.DotNet.UnifiedBuild.Tasks;
+
+public record AzureDevOpsArtifactInformation(int Id, string Name, string Source, AzdoArtifactResources Resource);
+public record AzdoArtifactResources(string Type, string Data, AzdoArtifactProperties Properties, string Url, string DownloadUrl);
+public record AzdoArtifactProperties(string RootId, string Artifactsize, string HashType, string DomainId);
\ No newline at end of file