Skip to content

Commit

Permalink
Join assets from verticals in the final join point (#43627)
Browse files Browse the repository at this point in the history
Co-authored-by: Matt Mitchell <[email protected]>
Co-authored-by: Viktor Hofer <[email protected]>
  • Loading branch information
3 people authored Oct 17, 2024
1 parent ce0abad commit ea38eba
Show file tree
Hide file tree
Showing 12 changed files with 755 additions and 3 deletions.
36 changes: 34 additions & 2 deletions eng/pipelines/templates/jobs/vmr-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,13 @@ jobs:
displayName: Publish Artifacts
sbomEnabled: true

# Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
- output: buildArtifacts
PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
ArtifactName: VerticalManifests
displayName: Publish Vertical Manifest
sbomEnabled: false

- ${{ if not(parameters.isBuiltFromVmr) }}:
- output: pipelineArtifact
displayName: Upload failed patches
Expand Down Expand Up @@ -271,7 +278,7 @@ jobs:
- script: |
set extraBuildProperties=
if not [${{ parameters.buildPass }}]==[] set extraBuildProperties=%extraBuildProperties% /p:DotNetBuildPass=${{ parameters.buildPass }}
call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} %extraBuildProperties% ${{ parameters.extraProperties }}
call build.cmd -ci -cleanWhileBuilding -prepareMachine %devArgument% /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) %extraBuildProperties% ${{ parameters.extraProperties }}
displayName: Build
workingDirectory: ${{ variables.sourcesPath }}
env:
Expand All @@ -282,7 +289,7 @@ jobs:
- ${{ if eq(parameters.runTests, 'True') }}:
- script: |
call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} ${{ parameters.extraProperties }}
call build.cmd -ci -prepareMachine -test -excludeCIBinarylog /bl:artifacts/log/Release/Test.binlog /p:TargetOS=${{ parameters.targetOS }} /p:TargetArchitecture=${{ parameters.targetArchitecture }} /p:VerticalName=$(Agent.JobName) ${{ parameters.extraProperties }}
displayName: Run Tests
workingDirectory: ${{ variables.sourcesPath }}
timeoutInMinutes: ${{ variables.runTestsTimeout }}
Expand Down Expand Up @@ -380,6 +387,8 @@ jobs:
extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
fi
extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)"
buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
# Only use Docker when a container is specified
Expand Down Expand Up @@ -455,6 +464,8 @@ jobs:
customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}"
fi
extraBuildProperties="$extraBuildProperties /p:VerticalName=$(Agent.JobName)"
if [[ -n "${{ parameters.extraProperties }}" ]]; then
extraBuildProperties="$extraBuildProperties ${{ parameters.extraProperties }}"
fi
Expand Down Expand Up @@ -585,6 +596,19 @@ jobs:
TargetFolder: $(Build.ArtifactStagingDirectory)/publishing
displayName: Copy artifacts to Artifact Staging Directory

- ${{ if eq(parameters.targetOS, 'windows') }}:
- powershell: |
$sourcePath = "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml"
$targetPath = "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/manifests" -Force | Out-Null
Copy-Item $sourcePath -Destination $targetPath -Force
displayName: Copy vertical manifest to Artifact Staging Directory
- ${{ else }}:
- script: |
mkdir -p "$(Build.ArtifactStagingDirectory)/manifests"
cp "$(sourcesPath)/artifacts/manifests/VerticalManifest.xml" "$(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml"
displayName: Copy vertical manifest to Artifact Staging Directory
# When building from source, the Private.SourceBuilt.Artifacts archive already contains the nuget packages
- ${{ if ne(parameters.buildSourceOnly, 'true') }}:
- task: CopyFiles@2
Expand All @@ -598,3 +622,11 @@ jobs:
artifact: $(Agent.JobName)_Artifacts
displayName: Publish Artifacts
continueOnError: true

# Using build artifacts to enable publishing the vertical manifests to a single artifact from different jobs
- task: PublishBuildArtifacts@1
inputs:
PathtoPublish: $(Build.ArtifactStagingDirectory)/manifests/$(Agent.JobName).xml
ArtifactName: VerticalManifests
displayName: Publish Vertical Manifest
condition: succeededOrFailed()
52 changes: 52 additions & 0 deletions eng/pipelines/templates/stages/vmr-final-join.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
parameters:
# Branch of the VMR to use (to push to for internal builds)
- name: vmrBranch
type: string
default: $(Build.SourceBranch)

- name: pool_Windows
type: object
default:
name: $(defaultPoolName)
image: $(poolImage_Windows)
demands: ImageOverride -equals $(poolImage_Windows)
os: windows

stages:
- stage: VMR_Final_Join
displayName: VMR Final Join
dependsOn: VMR_Vertical_Build
condition: succeededOrFailed()
variables:
- template: ../variables/vmr-build.yml
parameters:
vmrBranch: ${{ parameters.vmrBranch }}

jobs:
- job: FinalJoin
displayName: Final Build Pass
pool: ${{ parameters.pool_Windows }}
timeoutInMinutes: 240
templateContext:
outputs:
- output: buildArtifacts
PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/MergedManifest.xml
ArtifactName: AssetManifests
displayName: Publish Merged Manifest
sbomEnabled: false
- output: buildArtifacts
PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/assets
ArtifactName: BlobArtifacts
displayName: Publish Blob Artifacts
sbomEnabled: false
- output: buildArtifacts
PathtoPublish: $(Build.ArtifactStagingDirectory)/artifacts/packages
ArtifactName: PackageArtifacts
displayName: Publish Package Artifacts
sbomEnabled: false
steps:
- template: ../steps/vmr-join-verticals.yml
parameters:
dotNetBuildPass: final
primaryDependentJob: Windows_x64
outputFolder: $(Build.ArtifactStagingDirectory)/artifacts
37 changes: 37 additions & 0 deletions eng/pipelines/templates/steps/vmr-join-verticals.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
parameters:
- name: dotNetBuildPass
type: string
default: final

- name: primaryDependentJob
type: string
default: Windows_x64

- name: outputFolder
type: string
default: $(Build.ArtifactStagingDirectory)/artifacts

steps:
- task: DownloadBuildArtifacts@1
inputs:
artifactName: 'VerticalManifests'
downloadPath: $(Build.ArtifactStagingDirectory)
checkDownloadedFiles: true

- task: DownloadPipelineArtifact@2
inputs:
artifactName: ${{ parameters.primaryDependentJob }}_Artifacts
targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts
checkDownloadedFiles: true

- powershell: eng/join-verticals.ps1
/p:VerticalManifestsPath=$(Build.ArtifactStagingDirectory)/VerticalManifests
/p:MainVertical=${{ parameters.primaryDependentJob }}
/p:DotNetBuildPass=${{ parameters.dotNetBuildPass }}
/p:BuildId=$(Build.BuildId)
/p:AzureDevOpsToken=$(System.AccessToken)
/p:AzureDevOpsBaseUri=$(System.CollectionUri)
/p:AzureDevOpsProject=$(System.TeamProject)
/p:MainVerticalArtifactsFolder=$(Build.ArtifactStagingDirectory)/${{ parameters.primaryDependentJob }}_Artifacts
/p:OutputFolder=${{ parameters.outputFolder }}
displayName: Join Verticals
39 changes: 39 additions & 0 deletions src/SourceBuild/content/eng/join-verticals.proj
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
<Project Sdk="Microsoft.Build.Traversal">

<PropertyGroup>
<TargetFramework>$(NetCurrent)</TargetFramework>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="$(TasksDir)Microsoft.DotNet.UnifiedBuild.Tasks\Microsoft.DotNet.UnifiedBuild.Tasks.csproj" BuildInParallel="true" />
</ItemGroup>

<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.JoinVerticals" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
<Target Name="JoinVerticals" DependsOnTargets="ResolveProjectReferences" AfterTargets="Build">
<Error Condition="'$(MainVertical)' == ''" Text="MainVertical is not set." />
<Error Condition="'$(VerticalManifestsPath)' == ''" Text="VerticalManifestsPath is not set." />
<Error Condition="'$(BuildId)' == ''" Text="BuildId is not set." />
<Error Condition="'$(AzureDevOpsBaseUri)' == ''" Text="AzureDevOpsBaseUri is not set." />
<Error Condition="'$(AzureDevOpsProject)' == ''" Text="AzureDevOpsProject is not set." />
<Error Condition="'$(MainVerticalArtifactsFolder)' == ''" Text="MainVerticalArtifactsFolder is not set." />
<Error Condition="'$(OutputFolder)' == ''" Text="OutputFolder is not set." />

<ItemGroup>
<VerticalManifest
Include="$(VerticalManifestsPath)\*.xml"
Exclude="$(VerticalManifestsPath)\*Mono*.xml;$(VerticalManifestsPath)\*Pgo*.xml;$(VerticalManifestsPath)\*Shortstack*.xml;$(VerticalManifestsPath)\*DevVersions*.xml" />
</ItemGroup>

<!-- AzureDevOpsToken shouldn't be set when running in dnceng-public -->
<Microsoft.DotNet.UnifiedBuild.Tasks.JoinVerticals
VerticalManifest="@(VerticalManifest)"
MainVertical="$(MainVertical)"
BuildId="$(BuildId)"
AzureDevOpsToken="$(AzureDevOpsToken)"
AzureDevOpsBaseUri="$(AzureDevOpsBaseUri)"
AzureDevOpsProject="$(AzureDevOpsProject)"
MainVerticalArtifactsFolder="$(MainVerticalArtifactsFolder)"
OutputFolder="$(OutputFolder)" />
</Target>

</Project>
33 changes: 33 additions & 0 deletions src/SourceBuild/content/eng/join-verticals.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
[CmdletBinding(PositionalBinding=$false)]
Param(
[string][Alias('v')]$verbosity = "minimal",
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
)

$useGlobalNuGetCache=$false
$ci = $true

. $PSScriptRoot\common\tools.ps1

$project = Join-Path $EngRoot "join-verticals.proj"
$arguments = @()
$targets = "/t:JoinVerticals"

try {
$bl = '/bl:' + (Join-Path $LogDir 'JoinVerticals.binlog')

MSBuild -restore `
$project `
$bl `
$targets `
/p:Configuration=Release `
@properties `
@arguments
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Build' -Message $_
ExitWithExitCode 1
}

ExitWithExitCode 0
3 changes: 2 additions & 1 deletion src/SourceBuild/content/eng/merge-asset-manifests.proj
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@
<Microsoft.DotNet.UnifiedBuild.Tasks.MergeAssetManifests
AssetManifest="@(RepoAssetManifest)"
MergedAssetManifestOutputPath="$(MergedAssetManifestOutputPath)"
VmrBuildNumber="$(BUILD_BUILDNUMBER)" />
VmrBuildNumber="$(BUILD_BUILDNUMBER)"
VerticalName="$(VerticalName)" />
</Target>

</Project>
3 changes: 3 additions & 0 deletions src/SourceBuild/content/eng/pipelines/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,6 @@ extends:
scope: lite
${{ else }}:
scope: full

- ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}:
- template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self
3 changes: 3 additions & 0 deletions src/SourceBuild/content/eng/pipelines/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,6 @@ stages:
scope: lite
${{ else }}:
scope: full

- ${{ if ne(variables['isSourceOnlyBuild'], 'true') }}:
- template: /src/sdk/eng/pipelines/templates/stages/vmr-final-join.yml@self
Loading

0 comments on commit ea38eba

Please sign in to comment.