Skip to content

Commit

Permalink
Networking stress tests infrastructure improvements. (#108325)
Browse files Browse the repository at this point in the history
* Update HttpStress

* !fixup Uncomment

* Fix linux build-docker-sdk script

* Fix linux scripts

* Create artifact folders on windows

* Fixes

* fix permissions on bash scripts

* Apply same changes to SslStress

* revert some changes

* Improve build-local scripts

* Update SslStress

* Update sslstress yml files

* !fixup

* mkdir -p in yaml

* Revert cli arg change

* Reference original envvars

* code review changes
  • Loading branch information
rzikm authored Oct 4, 2024
1 parent f05bfc4 commit 2c5b609
Show file tree
Hide file tree
Showing 30 changed files with 613 additions and 630 deletions.
13 changes: 8 additions & 5 deletions eng/docker/build-docker-sdk.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@ Param(
[switch][Alias('w')]$buildWindowsContainers
)

$dotNetVersion="8.0"
$ErrorActionPreference = "Stop"

$REPO_ROOT_DIR=$(git -C "$PSScriptRoot" rev-parse --show-toplevel)
[xml]$xml = Get-Content (Join-Path $REPO_ROOT_DIR "eng\Versions.props")
$VERSION="$($xml.Project.PropertyGroup.MajorVersion[0]).$($xml.Project.PropertyGroup.MinorVersion[0])"

$dockerFilePrefix="$PSScriptRoot/libraries-sdk"

Expand All @@ -29,7 +30,7 @@ if ($buildWindowsContainers)
}

$dockerFile="$dockerFilePrefix.windows.Dockerfile"

# Collect the following artifacts to folder, that will be used as build context for the container,
# so projects can build and test against the live-built runtime:
# 1. Reference assembly pack (microsoft.netcore.app.ref)
Expand All @@ -54,15 +55,16 @@ if ($buildWindowsContainers)
-Destination $dockerContext\targetingpacks.targets
Copy-Item -Recurse -Path $REPO_ROOT_DIR\src\libraries\System.Net.Quic\src\System\Net\Quic\Interop `
-Destination $dockerContext\msquic-interop

# In case of non-CI builds, testhost may already contain Microsoft.AspNetCore.App (see build-local.ps1 in HttpStress):
$testHostAspNetCorePath="$dockerContext\testhost\net$dotNetVersion-windows-$configuration-x64/shared/Microsoft.AspNetCore.App"
$testHostAspNetCorePath="$dockerContext\testhost\net$VERSION-windows-$configuration-x64/shared/Microsoft.AspNetCore.App"
if (Test-Path $testHostAspNetCorePath) {
Remove-Item -Recurse -Force $testHostAspNetCorePath
}

docker build --tag $imageName `
--build-arg CONFIGURATION=$configuration `
--build-arg VERSION=$VERSION `
--file $dockerFile `
$dockerContext
}
Expand All @@ -73,6 +75,7 @@ else

docker build --tag $imageName `
--build-arg CONFIGURATION=$configuration `
--build-arg "VERSION=$VERSION" `
--file $dockerFile `
$REPO_ROOT_DIR
}
Expand Down
8 changes: 6 additions & 2 deletions eng/docker/build-docker-sdk.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"

imagename="dotnet-sdk-libs-current"
configuration="Release"
repo_root=$(git -C "$scriptroot" rev-parse --show-toplevel)
major_version=$(grep -oP '(?<=<MajorVersion>).*?(?=</MajorVersion>)' "$repo_root/eng/Versions.props")
minor_version=$(grep -oP '(?<=<MinorVersion>).*?(?=</MinorVersion>)' "$repo_root/eng/Versions.props")
version="$major_version.$minor_version"

while [[ $# > 0 ]]; do
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
Expand All @@ -41,12 +45,12 @@ while [[ $# > 0 ]]; do
esac
done

repo_root=$(git rev-parse --show-toplevel)
docker_file="$scriptroot/libraries-sdk.linux.Dockerfile"

docker build --tag $imagename \
--build-arg CONFIGURATION=$configuration \
--build-arg VERSION=$version \
--file $docker_file \
$repo_root

exit $?
exit $?
11 changes: 7 additions & 4 deletions eng/docker/libraries-sdk.linux.Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Builds and copies library artifacts into target dotnet sdk image
ARG BUILD_BASE_IMAGE=mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:8.0
ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:latest

FROM $BUILD_BASE_IMAGE as corefxbuild

Expand All @@ -12,10 +12,13 @@ RUN ./build.sh clr+libs -runtimeconfiguration Release -configuration $CONFIGURAT

FROM $SDK_BASE_IMAGE as target

ARG VERSION=9.0
ARG VERSION
ARG CONFIGURATION=Release
ENV _DOTNET_INSTALL_CHANNEL=$VERSION

# remove the existing SDK, we want to start from a clean slate with latest daily
RUN rm -rf /usr/share/dotnet

# Install latest daily SDK:
RUN wget https://dot.net/v1/dotnet-install.sh
RUN bash ./dotnet-install.sh --channel $_DOTNET_INSTALL_CHANNEL --quality daily --install-dir /usr/share/dotnet
Expand Down Expand Up @@ -48,8 +51,8 @@ COPY --from=corefxbuild \
/repo/src/libraries/System.Net.Quic/src/System/Net/Quic/Interop \
/live-runtime-artifacts/msquic-interop

# Add AspNetCore bits to testhost:
ENV _ASPNETCORE_SOURCE="/usr/share/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
# Add AspNetCore bits to testhost, there should be only one version since we started from an image without existing SDK:
ENV _ASPNETCORE_SOURCE="/usr/share/dotnet/shared/Microsoft.AspNetCore.App/*"
ENV _ASPNETCORE_DEST="/live-runtime-artifacts/testhost/net$VERSION-linux-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"
RUN mkdir -p $_ASPNETCORE_DEST
RUN cp -r $_ASPNETCORE_SOURCE $_ASPNETCORE_DEST
5 changes: 4 additions & 1 deletion eng/docker/libraries-sdk.windows.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ ARG CONFIGURATION=Release

USER ContainerAdministrator

# remove the existing ASP.NET SDK, we want to keep only the latest one we download later
RUN Remove-Item -Force -Recurse 'C:/Program Files/dotnet/shared/Microsoft.AspNetCore.App/*'

RUN Invoke-WebRequest -Uri https://dot.net/v1/dotnet-install.ps1 -OutFile .\dotnet-install.ps1
RUN & .\dotnet-install.ps1 -Channel $env:_DOTNET_INSTALL_CHANNEL -Quality daily -InstallDir 'C:/Program Files/dotnet'

Expand All @@ -19,7 +22,7 @@ USER ContainerUser
COPY . /live-runtime-artifacts

# Add AspNetCore bits to testhost:
ENV _ASPNETCORE_SOURCE="C:/Program Files/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
ENV _ASPNETCORE_SOURCE="C:/Program Files/dotnet/shared/Microsoft.AspNetCore.App/*"
ENV _ASPNETCORE_DEST="C:/live-runtime-artifacts/testhost/net$VERSION-windows-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"
RUN & New-Item -ItemType Directory -Path $env:_ASPNETCORE_DEST
RUN Copy-Item -Recurse -Path $env:_ASPNETCORE_SOURCE -Destination $env:_ASPNETCORE_DEST
58 changes: 21 additions & 37 deletions eng/pipelines/libraries/stress/http.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ extends:
timeoutInMinutes: 240
variables:
DUMPS_SHARE_MOUNT_ROOT: "/dumps-share"
DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
pool:
name: $(DncEngPublicBuildPool)
demands: ImageOverride -equals 1es-ubuntu-1804-open
Expand All @@ -58,31 +59,28 @@ extends:
- bash: |
cd '$(httpStressProject)'
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
mkdir -p $DUMPS_SHARE
docker-compose up --abort-on-container-exit --no-color
timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
- bash: |
cd '$(httpStressProject)'
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/2.0"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/2.0"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 2.0"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 2.0"
export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 2.0"
export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 2.0"
mkdir -p $DUMPS_SHARE
docker-compose up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 2.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
- bash: |
cd '$(httpStressProject)'
export CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/1.1"
export SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/1.1"
export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 1.1"
export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 1.1"
export STRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 1.1"
export STRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 1.1"
mkdir -p $DUMPS_SHARE
docker-compose up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 1.1
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
Expand All @@ -100,10 +98,8 @@ extends:
displayName: Docker NanoServer
timeoutInMinutes: 150
variables:
# This will get assigned to the DUMPS_SHARE_MOUNT_ROOT environment
# variable in the stress test script. We need to keep the
# DUMPS_SHARE_MOUNT_ROOT variable empty during the build step.
DUMPS_SHARE_MOUNT_ROOT_PATH: "C:/dumps-share"
DUMPS_SHARE_MOUNT_ROOT: "C:/dumps-share"
DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"

# The 1es-windows-2022-open image has an issue where the Chocolatey-installed V1 docker-compose takes precendence over the
# V2 docker-compose required by the stress tests, see: https://github.com/actions/runner-images/issues/7080
Expand Down Expand Up @@ -144,39 +140,27 @@ extends:
- powershell: |
cd '$(httpStressProject)'
$env:DUMPS_SHARE_MOUNT_ROOT = $env:DUMPS_SHARE_MOUNT_ROOT_PATH
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/3.0"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/3.0"
New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0"
$env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
$env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0"
New-Item -Force $env:DUMPS_SHARE -ItemType Directory
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 3.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
- powershell: |
cd '$(httpStressProject)'
$env:DUMPS_SHARE_MOUNT_ROOT = $env:DUMPS_SHARE_MOUNT_ROOT_PATH
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/2.0"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/2.0"
New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 2.0"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 2.0"
$env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 2.0"
$env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 2.0"
New-Item -Force $env:DUMPS_SHARE -ItemType Directory
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 2.0
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
- powershell: |
cd '$(httpStressProject)'
$env:DUMPS_SHARE_MOUNT_ROOT = $env:DUMPS_SHARE_MOUNT_ROOT_PATH
$env:CLIENT_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/client/1.1"
$env:SERVER_DUMPS_SHARE="$(Build.ArtifactStagingDirectory)/dumps/server/1.1"
New-Item -Force $env:CLIENT_DUMPS_SHARE -ItemType Directory
New-Item -Force $env:SERVER_DUMPS_SHARE -ItemType Directory
$env:HTTPSTRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 1.1"
$env:HTTPSTRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 1.1"
$env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 1.1"
$env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 1.1"
New-Item -Force $env:DUMPS_SHARE -ItemType Directory
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run HttpStress - HTTP 1.1
condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
Expand Down
19 changes: 19 additions & 0 deletions eng/pipelines/libraries/stress/ssl.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ extends:
- job: linux
displayName: Docker Linux
timeoutInMinutes: 120
variables:
DUMPS_SHARE_MOUNT_ROOT: "/dumps-share"
DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
pool:
name: $(DncEngPublicBuildPool)
demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
Expand All @@ -53,16 +56,25 @@ extends:
- bash: |
cd '$(sslStressProject)'
export STRESS_CLIENT_ARGS=$SSLSTRESS_CLIENT_ARGS
export STRESS_SERVER_ARGS=$SSLSTRESS_SERVER_ARGS
mkdir -p $DUMPS_SHARE
docker-compose up --abort-on-container-exit --no-color
displayName: Run SslStress
- publish: $(Build.ArtifactStagingDirectory)/dumps
artifact: DumpsLinux
condition: failed()

- job: windows
displayName: Docker NanoServer
timeoutInMinutes: 120
pool:
name: $(DncEngPublicBuildPool)
demands: ImageOverride -equals 1es-windows-2022-open
variables:
DUMPS_SHARE_MOUNT_ROOT: "C:/dumps-share"
DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
# The 1es-windows-2022-open image has an issue where the Chocolatey-installed V1 docker-compose takes precendence over the
# V2 docker-compose required by the stress tests, see: https://github.com/actions/runner-images/issues/7080
# This is worked around by handpicking the V2 executable.
Expand All @@ -87,5 +99,12 @@ extends:
- powershell: |
cd '$(sslStressProject)'
$env:STRESS_CLIENT_ARGS = $env:SSLSTRESS_CLIENT_ARGS
$env:STRESS_SERVER_ARGS = $env:SSLSTRESS_SERVER_ARGS
New-Item -Force $env:DUMPS_SHARE -ItemType Directory
& $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
displayName: Run SslStress
- publish: $(Build.ArtifactStagingDirectory)/dumps
artifact: DumpsWindows
condition: failed()
79 changes: 79 additions & 0 deletions src/libraries/Common/tests/System/Net/StressTests/build-local.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
## This is a helper script for non-containerized local build and test execution.
## It downloads and uses the daily SDK which contains the compatible AspNetCore bits.
## Usage:
## ./build-local.ps1 [TestProjectDir] [StressConfiguration] [LibrariesConfiguration]

$RepoRoot="$(git rev-parse --show-toplevel)"
[xml]$xml = Get-Content (Join-Path $RepoRoot "eng\Versions.props")
$Version="$($xml.Project.PropertyGroup.MajorVersion[0]).$($xml.Project.PropertyGroup.MinorVersion[0])"

function PrintUsageAndExit {
Write-Host "Usage:"
Write-Host "./build-local.ps1 [TestProjectDir] [StressConfiguration] [LibrariesConfiguration]"
Write-Host "StressConfiguration and LibrariesConfiguration default to Release!"
exit 1
}

if (-not ([string]::IsNullOrEmpty($args[0])) -and (Test-Path -Path $args[0])) {
$TestProjectDir = $args[0]
} else {
Write-Host "Valid TestProjectDir is required!"
PrintUsageAndExit
}

$ProjectName = (Get-Item $TestProjectDir).Name

$DailyDotnetRoot= Join-Path $TestProjectDir ".dotnet-daily"

$StressConfiguration = "Release"
if (-not ([string]::IsNullOrEmpty($args[1]))) {
$StressConfiguration = $args[1]
}

$LibrariesConfiguration = "Release"
if (-not ([string]::IsNullOrEmpty($args[2]))) {
$LibrariesConfiguration = $args[2]
}

$TestHostRoot="$RepoRoot/artifacts/bin/testhost/net$Version-windows-$LibrariesConfiguration-x64"

Write-Host "StressConfiguration: $StressConfiguration, LibrariesConfiguration: $LibrariesConfiguration, testhost: $TestHostRoot"

if (-not (Test-Path -Path $TestHostRoot)) {
Write-Host "Cannot find testhost in: $TestHostRoot"
Write-Host "Make sure libraries with the requested configuration are built!"
PrintUsageAndExit
exit 1
}

if (-not (Test-Path -Path $DailyDotnetRoot)) {
Write-Host "Downloading daily SDK to: $DailyDotnetRoot"
New-Item -ItemType Directory -Path $DailyDotnetRoot
Invoke-WebRequest -Uri https://dot.net/v1/dotnet-install.ps1 -OutFile "$DailyDotnetRoot\dotnet-install.ps1"
& "$DailyDotnetRoot\dotnet-install.ps1" -NoPath -Channel $Version -Quality daily -InstallDir $DailyDotnetRoot
} else {
Write-Host "Daily SDK found in $DailyDotnetRoot"
}

$env:DOTNET_ROOT=$DailyDotnetRoot
$env:PATH="$DailyDotnetRoot;$env:PATH"
$env:DOTNET_MULTILEVEL_LOOKUP=0

if (-not (Test-Path -Path "$TestHostRoot/shared/Microsoft.AspNetCore.App")) {
Write-Host "Copying Microsoft.AspNetCore.App bits from daily SDK to testhost: $TestHostRoot"
Copy-Item -Recurse -Path "$DailyDotnetRoot/shared/Microsoft.AspNetCore.App" -Destination "$TestHostRoot/shared"
} else {
Write-Host "Microsoft.AspNetCore.App found in testhost: $TestHostRoot"
}

Write-Host "Building solution."
dotnet build -c $StressConfiguration -f "net$Version"

$Runscript=".\run-stress-$StressConfiguration-$LibrariesConfiguration.ps1"
if (-not (Test-Path $Runscript)) {
Write-Host "Generating Runscript."
Add-Content -Path $Runscript -Value "& '$TestHostRoot/dotnet' exec --roll-forward Major ./bin/$StressConfiguration/net$Version/$ProjectName.dll `$args"
}

Write-Host "To run tests type:"
Write-Host "$Runscript [stress test args]"
Loading

0 comments on commit 2c5b609

Please sign in to comment.