System.Net Stress Tests Improvements 2 (#108703)

rzikm · web-flow · commit 1f40b0de528e · 2024-10-22T14:27:58.000+02:00
- replace docker-compose with docker compose on windows (removew workaround for actions/runner-images#7080) - Add -nobuild option to run-docker-compose.ps1/sh script to simplify running without building the images - remove version field from docker-compose.yml files to avoid triggering warnings - Add mount for crash dumps collections on ssl stress tests
diff --git a/eng/pipelines/libraries/stress/http.yml b/eng/pipelines/libraries/stress/http.yml
@@ -100,12 +100,6 @@ extends:
         variables:
           DUMPS_SHARE_MOUNT_ROOT: "C:/dumps-share"
           DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
-
-          # The 1es-windows-2022-open image has an issue where the Chocolatey-installed V1 docker-compose takes precendence over the
-          # V2 docker-compose required by the stress tests, see: https://github.com/actions/runner-images/issues/7080
-          # This is worked around by handpicking the V2 executable.
-          # The workaround should be removed when the official fix is propagated into 1es-windows-2022-open, or when we switch to another image.
-          DOCKER_COMPOSE_CMD: "C:/ProgramData/docker/cli-plugins/docker-compose.exe"
         pool:
           name: $(DncEngPublicBuildPool)
           demands: ImageOverride -equals 1es-windows-2022-open
@@ -140,7 +134,7 @@ extends:
             $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 3.0"
             $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 3.0"
             New-Item -Force $env:DUMPS_SHARE -ItemType Directory
-            & $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
+            docker compose up --abort-on-container-exit --no-color
           displayName: Run HttpStress - HTTP 3.0
           condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
@@ -149,7 +143,7 @@ extends:
             $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 2.0"
             $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 2.0"
             New-Item -Force $env:DUMPS_SHARE -ItemType Directory
-            & $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
+            docker compose up --abort-on-container-exit --no-color
           displayName: Run HttpStress - HTTP 2.0
           condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
@@ -158,7 +152,7 @@ extends:
             $env:STRESS_CLIENT_ARGS = "$env:HTTPSTRESS_CLIENT_ARGS -http 1.1"
             $env:STRESS_SERVER_ARGS = "$env:HTTPSTRESS_SERVER_ARGS -http 1.1"
             New-Item -Force $env:DUMPS_SHARE -ItemType Directory
-            & $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
+            docker compose up --abort-on-container-exit --no-color
           displayName: Run HttpStress - HTTP 1.1
           condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
diff --git a/eng/pipelines/libraries/stress/ssl.yml b/eng/pipelines/libraries/stress/ssl.yml
@@ -75,11 +75,6 @@ extends:
         variables:
           DUMPS_SHARE_MOUNT_ROOT: "C:/dumps-share"
           DUMPS_SHARE: "$(Build.ArtifactStagingDirectory)/dumps/"
-          # The 1es-windows-2022-open image has an issue where the Chocolatey-installed V1 docker-compose takes precendence over the
-          # V2 docker-compose required by the stress tests, see: https://github.com/actions/runner-images/issues/7080
-          # This is worked around by handpicking the V2 executable.
-          # The workaround should be removed when the official fix is propagated into 1es-windows-2022-open, or when we switch to another image.
-          DOCKER_COMPOSE_CMD: "C:/ProgramData/docker/cli-plugins/docker-compose.exe"
         steps:
         - checkout: self
           clean: true
@@ -99,7 +94,7 @@ extends:
             $env:STRESS_CLIENT_ARGS = $env:SSLSTRESS_CLIENT_ARGS
             $env:STRESS_SERVER_ARGS = $env:SSLSTRESS_SERVER_ARGS
             New-Item -Force $env:DUMPS_SHARE -ItemType Directory
-            & $env:DOCKER_COMPOSE_CMD up --abort-on-container-exit --no-color
+            docker compose up --abort-on-container-exit --no-color
           displayName: Run SslStress
 
         - publish: $(Build.ArtifactStagingDirectory)/dumps
diff --git a/src/libraries/Common/tests/System/Net/StressTests/run-docker-compose.ps1 b/src/libraries/Common/tests/System/Net/StressTests/run-docker-compose.ps1
@@ -9,6 +9,7 @@ Param(
     [switch][Alias('b')]$buildCurrentLibraries, # Drives the stress test using libraries built from current source
     [switch][Alias('pa')]$privateAspNetCore, # Drive the stress test using a private Asp.Net Core package, requires -b to be set
     [switch][Alias('o')]$buildOnly, # Build, but do not run the stress app
+    [switch][Alias('n')]$noBuild, # Do not build the docker image
     [string][Alias('t')]$sdkImageName = "dotnet-sdk-libs-current", # Name of the sdk image name, if built from source.
     [string]$clientStressArgs = "",
     [string]$serverStressArgs = "",
@@ -20,23 +21,12 @@ $COMPOSE_FILE = "$TestProjectDir/docker-compose.yml"
 [xml]$xml = Get-Content (Join-Path $RepoRoot "eng\Versions.props")
 $VERSION = "$($xml.Project.PropertyGroup.MajorVersion[0]).$($xml.Project.PropertyGroup.MinorVersion[0])"
 
-# This is a workaround for an issue with 1es-windows-2022-open, which should be eventually removed.
-# See comments in <repo>/eng/pipelines/libraries/stress/ssl.yml for more info.
-$dockerComposeCmd = $env:DOCKER_COMPOSE_CMD
-if (!(Test-Path $dockerComposeCmd -ErrorAction SilentlyContinue)) {
-    $dockerComposeCmd = "docker-compose"
-}
-
 if (!$dumpsSharePath) {
     $dumpsSharePath = "$TestProjectDir/dumps"
 }
 
 # Build runtime libraries and place in a docker image
 if ($buildCurrentLibraries) {
-    if ([string]::IsNullOrEmpty($sdkImageName)) {
-        $sdkImageName = "dotnet-sdk-libs-current"
-    }
-
     $LIBRARIES_BUILD_ARGS = " -t $sdkImageName -c $configuration"
     if ($useWindowsContainers) {
         $LIBRARIES_BUILD_ARGS += " -w"
@@ -55,46 +45,46 @@ elseif ($privateAspNetCore) {
     exit 1
 }
 
-# Dockerize the stress app using docker-compose
-$BuildArgs = @(
-    "--build-arg", "VERSION=$Version",
-    "--build-arg", "CONFIGURATION=$configuration"
-)
-if (![string]::IsNullOrEmpty($sdkImageName)) {
-    $BuildArgs += "--build-arg", "SDK_BASE_IMAGE=$sdkImageName"
-}
 if ($useWindowsContainers) {
     $env:DOCKERFILE = "windows.Dockerfile"
 }
 
-$originalErrorPreference = $ErrorActionPreference
-$ErrorActionPreference = 'Continue'
-try {
-    write-output "$dockerComposeCmd --log-level DEBUG --file $COMPOSE_FILE build $buildArgs"
-    & $dockerComposeCmd --log-level DEBUG --file $COMPOSE_FILE build @buildArgs 2>&1 | ForEach-Object { "$_" }
-    if ($LASTEXITCODE -ne 0) {
-        throw "docker-compose exited with error code $LASTEXITCODE"
+if (!$noBuild) {
+    # Dockerize the stress app using docker-compose
+    $BuildArgs = @(
+        "--build-arg", "VERSION=$Version",
+        "--build-arg", "CONFIGURATION=$configuration"
+    )
+    if ($sdkImageName) {
+        $BuildArgs += "--build-arg", "SDK_BASE_IMAGE=$sdkImageName"
+    }
+    $originalErrorPreference = $ErrorActionPreference
+    $ErrorActionPreference = 'Continue'
+    try {
+        write-output "docker compose --file $COMPOSE_FILE build $buildArgs"
+        docker compose --file $COMPOSE_FILE build @buildArgs 2>&1
+        if ($LASTEXITCODE -ne 0) {
+            throw "docker compose exited with error code $LASTEXITCODE"
+        }
+    }
+    finally {
+        $ErrorActionPreference = $originalErrorPreference
     }
-}
-finally {
-    $ErrorActionPreference = $originalErrorPreference
 }
 
 # Run the stress app
 if (!$buildOnly) {
-    if ($dumpsSharePath) {
-        if ($useWindowsContainers) {
-            $env:DUMPS_SHARE_MOUNT_ROOT = "C:/dumps-share"
-        }
-        else {
-            $env:DUMPS_SHARE_MOUNT_ROOT = "/dumps-share"
-        }
-
-        $env:DUMPS_SHARE = $dumpsSharePath
-        New-Item -Force $env:DUMPS_SHARE -ItemType Directory
+    if ($useWindowsContainers) {
+        $env:DUMPS_SHARE_MOUNT_ROOT = "C:/dumps-share"
     }
+    else {
+        $env:DUMPS_SHARE_MOUNT_ROOT = "/dumps-share"
+    }
+
+    $env:DUMPS_SHARE = $dumpsSharePath
+    New-Item -Force $env:DUMPS_SHARE -ItemType Directory
 
     $env:STRESS_CLIENT_ARGS = $clientStressArgs
     $env:STRESS_SERVER_ARGS = $serverStressArgs
-    & $dockerComposeCmd --file "$COMPOSE_FILE" up --abort-on-container-exit
+    docker compose --file "$COMPOSE_FILE" up --abort-on-container-exit
 }
diff --git a/src/libraries/Common/tests/System/Net/StressTests/run-docker-compose.sh b/src/libraries/Common/tests/System/Net/StressTests/run-docker-compose.sh
@@ -28,10 +28,12 @@ imagename="dotnet-sdk-libs-current"
 configuration="Release"
 buildcurrentlibraries=0
 buildonly=0
+nobuild=0
 clientstressargs=""
 serverstressargs=""
 
 projectdir=$1
+shift 1
 if [[ ! -d "$projectdir" ]]; then
     echo "First argument must be path to the stress project directory"
     exit 1
@@ -40,7 +42,7 @@ fi
 dumpssharepath="$projectdir/dumps"
 
 while [[ $# > 0 ]]; do
-  opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
+  opt="$(printf "%s" "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
   case "$opt" in
     -dumpssharepath|-d)
       dumpssharepath=$2
@@ -62,6 +64,10 @@ while [[ $# > 0 ]]; do
       buildonly=1
       shift 1
       ;;
+    -nobuild|-n)
+      nobuild=1
+      shift 1
+      ;;
     -clientstressargs)
       clientstressargs=$2
       shift 2
@@ -86,15 +92,17 @@ if [[ "$buildcurrentlibraries" -eq 1 ]]; then
     fi
 fi
 
-build_args="--build-arg VERSION=$version --build-arg CONFIGURATION=$configuration"
-if [[ -n "$imagename" ]]; then
-    build_args="$build_args --build-arg SDK_BASE_IMAGE=$imagename"
-fi
-
 compose_file="$projectdir/docker-compose.yml"
 
-if ! docker-compose --file "$compose_file" build $build_args; then
-    exit $?
+if [[ "$nobuild" -eq 0 ]]; then
+    build_args="--build-arg VERSION=$version --build-arg CONFIGURATION=$configuration"
+    if [[ -n "$imagename" ]]; then
+        build_args="$build_args --build-arg SDK_BASE_IMAGE=$imagename"
+    fi
+
+    if ! docker-compose --file "$compose_file" build $build_args; then
+        exit $?
+    fi
 fi
 
 if [[ "$buildonly" -eq 0 ]]; then
diff --git a/src/libraries/System.Net.Http/tests/StressTests/HttpStress/docker-compose.yml b/src/libraries/System.Net.Http/tests/StressTests/HttpStress/docker-compose.yml
@@ -1,4 +1,3 @@
-version: '3'
 services:
   client:
     build:
diff --git a/src/libraries/System.Net.Security/tests/StressTests/SslStress/docker-compose.yml b/src/libraries/System.Net.Security/tests/StressTests/SslStress/docker-compose.yml
@@ -1,9 +1,11 @@
-version: '3'
+version: '3' # Although the version attribute is obsolete and should be ignored, it's seemingly not the case on Build.Ubuntu.2204.Amd64.Open
 services:
   client:
     build:
       context: ../../../../ # ~> src/libraries
       dockerfile: ./System.Net.Security/tests/StressTests/SslStress/${DOCKERFILE:-Dockerfile}
+    volumes:
+      - "${DUMPS_SHARE}:${DUMPS_SHARE_MOUNT_ROOT}"
     links:
       - server
     environment:
@@ -12,6 +14,8 @@ services:
     build:
       context: ../../../../ # ~> src/libraries
       dockerfile: ./System.Net.Security/tests/StressTests/SslStress/${DOCKERFILE:-Dockerfile}
+    volumes:
+      - "${DUMPS_SHARE}:${DUMPS_SHARE_MOUNT_ROOT}"
     ports:
       - "5001:5001"
     environment:

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-version: '3'`
`2`	`1`	`services:`
`3`	`2`	`client:`
`4`	`3`	`build:`