diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index ac52965c2759..c794d98c96a2 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -414,6 +414,69 @@ stages: displayName: publish tracer and native loader test results artifact: build_linux_tracer_unit_tests_results_$(artifactSuffix)_$(System.JobAttempt) +- stage: build_linux_tracer_r2r + dependsOn: [merge_commit_id] + variables: + targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] + targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + jobs: + - template: steps/update-github-status-jobs.yml + parameters: + jobs: [build] + + - job: build + timeoutInMinutes: 60 #default value + dependsOn: [] + pool: + name: azure-linux-scale-set-2 + + steps: + - template: steps/clone-repo.yml + parameters: + targetShaId: $(targetShaId) + targetBranch: $(targetBranch) + + - template: steps/run-in-docker.yml + parameters: + build: true + target: builder + baseImage: debian + command: "Clean CompileManagedLoader" + retryCountForRunCommand: 1 + + - template: steps/run-in-docker.yml + parameters: + build: true + target: builder + baseImage: centos7 + useNativeSdkVersion: true + command: "BuildNativeTracerHome CompileTracerNativeTests RunTracerNativeTests" + retryCountForRunCommand: 1 + apiKey: $(DD_LOGGER_DD_API_KEY) + + - template: steps/run-in-docker.yml + parameters: + target: builder + baseImage: debian + command: "BuildManagedTracerHomeR2R ExtractDebugInfoLinux" + retryCountForRunCommand: 1 + + - publish: $(monitoringHome) + displayName: Uploading linux tracer home artifact + artifact: linux-tracer-home-linux-x64-r2r + + - publish: $(System.DefaultWorkingDirectory) + displayName: Upload working directory after the build + artifact: build-linux-x64-r2r-working-directory + + - publish: $(symbols) + displayName: Upload linux tracer symbols + artifact: linux-tracer-symbols-linux-x64-r2r + + - publish: artifacts/build_data/tests + displayName: publish tracer and native loader test results + artifact: build_linux_tracer_unit_tests_results_linux-x64-r2r_$(System.JobAttempt) + - stage: build_linux_universal dependsOn: [merge_commit_id] variables: @@ -657,6 +720,49 @@ stages: displayName: Upload linux tracer symbols artifact: linux-tracer-symbols-$(artifactSuffix) +- stage: build_arm64_tracer_r2r + dependsOn: [merge_commit_id] + variables: + targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']] + targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']] + jobs: + - template: steps/update-github-status-jobs.yml + parameters: + jobs: [build] + + - job: build + timeoutInMinutes: 60 #default value + dependsOn: [] + + pool: + name: aws-arm64-auto-scaling + workspace: + clean: all + steps: + - template: steps/clone-repo.yml + parameters: + targetShaId: $(targetShaId) + targetBranch: $(targetBranch) + - template: steps/run-in-docker.yml + parameters: + build: true + target: builder + baseImage: debian + command: "Clean CompileManagedLoader BuildNativeTracerHome BuildManagedTracerHomeR2R ExtractDebugInfoLinux" + retryCountForRunCommand: 1 + + - publish: $(monitoringHome) + displayName: Uploading linux tracer home artifact + artifact: linux-tracer-home-linux-arm64-r2r + + - publish: $(System.DefaultWorkingDirectory) + displayName: Upload working directory after the build + artifact: build-linux-arm64-r2r-working-directory + + - publish: $(symbols) + displayName: Upload linux tracer symbols + artifact: linux-tracer-symbols-linux-arm64-r2r + - stage: build_arm64_profiler dependsOn: [merge_commit_id] variables: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 949a52ff4261..6402883015e4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -115,6 +115,38 @@ deploy_to_reliability_env: rules: - when: never # dd-trace-dotnet does not use reliability environment +download-serverless-artifacts: + stage: package + image: registry.ddbuild.io/docker:20.10.13-gbi-focal + tags: [ "arch:amd64" ] + needs: [] + rules: + - if: $DOTNET_PACKAGE_VERSION + when: on_success + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+(-prerelease)?$/' # Manually triggered as artifacts are from the Github release + when: manual + allow_failure: false + - when: delayed # Artifacts come from Azure pipeline, wait a reasonable time before polling + start_in: 15 minutes + script: + - .gitlab/download-serverless-artifacts.sh + artifacts: + expire_in: 2 weeks + paths: + - artifacts + +aws-lambda-layer: + stage: package + needs: [download-serverless-artifacts] + when: manual + trigger: + project: DataDog/dd-trace-dotnet-aws-lambda-layer + strategy: depend + variables: + UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID + UPSTREAM_PIPELINE_REF: $CI_COMMIT_REF_NAME + allow_failure: true + benchmark-serverless: stage: benchmarks image: registry.ddbuild.io/ci/serverless-tools:1 diff --git a/.gitlab/download-serverless-artifacts.sh b/.gitlab/download-serverless-artifacts.sh new file mode 100755 index 000000000000..03bacd9d463b --- /dev/null +++ b/.gitlab/download-serverless-artifacts.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# +# This scripts downloads the necessary binaries to be used for the AWS Lambda Layer. +# This artifacts include: Tracer and ClrProfiler +# + +set -eo pipefail + +# Create a directory to store the files +target_dir=artifacts +mkdir -p $target_dir + +branchName="refs/heads/$CI_COMMIT_BRANCH" + +echo "Looking for azure devops PR builds for branch '$branchName' for commit '$CI_COMMIT_SHA' to start" + +# We should _definitely_ have the build by now, so if not, there probably won't be one +# Check for PR builds first (as more likely to be "full" builds) +allBuildsForPrUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds?api-version=7.1&definitions=54&\$top=100&queryOrder=queueTimeDescending&reasonFilter=pullRequest" +buildId=$(curl -sS $allBuildsForPrUrl | jq --arg version $CI_COMMIT_SHA --arg branch $CI_COMMIT_BRANCH '.value[] | select(.triggerInfo["pr.sourceBranch"] == $branch and .triggerInfo["pr.sourceSha"] == $version) | .id' | head -n 1) + +if [ -z "${buildId}" ]; then + echo "No PR builds found for commit '$CI_COMMIT_SHA' on branch '$branchName'. Checking for standalone builds..." + allBuildsForBranchUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds?api-version=7.1&definitions=54&\$top=10&queryOrder=queueTimeDescending&branchName=$branchName&reasonFilter=manual,individualCI" + buildId=$(curl -sS $allBuildsForBranchUrl | jq --arg version $CI_COMMIT_SHA '.value[] | select(.sourceVersion == $version and .reason != "schedule") | .id' | head -n 1) +fi + +if [ -z "${buildId}" ]; then + echo "No build found for commit '$CI_COMMIT_SHA' on branch '$branchName' (including PRs)" + exit 1 +fi + +echo "Found build with id '$buildId' for commit '$CI_COMMIT_SHA' on branch '$branchName'" + +architectures=("x64" "arm64") +for architecture in "${architectures[@]}"; do + echo "Looking for artifacts for architecture '$architecture'" + + artifacts=("linux-tracer-home-linux-$architecture-r2r" "linux-universal-home-linux-$architecture") + + # Now try to download the artifacts from the build + for artifactName in "${artifacts[@]}"; do + artifactsUrl="https://dev.azure.com/datadoghq/dd-trace-dotnet/_apis/build/builds/$buildId/artifacts?api-version=7.1&artifactName=$artifactName" + + # Keep trying to get the artifact for 30 minutes + downloadUrl="" + TIMEOUT=1800 + STARTED=0 + until (( STARTED == TIMEOUT )) || [ ! -z "${downloadUrl}" ] ; do + echo "Checking for '$artifactName' at '$artifactsUrl'..." + # If the artifact doesn't exist, .resource.downloadUrl will be null, so we filter that out + downloadUrl=$(curl -s $artifactsUrl | jq -r '.resource.downloadUrl | select( . != null )') + sleep 100 + (( STARTED += 100 )) + done + (( STARTED < TIMEOUT )) + + if [ -z "${downloadUrl}" ]; then + echo "No downloadUrl found after 30 minutes for commit '$CI_COMMIT_SHA' on branch '$branchName'" + exit 1 + fi + + echo "Downloading '$artifactName' from '$downloadUrl'..." + curl -o $target_dir/artifacts.zip "$downloadUrl" + unzip $target_dir/artifacts.zip -d $target_dir/$architecture + mv $target_dir/$architecture/$artifactName/* $target_dir/$architecture + rm -rf $target_dir/artifacts.zip + rmdir $target_dir/$architecture/$artifactName + done +done + +ls -l $target_dir \ No newline at end of file diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index 851ffb94dfe9..ddb21d1a74b4 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -263,6 +263,10 @@ "type": "string", "description": "Root directory during build execution" }, + "RuntimeIdentifier": { + "type": "string", + "description": "RuntimeIdentifier sets the target platform for ReadyToRun assemblies in 'PublishManagedTracerR2R'.See https://learn.microsoft.com/en-us/dotnet/core/rid-catalog" + }, "SampleName": { "type": "string", "description": "The sample name to execute when running or building sample apps" @@ -296,6 +300,7 @@ "BuildIisSampleApp", "BuildLinuxIntegrationTests", "BuildManagedTracerHome", + "BuildManagedTracerHomeR2R", "BuildMsi", "BuildNativeLoader", "BuildNativeTracerHome", @@ -405,6 +410,7 @@ "PublishDdDotnetSymbolsWindows", "PublishIisSamples", "PublishManagedTracer", + "PublishManagedTracerR2R", "PublishNativeLoader", "PublishNativeLoaderOsx", "PublishNativeLoaderUnix", @@ -515,6 +521,7 @@ "BuildIisSampleApp", "BuildLinuxIntegrationTests", "BuildManagedTracerHome", + "BuildManagedTracerHomeR2R", "BuildMsi", "BuildNativeLoader", "BuildNativeTracerHome", @@ -624,6 +631,7 @@ "PublishDdDotnetSymbolsWindows", "PublishIisSamples", "PublishManagedTracer", + "PublishManagedTracerR2R", "PublishNativeLoader", "PublishNativeLoaderOsx", "PublishNativeLoaderUnix", diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs index 27dfe8daad2b..7cf4857f3408 100644 --- a/tracer/build/_build/Build.Steps.cs +++ b/tracer/build/_build/Build.Steps.cs @@ -639,23 +639,50 @@ async Task DownloadWafVersion(string libddwafVersion = null, string uncompressFo .EnableNoRestore() .CombineWith(targetFrameworks, (p, framework) => p .SetFramework(framework) - .SetOutput(MonitoringHomeDirectory / framework))); + .SetOutput(MonitoringHomeDirectory / framework)) + ); }); + Target PublishManagedTracerR2R => _ => _ + .Unlisted() + .After(CompileManagedSrc) + .Executes(() => + { + var targetFramework = TargetFramework.NET6_0; + + // Needed as we need to restore with the RuntimeIdentifier + DotNetRestore(s => s + .SetProjectFile(Solution.GetProject(Projects.DatadogTraceMsBuild)) + .SetPublishReadyToRun(true) + .SetRuntime(RuntimeIdentifier) + ); + + DotNetPublish(s => s + .SetProject(Solution.GetProject(Projects.DatadogTraceMsBuild)) + .SetConfiguration(BuildConfiguration) + .SetTargetPlatformAnyCPU() + .SetPublishReadyToRun(true) + .SetRuntime(RuntimeIdentifier) + .SetSelfContained(false) + .SetFramework(targetFramework) + .SetOutput(MonitoringHomeDirectory / targetFramework) + ); + }); + Target PublishNativeSymbolsWindows => _ => _ - .Unlisted() - .OnlyWhenStatic(() => IsWin) - .After(CompileTracerNativeSrc, PublishManagedTracer) - .Executes(() => - { - foreach (var architecture in ArchitecturesForPlatformForTracer) - { + .Unlisted() + .OnlyWhenStatic(() => IsWin) + .After(CompileTracerNativeSrc, PublishManagedTracer) + .Executes(() => + { + foreach (var architecture in ArchitecturesForPlatformForTracer) + { var source = NativeTracerProject.Directory / "bin" / BuildConfiguration / architecture.ToString() / $"{NativeTracerProject.Name}.pdb"; var dest = SymbolsDirectory / $"win-{architecture}" / Path.GetFileName(source); CopyFile(source, dest, FileExistsPolicy.Overwrite); - } - }); + } + }); Target PublishDdDotnetSymbolsWindows => _ => _ .Unlisted() diff --git a/tracer/build/_build/Build.Utilities.cs b/tracer/build/_build/Build.Utilities.cs index 041f56b371a4..d8df7f0a44c5 100644 --- a/tracer/build/_build/Build.Utilities.cs +++ b/tracer/build/_build/Build.Utilities.cs @@ -492,6 +492,22 @@ private static MSBuildTargetPlatform GetDefaultTargetPlatform() return MSBuildTargetPlatform.x64; } + + private static string GetDefaultRuntimeIdentifier(bool isAlpine) + { + // https://learn.microsoft.com/en-us/dotnet/core/rid-catalog + return (Platform, (string)GetDefaultTargetPlatform()) switch + { + (PlatformFamily.Windows, "x86") => "win-x86", + (PlatformFamily.Windows, "x64") => "win-x64", + + (PlatformFamily.Linux, "x64") => isAlpine ? "linux-musl-x64" : "linux-x64", + (PlatformFamily.Linux, "ARM64" or "ARM64EC") => isAlpine ? "linux-musl-arm64" : "linux-arm64", + + (PlatformFamily.OSX, "ARM64" or "ARM64EC") => "osx-arm64", + _ => null + }; + } private static MSBuildTargetPlatform ARM64TargetPlatform = (MSBuildTargetPlatform)"ARM64"; private static MSBuildTargetPlatform ARM64ECTargetPlatform = (MSBuildTargetPlatform)"ARM64EC"; diff --git a/tracer/build/_build/Build.cs b/tracer/build/_build/Build.cs index b1ce0dedce57..157b87ca3cb7 100644 --- a/tracer/build/_build/Build.cs +++ b/tracer/build/_build/Build.cs @@ -102,6 +102,15 @@ partial class Build : NukeBuild [Parameter("Should we build native binaries as Universal. Default to false, so we can still build native libs outside of docker.")] readonly bool AsUniversal = false; + + [Parameter("RuntimeIdentifier sets the target platform for ReadyToRun assemblies in 'PublishManagedTracerR2R'." + + "See https://learn.microsoft.com/en-us/dotnet/core/rid-catalog")] + string RuntimeIdentifier { get; } + + public Build() + { + RuntimeIdentifier = GetDefaultRuntimeIdentifier(IsAlpine); + } Target Info => _ => _ .Description("Describes the current configuration") @@ -118,6 +127,7 @@ partial class Build : NukeBuild Logger.Information($"IncludeAllTestFrameworks: {IncludeAllTestFrameworks}"); Logger.Information($"IsAlpine: {IsAlpine}"); Logger.Information($"Version: {Version}"); + Logger.Information($"RuntimeIdentifier: {RuntimeIdentifier}"); }); Target Clean => _ => _ @@ -185,6 +195,19 @@ void DeleteReparsePoints(string path) .DependsOn(CopyLibDdwaf) .DependsOn(CreateMissingNullabilityFile) .DependsOn(CreateRootDescriptorsFile); + + Target BuildManagedTracerHomeR2R => _ => _ + .Unlisted() + .Description("Builds the native and managed src, and publishes the tracer home directory") + .After(Clean, BuildNativeTracerHome) + .DependsOn(CreateRequiredDirectories) + .DependsOn(Restore) + .DependsOn(CompileManagedSrc) + .DependsOn(PublishManagedTracerR2R) + .DependsOn(DownloadLibDdwaf) + .DependsOn(CopyLibDdwaf) + .DependsOn(CreateMissingNullabilityFile) + .DependsOn(CreateRootDescriptorsFile); Target BuildTracerHome => _ => _ .Description("Builds the native and managed src, and publishes the tracer home directory")