Upload Datadog average CPU usage metric in citool

Author: Kobzol

.github/workflows/ci.yml

@@ -183,7 +183,7 @@ jobs:
         run: src/ci/scripts/dump-environment.sh
 
       # Pre-build citool before the following step uninstalls rustup
-      # Build is into the build directory, to avoid modifying sources
+      # Build it into the build directory, to avoid modifying sources
       - name: build citool
         run: |
           cd src/ci/citool
@@ -238,13 +238,9 @@ jobs:
       - name: upload job metrics to DataDog
         if: needs.calculate_matrix.outputs.run_type != 'pr'
         env:
-          DATADOG_SITE: datadoghq.com
           DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
           DD_GITHUB_JOB_NAME: ${{ matrix.full_name }}
-        run: |
-          cd src/ci
-          npm ci
-          python3 scripts/upload-build-metrics.py ../../build/cpu-usage.csv
+        run: ./build/citool/debug/citool upload-build-metrics build/cpu-usage.csv
 
   # This job isused to tell bors the final status of the build, as there is no practical way to detect
   # when a workflow is successful listening to webhooks only in our current bors implementation (homu).

src/ci/citool/Cargo.toml

@@ -6,9 +6,11 @@ edition = "2021"
 [dependencies]
 anyhow = "1"
 clap = { version = "4.5", features = ["derive"] }
+csv = "1"
 serde = { version = "1", features = ["derive"] }
 serde_yaml = "0.9"
 serde_json = "1"
+ureq = { version = "3", features = ["json"] }
 
 build_helper = { path = "../../build_helper" }
 

src/ci/citool/src/cpu_usage.rs

@@ -0,0 +1,22 @@
+use std::path::Path;
+
+/// Loads CPU usage records from a CSV generated by the `src/ci/scripts/collect-cpu-stats.sh`
+/// script.
+pub fn load_cpu_usage(path: &Path) -> anyhow::Result<Vec<f64>> {
+    let reader = csv::ReaderBuilder::new().flexible(true).from_path(path)?;
+
+    let mut entries = vec![];
+    for row in reader.into_records() {
+        let row = row?;
+        let cols = row.into_iter().collect::<Vec<&str>>();
+
+        // The log might contain incomplete rows or some Python exception
+        if cols.len() == 2 {
+            if let Ok(idle) = cols[1].parse::<f64>() {
+                entries.push(100.0 - idle);
+            }
+        }
+    }
+
+    Ok(entries)
+}

src/ci/citool/src/datadog.rs

@@ -0,0 +1,43 @@
+use anyhow::Context;
+
+use crate::utils::load_env_var;
+
+/// Uploads a custom CI pipeline metric to Datadog.
+/// Expects to be executed from within the context of a GitHub Actions job.
+pub fn upload_datadog_metric(name: &str, value: f64) -> anyhow::Result<()> {
+    let datadog_api_key = load_env_var("DATADOG_API_KEY")?;
+    let github_server_url = load_env_var("GITHUB_SERVER_URL")?;
+    let github_repository = load_env_var("GITHUB_REPOSITORY")?;
+    let github_run_id = load_env_var("GITHUB_RUN_ID")?;
+    let github_run_attempt = load_env_var("GITHUB_RUN_ATTEMPT")?;
+    let github_job = load_env_var("GITHUB_JOB")?;
+    let dd_github_job_name = load_env_var("DD_GITHUB_JOB_NAME")?;
+
+    // This API endpoint is not documented in Datadog's API reference currently.
+    // It was reverse-engineered from the `datadog-ci measure` npm command.
+    ureq::post("https://api.datadoghq.com/api/v2/ci/pipeline/metrics")
+        .header("DD-API-KEY", datadog_api_key)
+        .send_json(serde_json::json!({
+            "data": {
+                "attributes": {
+                    "ci_env": {
+                        "GITHUB_SERVER_URL": github_server_url,
+                        "GITHUB_REPOSITORY": github_repository,
+                        "GITHUB_RUN_ID": github_run_id,
+                        "GITHUB_RUN_ATTEMPT": github_run_attempt,
+                        "GITHUB_JOB": github_job,
+                        "DD_GITHUB_JOB_NAME": dd_github_job_name
+                    },
+                    // Job level
+                    "ci_level": 1,
+                    "metrics": {
+                        name: value
+                    },
+                    "provider": "github"
+                },
+                "type": "ci_custom_metric"
+            }
+        }))
+        .context("cannot send metric to DataDog")?;
+    Ok(())
+}

src/ci/citool/src/main.rs

@@ -1,4 +1,7 @@
+mod cpu_usage;
+mod datadog;
 mod metrics;
+mod utils;
 
 use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
@@ -8,6 +11,10 @@ use anyhow::Context;
 use clap::Parser;
 use serde_yaml::Value;
 
+use crate::cpu_usage::load_cpu_usage;
+use crate::datadog::upload_datadog_metric;
+use crate::utils::load_env_var;
+
 use crate::metrics::postprocess_metrics;
 
 const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/..");
@@ -75,7 +82,7 @@ impl JobDatabase {
 }
 
 fn load_job_db(path: &Path) -> anyhow::Result<JobDatabase> {
-    let db = read_to_string(path)?;
+    let db = utils::read_to_string(path)?;
     let mut db: Value = serde_yaml::from_str(&db)?;
 
     // We need to expand merge keys (<<), because serde_yaml can't deal with them
@@ -148,10 +155,6 @@ impl GitHubContext {
     }
 }
 
-fn load_env_var(name: &str) -> anyhow::Result<String> {
-    std::env::var(name).with_context(|| format!("Cannot find variable {name}"))
-}
-
 fn load_github_ctx() -> anyhow::Result<GitHubContext> {
     let event_name = load_env_var("GITHUB_EVENT_NAME")?;
     let commit_message =
@@ -325,6 +328,18 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any
     if !result.success() { Err(anyhow::anyhow!("Job failed")) } else { Ok(()) }
 }
 
+fn upload_ci_metrics(cpu_usage_csv: &Path) -> anyhow::Result<()> {
+    let usage = load_cpu_usage(cpu_usage_csv).context("Cannot load CPU usage from input CSV")?;
+    eprintln!("CPU usage\n{usage:?}");
+
+    let avg = if !usage.is_empty() { usage.iter().sum::<f64>() / usage.len() as f64 } else { 0.0 };
+    eprintln!("CPU usage average: {avg}");
+
+    upload_datadog_metric("avg-cpu-usage", avg).context("Cannot upload Datadog metric")?;
+
+    Ok(())
+}
+
 #[derive(clap::Parser)]
 enum Args {
     /// Calculate a list of jobs that should be executed on CI.
@@ -350,6 +365,11 @@ enum Args {
         /// Usually, this will be GITHUB_STEP_SUMMARY on CI.
         summary_path: PathBuf,
     },
+    /// Upload CI metrics to Datadog.
+    UploadBuildMetrics {
+        /// Path to a CSV containing the CI job CPU usage.
+        cpu_usage_csv: PathBuf,
+    },
 }
 
 #[derive(clap::ValueEnum, Clone)]
@@ -370,7 +390,7 @@ fn main() -> anyhow::Result<()> {
             let jobs_path = jobs_file.as_deref().unwrap_or(default_jobs_file);
             let gh_ctx = load_github_ctx()
                 .context("Cannot load environment variables from GitHub Actions")?;
-            let channel = read_to_string(Path::new(CI_DIRECTORY).join("channel"))
+            let channel = utils::read_to_string(Path::new(CI_DIRECTORY).join("channel"))
                 .context("Cannot read channel file")?
                 .trim()
                 .to_string();
@@ -379,7 +399,10 @@ fn main() -> anyhow::Result<()> {
                 .context("Failed to calculate job matrix")?;
         }
         Args::RunJobLocally { job_type, name } => {
-            run_workflow_locally(load_db(default_jobs_file)?, job_type, name)?
+            run_workflow_locally(load_db(default_jobs_file)?, job_type, name)?;
+        }
+        Args::UploadBuildMetrics { cpu_usage_csv } => {
+            upload_ci_metrics(&cpu_usage_csv)?;
         }
         Args::PostprocessMetrics { metrics_path, summary_path } => {
             postprocess_metrics(&metrics_path, &summary_path)?;
@@ -388,8 +411,3 @@ fn main() -> anyhow::Result<()> {
 
     Ok(())
 }
-
-fn read_to_string<P: AsRef<Path>>(path: P) -> anyhow::Result<String> {
-    let error = format!("Cannot read file {:?}", path.as_ref());
-    std::fs::read_to_string(path).context(error)
-}

src/ci/citool/src/utils.rs

@@ -0,0 +1,12 @@
+use std::path::Path;
+
+use anyhow::Context;
+
+pub fn load_env_var(name: &str) -> anyhow::Result<String> {
+    std::env::var(name).with_context(|| format!("Cannot find environment variable `{name}`"))
+}
+
+pub fn read_to_string<P: AsRef<Path>>(path: P) -> anyhow::Result<String> {
+    let error = format!("Cannot read file {:?}", path.as_ref());
+    std::fs::read_to_string(path).context(error)
+}