Add first CI barista benchmark jobs.

(cherry picked from commit 9b3a5d7)

Author: akolic

compiler/ci/ci_common/benchmark-builders.jsonnet

@@ -14,6 +14,7 @@
     c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.dacapo + PR_bench_libgraal,
     c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.scala_dacapo + PR_bench_libgraal,
     c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.renaissance + PR_bench_libgraal,
+    c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.barista,
     c.daily + c.opt_post_merge + hw.e3 + jdk + cc.libgraal + bench.specjvm2008 + PR_bench_libgraal,
     c.on_demand                + hw.e3 + jdk + cc.libgraal + bench.dacapo_size_variants,
     c.on_demand                + hw.e3 + jdk + cc.libgraal + bench.scala_dacapo_size_variants,

compiler/ci/ci_common/benchmark-suites.libsonnet

@@ -8,7 +8,7 @@
 
   // convenient sets of benchmark suites for easy reuse
   groups:: {
-    open_suites:: unique_suites([$.awfy, $.dacapo, $.scala_dacapo, $.renaissance]),
+    open_suites:: unique_suites([$.awfy, $.dacapo, $.scala_dacapo, $.renaissance, $.barista]),
     spec_suites:: unique_suites([$.specjvm2008, $.specjbb2015]),
     jmh_micros_suites:: unique_suites([$.micros_graal_dist]),
     graal_internals_suites:: unique_suites([$.micros_graal_whitebox]),
@@ -112,6 +112,47 @@
 
   renaissance: self.renaissance_template(),
 
+  barista_template(suite_version=null, suite_name="barista", max_jdk_version=null, cmd_app_prefix=["hwloc-bind --cpubind node:0.core:0-3.pu:0 --membind node:0"], non_prefix_barista_args=[]):: cc.compiler_benchmark + {
+    suite:: suite_name,
+    local barista_version = "v0.2.0",
+    local suite_version_args = if suite_version != null then ["--bench-suite-version=" + suite_version] else [],
+    local prefix_barista_arg = if std.length(cmd_app_prefix) > 0 then [std.format("--cmd-app-prefix=%s", std.join(" ", cmd_app_prefix))] else [],
+    local all_barista_args = prefix_barista_arg + non_prefix_barista_args,
+    local barista_args_with_separator = if std.length(all_barista_args) > 0 then ["--"] + all_barista_args else [],
+    downloads+: {
+      "WRK": { "name": "wrk", "version": "a211dd5", platformspecific: true},
+      "WRK2": { "name": "wrk2", "version": "2.1", platformspecific: true},
+      "BARISTA_BENCHMARKS": { "name": "barista", "version": "0.2.0"}
+    },
+    packages+: {
+      maven: "==3.8.6",
+      "pip:toml": "==0.10.2"
+    },
+    setup: [
+      ["set-export", "PATH", "$WRK:$PATH"],
+      ["set-export", "PATH", "$WRK2:$PATH"],
+      ["git", "clone", "--depth", "1", "--branch", barista_version, ["mx", "urlrewrite", "https://github.com/graalvm/barista-suite.git"], "$BARISTA_HOME"],
+      ["cp", "-r", "$BARISTA_BENCHMARKS/*", "$BARISTA_HOME"] // copy the prebuilt jar/nib files
+    ] + super.setup,
+    run+: [
+      self.benchmark_cmd + ["barista:*"] + suite_version_args + ["--"] + self.extra_vm_args + barista_args_with_separator
+    ],
+    notify_emails+: ["[email protected]"],
+    timelimit: "1:20:00",
+    should_use_hwloc: false, // hwloc-bind is passed to barista with '--cmd-app-prefix'
+    environment+: {
+      BARISTA_HOME: "$BUILD_DIR/barista-suite",
+      XMX: "500m"
+    },
+    min_jdk_version:: 8,
+    max_jdk_version:: max_jdk_version,
+    forks_batches:: 3,
+    bench_forks_per_batch:: 4,
+    forks_timelimit:: "3:30:00"
+  },
+
+  barista: self.barista_template(),
+
   specjbb2015: cc.compiler_benchmark + c.heap.large_with_large_young_gen + bc.bench_max_threads + {
     suite:: "specjbb2015",
     downloads+: {

sdk/mx.sdk/mx_sdk_benchmark.py

@@ -1187,7 +1187,21 @@ def rules(self, out, benchmarks, bmSuiteArgs):
 
 
 _baristaConfig = {
-    "benchmarks": ["micronaut-hello-world", "micronaut-shopcart", "micronaut-similarity", "quarkus-hello-world", "quarkus-tika", "spring-hello-world", "spring-petclinic"],
+    "benchmarks": {
+        "micronaut-hello-world": {},
+        "micronaut-shopcart": {},
+        "micronaut-similarity": {},
+        "quarkus-hello-world": {},
+        "quarkus-tika-odt": {
+            "barista-bench-name": "quarkus-tika",
+        },
+        "quarkus-tika-pdf": {
+            "barista-bench-name": "quarkus-tika",
+            "workload": "pdf-workload.barista.json",
+        },
+        "spring-hello-world": {},
+        "spring-petclinic": {},
+    },
     "latency_percentiles": [50.0, 75.0, 90.0, 99.0, 99.9, 99.99, 99.999, 100.0],
     "rss_percentiles": [100, 99, 98, 97, 96, 95, 90, 75, 50, 25],
     "disable_trackers": [mx_benchmark.RssTracker, mx_benchmark.PsrecordTracker, mx_benchmark.PsrecordMaxrssTracker, mx_benchmark.RssPercentilesTracker, mx_benchmark.RssPercentilesAndMaxTracker],
@@ -1215,16 +1229,27 @@ def context(self):
     def context(self, value):
         self._context = value
 
+    def readBaristaVersionFromPyproject(self):
+        # tomllib was included in python standard library with version 3.11
+        try:
+            import tomllib
+            with open(self.baristaProjectConfigurationPath(), mode="rb") as pyproject:
+                return tomllib.load(pyproject)["project"]["version"]
+        except ImportError:
+            pass
+
+        # fallback to 'toml' library if tomllib is not present
+        try:
+            import toml
+            with open(self.baristaProjectConfigurationPath(), mode="rt") as pyproject:
+                return toml.loads(pyproject.read())["project"]["version"]
+        except ImportError:
+            mx.warn("Could not read the Barista version from the project's `pyproject.toml` file because there is no toml parser installed. Use python3.11+ or install `toml` with pip.")
+        return self.defaultSuiteVersion()
+
     def version(self):
         if self._version is None:
-            # tomllib was included in python standard library with version 3.11
-            try:
-                import tomllib
-                with open(self.baristaProjectConfigurationPath(), mode="rb") as pyproject:
-                    self._version = tomllib.load(pyproject)["project"]["version"]
-            except ImportError:
-                mx.warn("Could not read the Barista version from the project's `pyproject.toml` file because the `tomllib` python module is not installed. Use python3.11+ or install `tomllib` with pip.")
-                self._version = self.defaultSuiteVersion()
+            self._version = self.readBaristaVersionFromPyproject()
         return self._version
 
     def name(self):
@@ -1240,7 +1265,7 @@ def benchmarkList(self, bmSuiteArgs):
         return self.completeBenchmarkList(bmSuiteArgs)
 
     def completeBenchmarkList(self, bmSuiteArgs):
-        return _baristaConfig["benchmarks"]
+        return _baristaConfig["benchmarks"].keys()
 
     def baristaDirectoryPath(self):
         barista_home = mx.get_env("BARISTA_HOME")
@@ -1266,6 +1291,12 @@ def baristaBuilderPath(self):
     def baristaHarnessPath(self):
         return self.baristaFilePath("barista")
 
+    def baristaHarnessBenchmarkName(self):
+        return _baristaConfig["benchmarks"][self.context.benchmark].get("barista-bench-name", self.context.benchmark)
+
+    def baristaHarnessBenchmarkWorkload(self):
+        return _baristaConfig["benchmarks"][self.context.benchmark].get("workload")
+
     def validateEnvironment(self):
         self.baristaProjectConfigurationPath()
         self.baristaHarnessPath()
@@ -1290,7 +1321,7 @@ def rules(self, out, benchmarks, bmSuiteArgs):
 
         # Startup
         all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": ("<benchmark>", str),
+            "benchmark": self.context.benchmark,
             "metric.name": "request-time",
             "metric.type": "numeric",
             "metric.unit": "ms",
@@ -1299,11 +1330,11 @@ def rules(self, out, benchmarks, bmSuiteArgs):
             "metric.iteration": ("<startup.measurements.iteration>", int),
             "load-tester.id": ("<startup.id>", str),
             "load-tester.method-type": "requests"
-        }, ["benchmark", "startup.id", "startup.measurements.iteration", "startup.measurements.response_time"]))
+        }, ["startup.id", "startup.measurements.iteration", "startup.measurements.response_time"]))
 
         # Warmup
         all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": ("<benchmark>", str),
+            "benchmark": self.context.benchmark,
             "metric.name": "warmup",
             "metric.type": "numeric",
             "metric.unit": "op/s",
@@ -1312,11 +1343,11 @@ def rules(self, out, benchmarks, bmSuiteArgs):
             "metric.iteration": ("<warmup.measurements.iteration>", int),
             "load-tester.id": ("<warmup.id>", str),
             "load-tester.command": ("<warmup.measurements.command>", str)
-        }, ["benchmark", "warmup.id", "warmup.measurements.iteration", "warmup.measurements.throughput", "warmup.measurements.command"]))
+        }, ["warmup.id", "warmup.measurements.iteration", "warmup.measurements.throughput", "warmup.measurements.command"]))
 
         # Throughput
         all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": ("<benchmark>", str),
+            "benchmark": self.context.benchmark,
             "metric.name": "throughput",
             "metric.type": "numeric",
             "metric.unit": "op/s",
@@ -1325,11 +1356,11 @@ def rules(self, out, benchmarks, bmSuiteArgs):
             "metric.iteration": ("<throughput.measurements.iteration>", int),
             "load-tester.id": ("<throughput.id>", str),
             "load-tester.command": ("<throughput.measurements.command>", str)
-        }, ["benchmark", "throughput.id", "throughput.measurements.iteration", "throughput.measurements.throughput", "throughput.measurements.command"]))
+        }, ["throughput.id", "throughput.measurements.iteration", "throughput.measurements.throughput", "throughput.measurements.command"]))
 
         # Latency
         all_rules += [mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": ("<benchmark>", str),
+            "benchmark": self.context.benchmark,
             "metric.name": "latency",
             "metric.type": "numeric",
             "metric.unit": "ms",
@@ -1340,7 +1371,6 @@ def rules(self, out, benchmarks, bmSuiteArgs):
             "load-tester.id": ("<latency__id>", str),
             "load-tester.command": ("<latency__measurements__final_measurements__command>", str)
         }, [
-            "benchmark",
             "latency__id",
             "latency__measurements__final_measurements__iteration",
             f"latency__measurements__final_measurements__p_values__{float(percentile)}",
@@ -1349,24 +1379,26 @@ def rules(self, out, benchmarks, bmSuiteArgs):
 
         # Resource Usage
         all_rules += [mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": ("<benchmark>", str),
+            "benchmark": self.context.benchmark,
             "metric.name": "rss",
             "metric.type": "numeric",
             "metric.unit": "MB",
             "metric.value": (f"<resource_usage__rss__p{float(percentile)}>", float),
             "metric.percentile": float(percentile),
             "metric.better": "lower",
         }, [
-            "benchmark", f"resource_usage__rss__p{float(percentile)}"
+            f"resource_usage__rss__p{float(percentile)}"
         ], indexer_str="__") for percentile in _baristaConfig["rss_percentiles"]]
+        # Ensure we are reporting the analogous numbers across suites (p99 at the time of writing this comment)
+        percentile_to_copy_into_max_rss = float(mx_benchmark.RssPercentilesTracker.MaxRssCopyRule.percentile_to_copy_into_max_rss)
         all_rules.append(mx_benchmark.JsonArrayStdOutFileRule(json_file_pattern, json_file_group_name, {
-            "benchmark": ("<benchmark>", str),
+            "benchmark": self.context.benchmark,
             "metric.name": "max-rss",
             "metric.type": "numeric",
             "metric.unit": "MB",
-            "metric.value": ("<resource_usage__rss__p100.0>", float),
+            "metric.value": (f"<resource_usage__rss__p{percentile_to_copy_into_max_rss}>", float),
             "metric.better": "lower",
-        }, ["benchmark", f"resource_usage__rss__p100.0"], indexer_str="__"))
+        }, [f"resource_usage__rss__p{percentile_to_copy_into_max_rss}"], indexer_str="__"))
 
         return all_rules
 
@@ -1467,16 +1499,22 @@ def produceHarnessCommand(self, cmd, suite):
             if mode_match:
                 raise ValueError(f"You should not set the Barista '--mode' option manually! Found '{mode_match.group(0)}' in the run arguments!")
 
+            # Get bench name and workload to use in the barista harness - we might have custom named benchmarks that need to be mapped
+            barista_bench_name = suite.baristaHarnessBenchmarkName()
+            barista_workload = suite.baristaHarnessBenchmarkWorkload()
+
             # Construct the Barista command
             barista_cmd = [suite.baristaHarnessPath()]
             barista_cmd.append(f"--java-home={java_exe_match.group(1)}")
+            if barista_workload is not None:
+                barista_cmd.append(f"--config={barista_workload}")
             barista_cmd += run_args
             if jvm_vm_options:
                 self._updateCommandOption(barista_cmd, "--vm-options", "-v", " ".join(jvm_vm_options))
             if jvm_cmd_prefix:
                 self._updateCommandOption(barista_cmd, "--cmd-app-prefix", "-p", " ".join(jvm_cmd_prefix))
             barista_cmd += ["--mode", "jvm"]
-            barista_cmd.append(suite.context.benchmark)
+            barista_cmd.append(barista_bench_name)
             return barista_cmd
 
     class RuntimeContext():

substratevm/mx.substratevm/mx_substratevm_benchmark.py

@@ -295,7 +295,7 @@ def application_nib(self):
         if self.benchmarkName() not in self._application_nibs:
             # Run subprocess retrieving the application nib from the Barista 'build' script
             out = mx.OutputCapture()
-            mx.run([self.baristaBuilderPath(), "--get-nib", self.benchmarkName()], out=out)
+            mx.run([self.baristaBuilderPath(), "--get-nib", self.baristaHarnessBenchmarkName()], out=out)
             # Capture the application nib from the Barista 'build' script output
             nib_pattern = r"application nib file path is: ([^\n]+)\n"
             nib_match = re.search(nib_pattern, out.data)
@@ -435,8 +435,14 @@ def produceHarnessCommand(self, cmd, suite):
             nivm_cmd_prefix = cmd[:index_of_app_image]
             nivm_app_options = cmd[index_of_app_image + 1:]
 
+            # Get bench name and workload to use in the barista harness - we might have custom named benchmarks that need to be mapped
+            barista_bench_name = suite.baristaHarnessBenchmarkName()
+            barista_workload = suite.baristaHarnessBenchmarkWorkload()
+
             # Provide image built in the previous stage to the Barista harnesss using the `--app-executable` option
             ni_barista_cmd = [suite.baristaHarnessPath(), "--mode", "native", "--app-executable", app_image]
+            if barista_workload is not None:
+                ni_barista_cmd.append(f"--config={barista_workload}")
             ni_barista_cmd += suite.runArgs(suite.context.bmSuiteArgs)
             ni_barista_cmd += mx_sdk_benchmark.parse_prefixed_args("-Dnative-image.benchmark.extra-jvm-arg=", suite.context.bmSuiteArgs)
             if stage == mx_sdk_benchmark.Stage.INSTRUMENT_RUN:
@@ -451,7 +457,7 @@ def produceHarnessCommand(self, cmd, suite):
                 self._updateCommandOption(ni_barista_cmd, "--cmd-app-prefix", "-p", " ".join(nivm_cmd_prefix))
             if nivm_app_options:
                 self._updateCommandOption(ni_barista_cmd, "--app-args", "-a", " ".join(nivm_app_options))
-            ni_barista_cmd += [suite.benchmarkName()]
+            ni_barista_cmd += [barista_bench_name]
             return ni_barista_cmd