Read disposable performance tests

For: QubesOS/qubes-issues#1512

Author: ben-grande

qubes/tests/integ/dispvm.py

@@ -3,6 +3,7 @@
 #
 # Copyright (C) 2016 Marek Marczykowski-Górecki
 #                                        <[email protected]>
+# Copyright (C) 2025 Benjamin Grande <[email protected]>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -57,6 +58,8 @@ def setUp(self):
         )
         self.loop.run_until_complete(self.disp_base.create_on_disk())
         self.disp_base.template_for_dispvms = True
+        self.loop.run_until_complete(self.start_vm(self.disp_base))
+        self.shutdown_and_wait(self.disp_base)
         self.app.default_dispvm = self.disp_base
         self.testvm = self.app.add_new_vm(
             qubes.vm.appvm.AppVM,
@@ -220,6 +223,16 @@ def setUp(self):  # pylint: disable=invalid-name
             template_for_dispvms=True,
         )
         self.loop.run_until_complete(self.disp_base_alt.create_on_disk())
+        start_tasks = [
+            self.start_vm(self.disp_base),
+            self.start_vm(self.disp_base_alt),
+        ]
+        shutdown_tasks = [
+            self.disp_base.shutdown(wait=True),
+            self.disp_base_alt.shutdown(wait=True),
+        ]
+        self.loop.run_until_complete(asyncio.gather(*start_tasks))
+        self.loop.run_until_complete(asyncio.gather(*shutdown_tasks))
         # Setting "default_dispvm" fires the preload event before patches of
         # each test function is applied.
         if "_preload_" not in self._testMethodName:

qubes/tests/integ/dispvm_perf.py

@@ -1,8 +1,7 @@
 #
 # The Qubes OS Project, https://www.qubes-os.org/
 #
-# Copyright (C) 2025 Marek Marczykowski-Górecki
-#                           <[email protected]>
+# Copyright (C) 2025 Benjamin Grande <[email protected]>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU Lesser General Public License as published by
@@ -19,15 +18,25 @@
 
 import asyncio
 import os
+import shutil
 import sys
-import time
+import tempfile
 
 import qubes.tests
 
 
 class TC_00_DispVMPerfMixin:
     def setUp(self: qubes.tests.SystemTestCase):
         super().setUp()
+        self.results = os.getenv("QUBES_TEST_PERF_FILE")
+        self.test_dir = None
+        if "_reader" in self._testMethodName:
+            prefix = self.template + "_"
+            if self.results:
+                prefix = self.results + "_" + prefix
+            self.test_dir = tempfile.mkdtemp(prefix=prefix)
+            self.vm2 = None
+            return
         if "whonix-g" in self.template:
             self.skipTest(
                 "whonix gateway is not supported as DisposableVM Template"
@@ -53,32 +62,42 @@ def setUp(self: qubes.tests.SystemTestCase):
             template=self.app.domains[self.template],
             default_dispvm=self.dvm,
         )
+        # Necessary to be done post qube creation because of Whonix Admin Addon:
+        #   https://github.com/QubesOS/qubes-core-admin-addon-whonix/pull/25
+        for qube in [self.dvm, self.vm1, self.vm2]:
+            qube.default_dispvm = self.dvm
+            qube.netvm = None
         self.loop.run_until_complete(
             asyncio.gather(
                 self.dvm.create_on_disk(),
                 self.vm1.create_on_disk(),
                 self.vm2.create_on_disk(),
             )
         )
+        self.loop.run_until_complete(self.start_vm(self.dvm))
+        self.shutdown_and_wait(self.dvm)
         start_tasks = [self.vm1.start()]
-        if self._testMethodName.startswith("vm"):
+        if "_vm" in self._testMethodName:
             start_tasks.append(self.vm2.start())
         self.loop.run_until_complete(asyncio.gather(*start_tasks))
 
     def tearDown(self: qubes.tests.SystemTestCase):
         super().tearDown()
-        if self.vm2.is_running():
-            self.loop.run_until_complete(
-                asyncio.gather(
-                    self.vm2.shutdown(),
+        if self.test_dir and os.getenv("QUBES_TEST_PERF_GRAPH_DELETE"):
+            shutil.rmtree(self.test_dir)
+        if self.vm2:
+            if self.vm2.is_running():
+                self.loop.run_until_complete(
+                    asyncio.gather(
+                        self.vm2.shutdown(),
+                    )
                 )
-            )
 
     def run_test(self, name):
         dvm = self.dvm.name
         vm1 = self.vm1.name
         vm2 = ""
-        if name.startswith("vm"):
+        if "-vm" in name:
             vm2 = self.vm2.name
         cmd = [
             "/usr/lib/qubes/tests/dispvm_perf.py",
@@ -92,141 +111,169 @@ def run_test(self, name):
         if p.returncode:
             self.fail(f"'{' '.join(cmd)}' failed: {p.returncode}")
 
-    def test_000_dispvm(self):
+    def run_reader(self, args: list):
+        if not self.results:
+            self.skipTest("Did not set QUBES_TEST_PERF_FILE")
+        cmd = [
+            "/usr/lib/qubes/tests/dispvm_perf_reader.py",
+            f"--template={self.template}",
+            f"--output-dir={self.test_dir}",
+            *args,
+            "--",
+            self.results,
+        ]
+        p = self.loop.run_until_complete(asyncio.create_subprocess_exec(*cmd))
+        self.loop.run_until_complete(p.wait())
+        if p.returncode:
+            self.fail(f"'{' '.join(cmd)}' failed: {p.returncode}")
+
+    def test_000_vm_dispvm(self):
         """Latency of vm-dispvm calls"""
-        self.run_test("dispvm")
+        self.run_test("vm-dispvm")
 
-    def test_001_dispvm_gui(self):
+    def test_001_vm_dispvm_gui(self):
         """Latency of vm-dispvm GUI calls"""
-        self.run_test("dispvm-gui")
+        self.run_test("vm-dispvm-gui")
 
-    def test_002_dispvm_concurrent(self):
+    def test_002_vm_dispvm_concurrent(self):
         """Latency of vm-dispvm concurrent calls"""
-        self.run_test("dispvm-concurrent")
+        self.run_test("vm-dispvm-concurrent")
 
-    def test_003_dispvm_gui_concurrent(self):
+    def test_003_vm_dispvm_gui_concurrent(self):
         """Latency of vm-dispvm concurrent GUI calls"""
-        self.run_test("dispvm-gui-concurrent")
+        self.run_test("vm-dispvm-gui-concurrent")
 
-    def test_006_dispvm_from_dom0(self):
+    def test_006_dom0_dispvm(self):
         """Latency of dom0-dispvm calls"""
-        self.run_test("dispvm-dom0")
+        self.run_test("dom0-dispvm")
 
-    def test_007_dispvm_from_dom0_gui(self):
+    def test_007_dom0_dispvm_gui(self):
         """Latency of dom0-dispvm GUI calls"""
-        self.run_test("dispvm-dom0-gui")
+        self.run_test("dom0-dispvm-gui")
 
-    def test_008_dispvm_from_dom0_concurrent(self):
+    def test_008_dom0_dispvm_concurrent(self):
         """Latency of dom0-dispvm concurrent calls"""
-        self.run_test("dispvm-dom0-concurrent")
+        self.run_test("dom0-dispvm-concurrent")
 
-    def test_009_dispvm_from_dom0_gui_concurrent(self):
+    def test_009_dom0_dispvm_gui_concurrent(self):
         """Latency of dom0-dispvm concurrent GUI calls"""
-        self.run_test("dispvm-dom0-gui-concurrent")
+        self.run_test("dom0-dispvm-gui-concurrent")
 
-    def test_020_dispvm_preload(self):
+    def test_020_vm_dispvm_preload(self):
         """Latency of vm-dispvm (preload) calls"""
-        self.run_test("dispvm-preload")
+        self.run_test("vm-dispvm-preload")
 
-    def test_021_dispvm_preload_gui(self):
+    def test_021_vm_dispvm_preload_gui(self):
         """Latency of vm-dispvm (preload) GUI calls"""
-        self.run_test("dispvm-preload-gui")
+        self.run_test("vm-dispvm-preload-gui")
 
-    def test_022_dispvm_preload_concurrent(self):
+    def test_022_vm_dispvm_preload_concurrent(self):
         """Latency of vm-dispvm (preload) concurrent calls"""
-        self.run_test("dispvm-preload-concurrent")
+        self.run_test("vm-dispvm-preload-concurrent")
 
-    def test_023_dispvm_preload_gui_concurrent(self):
+    def test_023_vm_dispvm_preload_gui_concurrent(self):
         """Latency of vm-dispvm (preload) concurrent GUI calls"""
-        self.run_test("dispvm-preload-gui-concurrent")
+        self.run_test("vm-dispvm-preload-gui-concurrent")
 
-    def test_026_dispvm_from_dom0_preload(self):
+    def test_026_dom0_dispvm_preload(self):
         """Latency of dom0-dispvm (preload) calls"""
-        self.run_test("dispvm-preload-dom0")
+        self.run_test("dom0-dispvm-preload")
 
-    def test_027_dispvm_from_dom0_preload_gui(self):
+    def test_027_dom0_dispvm_preload_gui(self):
         """Latency of dom0-dispvm (preload) GUI calls"""
-        self.run_test("dispvm-preload-dom0-gui")
+        self.run_test("dom0-dispvm-preload-gui")
 
-    def test_028_dispvm_from_dom0_preload_concurrent(self):
+    def test_028_dom0_dispvm_preload_concurrent(self):
         """Latency of dom0-dispvm (preload) concurrent calls"""
-        self.run_test("dispvm-preload-dom0-concurrent")
+        self.run_test("dom0-dispvm-preload-concurrent")
 
-    def test_029_dispvm_from_dom0_preload_gui_concurrent(self):
+    def test_029_dom0_dispvm_preload_gui_concurrent(self):
         """Latency of dom0-dispvm (preload) concurrent GUI calls"""
-        self.run_test("dispvm-preload-dom0-gui-concurrent")
+        self.run_test("dom0-dispvm-preload-gui-concurrent")
 
-    def test_400_dispvm_api(self):
+    def test_400_dom0_dispvm_api(self):
         """Latency of dom0-dispvm API calls"""
-        self.run_test("dispvm-api")
+        self.run_test("dom0-dispvm-api")
 
-    def test_401_dispvm_gui_api(self):
+    def test_401_dom0_dispvm_gui_api(self):
         """Latency of dom0-dispvm GUI API calls"""
-        self.run_test("dispvm-gui-api")
+        self.run_test("dom0-dispvm-gui-api")
 
-    def test_402_dispvm_concurrent_api(self):
+    def test_402_dom0_dispvm_concurrent_api(self):
         """Latency of dom0-dispvm concurrent API calls"""
-        self.run_test("dispvm-concurrent-api")
+        self.run_test("dom0-dispvm-concurrent-api")
 
-    def test_403_dispvm_gui_concurrent_api(self):
+    def test_403_dom0_dispvm_gui_concurrent_api(self):
         """Latency of dom0-dispvm concurrent GUI API calls"""
-        self.run_test("dispvm-gui-concurrent-api")
+        self.run_test("dom0-dispvm-gui-concurrent-api")
 
-    def test_404_dispvm_preload_more_api(self):
-        """Latency of dom0-dispvm (preload more) API calls"""
-        self.run_test("dispvm-preload-more-api")
+    def test_404_dom0_dispvm_preload_less_less_api(self):
+        """Latency of dom0-dispvm (preload less less) API calls"""
+        self.run_test("dom0-dispvm-preload-less-less-api")
 
-    def test_404_dispvm_preload_less_api(self):
+    def test_405_dom0_dispvm_preload_less_api(self):
         """Latency of dom0-dispvm (preload less) API calls"""
-        self.run_test("dispvm-preload-less-api")
+        self.run_test("dom0-dispvm-preload-less-api")
 
-    def test_404_dispvm_preload_api(self):
+    def test_406_dom0_dispvm_preload_api(self):
         """Latency of dom0-dispvm (preload) API calls"""
-        self.run_test("dispvm-preload-api")
+        self.run_test("dom0-dispvm-preload-api")
+
+    def test_407_dom0_dispvm_preload_more_api(self):
+        """Latency of dom0-dispvm (preload more) API calls"""
+        self.run_test("dom0-dispvm-preload-more-api")
+
+    def test_408_dom0_dispvm_preload_more_more_api(self):
+        """Latency of dom0-dispvm (preload more more) API calls"""
+        self.run_test("dom0-dispvm-preload-more-more-api")
 
-    def test_405_dispvm_preload_gui_api(self):
+    def test_409_dom0_dispvm_preload_gui_api(self):
         """Latency of dom0-dispvm (preload) GUI API calls"""
-        self.run_test("dispvm-preload-gui-api")
+        self.run_test("dom0-dispvm-preload-gui-api")
 
-    def test_406_dispvm_preload_concurrent_api(self):
+    def test_410_dom0_dispvm_preload_concurrent_api(self):
         """Latency of dom0-dispvm (preload) concurrent GUI API calls"""
-        self.run_test("dispvm-preload-concurrent-api")
+        self.run_test("dom0-dispvm-preload-concurrent-api")
 
-    def test_407_dispvm_preload_gui_concurrent_api(self):
+    def test_411_dom0_dispvm_preload_gui_concurrent_api(self):
         """Latency of dom0-dispvm (preload) concurrent GUI API calls"""
-        self.run_test("dispvm-preload-gui-concurrent-api")
+        self.run_test("dom0-dispvm-preload-gui-concurrent-api")
 
-    def test_900_vm(self):
+    def test_800_vm_vm(self):
         """Latency of vm-vm calls"""
-        self.run_test("vm")
+        self.run_test("vm-vm")
 
-    def test_901_vm_gui(self):
+    def test_801_vm_vm_gui(self):
         """Latency of vm-vm GUI calls"""
-        self.run_test("vm-gui")
+        self.run_test("vm-vm-gui")
 
-    def test_902_vm_concurrent(self):
+    def test_802_vm_vm_concurrent(self):
         """Latency of vm-vm concurrent calls"""
-        self.run_test("vm-concurrent")
+        self.run_test("vm-vm-concurrent")
 
-    def test_903_vm_gui_concurrent(self):
+    def test_803_vm_vm_gui_concurrent(self):
         """Latency of vm-vm concurrent GUI calls"""
-        self.run_test("vm-gui-concurrent")
+        self.run_test("vm-vm-gui-concurrent")
 
-    def test_904_vm_api(self):
+    def test_804_dom0_vm_api(self):
         """Latency of dom0-vm API calls"""
-        self.run_test("vm-api")
+        self.run_test("dom0-vm-api")
 
-    def test_905_vm_gui_api(self):
+    def test_805_dom0_vm_gui_api(self):
         """Latency of dom0-vm GUI API calls"""
-        self.run_test("vm-gui-api")
+        self.run_test("dom0-vm-gui-api")
 
-    def test_906_vm_concurrent_api(self):
+    def test_806_dom0_vm_concurrent_api(self):
         """Latency of dom0-vm concurrent API calls"""
-        self.run_test("vm-concurrent-api")
+        self.run_test("dom0-vm-concurrent-api")
 
-    def test_907_vm_gui_concurrent_api(self):
+    def test_807_dom0_vm_gui_concurrent_api(self):
         """Latency of dom0-vm concurrent GUI API calls"""
-        self.run_test("vm-gui-concurrent-api")
+        self.run_test("dom0-vm-gui-concurrent-api")
+
+    def test_900_reader(self):
+        """Render performance graphs."""
+        self.run_reader(["--log-level=INFO", "--no-show"])
 
 
 def create_testcases_for_templates():

rpm_spec/core-dom0.spec.in

@@ -567,6 +567,7 @@ done
 /usr/lib/qubes/fix-dir-perms.sh
 /usr/lib/qubes/startup-misc.sh
 /usr/lib/qubes/tests/dispvm_perf.py
+/usr/lib/qubes/tests/dispvm_perf_reader.py
 /usr/lib/qubes/tests/qrexec_perf.py
 /usr/lib/qubes/tests/storage_perf.py
 %{_unitdir}/[email protected]/30_qubes.conf
@@ -585,6 +586,8 @@ done
 /usr/share/qubes/templates/libvirt/devices/pci.xml
 /usr/share/qubes/templates/libvirt/devices/net.xml
 /usr/share/qubes/tests-data/dispvm-open-thunderbird-attachment
+%dir /usr/share/qubes/tests-data/dispvm_perf/
+/usr/share/qubes/tests-data/dispvm_perf/*
 /usr/share/qubes/tests-data/sysfs
 /usr/lib/tmpfiles.d/qubes.conf
 %attr(0755,root,root) /usr/lib/qubes/create-snapshot