xenopsd: expose a best-effort mode that set the hard affinity mask (CP-54234)

This allows xapi to "hard-pin" instead of "soft-pin". This is useful to
restrict the scheduler to a single numa node, instead of letting it to move the
vpus across nodes. This must be used with care because the effect of
overprovisioning CPUs in this mode is unknown and will probalby have undesired
effects.

Signed-off-by: Pau Ruiz Safont <[email protected]>

Author: psafont

ocaml/xapi-idl/xen/xenops_interface.ml

@@ -496,9 +496,12 @@ module Host = struct
   [@@deriving rpcty]
 
   type numa_affinity_policy =
-    | Any  (** VMs may run on any NUMA nodes. This is the default in 8.2CU1 *)
+    | Any  (** VMs may run on any NUMA nodes. *)
     | Best_effort
-        (** best effort placement on the smallest number of NUMA nodes where possible *)
+        (** Best-effort placement. Assigns the memory of the VM to a single
+            node, and soft-pins its VCPUs to the node, if possible. Otherwise
+            behaves like Any. *)
+    | Best_effort_hard  (** Like Best_effort, but hard-pins the VCPUs *)
   [@@deriving rpcty]
 
   type numa_affinity_policy_opt = numa_affinity_policy option [@@deriving rpcty]

ocaml/xenopsd/lib/xenops_server.ml

@@ -3570,8 +3570,21 @@ let default_numa_affinity_policy = ref Xenops_interface.Host.Best_effort
 
 let numa_placement = ref !default_numa_affinity_policy
 
+type affinity = Soft | Hard
+
 let string_of_numa_affinity_policy =
-  Xenops_interface.Host.(function Any -> "any" | Best_effort -> "best-effort")
+  let open Xenops_interface.Host in
+  function
+  | Any ->
+      "any"
+  | Best_effort ->
+      "best-effort"
+  | Best_effort_hard ->
+      "best-effort-hard"
+
+let affinity_of_numa_affinity_policy =
+  let open Xenops_interface.Host in
+  function Any | Best_effort -> Soft | Best_effort_hard -> Hard
 
 module HOST = struct
   let stat _ dbg =

ocaml/xenopsd/xc/domain.ml

@@ -857,7 +857,13 @@ let numa_init () =
     )
     mem
 
-let numa_placement domid ~vcpus ~memory =
+let set_affinity = function
+  | Xenops_server.Hard ->
+      Xenctrlext.vcpu_setaffinity_hard
+  | Xenops_server.Soft ->
+      Xenctrlext.vcpu_setaffinity_soft
+
+let numa_placement domid ~vcpus ~memory affinity =
   let open Xenctrlext in
   let open Topology in
   with_lock numa_mutex (fun () ->
@@ -888,7 +894,7 @@ let numa_placement domid ~vcpus ~memory =
         | Some (cpu_affinity, mem_plan) ->
             let cpus = CPUSet.to_mask cpu_affinity in
             for i = 0 to vcpus - 1 do
-              Xenctrlext.vcpu_setaffinity_soft xcext domid i cpus
+              set_affinity affinity xcext domid i cpus
             done ;
             mem_plan
       in
@@ -978,14 +984,18 @@ let build_pre ~xc ~xs ~vcpus ~memory ~hard_affinity domid =
     match !Xenops_server.numa_placement with
     | Any ->
         None
-    | Best_effort ->
+    | (Best_effort | Best_effort_hard) as pin ->
         log_reraise (Printf.sprintf "NUMA placement") (fun () ->
             if hard_affinity <> [] then (
               D.debug "VM has hard affinity set, skipping NUMA optimization" ;
               None
             ) else
+              let affinity =
+                Xenops_server.affinity_of_numa_affinity_policy pin
+              in
               numa_placement domid ~vcpus
                 ~memory:(Int64.mul memory.xen_max_mib 1048576L)
+                affinity
               |> Option.map fst
         )
   in