CA-409482: Using computed delay for RRD loop

RRD loop is executed each 5 seconds. It delays fixed 5 seconds between each
loop. But the loop self also consumes time (The time consuming depends on CPU's
count. If there are many CPUs, the time consuming may be hundreds milliseconds).
This implementation leads RRD will take an offset after several loops. Then one
of RRD data lose and a gap can be observed on XenCenter performance graph.

The solution is to use a fixed deadline as each iteration start time and to use
a computed delay (timeslice - loop time consuming) instead of fixed delay.

Signed-off-by: Bengang Yuan <[email protected]>

Author: BengangY

ocaml/xcp-rrdd/bin/rrdd/dune

@@ -46,6 +46,8 @@
     http_lib
     httpsvr
     inotify
+    mtime
+    mtime.clock
     rpclib.core
     rpclib.json
     rpclib.xml

ocaml/xcp-rrdd/bin/rrdd/rrdd_server.ml

@@ -715,9 +715,25 @@ module Plugin = struct
          specified unique ID. If the plugin is not registered, -1 is returned. *)
       let next_reading (uid : P.uid) : float =
         let open Rrdd_shared in
+        let ( --- ) a b = Mtime.Span.abs_diff a b in
         if with_lock registered_m (fun _ -> Hashtbl.mem registered uid) then
-          with_lock last_loop_end_time_m (fun _ ->
-              !last_loop_end_time +. !timeslice -. Unix.gettimeofday ()
+          with_lock next_iteration_start_m (fun _ ->
+              let current = Mtime_clock.count from_loop_start in
+              let diff =
+                Mtime.Span.to_float_ns
+                  (!Rrdd_shared.next_iteration_start
+                  --- Mtime_clock.count from_loop_start
+                  )
+                /. 1_000_000_000.
+              in
+              if
+                Mtime.Span.is_longer
+                  ~than:!Rrdd_shared.next_iteration_start
+                  current
+              then
+                diff *. -1.
+              else
+                diff
           )
         else
           -1.

ocaml/xcp-rrdd/bin/rrdd/rrdd_shared.ml

@@ -20,14 +20,17 @@ module StringSet = Set.Make (String)
 (* Whether to enable all non-default datasources *)
 let enable_all_dss = ref false
 
-(* The time between each monitoring loop. *)
-let timeslice : float ref = ref 5.
+(* The expected time span between each monitoring loop. *)
+let timeslice : Mtime.span ref = ref Mtime.Span.(5 * s)
 
-(* Timestamp of the last monitoring loop end. *)
-let last_loop_end_time : float ref = ref neg_infinity
+(* The counter since the start of all monitoring loops. *)
+let from_loop_start : Mtime_clock.counter = Mtime_clock.counter ()
 
-(* The mutex that protects the last_loop_end_time against data corruption. *)
-let last_loop_end_time_m : Mutex.t = Mutex.create ()
+(* The time span of the next monitoring loop begin. *)
+let next_iteration_start : Mtime.Span.t ref = ref Mtime.Span.zero
+
+(* The mutex that protects the next_iteration_start against data corruption. *)
+let next_iteration_start_m : Mutex.t = Mutex.create ()
 
 (** Cache memory/target values *)
 let memory_targets : (int, int64) Hashtbl.t = Hashtbl.create 20

ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml

@@ -535,21 +535,46 @@ let monitor_write_loop writers =
   Debug.with_thread_named "monitor_write"
     (fun () ->
       Xenctrl.with_intf (fun xc ->
+          let ( --- ) = Mtime.Span.abs_diff in
+          let ( +++ ) = Mtime.Span.add in
           while true do
             try
               do_monitor_write xc writers ;
-              with_lock Rrdd_shared.last_loop_end_time_m (fun _ ->
-                  Rrdd_shared.last_loop_end_time := Unix.gettimeofday ()
+              with_lock Rrdd_shared.next_iteration_start_m (fun _ ->
+                  Rrdd_shared.next_iteration_start :=
+                    !Rrdd_shared.next_iteration_start +++ !Rrdd_shared.timeslice
               ) ;
-              Thread.delay !Rrdd_shared.timeslice
+              let last_iteration_end =
+                Mtime_clock.count Rrdd_shared.from_loop_start
+              in
+              let time_in_iteration =
+                !Rrdd_shared.next_iteration_start --- last_iteration_end
+              in
+              if
+                Mtime.Span.is_longer
+                  ~than:!Rrdd_shared.next_iteration_start
+                  last_iteration_end
+              then
+                warn
+                  "%s: Monitor write iteration took (%a), this is longer than \
+                   a full cycle, skipping the delay"
+                  __FUNCTION__ Debug.Pp.mtime_span
+                  (time_in_iteration +++ !Rrdd_shared.timeslice)
+              else
+                Thread.delay
+                  (Mtime.Span.to_float_ns time_in_iteration /. 1_000_000_000.)
             with e ->
               Backtrace.is_important e ;
               warn
-                "Monitor/write thread caught an exception. Pausing for 10s, \
-                 then restarting: %s"
-                (Printexc.to_string e) ;
+                "%s: Monitor/write thread caught an exception. Pausing for \
+                 10s, then restarting: %s"
+                __FUNCTION__ (Printexc.to_string e) ;
               log_backtrace e ;
-              Thread.delay 10.
+              Thread.delay 10. ;
+              with_lock Rrdd_shared.next_iteration_start_m (fun _ ->
+                  Rrdd_shared.next_iteration_start :=
+                    !Rrdd_shared.next_iteration_start +++ Mtime.Span.(10 * s)
+              )
           done
       )
     )