CA-409482: Using computed delay for RRD loop

RRD loop is executed each 5 seconds. It delays fixed 5 seconds between each
loop. But the loop self also consumes time (The time consuming depends on CPU's
count. If there are many CPUs, the time consuming may be hundreds milliseconds).
This implementation leads RRD will take an offset after several loops. Then one
of RRD data lose and a gap can be observed on XenCenter performance graph.

The solution is to use computed delay (timeslice - loop time consuming) instead
of fixed delay.

Signed-off-by: Bengang Yuan <[email protected]>

Author: BengangY

ocaml/xcp-rrdd/bin/rrdd/dune

@@ -46,6 +46,8 @@
     http_lib
     httpsvr
     inotify
+    mtime
+    mtime.clock
     rpclib.core
     rpclib.json
     rpclib.xml

ocaml/xcp-rrdd/bin/rrdd/rrdd_server.ml

@@ -715,9 +715,20 @@ module Plugin = struct
          specified unique ID. If the plugin is not registered, -1 is returned. *)
       let next_reading (uid : P.uid) : float =
         let open Rrdd_shared in
+        let ( --- ) a b = Mtime.Span.abs_diff a b in
         if with_lock registered_m (fun _ -> Hashtbl.mem registered uid) then
-          with_lock last_loop_end_time_m (fun _ ->
-              !last_loop_end_time +. !timeslice -. Unix.gettimeofday ()
+          with_lock last_iteration_end_m (fun _ ->
+              let time_in_iteration =
+                Mtime_clock.count from_loop_start --- !last_iteration_end
+              in
+              if
+                Mtime.Span.is_longer ~than:!Rrdd_shared.timeslice
+                  time_in_iteration
+              then
+                -1.
+              else
+                Mtime.Span.to_float_ns (!timeslice --- time_in_iteration)
+                /. 1_000_000_000.
           )
         else
           -1.

ocaml/xcp-rrdd/bin/rrdd/rrdd_shared.ml

@@ -20,14 +20,17 @@ module StringSet = Set.Make (String)
 (* Whether to enable all non-default datasources *)
 let enable_all_dss = ref false
 
-(* The time between each monitoring loop. *)
-let timeslice : float ref = ref 5.
+(* The expected time span between each monitoring loop. *)
+let timeslice : Mtime.span ref = ref Mtime.Span.(5 * s)
 
-(* Timestamp of the last monitoring loop end. *)
-let last_loop_end_time : float ref = ref neg_infinity
+(* The counter since the start of all monitoring loops. *)
+let from_loop_start : Mtime_clock.counter = Mtime_clock.counter ()
 
-(* The mutex that protects the last_loop_end_time against data corruption. *)
-let last_loop_end_time_m : Mutex.t = Mutex.create ()
+(* The time span of the last monitoring loop end. *)
+let last_iteration_end : Mtime.Span.t ref = ref Mtime.Span.zero
+
+(* The mutex that protects the last_iteration_end against data corruption. *)
+let last_iteration_end_m : Mutex.t = Mutex.create ()
 
 (** Cache memory/target values *)
 let memory_targets : (int, int64) Hashtbl.t = Hashtbl.create 20

ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml

@@ -535,19 +535,40 @@ let monitor_write_loop writers =
   Debug.with_thread_named "monitor_write"
     (fun () ->
       Xenctrl.with_intf (fun xc ->
+          let ( --- ) = Mtime.Span.abs_diff in
           while true do
             try
+              let last_iteration_start =
+                Mtime_clock.count Rrdd_shared.from_loop_start
+              in
               do_monitor_write xc writers ;
-              with_lock Rrdd_shared.last_loop_end_time_m (fun _ ->
-                  Rrdd_shared.last_loop_end_time := Unix.gettimeofday ()
+              with_lock Rrdd_shared.last_iteration_end_m (fun _ ->
+                  Rrdd_shared.last_iteration_end :=
+                    Mtime_clock.count Rrdd_shared.from_loop_start
               ) ;
-              Thread.delay !Rrdd_shared.timeslice
+              let time_in_iteration =
+                !Rrdd_shared.last_iteration_end --- last_iteration_start
+              in
+              if
+                Mtime.Span.is_longer ~than:!Rrdd_shared.timeslice
+                  time_in_iteration
+              then
+                warn
+                  "%s: Monitor write iteration took (%a), this is longer than \
+                   a full cycle, skipping the delay"
+                  __FUNCTION__ Debug.Pp.mtime_span time_in_iteration
+              else
+                Thread.delay
+                  (Mtime.Span.to_float_ns
+                     (!Rrdd_shared.timeslice --- time_in_iteration)
+                  /. 1_000_000_000.
+                  )
             with e ->
               Backtrace.is_important e ;
               warn
-                "Monitor/write thread caught an exception. Pausing for 10s, \
-                 then restarting: %s"
-                (Printexc.to_string e) ;
+                "%s: Monitor/write thread caught an exception. Pausing for \
+                 10s, then restarting: %s"
+                __FUNCTION__ (Printexc.to_string e) ;
               log_backtrace e ;
               Thread.delay 10.
           done