Merge branch 'crac' into crac-extract

Author: Roman Marchenko

src/hotspot/os/posix/os_posix.cpp

@@ -41,6 +41,7 @@
 #include "services/memTracker.hpp"
 #include "runtime/arguments.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/crac.hpp"
 #include "runtime/java.hpp"
 #include "runtime/orderAccess.hpp"
 #include "runtime/perfMemory.hpp"
@@ -1410,7 +1411,7 @@ jlong os::javaTimeNanos() {
   struct timespec tp;
   int status = clock_gettime(CLOCK_MONOTONIC, &tp);
   assert(status == 0, "clock_gettime error: %s", os::strerror(errno));
-  jlong result = jlong(tp.tv_sec) * NANOSECS_PER_SEC + jlong(tp.tv_nsec);
+  jlong result = jlong(tp.tv_sec) * NANOSECS_PER_SEC + jlong(tp.tv_nsec) + crac::monotonic_time_offset();
   return result;
 }
 

src/hotspot/share/runtime/crac.cpp

@@ -147,7 +147,7 @@ class CracRestoreParameters : public CHeapObj<mtInternal> {
 
   struct header {
     jlong _restore_time;
-    jlong _restore_counter;
+    jlong _restore_nanos;
     int _nflags;
     int _nprops;
     int _env_memory_size;
@@ -205,10 +205,10 @@ class CracRestoreParameters : public CHeapObj<mtInternal> {
       const SystemProperty* props,
       const char *args,
       jlong restore_time,
-      jlong restore_counter) {
+      jlong restore_nanos) {
     header hdr = {
       restore_time,
-      restore_counter,
+      restore_nanos,
       num_flags,
       system_props_length(props),
       env_vars_size(environ)
@@ -295,9 +295,17 @@ static char* _crengine_arg_str = NULL;
 static unsigned int _crengine_argc = 0;
 static const char* _crengine_args[32];
 static jlong _restore_start_time;
-static jlong _restore_start_counter;
+static jlong _restore_start_nanos;
 static FdsInfo _vm_inited_fds(false);
 
+// Timestamps recorded before checkpoint
+jlong crac::checkpoint_millis;
+jlong crac::checkpoint_nanos;
+char crac::checkpoint_bootid[UUID_LENGTH];
+// Value based on wall clock time difference that will guarantee monotonic
+// System.nanoTime() close to actual wall-clock time difference.
+jlong crac::javaTimeNanos_offset = 0;
+
 jlong crac::restore_start_time() {
   if (!_restore_start_time) {
     return -1;
@@ -306,10 +314,10 @@ jlong crac::restore_start_time() {
 }
 
 jlong crac::uptime_since_restore() {
-  if (!_restore_start_counter) {
+  if (!_restore_start_nanos) {
     return -1;
   }
-  return os::javaTimeNanos() - _restore_start_counter;
+  return os::javaTimeNanos() - _restore_start_nanos;
 }
 
 void VM_Crac::trace_cr(const char* msg, ...) {
@@ -600,6 +608,7 @@ class CracSHM {
 };
 
 static int checkpoint_restore(int *shmid) {
+  crac::record_time_before_checkpoint();
 
   int cres = call_crengine();
   if (cres < 0) {
@@ -617,6 +626,8 @@ static int checkpoint_restore(int *shmid) {
   } while (sig == -1 && errno == EINTR);
   assert(sig == RESTORE_SIGNAL, "got what requested");
 
+  crac::update_javaTimeNanos_offset();
+
   if (CRTraceStartupTime) {
     tty->print_cr("STARTUPTIME " JLONG_FORMAT " restore-native", os::javaTimeNanos());
   }
@@ -788,7 +799,9 @@ void VM_Crac::doit() {
 
   if (shmid <= 0 || !VM_Crac::read_shm(shmid)) {
     _restore_start_time = os::javaTimeMillis();
-    _restore_start_counter = os::javaTimeNanos();
+    _restore_start_nanos = os::javaTimeNanos();
+  } else {
+    _restore_start_nanos += crac::monotonic_time_offset();
   }
   PerfMemoryLinux::restore();
 
@@ -892,7 +905,7 @@ void crac::restore() {
   struct stat st;
 
   jlong restore_time = os::javaTimeMillis();
-  jlong restore_counter = os::javaTimeNanos();
+  jlong restore_nanos = os::javaTimeNanos();
 
   compute_crengine();
 
@@ -906,7 +919,7 @@ void crac::restore() {
           Arguments::system_properties(),
           Arguments::java_command() ? Arguments::java_command() : "",
           restore_time,
-          restore_counter)) {
+          restore_nanos)) {
       char strid[32];
       snprintf(strid, sizeof(strid), "%d", id);
       setenv("CRAC_NEW_ARGS_ID", strid, true);
@@ -1008,7 +1021,7 @@ bool CracRestoreParameters::read_from(int fd) {
   char* cursor = _raw_content + sizeof(header);
 
   ::_restore_start_time = hdr->_restore_time;
-  ::_restore_start_counter = hdr->_restore_counter;
+  ::_restore_start_nanos = hdr->_restore_nanos;
 
   for (int i = 0; i < hdr->_nflags; i++) {
     FormatBuffer<80> err_msg("%s", "");
@@ -1057,3 +1070,77 @@ bool CracRestoreParameters::read_from(int fd) {
   _args = cursor;
   return true;
 }
+
+void crac::record_time_before_checkpoint() {
+  checkpoint_millis = os::javaTimeMillis();
+  checkpoint_nanos = os::javaTimeNanos();
+  memset(checkpoint_bootid, 0, UUID_LENGTH);
+  read_bootid(checkpoint_bootid);
+}
+
+void crac::update_javaTimeNanos_offset() {
+  char buf[UUID_LENGTH];
+  // We will change the nanotime offset only if this is not the same boot
+  // to prevent reducing the accuracy of System.nanoTime() unnecessarily.
+  // It is possible that in a real-world case the boot_id does not change
+  // (containers keep the boot_id) - but the monotonic time changes. We will
+  // only guarantee that the nanotime does not go backwards in that case but
+  // won't offset the time based on wall-clock time as this change in monotonic
+  // time is likely intentional.
+  if (!read_bootid(buf) || memcmp(buf, checkpoint_bootid, UUID_LENGTH) != 0) {
+    assert(checkpoint_millis >= 0, "Restore without a checkpoint?");
+    long diff_millis = os::javaTimeMillis() - checkpoint_millis;
+    // If the wall clock has gone backwards we won't add it to the offset
+    if (diff_millis < 0) {
+      diff_millis = 0;
+    }
+    // javaTimeNanos() call on the second line below uses the *_offset, so we will zero
+    // it to make the call return true monotonic time rather than the adjusted value.
+    javaTimeNanos_offset = 0;
+    javaTimeNanos_offset = checkpoint_nanos - os::javaTimeNanos() + diff_millis * 1000000L;
+  } else {
+    // ensure monotonicity even if this looks like the same boot
+    jlong diff = os::javaTimeNanos() - checkpoint_nanos;
+    if (diff < 0) {
+      javaTimeNanos_offset -= diff;
+    }
+  }
+}
+
+static bool read_all(int fd, char *dest, size_t n) {
+  size_t rd = 0;
+  do {
+    ssize_t r = ::read(fd, dest + rd, n - rd);
+    if (r == 0) {
+      return false;
+    } else if (r < 0) {
+      if (errno == EINTR) {
+        continue;
+      }
+      return false;
+    }
+    rd += r;
+  } while (rd < n);
+  return true;
+}
+
+bool crac::read_bootid(char *dest) {
+  int fd = ::open("/proc/sys/kernel/random/boot_id", O_RDONLY);
+  if (fd < 0 || !read_all(fd, dest, UUID_LENGTH)) {
+    perror("CRaC: Cannot read system boot ID");
+    return false;
+  }
+  char c;
+  if (!read_all(fd, &c, 1) || c != '\n') {
+    perror("CRaC: system boot ID does not end with newline");
+    return false;
+  }
+  if (::read(fd, &c, 1) != 0) {
+    perror("CRaC: Unexpected data/error reading system boot ID");
+    return false;
+  }
+  if (::close(fd) != 0) {
+    perror("CRaC: Cannot close system boot ID file");
+  }
+  return true;
+}

src/hotspot/share/runtime/crac.hpp

@@ -28,6 +28,9 @@
 #include "runtime/handles.hpp"
 #include "utilities/macros.hpp"
 
+// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+#define UUID_LENGTH 36
+
 class crac: AllStatic {
 public:
   static void vm_create_start();
@@ -38,6 +41,21 @@ class crac: AllStatic {
 
   static jlong restore_start_time();
   static jlong uptime_since_restore();
+
+  static void record_time_before_checkpoint();
+  static void update_javaTimeNanos_offset();
+
+  static jlong monotonic_time_offset() {
+    return javaTimeNanos_offset;
+  }
+
+private:
+  static bool read_bootid(char *dest);
+
+  static jlong checkpoint_millis;
+  static jlong checkpoint_nanos;
+  static char checkpoint_bootid[UUID_LENGTH];
+  static jlong javaTimeNanos_offset;
 };
 
 #endif //SHARE_RUNTIME_CRAC_HPP

test/jdk/jdk/crac/java/lang/System/NanoTimeTest.java

@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2023, Azul Systems, Inc. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import jdk.crac.*;
+import jdk.crac.management.CRaCMXBean;
+import jdk.test.lib.Container;
+import jdk.test.lib.containers.docker.Common;
+import jdk.test.lib.containers.docker.DockerTestUtils;
+import jdk.test.lib.crac.CracBuilder;
+import jdk.test.lib.crac.CracTest;
+import jdk.test.lib.crac.CracTestArg;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import static jdk.test.lib.Asserts.*;
+
+/**
+ * @test NanoTimeTest
+ * @requires (os.family == "linux")
+ * @requires docker.support
+ * @library /test/lib
+ * @build NanoTimeTest
+ * @run driver jdk.test.lib.crac.CracTest      0 true
+ * @run driver jdk.test.lib.crac.CracTest  86400 true
+ * @run driver jdk.test.lib.crac.CracTest -86400 true
+ * @run driver jdk.test.lib.crac.CracTest  86400 false
+ * @run driver jdk.test.lib.crac.CracTest -86400 false
+ */
+public class NanoTimeTest implements CracTest {
+    private static final String imageName = Common.imageName("system-nanotime");
+
+    @CracTestArg(0)
+    long monotonicOffset;
+
+    @CracTestArg(1)
+    boolean changeBootId;
+
+    @Override
+    public void test() throws Exception {
+        if (!DockerTestUtils.canTestDocker()) {
+            return;
+        }
+        CracBuilder builder = new CracBuilder();
+        Path bootIdFile = Files.createTempFile("NanoTimeTest-", "-boot_id");
+        try {
+            // TODO: use more official image
+            builder.withBaseImage("ghcr.io/crac/test-base", "latest")
+                    .dockerOptions("-v", bootIdFile + ":/fake_boot_id")
+                    .inDockerImage(imageName);
+
+            Files.writeString(bootIdFile, "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx\n");
+            // We need to preload the library before checkpoint
+            builder.doCheckpoint(Container.ENGINE_COMMAND, "exec",
+                    "-e", "LD_PRELOAD=/opt/path-mapping-quiet.so",
+                    "-e", "PATH_MAPPING=/proc/sys/kernel/random/boot_id:/fake_boot_id",
+                    CracBuilder.CONTAINER_NAME,
+                    // In case we are trying to use negative monotonic offset we could
+                    // run into situation where we'd set it to negative value (prohibited).
+                    // Therefore, we'll rather offset it to the future before checkpoint
+                    // and set to 0 for restore.
+                    "unshare", "--fork", "--time", "--monotonic", String.valueOf(Math.max(-monotonicOffset, 0)),
+                    CracBuilder.DOCKER_JAVA);
+
+            if (changeBootId) {
+                Files.writeString(bootIdFile, "yyyyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy\n");
+            }
+
+            builder.doRestore(Container.ENGINE_COMMAND, "exec", CracBuilder.CONTAINER_NAME,
+                    "unshare", "--fork", "--time", "--boottime", "86400", "--monotonic", String.valueOf(Math.max(monotonicOffset, 0)),
+                    CracBuilder.DOCKER_JAVA);
+        } finally {
+            builder.ensureContainerKilled();
+            assertTrue(bootIdFile.toFile().delete());
+        }
+    }
+
+    @Override
+    public void exec() throws Exception {
+        System.out.println("Expected offset: " + monotonicOffset);
+        // We use uptime to assert that changing the clock worked
+        long boottimeBefore = readSystemUptime();
+
+        long before = System.nanoTime();
+        Core.checkpointRestore();
+        long after = System.nanoTime();
+        System.out.println("Before: " + before);
+        System.out.println("After: " + after);
+        assertLTE(before, after, "After < Before");
+        if (changeBootId || monotonicOffset <= 0) {
+            // Even though we have shifted the monotic offset by a day the difference
+            // is adjusted by difference between wall clock time before and after;
+            // the difference in monotonic time is considered "random"
+            assertLT(after, before + TimeUnit.HOURS.toNanos(1), "After too late");
+        } else {
+            assertGT(after, before + TimeUnit.HOURS.toNanos(1), "After too early");
+            assertLT(after, before + TimeUnit.HOURS.toNanos(25), "After too late");
+        }
+        long boottimeAfter = readSystemUptime();
+        assertGTE(boottimeAfter, boottimeBefore + 86_400_000, "Boottime was not changed");
+        RuntimeMXBean runtimeMX = ManagementFactory.getRuntimeMXBean();
+        assertGTE(runtimeMX.getUptime(), 0L, "VM Uptime is negative!");
+        CRaCMXBean cracBean = CRaCMXBean.getCRaCMXBean();
+        assertLT(cracBean.getUptimeSinceRestore(), 60_000L);
+        assertGTE(cracBean.getUptimeSinceRestore(), 0L);
+    }
+
+    private long readSystemUptime() throws IOException {
+        String uptimeStr = Files.readString(Path.of("/proc/uptime"));
+        String[] parts = uptimeStr.split(" ");
+        return (long)(Double.parseDouble(parts[0]) * 1000);
+    }
+}

test/lib/jdk/test/lib/containers/docker/DockerfileConfig.java

@@ -37,8 +37,11 @@
 // Note: base image version should not be an empty string. Use "latest" to get the latest version.
 
 public class DockerfileConfig {
-    static String getBaseImageName() {
-        String name = System.getProperty("jdk.test.docker.image.name");
+    public static final String BASE_IMAGE_NAME = "jdk.test.docker.image.name";
+    public static final String BASE_IMAGE_VERSION = "jdk.test.docker.image.version";
+
+    public static String getBaseImageName() {
+        String name = System.getProperty(BASE_IMAGE_NAME);
         if (name != null) {
             System.out.println("DockerfileConfig: using custom image name: " + name);
             return name;
@@ -56,8 +59,8 @@ static String getBaseImageName() {
         }
     }
 
-    static String getBaseImageVersion() {
-        String version = System.getProperty("jdk.test.docker.image.version");
+    public static String getBaseImageVersion() {
+        String version = System.getProperty(BASE_IMAGE_VERSION);
         if (version != null) {
             System.out.println("DockerfileConfig: using custom image version: " + version);
             return version;