8368251: Parallel: Refactor lgrp_id used in MutableNUMASpace

src/hotspot/os/linux/os_linux.cpp

@@ -851,11 +851,9 @@ static void *thread_native_entry(Thread *thread) {
   osthread->set_thread_id(checked_cast<pid_t>(os::current_thread_id()));
 
   if (UseNUMA) {
-    int lgrp_id = os::numa_get_group_id();
-    if (lgrp_id != -1) {
-      thread->set_lgrp_id(lgrp_id);
-    }
+    thread->update_lgrp_id();
   }
+
   // initialize signal mask for this thread
   PosixSignals::hotspot_sigmask(thread);
 
@@ -1182,10 +1180,7 @@ bool os::create_attached_thread(JavaThread* thread) {
   thread->set_osthread(osthread);
 
   if (UseNUMA) {
-    int lgrp_id = os::numa_get_group_id();
-    if (lgrp_id != -1) {
-      thread->set_lgrp_id(lgrp_id);
-    }
+    thread->update_lgrp_id();
   }
 
   if (os::is_primordial_thread()) {

src/hotspot/os/posix/os_posix.inline.hpp

@@ -34,9 +34,6 @@
 #include <sys/socket.h>
 #include <unistd.h>
 
-// Aix does not have NUMA support but need these for compilation.
-inline bool os::numa_has_group_homing()     { AIX_ONLY(ShouldNotReachHere();) return false;  }
-
 // Platform Mutex/Monitor implementation
 
 inline void PlatformMutex::lock() {

src/hotspot/os/windows/os_windows.cpp

@@ -535,10 +535,7 @@ static unsigned thread_native_entry(void* t) {
   assert(osthr->get_state() == RUNNABLE, "invalid os thread state");
 
   if (UseNUMA) {
-    int lgrp_id = os::numa_get_group_id();
-    if (lgrp_id != -1) {
-      thread->set_lgrp_id(lgrp_id);
-    }
+    thread->update_lgrp_id();
   }
 
   // Diagnostic code to investigate JDK-6573254
@@ -599,10 +596,7 @@ static OSThread* create_os_thread(Thread* thread, HANDLE thread_handle,
   osthread->set_thread_id(thread_id);
 
   if (UseNUMA) {
-    int lgrp_id = os::numa_get_group_id();
-    if (lgrp_id != -1) {
-      thread->set_lgrp_id(lgrp_id);
-    }
+    thread->update_lgrp_id();
   }
 
   // Initial thread state is INITIALIZED, not SUSPENDED

src/hotspot/os/windows/os_windows.inline.hpp

@@ -62,8 +62,6 @@ inline void os::map_stack_shadow_pages(address sp) {
   state->set_shadow_zone_growth_watermark(original_sp);
 }
 
-inline bool os::numa_has_group_homing()     { return false;  }
-
 // Platform Mutex/Monitor implementation
 
 inline void PlatformMutex::lock() {

src/hotspot/share/gc/parallel/mutableNUMASpace.cpp

@@ -118,73 +118,30 @@ size_t MutableNUMASpace::free_in_words() const {
   return s;
 }
 
-int MutableNUMASpace::lgrp_space_index(int lgrp_id) const {
-  return lgrp_spaces()->find_if([&](LGRPSpace* space) {
-    return space->lgrp_id() == checked_cast<uint>(lgrp_id);
+MutableNUMASpace::LGRPSpace *MutableNUMASpace::lgrp_space_for_thread(Thread* thr) const {
+  guarantee(thr != nullptr, "No thread");
+
+  int lgrp_id = thr->lgrp_id();
+  assert(lgrp_id != -1, "lgrp_id must be set during thread creation");
+
+  int lgrp_spaces_index = lgrp_spaces()->find_if([&](LGRPSpace* space) {
+    return space->lgrp_id() == (uint)lgrp_id;
   });
+
+  assert(lgrp_spaces_index != -1, "must have created spaces for all lgrp_ids");
+  return lgrp_spaces()->at(lgrp_spaces_index);
 }
 
 size_t MutableNUMASpace::tlab_capacity(Thread *thr) const {
-  guarantee(thr != nullptr, "No thread");
-  int lgrp_id = thr->lgrp_id();
-  if (lgrp_id == -1) {
-    // This case can occur after the topology of the system has
-    // changed. Thread can change their location, the new home
-    // group will be determined during the first allocation
-    // attempt. For now we can safely assume that all spaces
-    // have equal size because the whole space will be reinitialized.
-    if (lgrp_spaces()->length() > 0) {
-      return capacity_in_bytes() / lgrp_spaces()->length();
-    } else {
-      assert(false, "There should be at least one locality group");
-      return 0;
-    }
-  }
-  // That's the normal case, where we know the locality group of the thread.
-  int i = lgrp_space_index(lgrp_id);
-  if (i == -1) {
-    return 0;
-  }
-  return lgrp_spaces()->at(i)->space()->capacity_in_bytes();
+  return lgrp_space_for_thread(thr)->space()->capacity_in_bytes();
 }
 
 size_t MutableNUMASpace::tlab_used(Thread *thr) const {
-  // Please see the comments for tlab_capacity().
-  guarantee(thr != nullptr, "No thread");
-  int lgrp_id = thr->lgrp_id();
-  if (lgrp_id == -1) {
-    if (lgrp_spaces()->length() > 0) {
-      return (used_in_bytes()) / lgrp_spaces()->length();
-    } else {
-      assert(false, "There should be at least one locality group");
-      return 0;
-    }
-  }
-  int i = lgrp_space_index(lgrp_id);
-  if (i == -1) {
-    return 0;
-  }
-  return lgrp_spaces()->at(i)->space()->used_in_bytes();
+  return lgrp_space_for_thread(thr)->space()->used_in_bytes();
 }
 
-
 size_t MutableNUMASpace::unsafe_max_tlab_alloc(Thread *thr) const {
-  // Please see the comments for tlab_capacity().
-  guarantee(thr != nullptr, "No thread");
-  int lgrp_id = thr->lgrp_id();
-  if (lgrp_id == -1) {
-    if (lgrp_spaces()->length() > 0) {
-      return free_in_bytes() / lgrp_spaces()->length();
-    } else {
-      assert(false, "There should be at least one locality group");
-      return 0;
-    }
-  }
-  int i = lgrp_space_index(lgrp_id);
-  if (i == -1) {
-    return 0;
-  }
-  return lgrp_spaces()->at(i)->space()->free_in_bytes();
+  return lgrp_space_for_thread(thr)->space()->free_in_bytes();
 }
 
 // Bias region towards the first-touching lgrp. Set the right page sizes.
@@ -528,32 +485,13 @@ void MutableNUMASpace::clear(bool mangle_space) {
   }
 }
 
-/*
-   Linux supports static memory binding, therefore the most part of the
-   logic dealing with the possible invalid page allocation is effectively
-   disabled. Besides there is no notion of the home node in Linux. A
-   thread is allowed to migrate freely. Although the scheduler is rather
-   reluctant to move threads between the nodes. We check for the current
-   node every allocation. And with a high probability a thread stays on
-   the same node for some time allowing local access to recently allocated
-   objects.
- */
-
 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
-  Thread* thr = Thread::current();
-  int lgrp_id = thr->lgrp_id();
-  if (lgrp_id == -1 || !os::numa_has_group_homing()) {
-    lgrp_id = os::numa_get_group_id();
-    thr->set_lgrp_id(lgrp_id);
-  }
+  Thread *thr = Thread::current();
 
-  int i = lgrp_space_index(lgrp_id);
-  // It is possible that a new CPU has been hotplugged and
-  // we haven't reshaped the space accordingly.
-  if (i == -1) {
-    i = os::random() % lgrp_spaces()->length();
-  }
-  LGRPSpace *ls = lgrp_spaces()->at(i);
+  // Update the locality group to match where the thread actually is.
+  thr->update_lgrp_id();
+
+  LGRPSpace *ls = lgrp_space_for_thread(thr);
   MutableSpace *s = ls->space();
   HeapWord *p = s->cas_allocate(size);
   if (p != nullptr) {

src/hotspot/share/gc/parallel/mutableNUMASpace.hpp

@@ -152,7 +152,7 @@ class MutableNUMASpace : public MutableSpace {
   void select_tails(MemRegion new_region, MemRegion intersection,
                     MemRegion* bottom_region, MemRegion *top_region);
 
-  int lgrp_space_index(int lgrp_id) const;
+  LGRPSpace *lgrp_space_for_thread(Thread *thr) const;
 
 public:
   GrowableArray<LGRPSpace*>* lgrp_spaces() const     { return _lgrp_spaces;       }

src/hotspot/share/gc/parallel/psYoungGen.cpp

@@ -314,7 +314,7 @@ HeapWord* PSYoungGen::expand_and_allocate(size_t word_size) {
   }
 
   HeapWord* result = eden_space()->cas_allocate(word_size);
-  assert(result, "inv");
+  assert(result || UseNUMA, "inv");
   return result;
 }
 

src/hotspot/share/runtime/os.hpp

@@ -535,7 +535,6 @@ class os: AllStatic {
   static void   realign_memory(char *addr, size_t bytes, size_t alignment_hint);
 
   // NUMA-specific interface
-  static bool   numa_has_group_homing();
   static void   numa_make_local(char *addr, size_t bytes, int lgrp_hint);
   static void   numa_make_global(char *addr, size_t bytes);
   static size_t numa_get_groups_num();

src/hotspot/share/runtime/thread.cpp

@@ -65,7 +65,7 @@ Thread::Thread(MemTag mem_tag) {
   // stack and get_thread
   set_stack_base(nullptr);
   set_stack_size(0);
-  set_lgrp_id(-1);
+  _lgrp_id = -1;
   DEBUG_ONLY(clear_suspendible_thread();)
   DEBUG_ONLY(clear_indirectly_suspendible_thread();)
   DEBUG_ONLY(clear_indirectly_safepoint_thread();)

src/hotspot/share/runtime/thread.hpp

@@ -537,8 +537,8 @@ class Thread: public ThreadShadow {
   void    register_thread_stack_with_NMT();
   void    unregister_thread_stack_with_NMT();
 
-  int     lgrp_id() const        { return _lgrp_id; }
-  void    set_lgrp_id(int value) { _lgrp_id = value; }
+  int     lgrp_id() const  { return _lgrp_id; }
+  void    update_lgrp_id() { _lgrp_id = os::numa_get_group_id(); }
 
   // Printing
   void print_on(outputStream* st, bool print_extended_info) const;