Optimization: Zero-copy work creation

wenyuzhao · wenyuzhao · commit 12c3664855f7 · 2020-10-29T14:59:43.000+11:00
diff --git a/mmtk/src/api.rs b/mmtk/src/api.rs
@@ -17,6 +17,11 @@ use crate::UPCALLS;
 use crate::OpenJDK_Upcalls;
 use crate::SINGLETON;
 
+#[no_mangle]
+pub extern "C" fn release_buffer(ptr: *mut Address, length: usize, capacity: usize) {
+    let _vec = unsafe { Vec::<Address>::from_raw_parts(ptr, length, capacity) };
+}
+
 #[no_mangle]
 pub extern "C" fn openjdk_gc_init(calls: *const OpenJDK_Upcalls, heap_size: usize) {
     unsafe { UPCALLS = calls };
diff --git a/mmtk/src/lib.rs b/mmtk/src/lib.rs
@@ -1,6 +1,7 @@
 #![feature(specialization)]
 #![feature(const_fn)]
 #![feature(box_syntax)]
+#![feature(vec_into_raw_parts)]
 
 extern crate mmtk;
 extern crate libc;
@@ -26,6 +27,14 @@ mod abi;
 mod object_scanning;
 mod gc_works;
 
+#[repr(C)]
+pub struct NewBuffer {
+    pub ptr: *mut Address,
+    pub capacity: usize
+}
+
+type ProcessEdgesFn = *const extern "C" fn(buf: *mut Address, size: usize, cap: usize) -> NewBuffer;
+
 #[repr(C)]
 pub struct OpenJDK_Upcalls {
     pub stop_all_mutators: extern "C" fn(tls: OpaquePointer, create_stack_scan_work: *const extern "C" fn(&'static mut Mutator<SelectedPlan<OpenJDK>>)),
@@ -51,20 +60,20 @@ pub struct OpenJDK_Upcalls {
     pub referent_offset: extern "C" fn() -> i32,
     pub discovered_offset: extern "C" fn() -> i32,
     pub dump_object_string: extern "C" fn(object: ObjectReference) -> *const c_char,
-    pub scan_thread_roots: extern "C" fn(process_edges: *const extern "C" fn(buf: *const Address, size: usize), tls: OpaquePointer),
-    pub scan_thread_root: extern "C" fn(process_edges: *const extern "C" fn(buf: *const Address, size: usize), tls: OpaquePointer),
-    pub scan_universe_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_jni_handle_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_object_synchronizer_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_management_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_jvmti_export_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_aot_loader_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_system_dictionary_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_code_cache_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_string_table_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_class_loader_data_graph_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_weak_processor_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
-    pub scan_vm_thread_roots: extern "C" fn(process_edges: *const extern "C" fn (buf: *const Address, size: usize)),
+    pub scan_thread_roots: extern "C" fn(process_edges: ProcessEdgesFn, tls: OpaquePointer),
+    pub scan_thread_root: extern "C" fn(process_edges: ProcessEdgesFn, tls: OpaquePointer),
+    pub scan_universe_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_jni_handle_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_object_synchronizer_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_management_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_jvmti_export_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_aot_loader_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_system_dictionary_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_code_cache_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_string_table_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_class_loader_data_graph_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_weak_processor_roots: extern "C" fn(process_edges: ProcessEdgesFn),
+    pub scan_vm_thread_roots: extern "C" fn(process_edges: ProcessEdgesFn),
     pub number_of_mutators: extern "C" fn() -> usize,
 }
 
diff --git a/mmtk/src/object_model.rs b/mmtk/src/object_model.rs
@@ -20,6 +20,7 @@ impl ObjectModel<OpenJDK> for VMObjectModel {
             &*(o.to_address() + Self::GC_BYTE_OFFSET / 8).to_ptr::<AtomicU8>()
         }
     }
+    #[inline]
     fn copy(from: ObjectReference, allocator: Allocator, copy_context: &mut impl CopyContext) -> ObjectReference {
         let bytes = unsafe { ((*UPCALLS).get_object_size)(from) };
         let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::<usize>(), 0, allocator);
diff --git a/mmtk/src/scanning.rs b/mmtk/src/scanning.rs
@@ -5,7 +5,7 @@ use mmtk::util::{Address, ObjectReference, SynchronizedCounter};
 use mmtk::util::OpaquePointer;
 use mmtk::scheduler::gc_works::ProcessEdgesWork;
 use crate::OpenJDK;
-use super::{UPCALLS, SINGLETON};
+use super::{UPCALLS, SINGLETON, NewBuffer};
 use std::mem;
 use super::gc_works::*;
 use mmtk::MutatorContext;
@@ -15,12 +15,15 @@ static COUNTER: SynchronizedCounter = SynchronizedCounter::new(0);
 
 pub struct VMScanning {}
 
-pub extern fn create_process_edges_work<W: ProcessEdgesWork<VM=OpenJDK>>(ptr: *const Address, len: usize) {
-    let mut buf = Vec::with_capacity(len);
-    for i in 0..len {
-        buf.push(unsafe { *ptr.add(i) });
+pub extern fn create_process_edges_work<W: ProcessEdgesWork<VM=OpenJDK>>(ptr: *mut Address, length: usize, capacity: usize) -> NewBuffer {
+    if !ptr.is_null() {
+        let mut buf = unsafe { Vec::<Address>::from_raw_parts(ptr, length, capacity) };
+        SINGLETON.scheduler.closure_stage.add(W::new(buf, false));
+    }
+    let (ptr, _, capacity) =  Vec::with_capacity(W::CAPACITY).into_raw_parts();
+    NewBuffer {
+        ptr, capacity
     }
-    SINGLETON.scheduler.closure_stage.add(W::new(buf, false));
 }
 
 impl Scanning<OpenJDK> for VMScanning {
diff --git a/openjdk/mmtk.h b/openjdk/mmtk.h
@@ -47,6 +47,8 @@ extern void post_alloc(MMTk_Mutator mutator, void* refer, void* type_refer,
 extern void record_modified_node(MMTk_Mutator mutator, void* obj);
 extern void record_modified_edge(MMTk_Mutator mutator, void* slot);
 
+extern void release_buffer(void** buffer, size_t len, size_t cap);
+
 extern bool is_mapped_object(void* ref);
 extern bool is_mapped_address(void* addr);
 extern void modify_check(void* ref);
@@ -80,6 +82,13 @@ extern void start_worker(void *tls, void* worker);
 extern size_t free_bytes();
 extern size_t total_bytes();
 
+typedef struct {
+    void** buf;
+    size_t cap;
+} NewBuffer;
+
+typedef NewBuffer (*ProcessEdgesFn)(void** buf, size_t len, size_t cap);
+
 /**
  * OpenJDK-specific
  */
@@ -107,20 +116,20 @@ typedef struct {
     int (*referent_offset) ();
     int (*discovered_offset) ();
     char* (*dump_object_string) (void* object);
-    void (*scan_thread_roots)(void (*process_edges)(void** buf, size_t len), void* tls);
-    void (*scan_thread_root)(void (*process_edges)(void** buf, size_t len), void* tls);
-    void (*scan_universe_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_jni_handle_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_object_synchronizer_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_management_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_jvmti_export_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_aot_loader_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_system_dictionary_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_code_cache_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_string_table_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_class_loader_data_graph_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_weak_processor_roots) (void (*process_edges)(void** buf, size_t len));
-    void (*scan_vm_thread_roots) (void (*process_edges)(void** buf, size_t len));
+    void (*scan_thread_roots)(ProcessEdgesFn process_edges, void* tls);
+    void (*scan_thread_root)(ProcessEdgesFn process_edges, void* tls);
+    void (*scan_universe_roots) (ProcessEdgesFn process_edges);
+    void (*scan_jni_handle_roots) (ProcessEdgesFn process_edges);
+    void (*scan_object_synchronizer_roots) (ProcessEdgesFn process_edges);
+    void (*scan_management_roots) (ProcessEdgesFn process_edges);
+    void (*scan_jvmti_export_roots) (ProcessEdgesFn process_edges);
+    void (*scan_aot_loader_roots) (ProcessEdgesFn process_edges);
+    void (*scan_system_dictionary_roots) (ProcessEdgesFn process_edges);
+    void (*scan_code_cache_roots) (ProcessEdgesFn process_edges);
+    void (*scan_string_table_roots) (ProcessEdgesFn process_edges);
+    void (*scan_class_loader_data_graph_roots) (ProcessEdgesFn process_edges);
+    void (*scan_weak_processor_roots) (ProcessEdgesFn process_edges);
+    void (*scan_vm_thread_roots) (ProcessEdgesFn process_edges);
     size_t (*number_of_mutators)();
 } OpenJDK_Upcalls;
 
diff --git a/openjdk/mmtkRootsClosure.hpp b/openjdk/mmtkRootsClosure.hpp
@@ -45,8 +45,9 @@ class MMTkRootsClosure : public OopClosure {
 };
 
 class MMTkRootsClosure2 : public OopClosure {
-  void (*_process_edges)(void** buf, size_t len);
-  void* _buffer[ROOTS_BUFFER_SIZE];
+  ProcessEdgesFn _process_edges;
+  void** _buffer;
+  size_t _cap;
   size_t _cursor;
 
   template <class T>
@@ -58,21 +59,32 @@ class MMTkRootsClosure2 : public OopClosure {
     //   RawAccess<>::oop_store(p, fwd);
     // }
     _buffer[_cursor++] = (void*) p;
-    if (_cursor >= ROOTS_BUFFER_SIZE) {
+    if (_cursor >= _cap) {
       flush();
     }
   }
 
-  NOINLINE void flush() {
-    _process_edges(_buffer, _cursor);
-    _cursor = 0;
+  void flush() {
+    if (_cursor > 0) {
+      NewBuffer buf = _process_edges(_buffer, _cursor, _cap);
+      _buffer = buf.buf;
+      _cap = buf.cap;
+      _cursor = 0;
+    }
   }
 
 public:
-  MMTkRootsClosure2(void (*process_edges)(void** buf, size_t len)): _process_edges(process_edges), _cursor(0) {}
+  MMTkRootsClosure2(ProcessEdgesFn process_edges): _process_edges(process_edges), _cursor(0) {
+    NewBuffer buf = process_edges(NULL, 0, 0);
+    _buffer = buf.buf;
+    _cap = buf.cap;
+  }
 
   ~MMTkRootsClosure2() {
     if (_cursor > 0) flush();
+    if (_buffer != NULL) {
+      release_buffer(_buffer, _cursor, _cap);
+    }
   }
 
   virtual void do_oop(oop* p)       { do_oop_work(p); }
diff --git a/openjdk/mmtkUpcalls.cpp b/openjdk/mmtkUpcalls.cpp
@@ -155,25 +155,25 @@ static void mmtk_compute_thread_roots(void* trace, void* tls) {
     MMTkHeap::heap()->scan_thread_roots(cl);
 }
 
-static void mmtk_scan_thread_roots(void (*process_edges)(void** buf, size_t len), void* tls) {
+static void mmtk_scan_thread_roots(ProcessEdgesFn process_edges, void* tls) {
     MMTkRootsClosure2 cl(process_edges);
     MMTkHeap::heap()->scan_thread_roots(cl);
 }
 
-static void mmtk_scan_thread_root(void (*process_edges)(void** buf, size_t len), void* tls) {
+static void mmtk_scan_thread_root(ProcessEdgesFn process_edges, void* tls) {
     ResourceMark rm;
     JavaThread* thread = (JavaThread*) tls;
     MMTkRootsClosure2 cl(process_edges);
     CodeBlobToOopClosure cb_cl(&cl, false);
     thread->oops_do(&cl, &cb_cl);
 }
 
-static void mmtk_scan_static_roots(void (*process_edges)(void** buf, size_t len), void* tls) {
+static void mmtk_scan_static_roots(ProcessEdgesFn process_edges, void* tls) {
     MMTkRootsClosure2 cl(process_edges);
     MMTkHeap::heap()->scan_static_roots(cl);
 }
 
-static void mmtk_scan_global_roots(void (*process_edges)(void** buf, size_t len), void* tls) {
+static void mmtk_scan_global_roots(ProcessEdgesFn process_edges, void* tls) {
     MMTkRootsClosure2 cl(process_edges);
     MMTkHeap::heap()->scan_global_roots(cl);
 }
@@ -258,18 +258,18 @@ static char* dump_object_string(void* object) {
     return o->print_value_string();
 }
 
-static void mmtk_scan_universe_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_universe_roots(cl); }
-static void mmtk_scan_jni_handle_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_jni_handle_roots(cl); }
-static void mmtk_scan_object_synchronizer_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); }
-static void mmtk_scan_management_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_management_roots(cl); }
-static void mmtk_scan_jvmti_export_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_jvmti_export_roots(cl); }
-static void mmtk_scan_aot_loader_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_aot_loader_roots(cl); }
-static void mmtk_scan_system_dictionary_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_system_dictionary_roots(cl); }
-static void mmtk_scan_code_cache_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_code_cache_roots(cl); }
-static void mmtk_scan_string_table_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_string_table_roots(cl); }
-static void mmtk_scan_class_loader_data_graph_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl); }
-static void mmtk_scan_weak_processor_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_weak_processor_roots(cl); }
-static void mmtk_scan_vm_thread_roots(void (*process_edges)(void** buf, size_t len)) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_vm_thread_roots(cl); }
+static void mmtk_scan_universe_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_universe_roots(cl); }
+static void mmtk_scan_jni_handle_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_jni_handle_roots(cl); }
+static void mmtk_scan_object_synchronizer_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_object_synchronizer_roots(cl); }
+static void mmtk_scan_management_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_management_roots(cl); }
+static void mmtk_scan_jvmti_export_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_jvmti_export_roots(cl); }
+static void mmtk_scan_aot_loader_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_aot_loader_roots(cl); }
+static void mmtk_scan_system_dictionary_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_system_dictionary_roots(cl); }
+static void mmtk_scan_code_cache_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_code_cache_roots(cl); }
+static void mmtk_scan_string_table_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_string_table_roots(cl); }
+static void mmtk_scan_class_loader_data_graph_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_class_loader_data_graph_roots(cl); }
+static void mmtk_scan_weak_processor_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_weak_processor_roots(cl); }
+static void mmtk_scan_vm_thread_roots(ProcessEdgesFn process_edges) { MMTkRootsClosure2 cl(process_edges); MMTkHeap::heap()->scan_vm_thread_roots(cl); }
 
 static size_t mmtk_number_of_mutators() {
     return Threads::number_of_threads();

Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,7 @@ impl ObjectModel<OpenJDK> for VMObjectModel {`
`20`	`20`	`&*(o.to_address() + Self::GC_BYTE_OFFSET / 8).to_ptr::<AtomicU8>()`
`21`	`21`	`}`
`22`	`22`	`}`
	`23`	`+ #[inline]`
`23`	`24`	`fn copy(from: ObjectReference, allocator: Allocator, copy_context: &mut impl CopyContext) -> ObjectReference {`
`24`	`25`	`let bytes = unsafe { ((*UPCALLS).get_object_size)(from) };`
`25`	`26`	`let dst = copy_context.alloc_copy(from, bytes, ::std::mem::size_of::<usize>(), 0, allocator);`