Work Packets (#136)

* Execute a simple work packet

* Refactor

* More progress

* Progress to closure

* Progress to the end of closures

* Cleanup

* Can finish one GC (and crash)

* Working (with single-worker)

* Refactor

* Refactor: move scheduler code to a separate folder

* Refactor: Remove generic type `VM: VMBinding` from scheduler types

* Add scheduler tests

* Support multiple worker threads for OpenJDK

* WIP: Cleanup semispace

* Finer granularity for root-scan works

* Optimizations

* Support parallel stack scanning and stack scanning before safepoints start

* Update GC status correctly

* Cleanup

* Cleanup

* Cleanup: Reduce unsafe

* An implementation of GenCopy

* Fix synchronization bug

* Collect & report scheduler statistics

* Print scheduler statistics in `scheduler` test

* Only collect statistics for the timing iteration

* Fix nogc build error

* Fix buggy side mark-table

* WIP: GenCopy barriers

* GenCopy Sanity GC

* Fix several bugs

* Fix multi-worker synchronization error

* Remove `object_reference_write`

* Fix NoGC build error

* Minor fix

* Fix compile error

* WIP: Fix errors after rebase onto PR#146 -- nogc/semispace working

* Fix errors after rebase onto PR#146 -- nogc/semispace/gencopy working

* Fix `ci-test` warnings and errors

* WIP: Fix `ci-style`

* cargo fmt

* Cleanup

* Support Sanity GC

* Fix CI checks

* Remove gencopy sanity gc (we support `feature="sanity"` now)

* Optimizations

* Fix

* Store tls pointer to scheduler thread

* Fix `SFTMap`

* JikesRVM: Can finish one GC

* Add a method to get the total number of worker threads

* Clear references

* Fix review comments

* Add comments for gencopy's jikesrvm support

* Move `MUTATOR_ITERATOR_LOCK` to BasePlan

* Add comments to explain the use of raw pointers

* Fix incorrect work-packet comparison

* Fix cyclic reference: WorkerGroup -> Worker -> WorkerGroup

* Reduce unsafe when accessing `coordinator_worker`

* format

* Add a feature to force single-threaded collection

Author: wenyuzhao

.github/scripts/ci-style.sh

@@ -9,9 +9,11 @@ cargo clippy --features nogc
 cargo clippy --features nogc_lock_free
 cargo clippy --features nogc_no_zeroing
 cargo clippy --features semispace
+cargo clippy --features gencopy
 # check features
 cargo clippy --features nogc,sanity
 cargo clippy --features semispace,sanity
+cargo clippy --features gencopy,sanity
 cargo clippy --features nogc,vm_space,code_space,ro_space
 cargo clippy --features nogc,lockfreeimmortalspace
 cargo clippy --features semispace,vm_space,code_space,ro_space

Cargo.toml

@@ -28,16 +28,19 @@ enum-map = "0.6.2"
 downcast-rs = "1.1.1"
 atomic-traits = "0.2.0"
 atomic = "0.4.6"
+spin = "0.5.2"
 
 [dev-dependencies]
 crossbeam = "0.7.3"
+rand = "0.7.3"
 
 [features]
 default = []
 
 # plans
 nogc = ["immortalspace", "largeobjectspace"]
 semispace = ["immortalspace", "largeobjectspace", "copyspace"]
+gencopy = ["immortalspace", "largeobjectspace", "copyspace"]
 
 # spaces
 base_spaces = []
@@ -54,3 +57,5 @@ sanity = []
 force_32bit_heap_layout = []
 nogc_lock_free = ["nogc", "lockfreeimmortalspace"]
 nogc_no_zeroing = ["nogc_lock_free"]
+
+single_worker = []

src/lib.rs

@@ -1,3 +1,4 @@
+#![allow(incomplete_features)]
 #![feature(asm)]
 #![feature(const_fn)]
 #![feature(integer_atomics)]
@@ -6,6 +7,11 @@
 #![feature(box_syntax)]
 #![feature(maybe_uninit_ref)]
 #![feature(maybe_uninit_extra)]
+#![feature(get_mut_unchecked)]
+#![feature(arbitrary_self_types)]
+#![feature(associated_type_defaults)]
+#![feature(specialization)]
+#![feature(trait_alias)]
 
 //! Memory Management ToolKit (MMTk) is a portable and high performance memory manager
 //! that includes various garbage collection algorithms and provides clean and efficient
@@ -51,14 +57,12 @@ mod mm;
 mod mmtk;
 mod plan;
 pub mod policy;
+pub mod scheduler;
 pub mod vm;
 
 pub use crate::mm::memory_manager;
 pub use crate::mmtk::MMTK;
-pub use crate::plan::selected_plan::{
-    SelectedCollector, SelectedConstraints, SelectedPlan, SelectedTraceLocal,
-};
+pub use crate::plan::selected_plan::{SelectedConstraints, SelectedPlan};
 pub use crate::plan::{
-    Allocator, CollectorContext, Mutator, MutatorContext, ParallelCollector, Plan, TraceLocal,
-    TransitiveClosure,
+    Allocator, CopyContext, Mutator, MutatorContext, Plan, TraceLocal, TransitiveClosure,
 };

src/mm/memory_manager.rs

@@ -1,10 +1,8 @@
 use std::sync::atomic::Ordering;
 
 use crate::plan::mutator_context::{Mutator, MutatorContext};
-use crate::plan::transitive_closure::TransitiveClosure;
-use crate::plan::CollectorContext;
 use crate::plan::Plan;
-use crate::plan::TraceLocal;
+use crate::scheduler::GCWorker;
 
 use crate::vm::Collection;
 
@@ -14,7 +12,6 @@ use self::selected_plan::SelectedPlan;
 use crate::plan::selected_plan;
 use crate::util::alloc::allocators::AllocatorSelector;
 
-use self::selected_plan::{SelectedCollector, SelectedTraceLocal};
 use crate::mmtk::MMTK;
 use crate::plan::Allocator;
 use crate::util::constants::LOG_BYTES_IN_PAGE;
@@ -43,29 +40,28 @@ pub fn start_control_collector<VM: VMBinding>(mmtk: &MMTK<VM>, tls: OpaquePointe
     mmtk.plan.base().control_collector_context.run(tls);
 }
 
-pub fn gc_init<VM: VMBinding>(mmtk: &MMTK<VM>, heap_size: usize) {
+pub fn gc_init<VM: VMBinding>(mmtk: &'static mut MMTK<VM>, heap_size: usize) {
     crate::util::logger::init().unwrap();
-    mmtk.plan.gc_init(heap_size, &mmtk.vm_map);
-
-    // TODO: We should have an option so we know whether we should spawn the controller.
-    //    thread::spawn(|| {
-    //        SINGLETON.plan.common().control_collector_context.run(UNINITIALIZED_OPAQUE_POINTER )
-    //    });
+    mmtk.plan.gc_init(heap_size, &mmtk.vm_map, &mmtk.scheduler);
 }
 
 pub fn bind_mutator<VM: VMBinding>(
     mmtk: &'static MMTK<VM>,
     tls: OpaquePointer,
-) -> Box<Mutator<VM, SelectedPlan<VM>>> {
-    SelectedPlan::bind_mutator(&mmtk.plan, tls)
+) -> Box<Mutator<SelectedPlan<VM>>> {
+    SelectedPlan::bind_mutator(&mmtk.plan, tls, mmtk)
 }
 
-pub fn destroy_mutator<VM: VMBinding>(mutator: Box<Mutator<VM, SelectedPlan<VM>>>) {
+pub fn destroy_mutator<VM: VMBinding>(mutator: Box<Mutator<SelectedPlan<VM>>>) {
     drop(mutator);
 }
 
+pub fn flush_mutator<VM: VMBinding>(mutator: &mut Mutator<SelectedPlan<VM>>) {
+    mutator.flush()
+}
+
 pub fn alloc<VM: VMBinding>(
-    mutator: &mut Mutator<VM, SelectedPlan<VM>>,
+    mutator: &mut Mutator<SelectedPlan<VM>>,
     size: usize,
     align: usize,
     offset: isize,
@@ -75,7 +71,7 @@ pub fn alloc<VM: VMBinding>(
 }
 
 pub fn post_alloc<VM: VMBinding>(
-    mutator: &mut Mutator<VM, SelectedPlan<VM>>,
+    mutator: &mut Mutator<SelectedPlan<VM>>,
     refer: ObjectReference,
     type_refer: ObjectReference,
     bytes: usize,
@@ -92,102 +88,19 @@ pub fn get_allocator_mapping<VM: VMBinding>(
     mmtk.plan.get_allocator_mapping()[allocator]
 }
 
-// The parameter 'trace_local' could either be &mut SelectedTraceLocal or &mut SanityChecker.
-// Ideally we should make 'trace_local' as a trait object - &mut TraceLocal. However, this is a fat
-// pointer, and it would appear in our API (and possibly in native API), which imposes inconvenience
-// to store and pass around a fat pointer. Thus, we just assume it is &mut SelectedTraceLocal,
-// and use unsafe transmute when we know it is a SanityChcker ref.
-#[cfg(feature = "sanity")]
-pub fn report_delayed_root_edge<VM: VMBinding>(
-    mmtk: &MMTK<VM>,
-    trace_local: &mut SelectedTraceLocal<VM>,
-    addr: Address,
-) {
-    use crate::util::sanity::sanity_checker::SanityChecker;
-    if mmtk.plan.is_in_sanity() {
-        let sanity_checker: &mut SanityChecker<VM> =
-            unsafe { &mut *(trace_local as *mut SelectedTraceLocal<VM> as *mut SanityChecker<VM>) };
-        sanity_checker.report_delayed_root_edge(addr);
-    } else {
-        trace_local.report_delayed_root_edge(addr)
-    }
-}
-#[cfg(not(feature = "sanity"))]
-pub fn report_delayed_root_edge<VM: VMBinding>(
-    _: &MMTK<VM>,
-    trace_local: &mut SelectedTraceLocal<VM>,
-    addr: Address,
-) {
-    trace_local.report_delayed_root_edge(addr);
-}
-
-#[cfg(feature = "sanity")]
-pub fn will_not_move_in_current_collection<VM: VMBinding>(
-    mmtk: &MMTK<VM>,
-    trace_local: &mut SelectedTraceLocal<VM>,
-    obj: ObjectReference,
-) -> bool {
-    use crate::util::sanity::sanity_checker::SanityChecker;
-    if mmtk.plan.is_in_sanity() {
-        let sanity_checker: &mut SanityChecker<VM> =
-            unsafe { &mut *(trace_local as *mut SelectedTraceLocal<VM> as *mut SanityChecker<VM>) };
-        sanity_checker.will_not_move_in_current_collection(obj)
-    } else {
-        trace_local.will_not_move_in_current_collection(obj)
-    }
-}
-#[cfg(not(feature = "sanity"))]
-pub fn will_not_move_in_current_collection<VM: VMBinding>(
-    _: &MMTK<VM>,
-    trace_local: &mut SelectedTraceLocal<VM>,
-    obj: ObjectReference,
-) -> bool {
-    trace_local.will_not_move_in_current_collection(obj)
-}
-
-#[cfg(feature = "sanity")]
-pub fn process_interior_edge<VM: VMBinding>(
-    mmtk: &MMTK<VM>,
-    trace_local: &mut SelectedTraceLocal<VM>,
-    target: ObjectReference,
-    slot: Address,
-    root: bool,
-) {
-    use crate::util::sanity::sanity_checker::SanityChecker;
-    if mmtk.plan.is_in_sanity() {
-        let sanity_checker: &mut SanityChecker<VM> =
-            unsafe { &mut *(trace_local as *mut SelectedTraceLocal<VM> as *mut SanityChecker<VM>) };
-        sanity_checker.process_interior_edge(target, slot, root)
-    } else {
-        trace_local.process_interior_edge(target, slot, root)
-    }
-}
-#[cfg(not(feature = "sanity"))]
-pub fn process_interior_edge<VM: VMBinding>(
-    _: &MMTK<VM>,
-    trace_local: &mut SelectedTraceLocal<VM>,
-    target: ObjectReference,
-    slot: Address,
-    root: bool,
+pub fn start_worker<VM: VMBinding>(
+    tls: OpaquePointer,
+    worker: &'static mut GCWorker<VM>,
+    mmtk: &'static MMTK<VM>,
 ) {
-    trace_local.process_interior_edge(target, slot, root)
-}
-
-pub fn start_worker<VM: VMBinding>(tls: OpaquePointer, worker: &mut SelectedCollector<VM>) {
     worker.init(tls);
-    worker.run(tls);
+    worker.run(mmtk);
 }
 
 pub fn enable_collection<VM: VMBinding>(mmtk: &'static MMTK<VM>, tls: OpaquePointer) {
-    unsafe {
-        { &mut *mmtk.plan.base().control_collector_context.workers.get() }.init_group(mmtk, tls);
-        {
-            VM::VMCollection::spawn_worker_thread::<<SelectedPlan<VM> as Plan<VM>>::CollectorT>(
-                tls, None,
-            );
-        } // spawn controller thread
-        mmtk.plan.base().initialized.store(true, Ordering::SeqCst);
-    }
+    mmtk.scheduler.initialize(mmtk.options.threads, mmtk, tls);
+    VM::VMCollection::spawn_worker_thread(tls, None); // spawn controller thread
+    mmtk.plan.base().initialized.store(true, Ordering::SeqCst);
 }
 
 pub fn process<VM: VMBinding>(mmtk: &'static MMTK<VM>, name: &str, value: &str) -> bool {
@@ -219,41 +132,6 @@ pub fn scan_region() {
     crate::util::sanity::memory_scan::scan_region();
 }
 
-pub fn trace_get_forwarded_referent<VM: VMBinding>(
-    trace_local: &mut SelectedTraceLocal<VM>,
-    object: ObjectReference,
-) -> ObjectReference {
-    trace_local.get_forwarded_reference(object)
-}
-
-pub fn trace_get_forwarded_reference<VM: VMBinding>(
-    trace_local: &mut SelectedTraceLocal<VM>,
-    object: ObjectReference,
-) -> ObjectReference {
-    trace_local.get_forwarded_reference(object)
-}
-
-pub fn trace_root_object<VM: VMBinding>(
-    trace_local: &mut SelectedTraceLocal<VM>,
-    object: ObjectReference,
-) -> ObjectReference {
-    trace_local.trace_object(object)
-}
-
-pub extern "C" fn process_edge<VM: VMBinding>(
-    trace_local: &mut SelectedTraceLocal<VM>,
-    object: Address,
-) {
-    trace_local.process_edge(object);
-}
-
-pub fn trace_retain_referent<VM: VMBinding>(
-    trace_local: &mut SelectedTraceLocal<VM>,
-    object: ObjectReference,
-) -> ObjectReference {
-    trace_local.retain_referent(object)
-}
-
 pub fn handle_user_collection_request<VM: VMBinding>(mmtk: &MMTK<VM>, tls: OpaquePointer) {
     mmtk.plan.handle_user_collection_request(tls, false);
 }
@@ -305,6 +183,6 @@ pub fn harness_begin<VM: VMBinding>(mmtk: &MMTK<VM>, tls: OpaquePointer) {
     mmtk.harness_begin(tls);
 }
 
-pub fn harness_end<VM: VMBinding>(mmtk: &MMTK<VM>) {
+pub fn harness_end<VM: VMBinding>(mmtk: &'static MMTK<VM>) {
     mmtk.harness_end();
 }

src/mmtk.rs

@@ -1,19 +1,21 @@
-use crate::plan::phase::PhaseManager;
 use crate::plan::Plan;
 use crate::plan::SelectedPlan;
 use crate::policy::space::SFTMap;
+use crate::scheduler::Scheduler;
 use crate::util::heap::layout::heap_layout::Mmapper;
 use crate::util::heap::layout::heap_layout::VMMap;
-use crate::util::OpaquePointer;
-
+use crate::util::heap::layout::map::Map;
 use crate::util::options::{Options, UnsafeOptionsWrapper};
 use crate::util::reference_processor::ReferenceProcessors;
+#[cfg(feature = "sanity")]
+use crate::util::sanity::sanity_checker::SanityChecker;
+use crate::util::OpaquePointer;
+use crate::vm::VMBinding;
 use std::default::Default;
 use std::sync::atomic::{AtomicBool, Ordering};
-
-use crate::util::heap::layout::map::Map;
-use crate::vm::VMBinding;
 use std::sync::Arc;
+#[cfg(feature = "sanity")]
+use std::sync::Mutex;
 
 // TODO: remove this singleton at some point to allow multiple instances of MMTK
 // This helps refactoring.
@@ -32,29 +34,37 @@ lazy_static! {
 
 pub struct MMTK<VM: VMBinding> {
     pub plan: SelectedPlan<VM>,
-    pub phase_manager: PhaseManager,
     pub vm_map: &'static VMMap,
     pub mmapper: &'static Mmapper,
     pub sftmap: &'static SFTMap,
     pub reference_processors: ReferenceProcessors,
     pub options: Arc<UnsafeOptionsWrapper>,
-
+    pub scheduler: Arc<Scheduler<Self>>,
+    #[cfg(feature = "sanity")]
+    pub sanity_checker: Mutex<SanityChecker>,
     inside_harness: AtomicBool,
 }
 
+unsafe impl<VM: VMBinding> Send for MMTK<VM> {}
+unsafe impl<VM: VMBinding> Sync for MMTK<VM> {}
+
 impl<VM: VMBinding> MMTK<VM> {
     pub fn new() -> Self {
+        let scheduler = Scheduler::new();
         let options = Arc::new(UnsafeOptionsWrapper::new(Options::default()));
-        let plan = SelectedPlan::new(&VM_MAP, &MMAPPER, options.clone());
-        let phase_manager = PhaseManager::new(&plan.base().stats);
+        let plan = SelectedPlan::new(&VM_MAP, &MMAPPER, options.clone(), unsafe {
+            &*(scheduler.as_ref() as *const Scheduler<MMTK<VM>>)
+        });
         MMTK {
             plan,
-            phase_manager,
             vm_map: &VM_MAP,
             mmapper: &MMAPPER,
             sftmap: &SFT_MAP,
             reference_processors: ReferenceProcessors::new(),
             options,
+            scheduler,
+            #[cfg(feature = "sanity")]
+            sanity_checker: Mutex::new(SanityChecker::new()),
             inside_harness: AtomicBool::new(false),
         }
     }
@@ -64,10 +74,11 @@ impl<VM: VMBinding> MMTK<VM> {
         self.plan.handle_user_collection_request(tls, true);
         self.inside_harness.store(true, Ordering::SeqCst);
         self.plan.base().stats.start_all();
+        self.scheduler.enable_stat();
     }
 
-    pub fn harness_end(&self) {
-        self.plan.base().stats.stop_all();
+    pub fn harness_end(&'static self) {
+        self.plan.base().stats.stop_all(self);
         self.inside_harness.store(false, Ordering::SeqCst);
     }
 }