Refactor CI test scripts

.github/scripts/ci-build.sh

@@ -2,12 +2,15 @@ set -xe
 
 . $(dirname "$0")/common.sh
 
-unset JAVA_TOOL_OPTIONS
-unset MMTK_PLAN
+ensure_env OPENJDK_PATH
 
-# To OpenJDK folder
-cd $OPENJDK_PATH
+# Use fastdebug if DEBUG_LEVEL is unset
+DEBUG_LEVEL=${DEBUG_LEVEL:="fastdebug"}
+
+# Build target. Could be empty, or product-bundles.
+build_target=$1
 
-# Build
+# Build product bundle
+cd $OPENJDK_PATH
 sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL
-make CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$BINDING_PATH/openjdk
+make CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$BINDING_PATH/openjdk $OPENJDK_BUILD_TARGET

.github/scripts/ci-checkout.sh

@@ -6,6 +6,9 @@ set -ex
 OPENJDK_URL=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python3 -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["openjdk"]["openjdk_repo"])'`
 OPENJDK_VERSION=`cargo read-manifest --manifest-path=$BINDING_PATH/mmtk/Cargo.toml | python3 -c 'import json,sys; print(json.load(sys.stdin)["metadata"]["openjdk"]["openjdk_version"])'`
 
+# Use default openjdk path if it is not set (this is only need when MMTk core calls the script)
+OPENJDK_PATH=${OPENJDK_PATH:="$BINDING_PATH/repos/openjdk"}
+
 rm -rf $OPENJDK_PATH
 git clone $OPENJDK_URL $OPENJDK_PATH
 git -C $OPENJDK_PATH checkout $OPENJDK_VERSION

.github/scripts/ci-setup.sh

@@ -9,10 +9,10 @@ rustup component add clippy --toolchain $RUSTUP_TOOLCHAIN
 rustup component add rustfmt --toolchain $RUSTUP_TOOLCHAIN
 rustup override set $RUSTUP_TOOLCHAIN
 
-# Download dacapo
-mkdir -p $DACAPO_PATH
-wget https://downloads.sourceforge.net/project/dacapobench/archive/2006-10-MR2/dacapo-2006-10-MR2.jar -O $DACAPO_PATH/dacapo-2006-10-MR2.jar
+# Install running
+pip3 install running-ng
 
 # Install dependencies
 sudo apt-get update -y
+sudo apt-get install dos2unix
 sudo apt-get install build-essential libx11-dev libxext-dev libxrender-dev libxtst-dev libxt-dev libcups2-dev libasound2-dev libxrandr-dev

.github/scripts/ci-test-assertions.sh

@@ -4,57 +4,53 @@ set -xe
 
 unset JAVA_TOOL_OPTIONS
 
-export MMTK_EXTREME_ASSERTIONS=1
-. $(dirname "$0")/ci-build.sh
+run_subset() {
+    heap_multiplier=$1
 
-cd $OPENJDK_PATH
+    runbms_dacapo2006_with_heap_multiplier fop $heap_multiplier
+    runbms_dacapo2006_with_heap_multiplier luindex $heap_multiplier
+}
 
 # -- SemiSpace --
 export MMTK_PLAN=SemiSpace
 
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+run_subset 4
 
 # --- Immix ---
 export MMTK_PLAN=Immix
 
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+run_subset 4
 
 # --- GenImmix ---
 export MMTK_PLAN=GenImmix
 
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+run_subset 4
 
 # --- StickyImmix ---
 export MMTK_PLAN=StickyImmix
 
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+run_subset 4
 
 # -- GenCopy --
 export MMTK_PLAN=GenCopy
 
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+run_subset 4
 
 # -- NoGC --
 export MMTK_PLAN=NoGC
 
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms1G -Xmx1G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+runbms_dacapo2006_with_heap_size fop 1000 1000
+runbms_dacapo2006_with_heap_size luindex 1000 1000
 
 # --- MarkSweep ---
 export MMTK_PLAN=MarkSweep
 
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+run_subset 8
 
 # -- PageProtect --
 sudo sysctl -w vm.max_map_count=655300
 export MMTK_PLAN=PageProtect
 
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-# Note: Disable compressed pointers for luindex as it does not work well with GC plans that uses virtual memory excessively.
-build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -Xms4G -Xmx4G -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
+# Note: Disable compressed pointers as it does not work well with GC plans that uses virtual memory excessively.
+runbms_dacapo2006_with_heap_size fop 4000 4000 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers
+runbms_dacapo2006_with_heap_size luindex 4000 4000 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers

.github/scripts/ci-test-malloc-mark-sweep.sh

@@ -2,25 +2,11 @@ set -xe
 
 . $(dirname "$0")/common.sh
 
-build() {
-    cd $OPENJDK_PATH
-    export MMTK_MALLOC_MARK_SWEEP=1
-    sh configure --disable-warnings-as-errors --with-debug-level=$DEBUG_LEVEL
-    make CONF=linux-x86_64-normal-server-$DEBUG_LEVEL THIRD_PARTY_HEAP=$BINDING_PATH/openjdk
-    unset MMTK_MALLOC_MARK_SWEEP
-}
-
 run_test() {
     export MMTK_PLAN=MarkSweep
 
     # Malloc marksweep is horribly slow. We just run fop.
-
-    # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar antlr
-    build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:-UseCompressedOops -XX:-UseCompressedClassPointers -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms50M -Xmx50M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar fop
-    # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar luindex
-    # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar pmd
-    # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar hsqldb
-    # build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -server -XX:MetaspaceSize=100M -Xms500M -Xmx500M -jar $DACAPO_PATH/dacapo-2006-10-MR2.jar eclipse
+    runbms_dacapo2006_with_heap_multiplier fop 4 -XX:-UseCompressedOops -XX:-UseCompressedClassPointers
 
     unset MMTK_PLAN
 }
@@ -29,17 +15,4 @@ unset JAVA_TOOL_OPTIONS
 unset MMTK_PLAN
 
 # --- Normal test ---
-build
-run_test
-
-# --- Header mark bit ---
-export MARK_IN_HEADER=1
-build
-run_test
-unset MARK_IN_HEADER
-
-# --- Test assertions ---
-export MMTK_EXTREME_ASSERTIONS=1
-build
 run_test
-unset MMTK_EXTREME_ASSERTIONS

.github/scripts/ci-test-mark-in-header.sh

@@ -0,0 +1,30 @@
+set -xe
+
+. $(dirname "$0")/common.sh
+
+unset JAVA_TOOL_OPTIONS
+
+run_subset() {
+    heap_multiplier=$1
+
+    runbms_dacapo2006_with_heap_multiplier antlr $heap_multiplier
+    runbms_dacapo2006_with_heap_multiplier fop $heap_multiplier
+    runbms_dacapo2006_with_heap_multiplier luindex $heap_multiplier
+}
+
+# --- Immix ---
+# export MMTK_PLAN=Immix
+# run_subset 4
+
+# --- GenImmix ---
+# export MMTK_PLAN=GenImmix
+# run_subset 4
+
+# --- StickyImmix ---
+# export MMTK_PLAN=StickyImmix
+# run_subset 4
+
+# --- MarkSweep ---
+export MMTK_PLAN=MarkSweep
+
+run_subset 8