[wasm] Add custom mmap/munmap implementation for anonymous mappings (#101871)

* emscripten libc implements mmap/munmap as a broken adapter on top of malloc (not calloc), which means it has no choice but to invoke memset on every allocation in order to provide properly zeroed bytes for our allocation requests. this commit adds a custom mmap/munmap implementation that can skip zeroing already-zeroed pages
* re-enable freeing of pages in sgen on wasm if custom mmap is active
* add runtime option for custom mmap
* add warning switches to fix build on debian

Author: kg

src/mono/mono/sgen/sgen-internal.c

@@ -273,7 +273,7 @@ sgen_report_internal_mem_usage (void)
 void
 sgen_init_internal_allocator (void)
 {
-	int i, size;
+	int i;
 
 	for (i = 0; i < INTERNAL_MEM_MAX; ++i)
 		fixed_type_allocator_indexes [i] = -1;
@@ -284,7 +284,10 @@ sgen_init_internal_allocator (void)
 		mono_lock_free_allocator_init_allocator (&allocators [i], &size_classes [i], MONO_MEM_ACCOUNT_SGEN_INTERNAL);
 	}
 
-	for (size = mono_pagesize (); size <= LOCK_FREE_ALLOC_SB_MAX_SIZE; size <<= 1) {
+	// FIXME: This whole algorithm is broken on WASM due to its 64KB page size.
+	// Previously SB_MAX_SIZE was < mono_pagesize, so none of this ran.
+#ifndef HOST_WASM
+	for (int size = mono_pagesize (); size <= LOCK_FREE_ALLOC_SB_MAX_SIZE; size <<= 1) {
 		int max_size = (LOCK_FREE_ALLOC_SB_USABLE_SIZE (size) / 2) & ~(SIZEOF_VOID_P - 1);
 		/*
 		 * we assert that allocator_sizes contains the biggest possible object size
@@ -297,6 +300,7 @@ sgen_init_internal_allocator (void)
 		if (size < LOCK_FREE_ALLOC_SB_MAX_SIZE)
 			g_assert (block_size (max_size + 1) == size << 1);
 	}
+#endif
 }
 
 #endif

src/mono/mono/sgen/sgen-marksweep.c

@@ -33,6 +33,7 @@
 #include "mono/sgen/sgen-client.h"
 #include "mono/utils/mono-memory-model.h"
 #include "mono/utils/mono-proclib.h"
+#include "mono/utils/options.h"
 
 static int ms_block_size;
 
@@ -2133,12 +2134,15 @@ major_free_swept_blocks (size_t section_reserve)
 {
 	SGEN_ASSERT (0, sweep_state == SWEEP_STATE_SWEPT, "Sweeping must have finished before freeing blocks");
 
-#if defined(HOST_WIN32) || defined(HOST_ORBIS) || defined (HOST_WASM)
+#if defined(HOST_WIN32) || defined(HOST_ORBIS)
 		/*
 		 * sgen_free_os_memory () asserts in mono_vfree () because windows doesn't like freeing the middle of
 		 * a VirtualAlloc ()-ed block.
 		 */
 		return;
+#elif defined(HOST_WASM)
+	if (!mono_opt_wasm_mmap)
+		return;
 #endif
 
 	{

src/mono/mono/utils/CMakeLists.txt

@@ -206,7 +206,7 @@ set(utils_arch_sources "${utils_arch_sources};mono-hwcap-riscv.c")
 elseif(TARGET_S390X)
 set(utils_arch_sources "${utils_arch_sources};mono-hwcap-s390x.c")
 elseif(TARGET_WASM)
-set(utils_arch_sources "${utils_arch_sources};mono-hwcap-wasm.c;mono-mmap-wasm.c")
+set(utils_arch_sources "${utils_arch_sources};mono-hwcap-wasm.c;mono-mmap-wasm.c;mono-wasm-pagemgr.c")
 elseif(TARGET_WASI)
 set(utils_arch_sources "${utils_arch_sources};mono-hwcap-wasm.c")
 elseif(TARGET_POWERPC OR TARGET_POWERPC64)

src/mono/mono/utils/lock-free-alloc.h

@@ -45,6 +45,7 @@ typedef struct {
 	MonoMemAccountType account_type;
 } MonoLockFreeAllocator;
 
+// FIXME: On WASM the page size is 64KB, so this isn't enough.
 #define LOCK_FREE_ALLOC_SB_MAX_SIZE					16384
 #define LOCK_FREE_ALLOC_SB_HEADER_SIZE				(sizeof (gpointer))
 #define LOCK_FREE_ALLOC_SB_USABLE_SIZE(block_size)	((block_size) - LOCK_FREE_ALLOC_SB_HEADER_SIZE)

src/mono/mono/utils/mono-mmap-wasm.c

@@ -22,6 +22,9 @@
 #include "mono-proclib.h"
 #include <mono/utils/mono-threads.h>
 #include <mono/utils/atomic.h>
+#include <mono/utils/options.h>
+
+#include "mono-wasm-pagemgr.h"
 
 #define BEGIN_CRITICAL_SECTION do { \
 	MonoThreadInfo *__info = mono_thread_info_current_unchecked (); \
@@ -34,6 +37,9 @@
 int
 mono_pagesize (void)
 {
+	if (mono_opt_wasm_mmap)
+		return MWPM_PAGE_SIZE;
+
 	static int saved_pagesize = 0;
 
 	if (saved_pagesize)
@@ -108,7 +114,16 @@ valloc_impl (void *addr, size_t size, int flags, MonoMemAccountType type)
 	mflags |= MAP_PRIVATE;
 
 	BEGIN_CRITICAL_SECTION;
-	ptr = mmap (addr, size, prot, mflags, -1, 0);
+	if (mono_opt_wasm_mmap) {
+		// FIXME: Make this work if the requested address range is free
+		if ((flags & MONO_MMAP_FIXED) && addr)
+			return NULL;
+
+		ptr = mwpm_alloc_range (size, 1);
+		if (!ptr)
+			return NULL;
+	} else
+		ptr = mmap (addr, size, prot, mflags, -1, 0);
 	END_CRITICAL_SECTION;
 
 	if (ptr == MAP_FAILED)
@@ -142,6 +157,10 @@ typedef struct {
 void*
 mono_valloc_aligned (size_t size, size_t alignment, int flags, MonoMemAccountType type)
 {
+	// We don't need padding if the alignment is compatible with the page size
+	if (mono_opt_wasm_mmap && ((MWPM_PAGE_SIZE % alignment) == 0))
+		return valloc_impl (NULL, size, flags, type);
+
 	/* Allocate twice the memory to be able to put the block on an aligned address */
 	char *mem = (char *) valloc_impl (NULL, size + alignment, flags, type);
 	char *aligned;
@@ -175,13 +194,22 @@ mono_vfree (void *addr, size_t length, MonoMemAccountType type)
 		 * mono_valloc_align (), free the original mapping.
 		 */
 		BEGIN_CRITICAL_SECTION;
-		munmap (info->addr, info->size);
+		if (mono_opt_wasm_mmap)
+			mwpm_free_range (info->addr, info->size);
+		else
+			munmap (info->addr, info->size);
 		END_CRITICAL_SECTION;
 		g_free (info);
 		g_hash_table_remove (valloc_hash, addr);
 	} else {
+		// FIXME: We could be trying to unmap part of an aligned mapping, in which case the
+		//  hash lookup failed because addr isn't exactly the start of the mapping.
+		// Ideally if the custom page manager is enabled, we won't have done aligned alloc.
 		BEGIN_CRITICAL_SECTION;
-		munmap (addr, length);
+		if (mono_opt_wasm_mmap)
+			mwpm_free_range (addr, length);
+		else
+			munmap (addr, length);
 		END_CRITICAL_SECTION;
 	}
 

src/mono/mono/utils/mono-mmap.h

@@ -22,7 +22,8 @@ enum {
 	MONO_MMAP_ANON    = 1 << 6,
 	MONO_MMAP_FIXED   = 1 << 7,
 	MONO_MMAP_32BIT   = 1 << 8,
-	MONO_MMAP_JIT     = 1 << 9
+	MONO_MMAP_JIT     = 1 << 9,
+	MONO_MMAP_NOZERO  = 1 << 10,
 };
 
 typedef enum {