Switch 64-bit hashes to xxhash3

.gitmodules

@@ -22,3 +22,9 @@
     path = external/xamarin-android-tools
     url = https://github.com/xamarin/xamarin-android-tools
     branch = main
+[submodule "external/xxHash"]
+	path = external/xxHash
+	url = https://github.com/Cyan4973/xxHash.git
+[submodule "external/constexpr-xxh3"]
+	path = external/constexpr-xxh3
+	url = https://github.com/chys87/constexpr-xxh3.git

build-tools/scripts/generate-pinvoke-tables.sh

@@ -8,6 +8,7 @@ GENERATOR_BINARY="${MONODROID_SOURCE_DIR}/generate-pinvoke-tables"
 TARGET_FILE="${MONODROID_SOURCE_DIR}/pinvoke-tables.include"
 GENERATED_FILE="${TARGET_FILE}.generated"
 DIFF_FILE="${TARGET_FILE}.diff"
+EXTERNAL_DIR="${MY_DIR}/../../external/"
 
 function die()
 {
@@ -62,7 +63,7 @@ case ${HOST} in
 	*) die Unsupported OS ;;
 esac
 
-${COMPILER} -O2 -std=c++20 "${GENERATOR_SOURCE}" -o "${GENERATOR_BINARY}"
+${COMPILER} -O2 -std=c++20 -I${EXTERNAL_DIR} -I${EXTERNAL_DIR}/constexpr-xxh3 "${GENERATOR_SOURCE}" -o "${GENERATOR_BINARY}"
 "${GENERATOR_BINARY}" "${GENERATED_FILE}"
 
 FILES_DIFFER="no"

src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs

@@ -223,7 +223,7 @@ public static ulong GetMangledAssemblyNameSizeOverhead ()
 	public static ulong GetXxHash (byte[] stringBytes, bool is64Bit)
 	{
 		if (is64Bit) {
-			return XxHash64.HashToUInt64 (stringBytes);
+			return XxHash3.HashToUInt64 (stringBytes);
 		}
 
 		return (ulong)XxHash32.HashToUInt32 (stringBytes);

src/Xamarin.Android.Build.Tasks/Utilities/TypeMapHelper.cs

@@ -23,7 +23,7 @@ public static ulong HashJavaName (string name, bool is64Bit)
 	static ulong HashBytes (byte[] bytes, bool is64Bit)
 	{
 		if (is64Bit) {
-			return XxHash64.HashToUInt64 (bytes);
+			return XxHash3.HashToUInt64 (bytes);
 		}
 
 		return (ulong)XxHash32.HashToUInt32 (bytes);

src/monodroid/CMakeLists.txt

@@ -124,6 +124,8 @@ set(LZ4_INCLUDE_DIR ${LZ4_SRC_DIR})
 set(XA_BIN_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../bin/${XA_BUILD_CONFIGURATION}")
 set(XA_BUILD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../bin/Build${XA_BUILD_CONFIGURATION}")
 set(ROBIN_MAP_DIR "${EXTERNAL_DIR}/robin-map")
+set(XXHASH_DIR "${EXTERNAL_DIR}/xxHash")
+set(CONSTEXPR_XXH3_DIR "${EXTERNAL_DIR}/constexpr-xxh3")
 
 include("${XA_BUILD_DIR}/xa_build_configuration.cmake")
 
@@ -145,9 +147,11 @@ set(LZ4_SOURCES
 
 # Include directories
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/include/ ${CMAKE_SOURCE_DIR}/include)
+include_directories(${EXTERNAL_DIR})
 
 # The SYSTEM which will make clang skip warnings for the headers there. Since we can't do
 # much about them, we can just as well avoid cluttered build output.
+include_directories(SYSTEM ${CONSTEXPR_XXH3_DIR})
 include_directories(SYSTEM ${ROBIN_MAP_DIR}/include)
 include_directories(SYSTEM ${CMAKE_SYSROOT}/usr/include/c++/v1/)
 include_directories(SYSTEM ${LZ4_INCLUDE_DIR})

src/monodroid/jni/generate-pinvoke-tables.cc

@@ -688,9 +688,9 @@ struct constexpr_test {
 
 constexpr_test<xxhash32::hash<0> ("", 0), 0x2CC5D05U> constexprTest_1;
 constexpr_test<xxhash32::hash<2654435761U> ("", 0), 0x36B78AE7U> constexprTest_2;
-constexpr_test<xxhash64::hash<0> ("", 0), 0xEF46DB3751D8E999ULL> constexprTest_3;
-constexpr_test<xxhash64::hash<2654435761U> ("", 0), 0xAC75FDA2929B17EFULL> constexprTest_4;
+//constexpr_test<xxhash64::hash<0> ("", 0), 0xEF46DB3751D8E999ULL> constexprTest_3;
+//constexpr_test<xxhash64::hash<2654435761U> ("", 0), 0xAC75FDA2929B17EFULL> constexprTest_4;
 constexpr_test<xxhash32::hash<0> ("test", 4), 0x3E2023CFU> constexprTest32_5;
 constexpr_test<xxhash32::hash<2654435761U> ("test", 4), 0xA9C14438U> constexprTest32_6;
-constexpr_test<xxhash64::hash<0> ("test", 4), 0x4fdcca5ddb678139ULL> constexprTest64_7;
-constexpr_test<xxhash64::hash<2654435761U> ("test", 4), 0x5A183B8150E2F651ULL> constexprTest64_8;
+//constexpr_test<xxhash64::hash<0> ("test", 4), 0x4fdcca5ddb678139ULL> constexprTest64_7;
+//constexpr_test<xxhash64::hash<2654435761U> ("test", 4), 0x5A183B8150E2F651ULL> constexprTest64_8;

src/monodroid/jni/xxhash.hh

@@ -1,6 +1,16 @@
 #if !defined (__XXHASH_HH)
 #define __XXHASH_HH
 
+#include <type_traits>
+
+#if INTPTR_MAX == INT64_MAX
+#define XXH_NO_STREAM
+#define XXH_INLINE_ALL
+#define XXH_NAMESPACE xaInternal_
+#include <xxHash/xxhash.h>
+#include <constexpr-xxh3.h>
+#endif
+
 //
 // Based on original code at https://github.com/ekpyron/xxhashct
 //
@@ -142,122 +152,31 @@ namespace xamarin::android
 		}
 	};
 
+#if INTPTR_MAX == INT64_MAX
 	class xxhash64 final
 	{
-		static constexpr uint64_t PRIME1 = 11400714785074694791ULL;
-		static constexpr uint64_t PRIME2 = 14029467366897019727ULL;
-		static constexpr uint64_t PRIME3 =  1609587929392839161ULL;
-		static constexpr uint64_t PRIME4 =  9650029242287828579ULL;
-		static constexpr uint64_t PRIME5 =  2870177450012600261ULL;
-
 	public:
-		// We don't use any special seed in XA, the template parameter is just to keep the algorithm more easily
-		// understood and to run compile-time algorithm correctness tests
-		template<uint64_t Seed = 0>
-		force_inline static constexpr uint64_t hash (const char *p, size_t len) noexcept
-		{
-			return finalize ((len >= 32 ? h32bytes<Seed> (p, len) : Seed + PRIME5) + len, p + (len & ~0x1FU), len & 0x1F);
-		}
-
-		template<size_t Size, uint64_t Seed = 0>
-		force_inline static constexpr uint64_t hash (const char (&input)[Size]) noexcept
-		{
-			return hash<Seed> (input, Size - 1);
-		}
-
-		template<uint64_t Seed = 0>
-		force_inline static constexpr uint64_t hash (std::string_view const& input) noexcept
+		force_inline static XXH64_hash_t hash (const char *p, size_t len) noexcept
 		{
-			return hash<Seed> (input.data (), input.length ());
-		}
-
-	private:
-		template<int Bits>
-		force_inline static constexpr uint64_t rotl (uint64_t x) noexcept
-		{
-			return ((x << Bits) | (x >> (64 - Bits)));
+			return XXH3_64bits (static_cast<const void*>(p), len);
 		}
 
-		template<int RShift>
-		force_inline static constexpr uint64_t mix1 (const uint64_t h, const uint64_t prime) noexcept
+		force_inline static consteval XXH64_hash_t hash (std::string_view const& input) noexcept
 		{
-			return (h ^ (h >> RShift)) * prime;
+			return constexpr_xxh3::XXH3_64bits_const (input.data (), input.length ());
 		}
 
-		force_inline static constexpr uint64_t mix2 (const uint64_t p, const uint64_t v = 0) noexcept
+		// The C XXH64_64bits function from xxhash.h is not `constexpr` or `consteval`, so we cannot call it here.
+		// At the same time, at build time performance is not that important, so we call the "unoptmized" `consteval`
+		// C++  implementation here
+		template<size_t Size>
+		force_inline static consteval XXH64_hash_t hash (const char (&input)[Size]) noexcept
 		{
-			return rotl<31> (v + p * PRIME2) * PRIME1;
-		}
-
-		force_inline static constexpr uint64_t mix3 (const uint64_t h, const uint64_t v) noexcept
-		{
-			return (h ^ mix2 (v)) * PRIME1 + PRIME4;
-		}
-
-		// little-endian versions: all our target platforms are little-endian
-		force_inline static constexpr uint32_t endian32 (const char *v) noexcept
-		{
-			return
-				static_cast<uint32_t>(static_cast<uint8_t>(v[0])) |
-				(static_cast<uint32_t>(static_cast<uint8_t>(v[1])) << 8) |
-				(static_cast<uint32_t>(static_cast<uint8_t>(v[2])) << 16) |
-				(static_cast<uint32_t>(static_cast<uint8_t>(v[3])) << 24);
-		}
-
-		force_inline static constexpr uint64_t endian64 (const char *v)
-		{
-			return
-				static_cast<uint64_t>(static_cast<uint8_t>(v[0])) |
-				(static_cast<uint64_t>(static_cast<uint8_t>(v[1])) << 8) |
-				(static_cast<uint64_t>(static_cast<uint8_t>(v[2])) << 16) |
-				(static_cast<uint64_t>(static_cast<uint8_t>(v[3])) << 24) |
-				(static_cast<uint64_t>(static_cast<uint8_t>(v[4])) << 32) |
-				(static_cast<uint64_t>(static_cast<uint8_t>(v[5])) << 40) |
-				(static_cast<uint64_t>(static_cast<uint8_t>(v[6])) << 48) |
-				(static_cast<uint64_t>(static_cast<uint8_t>(v[7])) << 56);
-		}
-
-		force_inline static constexpr uint64_t fetch64 (const char *p, const uint64_t v = 0) noexcept
-		{
-			return mix2 (endian64 (p), v);
-		}
-
-		force_inline static constexpr uint64_t fetch32 (const char *p) noexcept
-		{
-			return static_cast<uint64_t>(endian32 (p)) * PRIME1;
-		}
-
-		force_inline static constexpr uint64_t fetch8 (const char *p) noexcept
-		{
-			return static_cast<uint8_t> (*p) * PRIME5;
-		}
-
-		force_inline static constexpr uint64_t finalize (const uint64_t h, const char *p, size_t len) noexcept
-		{
-			return
-				(len >= 8) ? (finalize (rotl<27> (h ^ fetch64 (p)) * PRIME1 + PRIME4, p + 8, len - 8)) :
-				((len >= 4) ? (finalize (rotl<23> (h ^ fetch32 (p)) * PRIME2 + PRIME3, p + 4, len - 4)) :
-				 ((len > 0) ? (finalize (rotl<11> (h ^ fetch8 (p)) * PRIME1, p + 1, len - 1)) :
-				  (mix1<32> (mix1<29> (mix1<33> (h, PRIME2), PRIME3), 1))));
-		}
-
-		force_inline static constexpr uint64_t h32bytes (const char *p, size_t len, const uint64_t v1,const uint64_t v2, const uint64_t v3, const uint64_t v4) noexcept
-		{
-			return (len >= 32) ? h32bytes (p + 32, len - 32, fetch64 (p, v1), fetch64 (p + 8, v2), fetch64 (p + 16, v3), fetch64 (p + 24, v4)) :
-				mix3 (mix3 (mix3 (mix3 (rotl<1> (v1) + rotl<7> (v2) + rotl<12> (v3) + rotl<18> (v4), v1), v2), v3), v4);
-		}
-
-		// We don't use any special seed in XA, the template parameter is just to keep the algorithm more easily
-		// understood
-		template<uint64_t Seed = 0>
-		force_inline static constexpr uint64_t h32bytes (const char *p, size_t len) noexcept
-		{
-			return h32bytes (p, len, Seed + PRIME1 + PRIME2, Seed + PRIME2, Seed, Seed - PRIME1);
+			return constexpr_xxh3::XXH3_64bits_const (input);
 		}
 	};
 
-#if INTPTR_MAX == INT64_MAX
-	using hash_t = uint64_t;
+	using hash_t = XXH64_hash_t;
 	using xxhash = xxhash64;
 #else
 	using hash_t = uint32_t;