System image compression with zstd (1.11 backport)

commit d062060
Author: Sam Schweigel <[email protected]>
Date:   Wed Aug 6 16:35:34 2025 -0700

    System image compression with zstd

    Co-authored-by: Gabriel Baraldi <[email protected]>

    madvise with MADV_WILLNEED before accessing sysimage pages

    MADV_WILLNEED sysimage pages before checksumming or decompressing

    Even without compression, this gives about an 8% improvement in load times.

    Fix using jl_page_size before it is initialized

    Add zstd to ANALYSIS_DEPS now that it is used in staticdata.c

    Fix jl_page_size being uninitialized in jl_prefetch_system_image

    Add --compress-sysimage flag; use zstd-15 when enabled

commit 56a6065
Author: Sam Schweigel <[email protected]>
Date:   Mon Aug 18 13:27:13 2025 -0700

    Janky zstd backport

commit 4b731bb
Author: Cody Tapscott <[email protected]>
Date:   Thu Feb 27 22:20:02 2025 -0500

    staticdata: Refactor sysimage loading (#57542)

    Introduce `jl_image_buf_t` to represent the in-memory details of an
    unparsed system image and clean-up several global variables and split
    loading logic in staticdata.c so that we get (almost) everything we need
    from the sysimage handle at once.

    This allows sysimage loading to be separated into three phases:
      1. Lookup the raw sysimage buffer as a `jl_image_buf_t`
    2. For .so sysimages, parse the sysimage and initialize JIT targets,
    etc.
      3. Finally load the sysimage into a `jl_image_t`

    Care was taken to preserve the existing behavior of calling
    `jl_set_sysimg_so` to configure the sysimage before initializing Julia,
    although this is likely to be next on the chopping block in a follow-up.

    Dependent on #57523.

Author: xal-0

Make.inc

@@ -65,6 +65,7 @@ USE_SYSTEM_LIBGIT2:=0
 USE_SYSTEM_PATCHELF:=0
 USE_SYSTEM_LIBWHICH:=0
 USE_SYSTEM_ZLIB:=0
+USE_SYSTEM_ZSTD:=0
 USE_SYSTEM_P7ZIP:=0
 USE_SYSTEM_LLD:=0
 
@@ -1397,7 +1398,7 @@ JLDFLAGS += -Wl,--stack,8388608 --disable-auto-import --disable-runtime-pseudo-r
 ifeq ($(ARCH),i686)
 JLDFLAGS += -Wl,--large-address-aware
 endif
-JCPPFLAGS += -D_WIN32_WINNT=0x0502
+JCPPFLAGS += -D_WIN32_WINNT=_WIN32_WINNT_WIN8
 UNTRUSTED_SYSTEM_LIBM := 1
 # Use hard links for files on windows, rather than soft links
 #   https://stackoverflow.com/questions/3648819/how-to-make-a-symbolic-link-with-cygwin-in-windows-7

Makefile

@@ -201,6 +201,12 @@ endif
 
 # private libraries, that are installed in $(prefix)/lib/julia
 JL_PRIVATE_LIBS-0 := libccalltest libccalllazyfoo libccalllazybar libllvmcalltest
+JL_PRIVATE_LIBS-1 := # libraries from USE_SYSTEM=1
+JL_PRIVATE_EXES := 7z
+ifeq ($(OS),WINNT)
+JL_PRIVATE_EXES += 7z.dll
+endif
+JL_PRIVATE_TOOLS :=
 ifeq ($(JULIA_BUILD_MODE),release)
 JL_PRIVATE_LIBS-0 += libjulia-internal libjulia-codegen
 else ifeq ($(JULIA_BUILD_MODE),debug)
@@ -225,9 +231,12 @@ JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += zlib
 else
 JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libz
 endif
+JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libzstd
+JL_PRIVATE_EXES += zstd$(EXE) zstdmt$(EXE)
 ifeq ($(USE_LLVM_SHLIB),1)
 JL_PRIVATE_LIBS-$(USE_SYSTEM_LLVM) += libLLVM $(LLVM_SHARED_LIB_NAME)
 endif
+JL_PRIVATE_TOOLS += lld$(EXE) dsymutil$(EXE)
 JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBUNWIND) += libunwind
 
 ifeq ($(USE_SYSTEM_LIBM),0)
@@ -284,45 +293,41 @@ install: $(build_depsbindir)/stringreplace $(BUILDROOT)/doc/_build/html/en/index
 
 	$(INSTALL_M) $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE)) $(DESTDIR)$(bindir)/
 ifeq ($(OS),WINNT)
-	-$(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/
+	$(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/
 ifeq ($(JULIA_BUILD_MODE),release)
-	-$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/
-	-$(INSTALL_M) $(build_libdir)/libjulia-internal.dll.a $(DESTDIR)$(libdir)/
+	$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/
+	$(INSTALL_M) $(build_libdir)/libjulia-internal.dll.a $(DESTDIR)$(libdir)/
 else ifeq ($(JULIA_BUILD_MODE),debug)
-	-$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/
-	-$(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/
+	$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/
+	$(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/
 endif
-	-$(INSTALL_M) $(wildcard $(build_private_libdir)/*.a) $(DESTDIR)$(private_libdir)/
-	-rm -f $(DESTDIR)$(private_libdir)/sys-o.a
+	$(INSTALL_M) $(filter-out %-bc.a %-o.a,$(wildcard $(build_private_libdir)/lib*.a)) $(DESTDIR)$(private_libdir)/
 
-	# We have a single exception; we want 7z.dll to live in private_libexecdir,
-	# not bindir, so that 7z.exe can find it.
-	-mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(private_libexecdir)/
-	-$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/
-	-$(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/
+	$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/
+	$(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/
 else
 
 # Copy over .dSYM directories directly for Darwin
 ifneq ($(DARWIN_FRAMEWORK),1)
 ifeq ($(OS),Darwin)
 ifeq ($(JULIA_BUILD_MODE),release)
-	-cp -a $(build_libdir)/libjulia.*.dSYM $(DESTDIR)$(libdir)
-	-cp -a $(build_libdir)/libjulia-internal.*.dSYM $(DESTDIR)$(private_libdir)
-	-cp -a $(build_libdir)/libjulia-codegen.*.dSYM $(DESTDIR)$(private_libdir)
-	-cp -a $(build_private_libdir)/sys.dylib.dSYM $(DESTDIR)$(private_libdir)
+	cp -a $(build_libdir)/libjulia.*.dSYM $(DESTDIR)$(libdir)
+	cp -a $(build_libdir)/libjulia-internal.*.dSYM $(DESTDIR)$(private_libdir)
+	cp -a $(build_libdir)/libjulia-codegen.*.dSYM $(DESTDIR)$(private_libdir)
+	cp -a $(build_private_libdir)/sys.dylib.dSYM $(DESTDIR)$(private_libdir)
 else ifeq ($(JULIA_BUILD_MODE),debug)
-	-cp -a $(build_libdir)/libjulia-debug.*.dSYM $(DESTDIR)$(libdir)
-	-cp -a $(build_libdir)/libjulia-internal-debug.*.dSYM $(DESTDIR)$(private_libdir)
-	-cp -a $(build_libdir)/libjulia-codegen-debug.*.dSYM $(DESTDIR)$(private_libdir)
-	-cp -a $(build_private_libdir)/sys-debug.dylib.dSYM $(DESTDIR)$(private_libdir)
+	cp -a $(build_libdir)/libjulia-debug.*.dSYM $(DESTDIR)$(libdir)
+	cp -a $(build_libdir)/libjulia-internal-debug.*.dSYM $(DESTDIR)$(private_libdir)
+	cp -a $(build_libdir)/libjulia-codegen-debug.*.dSYM $(DESTDIR)$(private_libdir)
+	cp -a $(build_private_libdir)/sys-debug.dylib.dSYM $(DESTDIR)$(private_libdir)
 endif
 endif
 
 # Copy over shared library file for libjulia.*
 	for suffix in $(JL_TARGETS) ; do \
 		for lib in $(build_libdir)/lib$${suffix}.*$(SHLIB_EXT)*; do \
 			if [ "$${lib##*.}" != "dSYM" ]; then \
-				$(INSTALL_M) $$lib $(DESTDIR)$(libdir) ; \
+				$(INSTALL_M) $$lib $(DESTDIR)$(libdir) || exit 1; \
 			fi \
 		done \
 	done
@@ -342,26 +347,24 @@ endif
 	for suffix in $(JL_PRIVATE_LIBS-0) ; do \
 		for lib in $(build_libdir)/$${suffix}.*$(SHLIB_EXT)*; do \
 			if [ "$${lib##*.}" != "dSYM" ]; then \
-				$(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \
+				$(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) || exit 1; \
 			fi \
 		done \
 	done
 	for suffix in $(JL_PRIVATE_LIBS-1) ; do \
 		for lib in $(build_private_libdir)/$${suffix}.$(SHLIB_EXT)*; do \
 			if [ "$${lib##*.}" != "dSYM" ]; then \
-				$(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) ; \
+				$(INSTALL_M) $$lib $(DESTDIR)$(private_libdir) || exit 1; \
 			fi \
 		done \
 	done
 endif
-	# Install `7z` into private_libexecdir
-	$(INSTALL_M) $(build_bindir)/7z$(EXE) $(DESTDIR)$(private_libexecdir)/
-
-	# Install `lld` into private_libexecdir
-	$(INSTALL_M) $(build_depsbindir)/lld$(EXE) $(DESTDIR)$(private_libexecdir)/
-
-	# Install `dsymutil` into private_libexecdir/
-	$(INSTALL_M) $(build_depsbindir)/dsymutil$(EXE) $(DESTDIR)$(private_libexecdir)/
+	for exe in $(JL_PRIVATE_EXES) ; do \
+		$(INSTALL_M) $(build_private_libexecdir)/$$exe $(DESTDIR)$(private_libexecdir) || exit 1; \
+	done
+	for exe in $(JL_PRIVATE_TOOLS) ; do \
+		$(INSTALL_M) $(build_depsbindir)/$$exe $(DESTDIR)$(private_libexecdir) || exit 1; \
+	done
 
 	# Copy public headers
 	cp -R -L $(build_includedir)/julia/* $(DESTDIR)$(includedir)/julia
@@ -413,13 +416,13 @@ ifneq ($(private_libdir_rel),$(build_private_libdir_rel))
 ifeq ($(OS), Darwin)
 ifneq ($(DARWIN_FRAMEWORK),1)
 	for j in $(JL_TARGETS) ; do \
-		install_name_tool -rpath @executable_path/$(build_private_libdir_rel) @executable_path/$(private_libdir_rel) $(DESTDIR)$(bindir)/$$j; \
-		install_name_tool -add_rpath @executable_path/$(build_libdir_rel) @executable_path/$(libdir_rel) $(DESTDIR)$(bindir)/$$j; \
+		install_name_tool -rpath @executable_path/$(build_private_libdir_rel) @executable_path/$(private_libdir_rel) $(DESTDIR)$(bindir)/$$j || exit 1; \
+		install_name_tool -rpath @executable_path/$(build_libdir_rel) @executable_path/$(libdir_rel) $(DESTDIR)$(bindir)/$$j || exit 1; \
 	done
 endif
 else ifneq (,$(findstring $(OS),Linux FreeBSD))
 	for j in $(JL_TARGETS) ; do \
-		$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(private_libdir_rel):$$ORIGIN/$(libdir_rel)' $(DESTDIR)$(bindir)/$$j; \
+		$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(private_libdir_rel):$$ORIGIN/$(libdir_rel)' $(DESTDIR)$(bindir)/$$j || exit 1; \
 	done
 endif
 
@@ -442,11 +445,11 @@ endif
 ifeq ($(OS), Darwin)
 ifneq ($(DARWIN_FRAMEWORK),1)
 ifeq ($(JULIA_BUILD_MODE),release)
-	install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-internal.$(SHLIB_EXT)
-	install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-codegen.$(SHLIB_EXT)
+	install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-internal.$(SHLIB_EXT)
+	install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-codegen.$(SHLIB_EXT)
 else ifeq ($(JULIA_BUILD_MODE),debug)
-	install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT)
-	install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel)/ $(DESTDIR)$(private_libdir)/libjulia-codegen-debug.$(SHLIB_EXT)
+	install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-internal-debug.$(SHLIB_EXT)
+	install_name_tool -add_rpath @loader_path/$(reverse_private_libdir_rel) $(DESTDIR)$(private_libdir)/libjulia-codegen-debug.$(SHLIB_EXT)
 endif
 endif
 else ifneq (,$(findstring $(OS),Linux FreeBSD))
@@ -483,7 +486,7 @@ ifeq ($(OS),FreeBSD)
 	# don't set libgfortran's RPATH, it won't be able to find its friends on systems
 	# that don't have the exact GCC port installed used for the build.
 	for lib in $(DESTDIR)$(private_libdir)/libgfortran*$(SHLIB_EXT)*; do \
-		$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $$lib; \
+		$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN' $$lib || exit 1; \
 	done
 endif
 

THIRDPARTY.md

@@ -55,6 +55,7 @@ Julia bundles the following external programs and libraries:
 
 - [7-Zip](https://www.7-zip.org/license.txt)
 - [ZLIB](https://zlib.net/zlib_license.html)
+- [ZSTD](https://github.com/facebook/zstd/blob/v1.5.7/LICENSE)
 
 On some platforms, distributions of Julia contain SSL certificate authority certificates,
 released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License).

base/options.jl

@@ -58,6 +58,7 @@ struct JLOptions
     strip_ir::Int8
     permalloc_pkgimg::Int8
     heap_size_hint::UInt64
+    compress_sysimage::Int8
 end
 
 # This runs early in the sysimage != is not defined yet

base/util.jl

@@ -245,6 +245,9 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Unio
     if opts.use_sysimage_native_code == 0
         push!(addflags, "--sysimage-native-code=no")
     end
+    if opts.compress_sysimage == 1
+        push!(addflags, "--compress-sysimage=yes")
+    end
     return `$julia -C $cpu_target -J$image_file $addflags`
 end
 

contrib/refresh_checksums.mk

@@ -24,7 +24,7 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS))
 NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS))
 
 # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded:
-BB_PROJECTS=mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient
+BB_PROJECTS=mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib zstd libsuitesparse openlibm blastrampoline libtracyclient
 BB_GCC_EXPANDED_PROJECTS=openblas csl
 BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld
 # These are non-BB source-only deps

deps/Makefile

@@ -155,6 +155,10 @@ ifeq ($(USE_SYSTEM_ZLIB), 0)
 DEP_LIBS += zlib
 endif
 
+ifeq ($(USE_SYSTEM_ZSTD), 0)
+DEP_LIBS += zstd
+endif
+
 ifeq ($(USE_SYSTEM_P7ZIP), 0)
 DEP_LIBS += p7zip
 endif
@@ -195,7 +199,7 @@ DEP_LIBS_STAGED := $(DEP_LIBS)
 # list all targets
 DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \
 	openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \
-	objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \
+	objconv mbedtls libssh2 nghttp2 curl libgit2 libwhich zlib zstd p7zip csl \
 	sanitizers libsuitesparse lld libtracyclient ittapi JuliaSyntax terminfo
 DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL)
 
@@ -244,6 +248,7 @@ include $(SRCDIR)/openblas.mk
 include $(SRCDIR)/utf8proc.mk
 include $(SRCDIR)/libsuitesparse.mk
 include $(SRCDIR)/zlib.mk
+include $(SRCDIR)/zstd.mk
 include $(SRCDIR)/unwind.mk
 include $(SRCDIR)/gmp.mk
 include $(SRCDIR)/mpfr.mk

deps/checksums/zstd

@@ -0,0 +1,38 @@
+Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/md5/d6b2fb32d705078dbc369986ac8b056b
+Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/sha512/5dfcf36087ce8540b1f6a04181adee962e2164a763e758ac5cc256c332756774b381ca58e26641a15ce555d59641690a6da72a67bf935d8611734f2006bde504
+Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/md5/0c627ec83e426383c25eb4bc297f3548
+Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/sha512/1fdcf77e877f0676fc26a05e0cc20a1d6e1df731d81e0bba9a5657131116bbea75da4d38953969d8d07dce0bf2d7654075dbb285ebe5f4588c446e88774336c8
+Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/md5/cc9ada74a19db50d7dd6edd05866c902
+Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/sha512/0b33c0df144bb1e95290685f01695b26da834a70a365c0362314cb001ba611962a0876bc5baac31f19c80bcb110e030fb9840a56761b4d29a7893ca65f95b111
+Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/md5/5daa5b2bf2b856c448feaa8329d0de1b
+Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/sha512/b39d025463b4bf21295fd5bbff91ba501506b3480363cdcfe6dd2f11d2e0afaf130f6c74d962e503fccb7a55bfcad0504ebb19f18b6b5c8b8103e7b9919df536
+Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/md5/f4218e8b4f8d415df49aeba9d43f0ba0
+Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/sha512/878d4f90160c6b0c341c61ecafbf5f5cb89c73db3175f272adc666bc25c88b127145d78946bc0fcb992489b54fbb48089bfcacf768397fc5d54d7cae4aeae9f9
+Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/md5/3c2e132ca47e6d1d23c149fdde9d8bd5
+Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/sha512/3745d99c9ca0ce9f98ff9393e405e8b382d05573a972067d57e800e282a9544fff7bc3d49b91eccc98d7736acdc3faa4c637911d79fab10f5a691d33ae775574
+Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/md5/926d765281bef388ecc25d04cbb66102
+Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/sha512/2d2c14587e2e7b2b147cb6423720cc30ed6aa57ed07372a1aa54e7f2e6badb5aa640b116e83371561d6f8f3a1b3f7fff7f6df137f8c7be788ee889bb30273eae
+Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/md5/c25420561ce254e57d74e30c88fc53dd
+Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/sha512/2f924e2089589057e8713d04db9a1cb2f2d571ad9e7eeda3b7f898c9a75f8fecf0647f2185d3c01fc3b399d3662ff3b1acb13429c8a953f0394a3ed9ca30b877
+Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/md5/3314bf1b52f2295555fb4ae44b1d9331
+Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/sha512/91502910a0c9b786d91499477fee2445b8f6de6bcb71af7d79c738ea2430c67cb1957866383ee3921ed1a23c53a80be19aea6abcf0e76056ffee69583728c3ed
+Zstd.v1.5.7+1.i686-linux-musl.tar.gz/md5/845eddc06527a4c4b196666f7ac64ba3
+Zstd.v1.5.7+1.i686-linux-musl.tar.gz/sha512/bb15b4327cef32be38c2fd68afedb3245c7db881ad66d3ece2198ff3034be9c12efa3d62bcba2b8e6056e7d8cb5f1b3e33726f7d1e1bead235c38f8fa985b557
+Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/md5/9bc0b3c951f5e66393fd5433bf60a2c8
+Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/sha512/550b0189097e569f98404aa836b76a5cbdc36428292214c4af8916dea2713440cf3ba94125b3e5fa0c65b2bcb916733094fdef906ad19f923d90dabfc961c75a
+Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/md5/468d930de7a27af961996e7c6ed35298
+Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/sha512/d680715b1ac9ff07d5662c499fbab67757509599335f861158b9dba32fe9b22da6e52d0db6b402dd4542799621ad3dccf254dfd9d3c8748bbd22f7446681539a
+Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/md5/b93fef8db2b0b4417f7836d73c5fbe86
+Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/sha512/9f3ee42c7952aba2d2c26252f058bb7ab96828fafc978c9273b500ef15ccd271c51399d4b93eebd4c832b087ab5ed8a4847104ce9c83c9483aaa13c22df681bb
+Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/md5/29a260789fae6f6b6df0e5cebdafd615
+Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/sha512/015045a1b7a477504057cb4c87428d42386218e48af38f83739dbe6b93961ca2c8dd4d794377a2d54b8cc284f5a467e3358d4f534cf8bcbcad886ef8cea038e9
+Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/md5/06656befb6ef9a8cc7f56e7152c2acc5
+Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/sha512/16aea0d95432a87d21d9a6f55d84e45df85caf1fda77c75b7e9a8bba519605168585f21a812773ddf1075d9bad68412e63b8cad1a143420e25ae4405bb41842e
+Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/md5/da13dd1cc0d20ba9a06e9e79a588cda4
+Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/sha512/cd4218fa92dcf8772390788d5654ca12132af7829fb0ada016f3c663e2045e29e7d7587f2f5a4f057020cacca17c188c8537f284b1456100d57e84bb47c40e77
+Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/md5/bce5f37e53e330bfe4df4a28cf5c223b
+Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/sha512/8f6bd7664efea537ac7815db0604ca1a07bcfb71b5152c22dc7f0a11b57643f059c341fa71d315407e2333e4c97e43e214471c73eed8b977680785302c7c2b3e
+Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/md5/7cf3a740fa174004b94125e8754f4a19
+Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/sha512/faac37ad4dacb0f083364c593cd3bd1c0b592947341a631bd2fbc4081361d97ef89482f4459c46ad37ae030aa900c62305a8525e64a2ad8e91204d76dda89db1
+zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/md5/a679d9aa86549b5851100ac5d4044c68
+zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/sha512/27c6fff165abea694d91311a6657a939433ba1d707147ed9072b5e4ecce259b929970306788e0c3e95db38ce85e894e5025936b1faa81cf67741b8464e24fc4e

deps/zstd.mk

@@ -0,0 +1,60 @@
+## Zstd ##
+ifneq ($(USE_BINARYBUILDER_ZSTD), 1)
+ZSTD_GIT_URL := https://github.com/facebook/zstd.git
+ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1
+$(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR)))
+
+ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)" bindir=$(build_private_libexecdir)
+
+$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted
+	echo 1 > $@
+
+$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured
+	$(MAKE) -C $(dir $<) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS)
+	echo 1 > $@
+
+$(eval $(call staged-install, \
+	zstd,$(ZSTD_SRC_DIR), \
+	MAKE_INSTALL,$(ZSTD_BUILD_OPTS) MT=1,, \
+	$(INSTALL_NAME_CMD)libzstd.$(SHLIB_EXT) $(build_shlibdir)/libzstd.$(SHLIB_EXT)))
+
+clean-zstd:
+	-rm -f $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled
+	-$(MAKE) -C $(BUILDDIR)/$(ZSTD_SRC_DIR) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS) clean
+
+get-zstd: $(ZSTD_SRC_FILE)
+extract-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted
+configure-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured
+compile-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled
+fastcheck-zstd: check-zstd
+check-zstd: compile-zstd
+
+else # USE_BINARYBUILDER_ZSTD
+
+$(eval $(call bb-install,zstd,ZSTD,false))
+# move from bindir to shlibdir, where we expect to install it
+install-zstd: post-install-zstd
+uninstall-zstd: pre-uninstall-zstd
+post-install-zstd: $(build_prefix)/manifest/zstd $(PATCHELF_MANIFEST)
+	mkdir -p $(build_private_libexecdir)/
+	[ ! -e $(build_bindir)/zstdmt$(EXE) ] || mv $(build_bindir)/zstdmt$(EXE) $(build_private_libexecdir)/zstdmt$(EXE)
+	[ ! -e $(build_bindir)/zstd$(EXE) ] || mv $(build_bindir)/zstd$(EXE) $(build_private_libexecdir)/zstd$(EXE)
+	[ -e $(build_private_libexecdir)/zstd$(EXE) ]
+	[ -e $(build_private_libexecdir)/zstdmt$(EXE) ]
+ifeq ($(OS), Darwin)
+	for j in zstd zstdmt ; do \
+		[ -L $(build_private_libexecdir)/$$j ] && continue; \
+		install_name_tool -rpath @executable_path/$(reverse_build_private_libexecdir_rel) @loader_path/$(build_libdir_rel) $(build_private_libexecdir)/$$j 2>/dev/null || true; \
+		install_name_tool -rpath @loader_path/$(build_libdir_rel) @executable_path/$(reverse_build_private_libexecdir_rel) $(build_private_libexecdir)/$$j || exit 1; \
+	done
+else ifneq (,$(findstring $(OS),Linux FreeBSD))
+	for j in zstd zstdmt ; do \
+		[ -L $(build_private_libexecdir)/$$j ] && continue; \
+		$(PATCHELF) $(PATCHELF_SET_RPATH_ARG) '$$ORIGIN/$(reverse_build_private_libexecdir_rel)' $(build_private_libexecdir)/$$j || exit 1; \
+	done
+endif
+
+pre-uninstall-zstd:
+	-rm -f $(build_private_libexecdir)/zstd$(EXE) $(build_private_libexecdir)/zstdmt$(EXE)
+
+endif # USE_BINARYBUILDER_ZSTD

deps/zstd.version

@@ -0,0 +1,8 @@
+# -*- makefile -*-
+## jll artifact
+ZSTD_JLL_NAME := Zstd
+
+## source build
+ZSTD_VER := 1.5.7
+ZSTD_BRANCH=v1.5.7
+ZSTD_SHA1=f8745da6ff1ad1e7bab384bd1f9d742439278e99