Skip to content

Commit 52e87ac

Browse files
vtjnashvtjnash
committed
Enable zstd support in llvm and elsewhere
1 parent dbc38d6 commit 52e87ac

File tree

21 files changed

+331
-137
lines changed

21 files changed

+331
-137
lines changed

‎Make.inc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ USE_SYSTEM_LIBGIT2:=0
5959
USE_SYSTEM_PATCHELF:=0
6060
USE_SYSTEM_LIBWHICH:=0
6161
USE_SYSTEM_ZLIB:=0
62+
USE_SYSTEM_ZSTD:=0
6263
USE_SYSTEM_P7ZIP:=0
6364
USE_SYSTEM_LLD:=0
6465

@@ -1417,7 +1418,7 @@ CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.34|GLIBCXX_3\.5\.|GLIBCXX_4\.
14171418
# Note: we explicitly _do not_ define `CSL` here, since it requires some more
14181419
# advanced techniques to decide whether it should be installed from a BB source
14191420
# or not. See `deps/csl.mk` for more detail.
1420-
BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP OPENSSL LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP LLD LIBTRACYCLIENT BOLT
1421+
BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP OPENSSL LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB ZSTD P7ZIP LLD LIBTRACYCLIENT BOLT
14211422

14221423
ifeq (${USE_THIRD_PARTY_GC},mmtk)
14231424
BB_PROJECTS += MMTK_JULIA

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,7 @@ JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += zlib
232232
else
233233
JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libz
234234
endif
235+
JL_PRIVATE_LIBS-$(USE_SYSTEM_ZLIB) += libzstd
235236
ifeq ($(USE_LLVM_SHLIB),1)
236237
JL_PRIVATE_LIBS-$(USE_SYSTEM_LLVM) += libLLVM $(LLVM_SHARED_LIB_NAME)
237238
endif

‎THIRDPARTY.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ Julia bundles the following external programs and libraries:
6767

6868
- [7-Zip](https://www.7-zip.org/license.txt)
6969
- [ZLIB](https://zlib.net/zlib_license.html)
70+
- [ZSTD](https://github.com/facebook/zstd/blob/v1.5.7/LICENSE)
7071

7172
On some platforms, distributions of Julia contain SSL certificate authority certificates,
7273
released under the [Mozilla Public License](https://en.wikipedia.org/wiki/Mozilla_Public_License).

‎contrib/refresh_checksums.mk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ CLANG_TRIPLETS=$(filter %-darwin %-freebsd,$(TRIPLETS))
2424
NON_CLANG_TRIPLETS=$(filter-out %-darwin %-freebsd,$(TRIPLETS))
2525

2626
# These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded:
27-
BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia
27+
BB_PROJECTS=openssl libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind llvmunwind dsfmt objconv p7zip zlib zstd libsuitesparse openlibm blastrampoline libtracyclient mmtk_julia
2828
BB_GCC_EXPANDED_PROJECTS=openblas csl
2929
BB_CXX_EXPANDED_PROJECTS=gmp llvm clang llvm-tools lld
3030
# These are non-BB source-only deps

‎deps/Makefile‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,10 @@ ifeq ($(USE_SYSTEM_ZLIB), 0)
157157
DEP_LIBS += zlib
158158
endif
159159

160+
ifeq ($(USE_SYSTEM_ZSTD), 0)
161+
DEP_LIBS += zstd
162+
endif
163+
160164
ifeq ($(USE_SYSTEM_P7ZIP), 0)
161165
DEP_LIBS += p7zip
162166
endif
@@ -204,7 +208,7 @@ DEP_LIBS_STAGED := $(DEP_LIBS)
204208
# list all targets
205209
DEP_LIBS_STAGED_ALL := llvm llvm-tools clang llvmunwind unwind libuv pcre \
206210
openlibm dsfmt blastrampoline openblas lapack gmp mpfr patchelf utf8proc \
207-
objconv openssl libssh2 nghttp2 curl libgit2 libwhich zlib p7zip csl \
211+
objconv openssl libssh2 nghttp2 curl libgit2 libwhich zlib zstd p7zip csl \
208212
sanitizers libsuitesparse lld libtracyclient ittapi nvtx JuliaSyntax \
209213
terminfo mmtk_julia
210214
DEP_LIBS_ALL := $(DEP_LIBS_STAGED_ALL)
@@ -256,6 +260,7 @@ include $(SRCDIR)/openblas.mk
256260
include $(SRCDIR)/utf8proc.mk
257261
include $(SRCDIR)/libsuitesparse.mk
258262
include $(SRCDIR)/zlib.mk
263+
include $(SRCDIR)/zstd.mk
259264
include $(SRCDIR)/unwind.mk
260265
include $(SRCDIR)/gmp.mk
261266
include $(SRCDIR)/mpfr.mk

‎deps/checksums/llvm‎

Lines changed: 120 additions & 120 deletions
Large diffs are not rendered by default.

‎deps/checksums/zstd‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/md5/d6b2fb32d705078dbc369986ac8b056b
2+
Zstd.v1.5.7+1.aarch64-apple-darwin.tar.gz/sha512/5dfcf36087ce8540b1f6a04181adee962e2164a763e758ac5cc256c332756774b381ca58e26641a15ce555d59641690a6da72a67bf935d8611734f2006bde504
3+
Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/md5/0c627ec83e426383c25eb4bc297f3548
4+
Zstd.v1.5.7+1.aarch64-linux-gnu.tar.gz/sha512/1fdcf77e877f0676fc26a05e0cc20a1d6e1df731d81e0bba9a5657131116bbea75da4d38953969d8d07dce0bf2d7654075dbb285ebe5f4588c446e88774336c8
5+
Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/md5/cc9ada74a19db50d7dd6edd05866c902
6+
Zstd.v1.5.7+1.aarch64-linux-musl.tar.gz/sha512/0b33c0df144bb1e95290685f01695b26da834a70a365c0362314cb001ba611962a0876bc5baac31f19c80bcb110e030fb9840a56761b4d29a7893ca65f95b111
7+
Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/md5/5daa5b2bf2b856c448feaa8329d0de1b
8+
Zstd.v1.5.7+1.aarch64-unknown-freebsd.tar.gz/sha512/b39d025463b4bf21295fd5bbff91ba501506b3480363cdcfe6dd2f11d2e0afaf130f6c74d962e503fccb7a55bfcad0504ebb19f18b6b5c8b8103e7b9919df536
9+
Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/md5/f4218e8b4f8d415df49aeba9d43f0ba0
10+
Zstd.v1.5.7+1.armv6l-linux-gnueabihf.tar.gz/sha512/878d4f90160c6b0c341c61ecafbf5f5cb89c73db3175f272adc666bc25c88b127145d78946bc0fcb992489b54fbb48089bfcacf768397fc5d54d7cae4aeae9f9
11+
Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/md5/3c2e132ca47e6d1d23c149fdde9d8bd5
12+
Zstd.v1.5.7+1.armv6l-linux-musleabihf.tar.gz/sha512/3745d99c9ca0ce9f98ff9393e405e8b382d05573a972067d57e800e282a9544fff7bc3d49b91eccc98d7736acdc3faa4c637911d79fab10f5a691d33ae775574
13+
Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/md5/926d765281bef388ecc25d04cbb66102
14+
Zstd.v1.5.7+1.armv7l-linux-gnueabihf.tar.gz/sha512/2d2c14587e2e7b2b147cb6423720cc30ed6aa57ed07372a1aa54e7f2e6badb5aa640b116e83371561d6f8f3a1b3f7fff7f6df137f8c7be788ee889bb30273eae
15+
Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/md5/c25420561ce254e57d74e30c88fc53dd
16+
Zstd.v1.5.7+1.armv7l-linux-musleabihf.tar.gz/sha512/2f924e2089589057e8713d04db9a1cb2f2d571ad9e7eeda3b7f898c9a75f8fecf0647f2185d3c01fc3b399d3662ff3b1acb13429c8a953f0394a3ed9ca30b877
17+
Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/md5/3314bf1b52f2295555fb4ae44b1d9331
18+
Zstd.v1.5.7+1.i686-linux-gnu.tar.gz/sha512/91502910a0c9b786d91499477fee2445b8f6de6bcb71af7d79c738ea2430c67cb1957866383ee3921ed1a23c53a80be19aea6abcf0e76056ffee69583728c3ed
19+
Zstd.v1.5.7+1.i686-linux-musl.tar.gz/md5/845eddc06527a4c4b196666f7ac64ba3
20+
Zstd.v1.5.7+1.i686-linux-musl.tar.gz/sha512/bb15b4327cef32be38c2fd68afedb3245c7db881ad66d3ece2198ff3034be9c12efa3d62bcba2b8e6056e7d8cb5f1b3e33726f7d1e1bead235c38f8fa985b557
21+
Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/md5/9bc0b3c951f5e66393fd5433bf60a2c8
22+
Zstd.v1.5.7+1.i686-w64-mingw32.tar.gz/sha512/550b0189097e569f98404aa836b76a5cbdc36428292214c4af8916dea2713440cf3ba94125b3e5fa0c65b2bcb916733094fdef906ad19f923d90dabfc961c75a
23+
Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/md5/468d930de7a27af961996e7c6ed35298
24+
Zstd.v1.5.7+1.powerpc64le-linux-gnu.tar.gz/sha512/d680715b1ac9ff07d5662c499fbab67757509599335f861158b9dba32fe9b22da6e52d0db6b402dd4542799621ad3dccf254dfd9d3c8748bbd22f7446681539a
25+
Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/md5/b93fef8db2b0b4417f7836d73c5fbe86
26+
Zstd.v1.5.7+1.riscv64-linux-gnu.tar.gz/sha512/9f3ee42c7952aba2d2c26252f058bb7ab96828fafc978c9273b500ef15ccd271c51399d4b93eebd4c832b087ab5ed8a4847104ce9c83c9483aaa13c22df681bb
27+
Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/md5/29a260789fae6f6b6df0e5cebdafd615
28+
Zstd.v1.5.7+1.x86_64-apple-darwin.tar.gz/sha512/015045a1b7a477504057cb4c87428d42386218e48af38f83739dbe6b93961ca2c8dd4d794377a2d54b8cc284f5a467e3358d4f534cf8bcbcad886ef8cea038e9
29+
Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/md5/06656befb6ef9a8cc7f56e7152c2acc5
30+
Zstd.v1.5.7+1.x86_64-linux-gnu.tar.gz/sha512/16aea0d95432a87d21d9a6f55d84e45df85caf1fda77c75b7e9a8bba519605168585f21a812773ddf1075d9bad68412e63b8cad1a143420e25ae4405bb41842e
31+
Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/md5/da13dd1cc0d20ba9a06e9e79a588cda4
32+
Zstd.v1.5.7+1.x86_64-linux-musl.tar.gz/sha512/cd4218fa92dcf8772390788d5654ca12132af7829fb0ada016f3c663e2045e29e7d7587f2f5a4f057020cacca17c188c8537f284b1456100d57e84bb47c40e77
33+
Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/md5/bce5f37e53e330bfe4df4a28cf5c223b
34+
Zstd.v1.5.7+1.x86_64-unknown-freebsd.tar.gz/sha512/8f6bd7664efea537ac7815db0604ca1a07bcfb71b5152c22dc7f0a11b57643f059c341fa71d315407e2333e4c97e43e214471c73eed8b977680785302c7c2b3e
35+
Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/md5/7cf3a740fa174004b94125e8754f4a19
36+
Zstd.v1.5.7+1.x86_64-w64-mingw32.tar.gz/sha512/faac37ad4dacb0f083364c593cd3bd1c0b592947341a631bd2fbc4081361d97ef89482f4459c46ad37ae030aa900c62305a8525e64a2ad8e91204d76dda89db1
37+
zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/md5/a679d9aa86549b5851100ac5d4044c68
38+
zstd-f8745da6ff1ad1e7bab384bd1f9d742439278e99.tar.gz/sha512/27c6fff165abea694d91311a6657a939433ba1d707147ed9072b5e4ecce259b929970306788e0c3e95db38ce85e894e5025936b1faa81cf67741b8464e24fc4e

‎deps/llvm.mk‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ LLVM_CMAKE += -DLLVM_TARGETS_TO_BUILD:STRING="$(LLVM_TARGETS)" -DCMAKE_BUILD_TYP
9999
LLVM_CMAKE += -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD:STRING="$(LLVM_EXPERIMENTAL_TARGETS)"
100100
LLVM_CMAKE += -DLLVM_ENABLE_LIBXML2=OFF -DLLVM_HOST_TRIPLE="$(or $(XC_HOST),$(BUILD_MACHINE))"
101101
LLVM_CMAKE += -DLLVM_ENABLE_ZLIB=FORCE_ON -DZLIB_ROOT="$(build_prefix)"
102-
LLVM_CMAKE += -DLLVM_ENABLE_ZSTD=OFF
102+
LLVM_CMAKE += -DLLVM_ENABLE_ZSTD=FORCE_ON -DZSTD_ROOT="$(build_prefix)"
103103
ifeq ($(USE_POLLY_ACC),1)
104104
LLVM_CMAKE += -DPOLLY_ENABLE_GPGPU_CODEGEN=ON
105105
endif
@@ -251,6 +251,11 @@ ifeq ($(USE_SYSTEM_ZLIB), 0)
251251
$(LLVM_BUILDDIR_withtype)/build-configured: | $(build_prefix)/manifest/zlib
252252
endif
253253

254+
ifeq ($(USE_SYSTEM_ZSTD), 0)
255+
$(LLVM_BUILDDIR_withtype)/build-configured: | $(build_prefix)/manifest/zstd
256+
endif
257+
258+
254259
# NOTE: LLVM 12 and 13 have their patches applied to JuliaLang/llvm-project
255260

256261
# declare that all patches must be applied before running ./configure

‎deps/llvm.version‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
## jll artifact
44
LLVM_JLL_NAME := libLLVM
5-
LLVM_ASSERT_JLL_VER := 20.1.2+0
5+
LLVM_ASSERT_JLL_VER := 20.1.2+1
66
## source build
77
# Version number of LLVM
88
LLVM_VER := 20.1.2

‎deps/zstd.mk‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
## Zstd ##
2+
ifneq ($(USE_BINARYBUILDER_ZSTD), 1)
3+
ZSTD_GIT_URL := https://github.com/facebook/zstd.git
4+
ZSTD_TAR_URL = https://api.github.com/repos/facebook/zstd/tarball/$1
5+
$(eval $(call git-external,zstd,ZSTD,,,$(BUILDDIR)))
6+
7+
ZSTD_BUILD_OPTS := MOREFLAGS="-DZSTD_MULTITHREAD $(fPIC)"
8+
9+
$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted
10+
echo 1 > $@
11+
12+
$(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured
13+
$(MAKE) -C $(dir $<) $(MAKE_COMMON) $(ZSTD_BUILD_OPTS)
14+
echo 1 > $@
15+
16+
ZSTD_INSTALL=$(MAKE_INSTALL)
17+
18+
$(eval $(call staged-install, \
19+
zstd,$(ZSTD_SRC_DIR), \
20+
ZSTD_INSTALL,$(ZSTD_BUILD_OPTS) MT=1,, \
21+
$(INSTALL_NAME_CMD)libzstd.$(SHLIB_EXT) $(build_shlibdir)/libzstd.$(SHLIB_EXT)))
22+
23+
clean-zstd:
24+
-rm -f $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled
25+
-$(MAKE) -C $(BUILDDIR)/$(ZSTD_SRC_DIR) $(ZSTD_BUILD_OPTS) clean
26+
27+
get-zstd: $(ZSTD_SRC_FILE)
28+
extract-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/source-extracted
29+
configure-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-configured
30+
compile-zstd: $(BUILDDIR)/$(ZSTD_SRC_DIR)/build-compiled
31+
fastcheck-zstd: check-zstd
32+
check-zstd: compile-zstd
33+
34+
else # USE_BINARYBUILDER_ZSTD
35+
36+
$(eval $(call bb-install,zstd,ZSTD,false))
37+
38+
endif # USE_BINARYBUILDER_ZSTD

0 commit comments

Comments
 (0)