Tdrz/ccoderefactoring

build-pglite.sh

@@ -21,8 +21,28 @@ fi
 
 echo "pglite: PGLITE_CFLAGS=$PGLITE_CFLAGS"
 
+# run ./configure only if config.status is older than this file
+# TODO: we should ALSO check if any of the PGLITE_CFLAGS have changed and trigger a ./configure if they did!!!
+REF_FILE="build-pglite.sh"
+CONFIG_STATUS="config.status"
+RUN_CONFIGURE=false
+
+if [ ! -f "$CONFIG_STATUS" ]; then
+    echo "$CONFIG_STATUS does not exist, need to run ./configure"
+    RUN_CONFIGURE=true
+elif [ "$REF_FILE" -nt "$CONFIG_STATUS" ]; then
+    echo "$CONFIG_STATUS is older than $REF_FILE. Need to run ./configure."
+    RUN_CONFIGURE=true
+else
+    echo "$CONFIG_STATUS exists and is newer than $REF_FILE. ./configure will NOT be run."
+fi
+
 # Step 1: configure the project
-LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DPYDK=1 -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm  --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; }
+if [ "$RUN_CONFIGURE" = true ]; then
+    LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm  --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; }
+else
+    echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}"
+fi
 
 # Step 2: make and install all except pglite
 emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscripten -j' ; exit 21; }
@@ -42,5 +62,16 @@ emmake make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pgli
 PATH=$SAVE_PATH
 
 # Step 5: make and install pglite
+# we define here "all" emscripten flags in order to allow native builds (like libpglite)
+EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction"
+PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \
+-sSUPPORT_LONGJMP=emscripten \
+-sFORCE_FILESYSTEM=1 \
+-sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \
+-sMAIN_MODULE=2 -sMODULARIZE=1 -sEXPORT_ES6=1 \
+-sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \
+-sERROR_ON_UNDEFINED_SYMBOLS=1 \
+-sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS"
+
 # Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available.
-PGLITE_CFLAGS=$PGLITE_CFLAGS emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; }
+PGLITE_CFLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; }

build-with-docker.sh

@@ -1,13 +1,23 @@
 # although we could use any path inside docker, using the same path as on the host
 # allows the DWARF info (when building in DEBUG) to contain the correct file paths
 DOCKER_WORKSPACE=$(pwd)
+GITHUB_ACTIONS=${GITHUB_ACTIONS:-"false"}
+ADDITIONAL_DOCKER_PARAMS=""
+if [ "$GITHUB_ACTIONS" = "true" ]; then
+    echo "Running inside GitHub Actions"
+    ADDITIONAL_DOCKER_PARAMS=""
+else
+    echo "Not running inside GitHub Actions"
+    ADDITIONAL_DOCKER_PARAMS="-it -u $(id -u):$(id -g)"
+fi
 
 docker run $@ \
+  $ADDITIONAL_DOCKER_PARAMS \
   --rm \
   -e DEBUG=${DEBUG:-false} \
   --workdir=${DOCKER_WORKSPACE} \
   -v .:${DOCKER_WORKSPACE}:rw \
   -v ./dist:/install/pglite:rw \
-  electricsql/pglite-builder:3.1.74_1 \
+  electricsql/pglite-builder:3.1.74_2 \
   ./build-pglite.sh
 

clean-pglite.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
 
-emmake make -C src/backend uninstall; emmake make -C src/backend clean;
-emmake make -C pglite/ clean; emmake make -C pglite/ uninstall;
-emmake make -C contrib/ clean; emmake make -C contrib/ uninstall; emmake make -C pglite clean; emmake make -C pglite uninstall;
-emmake make clean; emmake make uninstal
+emmake make PORTNAME=emscripten -C src/backend uninstall; emmake make PORTNAME=emscripten -C src/backend clean;
+emmake make PORTNAME=emscripten -C pglite/ clean; emmake make PORTNAME=emscripten -C pglite/ uninstall;
+emmake make PORTNAME=emscripten -C contrib/ clean; emmake make PORTNAME=emscripten -C contrib/ uninstall; 
+emmake make PORTNAME=emscripten -C pglite clean; emmake make PORTNAME=emscripten -C pglite uninstall;
+emmake make PORTNAME=emscripten clean; emmake make PORTNAME=emscripten uninstall;
+
+echo "removing config.status"
+rm config.status

other/empty

@@ -0,0 +1 @@
+PGlite is the best!

pglite-wasm/builder/Dockerfile

@@ -3,7 +3,7 @@ ARG BUILDPLATFORM
 # we only need to build on one platform, since we're interested in the WASM output
 # building on the native (BUILDPLATFORM) is much faster, so use that
 # remove "-arm64" suffix if building on x86_64
-FROM --platform=$BUILDPLATFORM emscripten/emsdk:${EMSDK_VER}-arm64 AS builder
+FROM --platform=$BUILDPLATFORM emscripten/emsdk:${EMSDK_VER} AS builder
 
 ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm
 
@@ -21,7 +21,7 @@ WORKDIR /install/libs
 
 WORKDIR /src
 
-# ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -D__PYDK__=1 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn"
+# ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn"
 ENV EMCC_COMMON_FLAGS="-O2 -fPIC"
 
 WORKDIR /src
@@ -60,6 +60,8 @@ RUN ${LLVM_NM} /install/libs/lib/libssl.a | awk '$2 ~ /^[TDB]$/ {print $3}' | se
 
 WORKDIR /src
 RUN curl -L ftp://ftp.ossp.org/pkg/lib/uuid/uuid-1.6.2.tar.gz | tar xz
+# COPY . .
+# RUN tar xvf uuid-1.6.2.tar.gz
 WORKDIR /src/uuid-1.6.2
 RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --build=aarch64-unknown-linux-gnu --enable-shared=no --enable-static=yes --with-perl=no --with-perl-compat=no --prefix=/install/libs --with-php=no --with-pic=yes
 RUN emmake make -j && emmake make install || true # install tries to strip the wasm, but it doesnt recognize the format, so ignore atm
@@ -85,5 +87,8 @@ ENV LIB_EXPORTS_DIR=${LIB_EXPORTS_DIR}
 COPY --from=builder /install/libs ${INSTALL_PREFIX}
 COPY --from=builder /install/exports ${LIB_EXPORTS_DIR}
 
+# allow access to anyone 
+RUN chmod -R 777 /install
+
 # needed in building pglite.wasm
 ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm

pglite-wasm/included.pglite.imports

@@ -1,15 +1,10 @@
 close
 fcntl
 free
-get_buffer_addr
-get_buffer_size
-get_channel
 getpid
 gettimeofday
 gmtime
 interactive_one
-interactive_read
-interactive_write
 ioctl
 isalnum
 isxdigit
@@ -29,10 +24,10 @@ read
 readstoplist
 realloc
 searchstoplist
+set_read_write_cbs
 socket
 srand
 strcmp
 strftime
 strlen
-strtoul
-use_wire
+strtoul