.buildconfig

@@ -1,2 +1,4 @@
 PG_VERSION=16.4
-SDK_VERSION=3.1.68.3bi
+SDK_VERSION=3.1.72.3bi
+WASI_SDK_VERSION=24.0.4
+SDKROOT=/opt/python-wasm-sdk

.github/workflows/build_and_test.yml

@@ -207,7 +207,7 @@ jobs:
 
       - name: Build demo site
         run: |
-          bash ./cibuild.sh demo-site
+          NO_SDK_CHECK=true SDKROOT=/opt/python-wasm-sdk bash ./cibuild.sh demo-site
 
       - name: Build docs
         working-directory: ./docs

.github/workflows/build_wasm_postgres.yml

@@ -14,14 +14,18 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       BUILD_CONFIG_FILE: .buildconfig
-      SDK_ARCHIVE: python3.12-wasm-sdk-Ubuntu-22.04.tar.lz4
+      SDK_ARCHIVE: python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4
+      WASI_SDK_ARCHIVE: python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4
       SDKROOT: /opt/python-wasm-sdk
       SYS_PYTHON: /usr/bin/python3
       PGROOT: /tmp/pglite
       TOTAL_MEMORY: 128MB
+      CMA_MB: 16
       DEBUG: false
       OBJDUMP: true
-      EXTRA_EXT: vector
+      contrib: contrib
+      extra: extra
+      EXTRA_EXT: vector postgis
 
     steps:
       - uses: actions/checkout@v4
@@ -43,15 +47,17 @@ jobs:
             /tmp/sdk/postgres-*.tar.gz
             postgres
             postgresql-*
-          key: build-cache-${{ hashFiles('.buildconfig', 'cibuild.sh', '.github/workflows/build_wasm_postgres.yml', 'package.json', 'cibuild/**', 'patches/*.c', 'patches/*.h', 'patches/**/*.diff') }}
+          key: build-cache-${{ hashFiles('.buildconfig', 'cibuild.sh', '.github/workflows/build_wasm_postgres.yml', 'package.json', 'cibuild/**', 'extra/**', 'patches/*.c', 'patches/*.h', 'patches/**/*.diff') }}
 
       - name: Install python-wasm-sdk for emsdk/wasi+prebuilts
         if: steps.cache-restore.outputs.cache-hit != 'true'
         working-directory: /
         run: |
           sudo apt-get install -y lz4 wget pv bash
           echo https://github.com/pygame-web/python-wasm-sdk/releases/download/$SDK_VERSION/$SDK_ARCHIVE
-          curl -sL --retry 5 https://github.com/pygame-web/python-wasm-sdk/releases/download/$SDK_VERSION/$SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 24400 >/dev/null
+          curl -sL --retry 5 https://github.com/pygame-web/python-wasm-sdk/releases/download/$SDK_VERSION/$SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 34500 >/dev/null
+          echo https://github.com/pygame-web/python-wasi-sdk/releases/download/$WASI_SDK_VERSION/$WASI_SDK_ARCHIVE
+          curl -sL --retry 5 https://github.com/pygame-web/python-wasi-sdk/releases/download/$WASI_SDK_VERSION/$WASI_SDK_ARCHIVE | tar xvP --use-compress-program=lz4 | pv -p -l -s 23000 >/dev/null
 
       - name: Build postgres WASM via emsdk
         if: steps.cache-restore.outputs.cache-hit != 'true'

README.md

@@ -124,20 +124,51 @@ Fortunately, PostgreSQL includes a "single user mode" primarily intended for com
 
 - PGlite is single user/connection.
 
-## How to contribute
+## How to build PGlite and contribute
 
-You will need [pnpm](https://pnpm.io/) installed, and a recent version of Node.js (v20 and above).
+The build process of PGlite is split into two parts:
 
-You will also need the Postgres WASM build files, which you download from a comment under the most recently merged PR, labeled as _interim build files_, and place them under `packages/pglite/release`. These are necessary to build PGlite and the dependent workspace projects. We plan to enable a local build in the future to streamline this step.
+1. Building the Postgres WASM module.
+2. Building the PGlite client library and other TypeScript packages.
+
+Docker is required to build the WASM module, along with Node (v20 or above) and [pnpm](https://pnpm.io/) for package management and building the TypeScript packages.
+
+To start checkout the repository and install dependencies:
 
-Once the requirements are met, you can install dependencies and build the workspace projects:
 ```bash
+git clone https://github.com/electric-sql/pglite
+cd pglite
 pnpm install
-pnpm build
+```
+
+To build everything, we have the convenient `pnpm build:all` command in the root of the repository. This command will:
+
+1. Use Docker to build the Postgres WASM module. The artifacts from this are then copied to `/packages/pglite/release`.
+2. Build the PGlite client library and other TypeScript packages.
+
+To _only_ build the Postgres WASM module (i.e. point 1 above), run
+
+```bash
+pnpm wasm:build
+```
+
+If you don't want to build the WASM module and assorted WASM binaries from scratch, you can download them from a comment under the most recently merged PR, labeled as _interim build files_, and place them under `packages/pglite/release`. 
+
+To build all TypeScript packages (i.e. point 2 of the above), run:
+
+```bash
+pnpm ts:build
 ```
 
 This will build all packages in the correct order based on their dependency relationships. You can now develop any individual package using the `build` and `test` scripts, as well as the `stylecheck` and `typecheck` scripts to ensure style and type validity.
 
+Or alternatively to build a single package, move into the package directory and run:
+
+```bash
+cd packages/pglite
+pnpm build
+```
+
 When ready to open a PR, run the following command at the root of the repository:
 ```bash
 pnpm changeset

cibuild.sh

@@ -6,10 +6,11 @@ export CMA_MB=${CMA_MB:-64}
 
 export CI=${CI:-false}
 
-if $CI
-then
-    . .buildconfig
-fi
+chmod +x ./extra/*.sh cibuild/*.sh
+
+. .buildconfig
+
+export PG_VERSION SDK_VERSION WASI_SDK_VERSION SDKROOT
 
 export PG_VERSION=${PG_VERSION:-16.4}
 export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)}
@@ -26,7 +27,15 @@ export WASI=${WASI:-false}
 # exit on error
 EOE=false
 
-mkdir -p /tmp/sdk
+
+if ./cibuild/sdk.sh
+then
+    echo "sdk check passed (emscripten)"
+else
+    echo sdk failed
+    exit 44
+fi
+
 
 # the default is a user writeable path.
 if mkdir -p ${PGROOT}/sdk
@@ -369,7 +378,7 @@ then
                 . /etc/lsb-release
                 DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}"
                 CIVER=${CIVER:-$DISTRIB}
-                SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python-emsdk-sdk-extra-${CIVER}.tar.lz4
+                SDK_URL=https://github.com/pygame-web/python-wasm-sdk-extra/releases/download/$SDK_VERSION/python3.13-emsdk-sdk-extra-${CIVER}.tar.lz4
                 echo "Installing $SDK_URL"
                 curl -sL --retry 5 $SDK_URL | tar xvP --use-compress-program=lz4 | pv -p -l -s 15000 >/dev/null
                 chmod +x ./extra/*.sh
@@ -463,8 +472,8 @@ ________________________________________________________________________________
 
                 # debug CI does not use pnpm/npm for building pg, so call the typescript build
                 # part from here
-                pnpm --filter "pglite^..." build || exit 450
-
+                #pnpm --filter "pglite^..." build || exit 450
+                pnpm run build:js || exit 476
                 pnpm pack || exit 31
                 packed=$(echo -n electric-sql-pglite-*.tgz)
 

cibuild/build-with-docker.sh

@@ -14,14 +14,20 @@ fi
 
 IMG_NAME="electricsql/pglite-builder"
 IMG_TAG="${PG_VERSION}_${SDK_VERSION}"
+SDK_ARCHIVE="${SDK_ARCHIVE:-python3.13-wasm-sdk-Ubuntu-22.04.tar.lz4}"
+WASI_SDK_ARCHIVE="${WASI_SDK_ARCHIVE:-python3.13-wasi-sdk-Ubuntu-22.04.tar.lz4}"
 
 docker run \
   --rm \
   -e OBJDUMP=${OBJDUMP:-true} \
-  -v ./cibuild.sh:/workspace/cibuild.sh \
-  -v ./cibuild:/workspace/cibuild \
-  -v ./patches:/opt/patches \
-  -v ./tests:/workspace/tests \
-  -v ./packages/pglite:/workspace/packages/pglite \
+  -e SDK_ARCHIVE \
+  -e WASI_SDK_ARCHIVE \
+  -v ./cibuild.sh:/workspace/cibuild.sh:rw \
+  -v ./.buildconfig:/workspace/.buildconfig:rw \
+  -v ./extra:/workspace/extra:rw \
+  -v ./cibuild:/workspace/cibuild:rw \
+  -v ./patches:/opt/patches:rw \
+  -v ./tests:/workspace/tests:rw \
+  -v ./packages/pglite:/workspace/packages/pglite:rw \
   $IMG_NAME:$IMG_TAG \
   bash ./cibuild/build-all.sh