chore: build system improvements (#6811)

* build system improvements

* public the secret download link

* typo

* i think i fixed it

* fix ci

* un-bump the cmake version

13 files changed, 155 insertions, 80 deletions

diff --git a/.github/workflows/bun-mac-aarch64.yml b/.github/workflows/bun-mac-aarch64.yml
index a894b4426..d7123131d 100644
--- a/.github/workflows/bun-mac-aarch64.yml
+++ b/.github/workflows/bun-mac-aarch64.yml
@@ -6,6 +6,7 @@ concurrency:
 
 env:
   LLVM_VERSION: 16
+  BUN_DOWNLOAD_URL_BASE: https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/latest
 
 on:
   push:
@@ -210,9 +211,10 @@ jobs:
           cmake -S $SOURCE_DIR -B $OBJ_DIR \
             -G Ninja \
             -DCMAKE_BUILD_TYPE=Release \
-            -DBUN_CPP_ONLY=1
+            -DBUN_CPP_ONLY=1 \
+            -DNO_CONFIGURE_DEPENDS=1
 
-          bash compile-cpp-only.sh
+          bash compile-cpp-only.sh -v
 
       - name: Upload C++
         uses: actions/upload-artifact@v3
@@ -253,7 +255,7 @@ jobs:
           echo "$(brew --prefix llvm@$LLVM_VERSION)/bin" >> $GITHUB_PATH
           brew link --overwrite llvm@$LLVM_VERSION
 
-          curl -LO "https://github.com/oven-sh/bun/releases/download/bun-v1.0.7/bun-darwin-${{matrix.arch}}.zip"
+          curl -LO "$BUN_DOWNLOAD_URL_BASE/bun-darwin-${{matrix.arch}}.zip"
           unzip bun-darwin-${{matrix.arch}}.zip
           mkdir -p ${{ runner.temp }}/.bun/bin
           mv bun-darwin-${{matrix.arch}}/bun ${{ runner.temp }}/.bun/bin/bun
@@ -291,8 +293,9 @@ jobs:
             -DBUN_LINK_ONLY=1 \
             -DBUN_ZIG_OBJ="${{ runner.temp }}/release/bun-zig.o" \
             -DBUN_CPP_ARCHIVE="${{ runner.temp }}/bun-cpp-obj/bun-cpp-objects.a" \
-            -DBUN_DEPS_OUT_DIR="${{runner.temp}}/bun-deps"
-          ninja
+            -DBUN_DEPS_OUT_DIR="${{runner.temp}}/bun-deps" \
+            -DNO_CONFIGURE_DEPENDS=1
+          ninja -v
       - name: Zip
         run: |
           cd ${{runner.temp}}/link-build
diff --git a/.github/workflows/bun-mac-x64-baseline.yml b/.github/workflows/bun-mac-x64-baseline.yml
index 5dca35efb..479521811 100644
--- a/.github/workflows/bun-mac-x64-baseline.yml
+++ b/.github/workflows/bun-mac-x64-baseline.yml
@@ -6,6 +6,7 @@ concurrency:
 
 env:
   LLVM_VERSION: 16
+  BUN_DOWNLOAD_URL_BASE: https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/latest
 
 on:
   push:
@@ -202,7 +203,7 @@ jobs:
           echo "$(brew --prefix llvm@$LLVM_VERSION)/bin" >> $GITHUB_PATH
           brew link --overwrite llvm@$LLVM_VERSION
 
-          curl -LO "https://github.com/oven-sh/bun/releases/download/bun-v1.0.7/bun-darwin-x64.zip"
+          curl -LO "$BUN_DOWNLOAD_URL_BASE/bun-darwin-x64.zip"
           unzip bun-darwin-x64.zip
           mkdir -p ${{ runner.temp }}/.bun/bin
           mv bun-darwin-x64/bun ${{ runner.temp }}/.bun/bin/bun
@@ -229,9 +230,10 @@ jobs:
           cmake -S $SOURCE_DIR -B $OBJ_DIR \
             -G Ninja \
             -DCMAKE_BUILD_TYPE=Release \
-            -DBUN_CPP_ONLY=1
+            -DBUN_CPP_ONLY=1 \
+            -DNO_CONFIGURE_DEPENDS=1
 
-          bash compile-cpp-only.sh
+          bash compile-cpp-only.sh -v
 
       - name: Upload C++
         uses: actions/upload-artifact@v3
@@ -272,7 +274,7 @@ jobs:
           echo "$(brew --prefix llvm@$LLVM_VERSION)/bin" >> $GITHUB_PATH
           brew link --overwrite llvm@$LLVM_VERSION
 
-          curl -LO "https://github.com/oven-sh/bun/releases/download/bun-v1.0.7/bun-darwin-x64.zip"
+          curl -LO "$BUN_DOWNLOAD_URL_BASE/bun-darwin-x64.zip"
           unzip bun-darwin-x64.zip
           mkdir -p ${{ runner.temp }}/.bun/bin
           mv bun-darwin-x64/bun ${{ runner.temp }}/.bun/bin/bun
@@ -310,8 +312,9 @@ jobs:
             -DBUN_LINK_ONLY=1 \
             -DBUN_ZIG_OBJ="${{ runner.temp }}/release/bun-zig.o" \
             -DBUN_CPP_ARCHIVE="${{ runner.temp }}/bun-cpp-obj/bun-cpp-objects.a" \
-            -DBUN_DEPS_OUT_DIR="${{runner.temp}}/bun-deps"
-          ninja
+            -DBUN_DEPS_OUT_DIR="${{runner.temp}}/bun-deps" \
+            -DNO_CONFIGURE_DEPENDS=1
+          ninja -v
       - name: Zip
         run: |
           cd ${{runner.temp}}/link-build
diff --git a/.github/workflows/bun-mac-x64.yml b/.github/workflows/bun-mac-x64.yml
index 47c252eb1..57f3d1781 100644
--- a/.github/workflows/bun-mac-x64.yml
+++ b/.github/workflows/bun-mac-x64.yml
@@ -6,6 +6,7 @@ concurrency:
 
 env:
   LLVM_VERSION: 16
+  BUN_DOWNLOAD_URL_BASE: https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/latest
 
 on:
   push:
@@ -200,7 +201,7 @@ jobs:
           echo "$(brew --prefix llvm@$LLVM_VERSION)/bin" >> $GITHUB_PATH
           brew link --overwrite llvm@$LLVM_VERSION
 
-          curl -LO "https://github.com/oven-sh/bun/releases/download/bun-v1.0.7/bun-darwin-x64.zip"
+          curl -LO "$BUN_DOWNLOAD_URL_BASE/bun-darwin-x64.zip"
           unzip bun-darwin-x64.zip
           mkdir -p ${{ runner.temp }}/.bun/bin
           mv bun-darwin-x64/bun ${{ runner.temp }}/.bun/bin/bun
@@ -227,9 +228,10 @@ jobs:
           cmake -S $SOURCE_DIR -B $OBJ_DIR \
             -G Ninja \
             -DCMAKE_BUILD_TYPE=Release \
-            -DBUN_CPP_ONLY=1
+            -DBUN_CPP_ONLY=1 \
+            -DNO_CONFIGURE_DEPENDS=1
 
-          bash compile-cpp-only.sh
+          bash compile-cpp-only.sh -v
 
       - name: Upload C++
         uses: actions/upload-artifact@v3
@@ -271,7 +273,7 @@ jobs:
           echo "$(brew --prefix llvm@$LLVM_VERSION)/bin" >> $GITHUB_PATH
           brew link --overwrite llvm@$LLVM_VERSION
 
-          curl -LO "https://github.com/oven-sh/bun/releases/download/bun-v1.0.7/bun-darwin-x64.zip"
+          curl -LO "$BUN_DOWNLOAD_URL_BASE/bun-darwin-x64.zip"
           unzip bun-darwin-x64.zip
           mkdir -p ${{ runner.temp }}/.bun/bin
           mv bun-darwin-x64/bun ${{ runner.temp }}/.bun/bin/bun
@@ -309,8 +311,9 @@ jobs:
             -DBUN_LINK_ONLY=1 \
             -DBUN_ZIG_OBJ="${{ runner.temp }}/release/bun-zig.o" \
             -DBUN_CPP_ARCHIVE="${{ runner.temp }}/bun-cpp-obj/bun-cpp-objects.a" \
-            -DBUN_DEPS_OUT_DIR="${{runner.temp}}/bun-deps"
-          ninja
+            -DBUN_DEPS_OUT_DIR="${{runner.temp}}/bun-deps" \
+            -DNO_CONFIGURE_DEPENDS=1
+          ninja -v
       - name: Zip
         run: |
           cd ${{runner.temp}}/link-build
diff --git a/.github/workflows/bun-windows-x64.yml b/.github/workflows/bun-windows-x64.yml
index c787f7d04..9eb6d73a5 100644
--- a/.github/workflows/bun-windows-x64.yml
+++ b/.github/workflows/bun-windows-x64.yml
@@ -117,7 +117,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - run: |
-          curl -fsSL https://github.com/oven-sh/bun/releases/download/bun-v1.0.7/bun-linux-x64.zip > bun.zip
+          curl -fsSL ${{ secrets.BUN_DOWNLOAD_URL_BASE }}/bun-linux-x64.zip > bun.zip
           unzip bun.zip
           export PATH="$PWD/bun-linux-x64:$PATH"
           ./scripts/cross-compile-codegen.sh win32 x64
@@ -152,7 +152,7 @@ jobs:
         run: |
           git submodule update --init --recursive --progress --depth=1 --checkout
           cd build
-          cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DNO_CODEGEN=1
+          cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release -DNO_CODEGEN=1 -DNO_CONFIGURE_DEPENDS=1
       - uses: actions/upload-artifact@v3
         with:
           name: bun-deps/
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8c265ad23..f03b9a218 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -573,6 +573,7 @@ if(NOT NO_CODEGEN)
         COMMAND ${ZIG_COMPILER} run "${BUN_SRC}/js_lexer/identifier_data.zig"
         VERBATIM
         COMMENT "Building Identifier Cache"
+        DEPENDS_EXPLICIT_ONLY
     )
 endif()
 
@@ -674,12 +675,11 @@ file(GLOB ZIG_FILES
 
 if(NOT BUN_ZIG_OBJ)
     set(BUN_ZIG_OBJ "${BUN_WORKDIR}/CMakeFiles/bun-zig.o")
+else()
+    get_filename_component(BUN_ZIG_OBJ "${BUN_ZIG_OBJ}" REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
 endif()
 
-get_filename_component(BUN_ZIG_OBJ "${BUN_ZIG_OBJ}" REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
-
 set(USES_TERMINAL_NOT_IN_CI "")
-
 if(NOT CI)
     set(USES_TERMINAL_NOT_IN_CI "USES_TERMINAL")
 endif()
@@ -711,6 +711,7 @@ if(NOT BUN_LINK_ONLY AND NOT BUN_CPP_ONLY)
         ${USES_TERMINAL_NOT_IN_CI}
     )
 endif()
+add_custom_target(zig DEPENDS "${BUN_ZIG_OBJ}")
 
 if(WIN32)
     list(APPEND BUN_RAW_SOURCES "${BUN_SRC}/bun.js/bindings/windows/musl-memmem.c")
@@ -752,7 +753,7 @@ if(NOT BUN_CPP_ARCHIVE)
             "#!/usr/bin/env bash\n"
             "set -e\n"
             "OBJ_LIST=(\n  ${BUN_OBJECT_LIST}\n)\n"
-            "ninja \${OBJ_LIST[@]}\n"
+            "ninja \${OBJ_LIST[@]} $@\n"
             "${AR} rcvs bun-cpp-objects.a \${OBJ_LIST[@]}\n"
             "echo '-> bun-cpp-objects.a'\n"
         )
diff --git a/Dockerfile b/Dockerfile
index 53feecde9..80a0aa6cb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,6 +16,7 @@ ARG BUILD_MACHINE_ARCH=x86_64
 ARG BUILDARCH=amd64
 ARG TRIPLET=${ARCH}-linux-gnu
 ARG GIT_SHA="unknown"
+ARG BUN_DOWNLOAD_URL_BASE="https://pub-5e11e972747a44bf9aaf9394f185a982.r2.dev/releases/latest"
 
 ARG BUN_VERSION="1.0.7"
 ARG NODE_VERSION="20"
@@ -31,6 +32,7 @@ ARG AWS_SECRET_ACCESS_KEY
 
 FROM bitnami/minideb:bullseye as bun-base
 
+ARG BUN_DOWNLOAD_URL_BASE
 ARG DEBIAN_FRONTEND
 ARG BUN_VERSION
 ARG NODE_VERSION
@@ -110,7 +112,7 @@ RUN apt-get update -y \
   arm64) variant="aarch64";; \
   *) echo "error: unsupported architecture: $arch"; exit 1 ;; \
   esac \
-  && wget "https://github.com/oven-sh/bun/releases/download/bun-v${BUN_VERSION}/bun-linux-${variant}.zip" \
+  && wget "${BUN_DOWNLOAD_URL_BASE}/bun-linux-${variant}.zip" \
   && unzip bun-linux-${variant}.zip \
   && mv bun-linux-${variant}/bun /usr/bin/bun \
   && ln -s /usr/bin/bun /usr/bin/bunx \
diff --git a/docs/project/building-windows.md b/docs/project/building-windows.md
index e69de29bb..8f6f0ec91 100644
--- a/docs/project/building-windows.md
+++ b/docs/project/building-windows.md
@@ -0,0 +1,61 @@
+## Prerequisites
+
+### System Dependencies
+
+- [Visual Studio](https://visualstudio.microsoft.com) with the "Desktop development with C++" workload. You should install Git and CMake from here, if not already installed.
+- Ninja
+- Go
+- Rust
+- NASM
+- Perl
+  - Do not use Strawberry Perl, it includes tools from MSys2 which will break your build.
+- Ruby
+
+### Zig
+
+Bun pins a version of Zig. As the compiler is still in development, breaking changes happen often that will break the build. It is recommended to use [Zigup](https://github.com/marler8997/zigup/releases) as it can quickly switch to any version by name, but you can also [manually download Zig](https://ziglang.org/download/).
+
+```bash
+$ zigup 0.12.0-dev.1297+a9e66ed73
+```
+
+{% callout %}
+We last updated Zig on **October 26th, 2023**
+{% /callout %}
+
+### Codegen
+
+On Unix platforms, we depend on an existing build of Bun to generate code for itself. Since the Windows branch is not stable enough for this to pass, you currently need to generate the code.
+
+On a system with Bun installed, run:
+
+```bash
+$ bash ./scripts/cross-compile-codegen.sh win32 x64
+# -> build-codegen-win32-x64
+```
+
+Copy the contents of this to the Windows machine into a folder named `build`
+
+TODO: Use WSL to automatically run codegen without a separate machine.
+
+## Building
+
+```ps1
+npm install
+
+.\scripts\env.ps1
+
+.\scripts\update-submodules.ps1
+.\scripts\all-dependencies.ps1
+
+cd build # this was created by the codegen script in the prerequisites
+
+cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug
+ninja
+```
+
+If this was successful, you should have a `bun-debug.exe` in the `build` folder.
+
+```ps1
+.\bun-debug.exe --version
+```
diff --git a/scripts/all-dependencies.sh b/scripts/all-dependencies.sh
index bd6a015a2..48c367b9b 100755
--- a/scripts/all-dependencies.sh
+++ b/scripts/all-dependencies.sh
@@ -30,7 +30,7 @@ dep() {
                 break
             fi
         done
-        if [ "$HAS_ALL_DEPS" -eq 1 ]; then
+        if [ "$HAS_ALL_DEPS" == "1" ]; then
             printf "%s - already built\n" "$script"
             return
         fi
@@ -56,8 +56,8 @@ dep boringssl libcrypto.a libssl.a libdecrepit.a
 dep cares libcares.a
 dep libarchive libarchive.a
 dep lolhtml liblolhtml.a
-dep mimalloc-debug libmimalloc-debug.a
-dep mimalloc libmimalloc.a
+dep mimalloc-debug libmimalloc-debug.a libmimalloc-debug.o
+dep mimalloc libmimalloc.a libmimalloc.o
 dep tinycc libtcc.a
 dep zlib libz.a
 dep zstd libzstd.a
diff --git a/scripts/build-lolhtml.ps1 b/scripts/build-lolhtml.ps1
index e7ee42fa0..dc3ca456e 100644
--- a/scripts/build-lolhtml.ps1
+++ b/scripts/build-lolhtml.ps1
@@ -8,4 +8,4 @@ try {
   Copy-Item target/x86_64-pc-windows-msvc/release/lolhtml.lib $BUN_DEPS_OUT_DIR
   Copy-Item target/x86_64-pc-windows-msvc/release/lolhtml.pdb $BUN_DEPS_OUT_DIR
   Write-Host "-> lolhtml.lib"
-} finally { Pop-Location }
\ No newline at end of file
+} finally { Pop-Location }
diff --git a/scripts/build-mimalloc-debug.sh b/scripts/build-mimalloc-debug.sh
index ebb82ccbc..c14740bf6 100755
--- a/scripts/build-mimalloc-debug.sh
+++ b/scripts/build-mimalloc-debug.sh
@@ -27,3 +27,4 @@ cmake "${CMAKE_FLAGS[@]}" . \
 ninja
 
 cp libmimalloc-debug.a $BUN_DEPS_OUT_DIR/libmimalloc-debug.a
+cp CMakeFiles/mimalloc-obj.dir/src/static.c.o $BUN_DEPS_OUT_DIR/libmimalloc-debug.o
diff --git a/scripts/build-mimalloc.sh b/scripts/build-mimalloc.sh
index 616e45186..b661e9b10 100755
--- a/scripts/build-mimalloc.sh
+++ b/scripts/build-mimalloc.sh
@@ -26,3 +26,4 @@ cmake "${CMAKE_FLAGS[@]}" . \
 ninja
 
 cp libmimalloc.a $BUN_DEPS_OUT_DIR/libmimalloc.a
+cp CMakeFiles/mimalloc-obj.dir/src/static.c.o $BUN_DEPS_OUT_DIR/libmimalloc.o
diff --git a/src/bun.js/WebKit b/src/bun.js/WebKit
-Subproject b82708ff8c18785f0f8c210f27cef316c7e40fd
+Subproject af34f971a48a8a689f9858731b0e6478d946bd4
diff --git a/src/codegen/generate-jssink.ts b/src/codegen/generate-jssink.ts
index 31a475e0b..f30752c53 100644
--- a/src/codegen/generate-jssink.ts
+++ b/src/codegen/generate-jssink.ts
@@ -484,51 +484,6 @@ JSC_DEFINE_HOST_FUNCTION(${name}__doClose, (JSC::JSGlobalObject * lexicalGlobalO
   `;
 
   for (let name of classes) {
-    const {
-      className,
-      controller,
-      prototypeName,
-      controllerName,
-      controllerPrototypeName,
-      constructor,
-      writableStreamSourcePrototype,
-      writableStreamName,
-    } = names(name);
-    const protopad = `${controller}__close`.length;
-    const padding = `${name}__doClose`.length;
-    templ += `
-/* Source for JS${name}PrototypeTableValues.lut.h
-@begin JS${name}PrototypeTable
-  close      ${`${name}__doClose`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 0
-  flush      ${`${name}__flush`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 1
-  end        ${`${name}__end`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 0
-  start      ${`${name}__start`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 1
-  write      ${`${name}__write`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 1
-  ref        ${`${name}__ref`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 0
-  unref      ${`${name}__unref`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 0
-@end
-*/
-
-
-/* Source for ${controllerPrototypeName}TableValues.lut.h
-@begin ${controllerPrototypeName}Table
-  close        ${`${controller}__close`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 0
-  flush        ${`${name}__flush`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 1
-  end          ${`${controller}__end`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 0
-  start        ${`${name}__start`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 1
-  write        ${`${name}__write`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 1
-@end
-*/
-
-  `;
-  }
-
-  const footer = `
-} // namespace WebCore
-
-`;
-
-  for (let name of classes) {
     const { className, controller, prototypeName, controllerPrototypeName, constructor, controllerName } = names(name);
     templ += `
 #pragma mark - ${name}
@@ -854,6 +809,10 @@ default:
   }
 }`;
 
+  const footer = `
+} // namespace WebCore
+
+`;
   templ += footer;
 
   for (let name of classes) {
@@ -951,14 +910,55 @@ extern "C" void ${name}__onClose(JSC__JSValue controllerValue, JSC__JSValue reas
   return templ;
 }
 
+function lutInput() {
+  let templ = "";
+  for (let name of classes) {
+    const { controller, controllerPrototypeName } = names(name);
+    const protopad = `${controller}__close`.length;
+    const padding = `${name}__doClose`.length;
+    templ += `
+/* Source for JS${name}PrototypeTableValues.lut.h
+@begin JS${name}PrototypeTable
+    close      ${`${name}__doClose`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 0
+    flush      ${`${name}__flush`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 1
+    end        ${`${name}__end`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 0
+    start      ${`${name}__start`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 1
+    write      ${`${name}__write`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 1
+    ref        ${`${name}__ref`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 0
+    unref      ${`${name}__unref`.padEnd(padding + 8)} ReadOnly|DontDelete|Function 0
+@end
+*/
+
+
+/* Source for ${controllerPrototypeName}TableValues.lut.h
+@begin ${controllerPrototypeName}Table
+    close        ${`${controller}__close`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 0
+    flush        ${`${name}__flush`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 1
+    end          ${`${controller}__end`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 0
+    start        ${`${name}__start`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 1
+    write        ${`${name}__write`.padEnd(protopad + 4)}  ReadOnly|DontDelete|Function 1
+@end
+*/
+`;
+  }
+
+  return templ;
+}
+
 const outDir = resolve(process.argv[2]);
 
 await Bun.write(resolve(outDir + "/JSSink.h"), header());
 await Bun.write(resolve(outDir + "/JSSink.cpp"), await implementation());
-
-Bun.spawnSync([
-  process.execPath,
-  join(import.meta.dir, "create-hash-table.ts"),
-  resolve(outDir + "/JSSink.cpp"),
-  join(outDir, "JSSink.lut.h"),
-]);
+await Bun.write(resolve(outDir + "/JSSink.lut.txt"), lutInput());
+
+Bun.spawnSync(
+  [
+    process.execPath,
+    join(import.meta.dir, "create-hash-table.ts"),
+    resolve(outDir + "/JSSink.lut.txt"),
+    join(outDir, "JSSink.lut.h"),
+  ],
+  {
+    stdio: ["inherit", "inherit", "inherit"],
+  },
+);