Rewrite Bun's runtime CommonJS loader (#3379)

* wip changes for CommonJS

* this rewrite is almost complete

* even more code

* wip

* Remove usages of `import.meta.require` from builtins

* Remove usages of require

* Regenerate

* :scissors: builtin rewrite commonjs in printer

* Use lazy custom getters for import.meta

* fixups

* Remove depd

* ugh

* still crashing

* fixup undici

* comment out import.meta.require.resolve temporarily

not a real solution but it stops the crashes

* Redo import.meta.primordials

* Builtins now have a `builtin://` protocol in source origin

* Seems to work?

* Finsih getting rid of primordials

* switcharoo

* No more function

* just one more bug

* Update launch.json

* Implement `require.main`

* :scissors:

* Bump WebKit

* Fixup import cycles

* Fixup improt cycles

* export more things

* Implement `createCommonJSModule` builtin

* More exports

* regenerate

* i broke some stuff

* some of these tests work now

* We lost the encoding

* Sort of fix zlib

* Sort of fix util

* Update events.js

* bump

* bump

* bump

* Fix missing export in fs

* fix some bugs with builtin esm modules (stream, worker_threads, events). its not perfect yet.

* fix some other internal module bugs

* oops

* fix some extra require default stuff

* uncomment this file but it crsahes on my machine

* tidy code here

* fixup tls exports

* make simdutf happier

* Add hasPrefix binding

* Add test for `require.main`

* Fix CommonJS evaluation order race condition

* Make node:http load faster

* Add missing exports to tls.js

* Use the getter

* Regenerate builtins

* Fix assertion failure in Bun.write()

* revamp dotEnv parser (#3347)

- fixes `strings.indexOfAny()`
- fixes OOB array access

fixes #411
fixes #2823
fixes #3042

* fix tests for `expect()` (#3384)

- extend test job time-out for `darwin-aarch64`

* `expect().resolves` and `expect().rejects` (#3318)

* Move expect and snapshots to their own files

* expect().resolves and expect().rejects

* Fix promise being added to unhandled rejection list

* Handle timeouts in expect(<promise>)

* wip merge

* Fix merge issue

---------

Co-authored-by: Jarred Sumner <jarred@jarredsumner.com>
Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

* fixup min/memcopy (#3388)

* Fix crash in builtins

* Don't attempt to evaluate modules with no source code

* Update WebCoreJSBuiltins.cpp

* Update WebCoreJSBuiltins.cpp

* Update WebCoreJSBuiltins.cpp

* Fix crash

* cleanup

* Fix test

cc @paperdave

* Fixup Undici

* Fix issue in node:http

* Create util-deprecate.mjs

* Fix several bugs

* Use the identifier

* Support error.code in `util.deprecate`

* make the CJs loader slightly more resilient

* Update WebCoreJSBuiltins.cpp

* Fix macros

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>
Co-authored-by: dave caruso <me@paperdave.net>
Co-authored-by: Alex Lam S.L <alexlamsl@gmail.com>
Co-authored-by: Ashcon Partovi <ashcon@partovi.net>
Co-authored-by: Ciro Spaciari <ciro.spaciari@gmail.com>

1 files changed, 128 insertions, 3 deletions

diff --git a/src/bun.js/bindings/ModuleLoader.cpp b/src/bun.js/bindings/ModuleLoader.cpp
index ed1e5702b..0ccbb7dbb 100644
--- a/src/bun.js/bindings/ModuleLoader.cpp
+++ b/src/bun.js/bindings/ModuleLoader.cpp
@@ -36,6 +36,11 @@
 #include "../modules/TTYModule.h"
 #include "node_util_types.h"
 #include "CommonJSModuleRecord.h"
+#include <JavaScriptCore/JSModuleLoader.h>
+#include <JavaScriptCore/Completion.h>
+#include <JavaScriptCore/JSModuleNamespaceObject.h>
+#include <JavaScriptCore/JSMap.h>
+#include <JavaScriptCore/JSMapInlines.h>
 
 namespace Bun {
 using namespace Zig;
@@ -350,6 +355,110 @@ extern "C" void Bun__onFulfillAsyncModule(
     promise->resolve(promise->globalObject(), JSC::JSSourceCode::create(vm, JSC::SourceCode(provider)));
 }
 
+JSValue fetchCommonJSModule(
+    Zig::GlobalObject* globalObject,
+    JSCommonJSModule* target,
+    JSValue specifierValue,
+    BunString* specifier,
+    BunString* referrer)
+{
+    void* bunVM = globalObject->bunVM();
+    auto& vm = globalObject->vm();
+    auto scope = DECLARE_THROW_SCOPE(vm);
+    ErrorableResolvedSource resValue;
+    ErrorableResolvedSource* res = &resValue;
+
+    auto& builtinNames = WebCore::clientData(vm)->builtinNames();
+
+    if (Bun__fetchBuiltinModule(bunVM, globalObject, specifier, referrer, res)) {
+        if (!res->success) {
+            throwException(scope, res->result.err, globalObject);
+            return JSValue();
+        }
+
+        switch (res->result.value.tag) {
+        case SyntheticModuleType::Module: {
+            target->evaluate(globalObject, Bun::toWTFString(*specifier), generateNodeModuleModule);
+            RETURN_IF_EXCEPTION(scope, {});
+            RELEASE_AND_RETURN(scope, target);
+        }
+
+        case SyntheticModuleType::Buffer: {
+            target->evaluate(globalObject, Bun::toWTFString(*specifier), generateBufferSourceCode);
+            RETURN_IF_EXCEPTION(scope, {});
+            RELEASE_AND_RETURN(scope, target);
+        }
+        case SyntheticModuleType::TTY: {
+            target->evaluate(globalObject, Bun::toWTFString(*specifier), generateTTYSourceCode);
+            RETURN_IF_EXCEPTION(scope, {});
+            RELEASE_AND_RETURN(scope, target);
+        }
+        case SyntheticModuleType::NodeUtilTypes: {
+            target->evaluate(globalObject, Bun::toWTFString(*specifier), Bun::generateNodeUtilTypesSourceCode);
+            RETURN_IF_EXCEPTION(scope, {});
+            RELEASE_AND_RETURN(scope, target);
+        }
+        case SyntheticModuleType::Process: {
+            target->evaluate(globalObject, Bun::toWTFString(*specifier), generateProcessSourceCode);
+            RETURN_IF_EXCEPTION(scope, {});
+            RELEASE_AND_RETURN(scope, target);
+        }
+        case SyntheticModuleType::Events: {
+            target->evaluate(globalObject, Bun::toWTFString(*specifier), generateEventsSourceCode);
+            RETURN_IF_EXCEPTION(scope, {});
+            RELEASE_AND_RETURN(scope, target);
+        }
+        case SyntheticModuleType::StringDecoder: {
+            target->evaluate(globalObject, Bun::toWTFString(*specifier), generateStringDecoderSourceCode);
+            RETURN_IF_EXCEPTION(scope, {});
+            RELEASE_AND_RETURN(scope, target);
+        }
+        default: {
+            RELEASE_AND_RETURN(scope, jsNumber(-1));
+        }
+        }
+    }
+
+    // if (JSC::JSValue virtualModuleResult = JSValue::decode(Bun__runVirtualModule(globalObject, specifier))) {
+    //     return handleVirtualModuleResult<allowPromise>(globalObject, virtualModuleResult, res, specifier, referrer);
+    // }
+    auto* loader = globalObject->moduleLoader();
+    JSMap* registry = jsCast<JSMap*>(loader->getDirect(vm, Identifier::fromString(vm, "registry"_s)));
+
+    auto hasAlreadyLoadedESMVersionSoWeShouldntTranspileItTwice = [&]() -> bool {
+        JSValue entry = registry->get(globalObject, specifierValue);
+
+        if (!entry || !entry.isObject()) {
+            return false;
+        }
+
+        int status = entry.getObject()->getDirect(vm, WebCore::clientData(vm)->builtinNames().statePublicName()).asInt32();
+        return status > JSModuleLoader::Status::Fetch;
+    };
+
+    if (hasAlreadyLoadedESMVersionSoWeShouldntTranspileItTwice()) {
+        RELEASE_AND_RETURN(scope, jsNumber(-1));
+    }
+
+    Bun__transpileFile(bunVM, globalObject, specifier, referrer, res, false);
+
+    if (res->success && res->result.value.commonJSExportsLen) {
+        target->evaluate(globalObject, Bun::toWTFString(*specifier).isolatedCopy(), res->result.value);
+        RETURN_IF_EXCEPTION(scope, {});
+        RELEASE_AND_RETURN(scope, target);
+    }
+
+    if (!res->success) {
+        throwException(scope, res->result.err, globalObject);
+        RELEASE_AND_RETURN(scope, {});
+    }
+
+    auto&& provider = Zig::SourceProvider::create(globalObject, res->result.value);
+    globalObject->moduleLoader()->provideFetch(globalObject, specifierValue, JSC::SourceCode(provider));
+    RETURN_IF_EXCEPTION(scope, {});
+    RELEASE_AND_RETURN(scope, jsNumber(-1));
+}
+
 template<bool allowPromise>
 static JSValue fetchSourceCode(
     Zig::GlobalObject* globalObject,
@@ -382,6 +491,11 @@ static JSValue fetchSourceCode(
 
     auto rejectOrResolve = [&](JSValue code) -> JSValue {
         if (auto* exception = scope.exception()) {
+            if constexpr (!allowPromise) {
+                scope.release();
+                return {};
+            }
+
             scope.clearException();
             return rejectedInternalPromise(globalObject, exception);
         }
@@ -457,7 +571,7 @@ static JSValue fetchSourceCode(
             return rejectOrResolve(JSSourceCode::create(vm, WTFMove(source)));
         }
         default: {
-            auto&& provider = Zig::SourceProvider::create(globalObject, res->result.value);
+            auto&& provider = Zig::SourceProvider::create(globalObject, res->result.value, JSC::SourceProviderSourceType::Module, true);
             return rejectOrResolve(JSC::JSSourceCode::create(vm, JSC::SourceCode(provider)));
         }
         }
@@ -477,8 +591,19 @@ static JSValue fetchSourceCode(
     }
 
     if (res->success && res->result.value.commonJSExportsLen) {
-        auto source = Bun::createCommonJSModule(globalObject, res->result.value);
-        return rejectOrResolve(JSSourceCode::create(vm, WTFMove(source)));
+        auto created = Bun::createCommonJSModule(globalObject, res->result.value);
+
+        if (created.has_value()) {
+            return rejectOrResolve(JSSourceCode::create(vm, WTFMove(created.value())));
+        }
+
+        if constexpr (allowPromise) {
+            auto* exception = scope.exception();
+            scope.clearException();
+            return rejectedInternalPromise(globalObject, exception);
+        } else {
+            return JSC::jsUndefined();
+        }
     }
 
     if (!res->success) {