diff options
author | 2023-08-02 16:27:36 -0700 | |
---|---|---|
committer | 2023-08-02 16:27:36 -0700 | |
commit | c2a77cf7ec9de9eadf938046bdf78e58561c8a6d (patch) | |
tree | 0f90f1b323061455875333c9f40592b303585973 /src/js/builtins | |
parent | 7656b4b17e91f15b58eeab8f45b78c416ec6a045 (diff) | |
download | bun-c2a77cf7ec9de9eadf938046bdf78e58561c8a6d.tar.gz bun-c2a77cf7ec9de9eadf938046bdf78e58561c8a6d.tar.zst bun-c2a77cf7ec9de9eadf938046bdf78e58561c8a6d.zip |
Rewrite built-in modules to use CommonJS over ESM (#3814)
* stfdsafsd
sadffdsa
stuff
finish commonjs stuff
asdf
not done but work
not done but work
not done yet but this is how far i am
remove files
lol
update built files
uncomment everything in events lol
export default
stuff
* afdsafsd
* its not perfect but almost done
* okay
* cool
* remove temp file
* finish rebase
* revert settings.json
* a
* ch-ch-ch-ch-changes
* okay
* remove this check in release for now
* sxdcfghnjm,
* lkjhgf
* fmt
* filename can be null
* Update NodeModuleModule.h
* weee
* fmt
---------
Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>
Diffstat (limited to 'src/js/builtins')
-rw-r--r-- | src/js/builtins/AsyncContext.ts | 9 | ||||
-rw-r--r-- | src/js/builtins/BunBuiltinNames.h | 13 | ||||
-rw-r--r-- | src/js/builtins/BundlerPlugin.ts | 1 | ||||
-rw-r--r-- | src/js/builtins/EventSource.ts | 2 | ||||
-rw-r--r-- | src/js/builtins/ImportMetaObject.ts | 13 | ||||
-rw-r--r-- | src/js/builtins/Module.ts | 48 | ||||
-rw-r--r-- | src/js/builtins/ProcessObjectInternals.ts | 53 | ||||
-rw-r--r-- | src/js/builtins/README.md | 53 | ||||
-rw-r--r-- | src/js/builtins/builtins.d.ts | 513 | ||||
-rw-r--r-- | src/js/builtins/codegen/builtin-parser.ts | 89 | ||||
-rw-r--r-- | src/js/builtins/codegen/helpers.ts | 25 | ||||
-rw-r--r-- | src/js/builtins/codegen/index.ts | 661 | ||||
-rw-r--r-- | src/js/builtins/codegen/replacements.ts | 99 | ||||
-rw-r--r-- | src/js/builtins/tsconfig.json | 5 |
14 files changed, 48 insertions, 1536 deletions
diff --git a/src/js/builtins/AsyncContext.ts b/src/js/builtins/AsyncContext.ts deleted file mode 100644 index 1c55feeba..000000000 --- a/src/js/builtins/AsyncContext.ts +++ /dev/null @@ -1,9 +0,0 @@ -// Used by async_hooks to manipulate the async context - -export function getAsyncContext(): ReadonlyArray<any> | undefined { - return $getInternalField($asyncContext, 0); -} - -export function setAsyncContext(contextValue: ReadonlyArray<any> | undefined) { - return $putInternalField($asyncContext, 0, contextValue); -} diff --git a/src/js/builtins/BunBuiltinNames.h b/src/js/builtins/BunBuiltinNames.h index 1897f939e..1c34f2726 100644 --- a/src/js/builtins/BunBuiltinNames.h +++ b/src/js/builtins/BunBuiltinNames.h @@ -1,6 +1,6 @@ +// clang-format off #pragma once - #ifdef ASSERT_ENABLED #if ASSERT_ENABLED #define ORIGINAL_ASSERT_ENABLED 1 @@ -9,7 +9,6 @@ #endif #endif - #include "JavaScriptCore/BuiltinUtils.h" #include "root.h" @@ -17,10 +16,6 @@ namespace WebCore { using namespace JSC; -#if !defined(BUN_ADDITIONAL_PRIVATE_IDENTIFIERS) -#define BUN_ADDITIONAL_PRIVATE_IDENTIFIERS(macro) -#endif - #define BUN_COMMON_PRIVATE_IDENTIFIERS_EACH_PROPERTY_NAME(macro) \ macro(AbortSignal) \ macro(Buffer) \ @@ -66,7 +61,6 @@ using namespace JSC; macro(closedPromise) \ macro(closedPromiseCapability) \ macro(code) \ - macro(commonJSSymbol) \ macro(connect) \ macro(consumeReadableStream) \ macro(controlledReadableStream) \ @@ -168,6 +162,7 @@ using namespace JSC; macro(ppid) \ macro(prependEventListener) \ macro(process) \ + macro(processBindingConstants) \ macro(protocol) \ macro(pull) \ macro(pullAgain) \ @@ -250,7 +245,9 @@ using namespace JSC; macro(writer) \ macro(writing) \ macro(written) \ - BUN_ADDITIONAL_PRIVATE_IDENTIFIERS(macro) \ + macro(createInternalModuleById) \ + macro(internalModuleRegistry) \ + macro(requireNativeModule) \ class BunBuiltinNames { public: diff --git a/src/js/builtins/BundlerPlugin.ts b/src/js/builtins/BundlerPlugin.ts index 831a6614e..7be030ee8 100644 --- a/src/js/builtins/BundlerPlugin.ts +++ b/src/js/builtins/BundlerPlugin.ts @@ -3,7 +3,6 @@ import type { BuildConfig, BunPlugin, OnLoadCallback, - OnLoadResult, OnLoadResultObject, OnLoadResultSourceCode, OnResolveCallback, diff --git a/src/js/builtins/EventSource.ts b/src/js/builtins/EventSource.ts index a26c8d4ce..5a6564323 100644 --- a/src/js/builtins/EventSource.ts +++ b/src/js/builtins/EventSource.ts @@ -494,7 +494,5 @@ export function getEventSource() { value: 2, }); - EventSource[Symbol.for("CommonJS")] = 0; - return EventSource; } diff --git a/src/js/builtins/ImportMetaObject.ts b/src/js/builtins/ImportMetaObject.ts index 4a08524c6..9409bb0f1 100644 --- a/src/js/builtins/ImportMetaObject.ts +++ b/src/js/builtins/ImportMetaObject.ts @@ -103,11 +103,6 @@ export function requireESM(this: ImportMetaObject, resolved) { throw new TypeError(`require() failed to evaluate module "${resolved}". This is an internal consistentency error.`); } var exports = Loader.getModuleNamespaceObject(entry.module); - if (exports[$commonJSSymbol] === 0) { - // CommonJS module created via `Bun::CommonJSModuleRecord` - // We will refer to the requireMap to get the exports - return; - } return exports; } @@ -141,10 +136,6 @@ export function internalRequire(this: ImportMetaObject, id) { if (cachedModule) { return cachedModule.exports; } - var defaultExport = exports?.default; - if (defaultExport?.[$commonJSSymbol] === 0) { - exports = defaultExport; - } $requireMap.$set(id, $createCommonJSModule(id, exports, true)); return exports; } @@ -161,9 +152,7 @@ export function createRequireCache() { const esm = Loader.registry.$get(key); if (esm?.evaluated) { const namespace = Loader.getModuleNamespaceObject(esm.module); - const exports = - namespace[$commonJSSymbol] === 0 || namespace.default?.[$commonJSSymbol] ? namespace.default : namespace; - const mod = $createCommonJSModule(key, exports, true); + const mod = $createCommonJSModule(key, namespace, true); $requireMap.$set(key, mod); return mod; } diff --git a/src/js/builtins/Module.ts b/src/js/builtins/Module.ts index 0b5fcafe8..5cf7290ae 100644 --- a/src/js/builtins/Module.ts +++ b/src/js/builtins/Module.ts @@ -1,9 +1,13 @@ -interface Module { +interface CommonJSModuleRecord { + $require(id: string, mod: any): any; + children: CommonJSModuleRecord[]; + exports: any; id: string; + loaded: boolean; + parent: undefined; path: string; - - $require(id: string, mod: any): any; - children: Module[]; + paths: string[]; + require: typeof require; } $getter; @@ -11,7 +15,7 @@ export function main() { return $requireMap.$get(Bun.main); } -export function require(this: Module, id: string) { +export function require(this: CommonJSModuleRecord, id: string) { const existing = $requireMap.$get(id) || $requireMap.$get((id = $resolveSync(id, this.path, false))); if (existing) { // Scenario where this is necessary: @@ -39,20 +43,6 @@ export function require(this: Module, id: string) { return $internalRequire(id); } - let esm = Loader.registry.$get(id); - if (esm?.evaluated && (esm.state ?? 0) >= $ModuleReady) { - const mod = esm.module; - const namespace = Loader.getModuleNamespaceObject(mod); - const exports = - namespace?.[$commonJSSymbol] === 0 || namespace?.default?.[$commonJSSymbol] === 0 - ? namespace.default - : namespace.__esModule - ? namespace - : Object.create(namespace, { __esModule: { value: true } }); - $requireMap.$set(id, $createCommonJSModule(id, exports, true)); - return exports; - } - // To handle import/export cycles, we need to create a module object and put // it into the map before we import it. const mod = $createCommonJSModule(id, {}, false); @@ -76,18 +66,14 @@ export function require(this: Module, id: string) { throw exception; } - esm = Loader.registry.$get(id); + const esm = Loader.registry.$get(id); // If we can pull out a ModuleNamespaceObject, let's do it. if (esm?.evaluated && (esm.state ?? 0) >= $ModuleReady) { const namespace = Loader.getModuleNamespaceObject(esm!.module); return (mod.exports = // if they choose a module - namespace?.[$commonJSSymbol] === 0 || namespace?.default?.[$commonJSSymbol] === 0 - ? namespace.default - : namespace.__esModule - ? namespace - : Object.create(namespace, { __esModule: { value: true } })); + namespace.__esModule ? namespace : Object.create(namespace, { __esModule: { value: true } })); } } @@ -95,6 +81,16 @@ export function require(this: Module, id: string) { return mod.exports; } -export function requireResolve(this: Module, id: string) { +export function requireResolve(this: CommonJSModuleRecord, id: string) { return $resolveSync(id, this.path, false); } + +export function requireNativeModule(id: string) { + // There might be a race condition here? + let esm = Loader.registry.$get(id); + if (esm?.evaluated && (esm.state ?? 0) >= $ModuleReady) { + const exports = Loader.getModuleNamespaceObject(esm.module); + return exports.default; + } + return $requireESM(id).default; +} diff --git a/src/js/builtins/ProcessObjectInternals.ts b/src/js/builtins/ProcessObjectInternals.ts index f6990dc5e..fc6b3800f 100644 --- a/src/js/builtins/ProcessObjectInternals.ts +++ b/src/js/builtins/ProcessObjectInternals.ts @@ -23,38 +23,29 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +// TODO: move this to native code? export function binding(bindingName) { - if (bindingName !== "constants") - throw new TypeError( - "process.binding() is not supported in Bun. If that breaks something, please file an issue and include a reproducible code sample.", + if (bindingName === "constants") { + return $processBindingConstants; + } + const issue = { + fs: 3546, + buffer: 2020, + natives: 2254, + uv: 2891, + }[bindingName]; + if (issue) { + throw new Error( + `process.binding("${bindingName}") is not implemented in Bun. Track the status & thumbs up the issue: https://github.com/oven-sh/bun/issues/${issue}`, ); - - var cache = globalThis.Symbol.for("process.bindings.constants"); - var constants = globalThis[cache]; - if (!constants) { - // TODO: make this less hacky. - // This calls require("node:fs").constants - // except, outside an ESM module. - const { constants: fs } = $lazy("createImportMeta", "node:process").require("node:fs"); - constants = { - fs, - zlib: {}, - crypto: {}, - os: Bun._Os().constants, - }; - globalThis[cache] = constants; } - return constants; + throw new TypeError( + `process.binding("${bindingName}") is not implemented in Bun. If that breaks something, please file an issue and include a reproducible code sample.`, + ); } export function getStdioWriteStream(fd_, getWindowSize) { - var require = path => { - var existing = $requireMap.get(path); - if (existing) return existing.exports; - - return $internalRequire(path); - }; - var module = { path: "node:process", require }; + var EventEmitter = require("node:events"); function createStdioWriteStream(fd_) { var { Duplex, eos, destroy } = require("node:stream"); @@ -213,8 +204,6 @@ export function getStdioWriteStream(fd_, getWindowSize) { return new StdioWriteStream(fd_); } - var { EventEmitter } = require("node:events"); - function isFastEncoding(encoding) { if (!encoding) return true; @@ -499,14 +488,6 @@ export function getStdioWriteStream(fd_, getWindowSize) { } export function getStdinStream(fd_) { - var require = path => { - var existing = $requireMap.get(path); - if (existing) return existing.exports; - - return $internalRequire(path); - }; - - var module = { path: "node:process", require: require }; var { Duplex, eos, destroy } = require("node:stream"); var StdinStream = class StdinStream extends Duplex { diff --git a/src/js/builtins/README.md b/src/js/builtins/README.md deleted file mode 100644 index 67b8882ee..000000000 --- a/src/js/builtins/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# JavaScript Builtins - -**TLDR** — When files in this directory change, run: - -```bash -# Delete the built files -$ make regenerate-bindings -# Re-link the binary without compiling zig (so it's faster) -$ make bun-link-lld-debug -``` - -TypeScript files in [./ts](./ts) are bundled into C++ Headers that can access JavaScriptCore intrinsics. These files use special globals that are prefixed with `$`. - -```js -$getter -export function foo() { - return $getByIdDirectPrivate(this, "superSecret"); -} -``` - -It looks kind of like decorators but they're not. They let you directly call engine intrinsics and help with avoiding prototype pollution issues. - -V8 has a [similar feature](https://v8.dev/blog/embedded-builtins) (they use `%` instead of `@`) - -They usually are accompanied by a C++ file. - -We use a custom code generator located in `./codegen` which contains a regex-based parser that separates each function into it's own bundling context, so syntax like top level variables / functions will not work. - -You can also use `process.platform` and `process.arch` in these files. The values are inlined and DCE'd. - -## Generating builtins - -To regenerate the builtins, run this from Bun's project root (where the `Makefile` is) - -```bash -$ make builtins -``` - -You'll want to also rebuild all the C++ bindings or you will get strange crashes on start - -```bash -$ make clean-bindings -``` - -The `make regenerate-bindings` command will clean and rebuild the bindings. - -Also, you can run the code generator manually. - -```bash -$ bun ./codegen/index.ts -# pass --minify to minify (make passes this by default) -# pass --keep-tmp to keep the temporary ./tmp folder, which contains processed pre-bundled .ts files -``` diff --git a/src/js/builtins/builtins.d.ts b/src/js/builtins/builtins.d.ts deleted file mode 100644 index 1b585ae7a..000000000 --- a/src/js/builtins/builtins.d.ts +++ /dev/null @@ -1,513 +0,0 @@ -// Typedefs for JSC intrinsics. Instead of @, we use $ -type TODO = any; - -/** Place this directly above a function declaration (like a decorator) to make it a getter. */ -declare const $getter: never; -/** Assign to this directly above a function declaration (like a decorator) to override the function's display name. */ -declare var $overriddenName: string; -/** ??? */ -declare var $linkTimeConstant: never; -/** Assign to this directly above a function declaration (like a decorator) to set visibility */ -declare var $visibility: "Public" | "Private"; -/** ??? */ -declare var $nakedConstructor: never; -/** Assign to this directly above a function declaration (like a decorator) to set intrinsic */ -declare var $intrinsic: string; -declare var $constructor; -/** Place this directly above a function declaration (like a decorator) to NOT include "use strict" */ -declare var $sloppy; - -declare function $extractHighWaterMarkFromQueuingStrategyInit(obj: any): any; - -// JSC defines their intrinsics in a nice list here: -// https://github.com/WebKit/WebKit/blob/main/Source/JavaScriptCore/bytecode/BytecodeIntrinsicRegistry.h -// -// And implemented here: (search for "emit_intrinsic_<name>", like "emit_intrinsic_arrayPush") -// https://github.com/WebKit/WebKit/blob/main/Source/JavaScriptCore/bytecompiler/NodesCodegen.cpp - -/** Assert a value is true */ -declare function $assert(index: any): void; -/** returns `arguments[index]` */ -declare function $argument<T = any>(index: number): any; -/** returns number of arguments */ -declare function $argumentCount(): number; -/** array.push(item) */ -declare function $arrayPush(array: T[], item: T): void; -/** gets a property on an object */ -declare function $getByIdDirect<T = any>(obj: any, key: string): T; -/** - * gets a private property on an object. translates to the `op_get_by_id_direct` bytecode. - * - * TODO: clarify what private means exactly. - */ -declare function $getByIdDirectPrivate<T = any>(obj: any, key: string): T; -/** - * gets a property on an object - */ -declare function $getByValWithThis(target: any, receiver: any, propertyKey: string): void; -/** gets the prototype of an object */ -declare function $getPrototypeOf(value: any): any; -/** gets an internal property on a promise - * - * You can pass - * - $promiseFieldFlags - get a number with flags - * - $promiseFieldReactionsOrResult - get the result (like Bun.peek) - */ -declare function $getPromiseInternalField<K extends PromiseFieldType, V>( - promise: Promise<V>, - key: K, -): PromiseFieldToValue<K, V>; -declare function $getInternalField<Fields extends any[], N extends keyof Fields>( - base: InternalFieldObject<Fields>, - number: N, -): Fields[N]; -declare function $fulfillPromise(...args: any[]): TODO; -declare function $evaluateCommonJSModule(...args: any[]): TODO; -declare function $loadCJS2ESM(...args: any[]): TODO; -declare function $getGeneratorInternalField(): TODO; -declare function $getAsyncGeneratorInternalField(): TODO; -declare function $getAbstractModuleRecordInternalField(): TODO; -declare function $getArrayIteratorInternalField(): TODO; -declare function $getStringIteratorInternalField(): TODO; -declare function $getMapIteratorInternalField(): TODO; -declare function $getSetIteratorInternalField(): TODO; -declare function $getProxyInternalField(): TODO; -declare function $idWithProfile(): TODO; -declare function $isObject(obj: unknown): obj is object; -declare function $isCallable(fn: unknown): fn is CallableFunction; -declare function $isConstructor(fn: unknown): fn is { new (...args: any[]): any }; -declare function $isJSArray(obj: unknown): obj is any[]; -declare function $isProxyObject(obj: unknown): obj is Proxy; -declare function $isDerivedArray(): TODO; -declare function $isGenerator(obj: unknown): obj is Generator<any, any, any>; -declare function $isAsyncGenerator(obj: unknown): obj is AsyncGenerator<any, any, any>; -declare function $isPromise(obj: unknown): obj is Promise<any>; -declare function $isRegExpObject(obj: unknown): obj is RegExp; -declare function $isMap<K, V>(obj: unknown): obj is Map<K, V>; -declare function $isSet<V>(obj: unknown): obj is Set<V>; -declare function $isShadowRealm(obj: unknown): obj is ShadowRealm; -declare function $isStringIterator(obj: unknown): obj is Iterator<string>; -declare function $isArrayIterator(obj: unknown): obj is Iterator<any>; -declare function $isMapIterator(obj: unknown): obj is Iterator<any>; -declare function $isSetIterator(obj: unknown): obj is Iterator<any>; -declare function $isUndefinedOrNull(obj: unknown): obj is null | undefined; -declare function $tailCallForwardArguments(): TODO; -/** - * **NOTE** - use `throw new TypeError()` instead. it compiles to the same builtin - * @deprecated - */ -declare function $throwTypeError(message: string): never; -/** - * **NOTE** - use `throw new RangeError()` instead. it compiles to the same builtin - * @deprecated - */ -declare function $throwRangeError(message: string): never; -/** - * **NOTE** - use `throw new OutOfMemoryError()` instead. it compiles to the same builtin - * @deprecated - */ -declare function $throwOutOfMemoryError(): never; -declare function $tryGetById(): TODO; -declare function $tryGetByIdWithWellKnownSymbol(obj: any, key: WellKnownSymbol): any; -declare function $putByIdDirect(obj: any, key: PropertyKey, value: any): void; -declare function $putByIdDirectPrivate(obj: any, key: PropertyKey, value: any): void; -declare function $putByValDirect(obj: any, key: PropertyKey, value: any): void; -declare function $putByValWithThisSloppy(): TODO; -declare function $putByValWithThisStrict(): TODO; -declare function $putInternalField<Fields extends any[], N extends keyof Fields>( - base: InternalFieldObject<Fields>, - number: N, - value: Fields[N], -): void; -declare function $putPromiseInternalField<T extends PromiseFieldType, P extends Promise<any>>( - promise: P, - key: T, - value: PromiseFieldToValue<T, P>, -): void; -declare function $putGeneratorInternalField(): TODO; -declare function $putAsyncGeneratorInternalField(): TODO; -declare function $putArrayIteratorInternalField(): TODO; -declare function $putStringIteratorInternalField(): TODO; -declare function $putMapIteratorInternalField(): TODO; -declare function $putSetIteratorInternalField(): TODO; -declare function $superSamplerBegin(): TODO; -declare function $superSamplerEnd(): TODO; -declare function $toNumber(x: any): number; -declare function $toString(x: any): string; -declare function $toPropertyKey(x: any): PropertyKey; -/** - * Often used like - * `$toObject(this, "Class.prototype.method requires that |this| not be null or undefined");` - */ -declare function $toObject(object: any, errorMessage?: string): object; -declare function $newArrayWithSize<T>(size: number): T[]; -declare function $newArrayWithSpecies(): TODO; -declare function $newPromise(): TODO; -declare function $createPromise(): TODO; -declare const $iterationKindKey: TODO; -declare const $iterationKindValue: TODO; -declare const $iterationKindEntries: TODO; -declare const $MAX_ARRAY_INDEX: number; -declare const $MAX_STRING_LENGTH: number; -declare const $MAX_SAFE_INTEGER: number; -declare const $ModuleFetch: number; -declare const $ModuleTranslate: number; -declare const $ModuleInstantiate: number; -declare const $ModuleSatisfy: number; -declare const $ModuleLink: number; -declare const $ModuleReady: number; -declare const $promiseRejectionReject: TODO; -declare const $promiseRejectionHandle: TODO; -declare const $promiseStatePending: number; -declare const $promiseStateFulfilled: number; -declare const $promiseStateRejected: number; -declare const $promiseStateMask: number; -declare const $promiseFlagsIsHandled: number; -declare const $promiseFlagsIsFirstResolvingFunctionCalled: number; -declare const $promiseFieldFlags: 0; -declare const $promiseFieldReactionsOrResult: 1; -declare const $proxyFieldTarget: TODO; -declare const $proxyFieldHandler: TODO; -declare const $generatorFieldState: TODO; -declare const $generatorFieldNext: TODO; -declare const $generatorFieldThis: TODO; -declare const $generatorFieldFrame: TODO; -declare const $generatorFieldContext: TODO; -declare const $GeneratorResumeModeNormal: TODO; -declare const $GeneratorResumeModeThrow: TODO; -declare const $GeneratorResumeModeReturn: TODO; -declare const $GeneratorStateCompleted: TODO; -declare const $GeneratorStateExecuting: TODO; -declare const $arrayIteratorFieldIndex: TODO; -declare const $arrayIteratorFieldIteratedObject: TODO; -declare const $arrayIteratorFieldKind: TODO; -declare const $mapIteratorFieldMapBucket: TODO; -declare const $mapIteratorFieldKind: TODO; -declare const $setIteratorFieldSetBucket: TODO; -declare const $setIteratorFieldKind: TODO; -declare const $stringIteratorFieldIndex: TODO; -declare const $stringIteratorFieldIteratedString: TODO; -declare const $asyncGeneratorFieldSuspendReason: TODO; -declare const $asyncGeneratorFieldQueueFirst: TODO; -declare const $asyncGeneratorFieldQueueLast: TODO; -declare const $AsyncGeneratorStateCompleted: TODO; -declare const $AsyncGeneratorStateExecuting: TODO; -declare const $AsyncGeneratorStateAwaitingReturn: TODO; -declare const $AsyncGeneratorStateSuspendedStart: TODO; -declare const $AsyncGeneratorStateSuspendedYield: TODO; -declare const $AsyncGeneratorSuspendReasonYield: TODO; -declare const $AsyncGeneratorSuspendReasonAwait: TODO; -declare const $AsyncGeneratorSuspendReasonNone: TODO; -declare const $abstractModuleRecordFieldState: TODO; -declare const $asyncContext: InternalFieldObject<[ReadonlyArray<any> | undefined]>; - -// We define our intrinsics in ./BunBuiltinNames.h. Some of those are globals. - -declare var $_events: TODO; -declare function $abortAlgorithm(): TODO; -declare function $abortSteps(): TODO; -declare function $addEventListener(): TODO; -declare function $appendFromJS(): TODO; -declare function $argv(): TODO; -declare function $assignToStream(): TODO; -declare function $associatedReadableByteStreamController(): TODO; -declare function $autoAllocateChunkSize(): TODO; -declare function $backpressure(): TODO; -declare function $backpressureChangePromise(): TODO; -declare function $basename(): TODO; -declare function $body(): TODO; -declare function $bunNativePtr(): TODO; -declare function $bunNativeType(): TODO; -declare function $byobRequest(): TODO; -declare function $cancel(): TODO; -declare function $cancelAlgorithm(): TODO; -declare function $chdir(): TODO; -declare function $cloneArrayBuffer(a, b, c): TODO; -declare function $close(): TODO; -declare function $closeAlgorithm(): TODO; -declare function $closeRequest(): TODO; -declare function $closeRequested(): TODO; -declare function $closed(): TODO; -declare function $closedPromise(): TODO; -declare function $closedPromiseCapability(): TODO; -declare function $code(): TODO; -declare const $commonJSSymbol: unique symbol; -declare function $connect(): TODO; -declare function $consumeReadableStream(): TODO; -declare function $controlledReadableStream(): TODO; -declare function $controller(): TODO; -declare function $cork(): TODO; -declare function $createEmptyReadableStream(): TODO; -declare function $createFIFO(): TODO; -declare function $createNativeReadableStream(): TODO; -declare function $createReadableStream(): TODO; -declare function $createUninitializedArrayBuffer(size: number): ArrayBuffer; -declare function $createWritableStreamFromInternal(...args: any[]): TODO; -declare function $cwd(): TODO; -declare function $data(): TODO; -declare function $dataView(): TODO; -declare function $decode(): TODO; -declare function $delimiter(): TODO; -declare function $destroy(): TODO; -declare function $dir(): TODO; -declare function $direct(): TODO; -declare function $dirname(): TODO; -declare function $disturbed(): TODO; -declare function $document(): TODO; -declare function $encode(): TODO; -declare function $encoding(): TODO; -declare function $end(): TODO; -declare function $errno(): TODO; -declare function $errorSteps(): TODO; -declare function $execArgv(): TODO; -declare function $extname(): TODO; -declare function $failureKind(): TODO; -declare function $fatal(): TODO; -declare function $fetch(): TODO; -declare function $fetchRequest(): TODO; -declare function $file(): TODO; -declare function $filePath(): TODO; -declare function $fillFromJS(): TODO; -declare function $filter(): TODO; -declare function $finishConsumingStream(): TODO; -declare function $flush(): TODO; -declare function $flushAlgorithm(): TODO; -declare function $format(): TODO; -declare function $fulfillModuleSync(key: string): void; -declare function $get(): TODO; -declare function $getInternalWritableStream(writable: WritableStream): TODO; -declare function $handleEvent(): TODO; -declare function $hash(): TODO; -declare function $header(): TODO; -declare function $headers(): TODO; -declare function $highWaterMark(): TODO; -declare function $host(): TODO; -declare function $hostname(): TODO; -declare function $href(): TODO; -declare function $ignoreBOM(): TODO; -declare function $importer(): TODO; -declare function $inFlightCloseRequest(): TODO; -declare function $inFlightWriteRequest(): TODO; -declare function $initializeWith(): TODO; -declare function $internalRequire(path: string): TODO; -declare function $internalStream(): TODO; -declare function $internalWritable(): TODO; -declare function $isAbortSignal(signal: unknown): signal is AbortSignal; -declare function $isAbsolute(): TODO; -declare function $isDisturbed(): TODO; -declare function $isPaused(): TODO; -declare function $isWindows(): TODO; -declare function $join(): TODO; -declare function $kind(): TODO; -declare function $lazy(): TODO; -declare function $lazyLoad(): TODO; -declare function $lazyStreamPrototypeMap(): TODO; -declare function $loadModule(): TODO; -declare function $localStreams(): TODO; -declare function $main(): TODO; -declare function $makeDOMException(): TODO; -declare function $makeGetterTypeError(className: string, prop: string): Error; -declare function $makeThisTypeError(className: string, method: string): Error; -declare function $map(): TODO; -declare function $method(): TODO; -declare function $nextTick(): TODO; -declare function $normalize(): TODO; -declare function $on(): TODO; -declare function $once(): TODO; -declare function $options(): TODO; -declare function $origin(): TODO; -declare function $ownerReadableStream(): TODO; -declare function $parse(): TODO; -declare function $password(): TODO; -declare function $patch(): TODO; -declare function $path(): TODO; -declare function $pathname(): TODO; -declare function $pause(): TODO; -declare function $pendingAbortRequest(): TODO; -declare function $pendingPullIntos(): TODO; -declare function $pid(): TODO; -declare function $pipe(): TODO; -declare function $port(): TODO; -declare function $post(): TODO; -declare function $ppid(): TODO; -declare function $prependEventListener(): TODO; -declare function $process(): TODO; -declare function $protocol(): TODO; -declare function $pull(): TODO; -declare function $pullAgain(): TODO; -declare function $pullAlgorithm(): TODO; -declare function $pulling(): TODO; -declare function $put(): TODO; -declare function $queue(): TODO; -declare function $read(): TODO; -declare function $readIntoRequests(): TODO; -declare function $readRequests(): TODO; -declare function $readable(): TODO; -declare function $readableByteStreamControllerGetDesiredSize(...args: any): TODO; -declare function $readableStreamController(): TODO; -declare function $readableStreamToArray(): TODO; -declare function $reader(): TODO; -declare function $readyPromise(): TODO; -declare function $readyPromiseCapability(): TODO; -declare function $redirect(): TODO; -declare function $relative(): TODO; -declare function $releaseLock(): TODO; -declare function $removeEventListener(): TODO; -declare function $require(): TODO; -declare function $requireESM(path: string): any; -declare const $requireMap: Map<string, NodeModule>; -declare function $resolve(name: string, from: string): Promise<string>; -declare function $resolveSync(name: string, from: string, isESM?: boolean): string; -declare function $resume(): TODO; -declare function $search(): TODO; -declare function $searchParams(): TODO; -declare function $self(): TODO; -declare function $sep(): TODO; -declare function $setBody(): TODO; -declare function $setStatus(): TODO; -declare function $setup(): TODO; -declare function $sink(): TODO; -declare function $size(): TODO; -declare function $start(): TODO; -declare function $startAlgorithm(): TODO; -declare function $startConsumingStream(): TODO; -declare function $startDirectStream(): TODO; -declare function $started(): TODO; -declare function $startedPromise(): TODO; -declare function $state(): TODO; -declare function $status(): TODO; -declare function $storedError(): TODO; -declare function $strategy(): TODO; -declare function $strategyHWM(): TODO; -declare function $strategySizeAlgorithm(): TODO; -declare function $stream(): TODO; -declare function $streamClosed(): TODO; -declare function $streamClosing(): TODO; -declare function $streamErrored(): TODO; -declare function $streamReadable(): TODO; -declare function $streamWaiting(): TODO; -declare function $streamWritable(): TODO; -declare function $structuredCloneForStream(): TODO; -declare function $syscall(): TODO; -declare function $textDecoderStreamDecoder(): TODO; -declare function $textDecoderStreamTransform(): TODO; -declare function $textEncoderStreamEncoder(): TODO; -declare function $textEncoderStreamTransform(): TODO; -declare function $toNamespacedPath(): TODO; -declare function $trace(): TODO; -declare function $transformAlgorithm(): TODO; -declare function $uncork(): TODO; -declare function $underlyingByteSource(): TODO; -declare function $underlyingSink(): TODO; -declare function $underlyingSource(): TODO; -declare function $unpipe(): TODO; -declare function $unshift(): TODO; -declare function $url(): TODO; -declare function $username(): TODO; -declare function $version(): TODO; -declare function $versions(): TODO; -declare function $view(): TODO; -declare function $whenSignalAborted(signal: AbortSignal, cb: (reason: any) => void): TODO; -declare function $writable(): TODO; -declare function $write(): TODO; -declare function $writeAlgorithm(): TODO; -declare function $writeRequests(): TODO; -declare function $writer(): TODO; -declare function $writing(): TODO; -declare function $written(): TODO; - -declare function $createCommonJSModule(id: string, exports: any, hasEvaluated: boolean): NodeModule; - -// The following I cannot find any definitions of, but they are functional. -declare function $toLength(length: number): number; -declare function $isTypedArrayView(obj: unknown): obj is ArrayBufferView | DataView | Uint8Array; -declare function $setStateToMax(target: any, state: number): void; -declare function $trunc(target: number): number; -declare function $newPromiseCapability(C: PromiseConstructor): TODO; -/** @deprecated, use new TypeError instead */ -declare function $makeTypeError(message: string): TypeError; -declare function $newHandledRejectedPromise(error: unknown): Promise<never>; - -declare const __internal: unique symbol; -interface InternalFieldObject<T extends any[]> { - [__internal]: T; -} - -// Types used in the above functions -type PromiseFieldType = typeof $promiseFieldFlags | typeof $promiseFieldReactionsOrResult; -type PromiseFieldToValue<X extends PromiseFieldType, V> = X extends typeof $promiseFieldFlags - ? number - : X extends typeof $promiseFieldReactionsOrResult - ? V | any - : any; -type WellKnownSymbol = keyof { [K in keyof SymbolConstructor as SymbolConstructor[K] extends symbol ? K : never]: K }; - -// You can also `@` on any method on a classes to avoid prototype pollution and secret internals -type ClassWithIntrinsics<T> = { [K in keyof T as T[K] extends Function ? `$${K}` : never]: T[K] }; - -declare interface Map<K, V> extends ClassWithIntrinsics<Map<K, V>> {} -declare interface CallableFunction extends ClassWithIntrinsics<CallableFunction> {} -declare interface Promise<T> extends ClassWithIntrinsics<Promise<T>> {} -declare interface ArrayBufferConstructor<T> extends ClassWithIntrinsics<ArrayBufferConstructor<T>> {} -declare interface PromiseConstructor<T> extends ClassWithIntrinsics<PromiseConstructor<T>> {} - -declare interface UnderlyingSource { - $lazy?: boolean; - $bunNativeType?: number; - $bunNativePtr?: number; - autoAllocateChunkSize?: number; -} - -declare class OutOfMemoryError { - constructor(); -} - -declare class ReadableStreamDefaultController { - constructor( - stream: unknown, - underlyingSource: unknown, - size: unknown, - highWaterMark: unknown, - $isReadableStream: typeof $isReadableStream, - ); -} -declare class ReadableByteStreamController { - constructor( - stream: unknown, - underlyingSource: unknown, - strategy: unknown, - $isReadableStream: typeof $isReadableStream, - ); -} -declare class ReadableStreamBYOBRequest { - constructor(stream: unknown, view: unknown, $isReadableStream: typeof $isReadableStream); -} -declare class ReadableStreamBYOBReader { - constructor(stream: unknown); -} - -// Inlining our enum types -declare const $ImportKindIdToLabel: Array<import("bun").ImportKind>; -declare const $ImportKindLabelToId: Record<import("bun").ImportKind, number>; -declare const $LoaderIdToLabel: Array<import("bun").Loader>; -declare const $LoaderLabelToId: Record<import("bun").Loader, number>; - -// not a builtin, but a build-time macro of our own -/** Returns a not implemented error that points to a github issue. */ -declare function notImplementedIssue(issueNumber: number, description: string): Error; -/** Return a function that throws a not implemented error that points to a github issue */ -declare function notImplementedIssueFn(issueNumber: number, description: string): (...args: any[]) => never; - -declare type JSCSourceCodeObject = unique symbol; - -declare interface Function { - path: string; -} - -declare var $Buffer: { - new (a: any, b?: any, c?: any): Buffer; -}; - -declare interface Error { - code?: string; -} diff --git a/src/js/builtins/codegen/builtin-parser.ts b/src/js/builtins/codegen/builtin-parser.ts deleted file mode 100644 index e96d79c63..000000000 --- a/src/js/builtins/codegen/builtin-parser.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { applyReplacements } from "./replacements"; - -/** - * Slices a string until it hits a }, but keeping in mind JS comments, - * regex, template literals, comments, and matching { - * - * Used to extract function bodies without parsing the code. - * - * If you pass replace=true, it will run replacements on the code - */ -export function sliceSourceCode( - contents: string, - replace: boolean, -): { result: string; rest: string; usesThis: boolean } { - let bracketCount = 0; - let i = 0; - let result = ""; - let usesThis = false; - while (contents.length) { - // TODO: template literal, regexp - // these are important because our replacement logic would replace intrinsics - // within these, when it should remain as the literal dollar. - // but this isn't used in the codebase - i = contents.match(/\/\*|\/\/|'|"|{|}|`/)?.index ?? contents.length; - const chunk = replace ? applyReplacements(contents.slice(0, i)) : contents.slice(0, i); - if (chunk.includes("this")) usesThis = true; - result += chunk; - contents = contents.slice(i); - if (!contents.length) break; - if (contents.startsWith("/*")) { - i = contents.slice(2).indexOf("*/") + 2; - } else if (contents.startsWith("//")) { - i = contents.slice(2).indexOf("\n") + 2; - } else if (contents.startsWith("'")) { - i = contents.slice(1).match(/(?<!\\)'/)!.index! + 2; - } else if (contents.startsWith('"')) { - i = contents.slice(1).match(/(?<!\\)"/)!.index! + 2; - } else if (contents.startsWith("`")) { - const { result: result2, rest } = sliceTemplateLiteralSourceCode(contents.slice(1), replace); - result += "`" + result2; - contents = rest; - continue; - } else if (contents.startsWith("{")) { - bracketCount++; - i = 1; - } else if (contents.startsWith("}")) { - bracketCount--; - if (bracketCount <= 0) { - result += "}"; - contents = contents.slice(1); - break; - } - i = 1; - } else { - throw new Error("TODO"); - } - result += contents.slice(0, i); - contents = contents.slice(i); - } - - return { result, rest: contents, usesThis }; -} - -function sliceTemplateLiteralSourceCode(contents: string, replace: boolean) { - let i = 0; - let result = ""; - let usesThis = false; - while (contents.length) { - i = contents.match(/`|\${/)!.index!; - result += contents.slice(0, i); - contents = contents.slice(i); - if (!contents.length) break; - if (contents.startsWith("`")) { - result += "`"; - contents = contents.slice(1); - break; - } else if (contents.startsWith("$")) { - const { result: result2, rest, usesThis: usesThisVal } = sliceSourceCode(contents.slice(1), replace); - result += "$" + result2; - contents = rest; - usesThis ||= usesThisVal; - continue; - } else { - throw new Error("TODO"); - } - } - - return { result, rest: contents, usesThis }; -} diff --git a/src/js/builtins/codegen/helpers.ts b/src/js/builtins/codegen/helpers.ts deleted file mode 100644 index 6345f8ffa..000000000 --- a/src/js/builtins/codegen/helpers.ts +++ /dev/null @@ -1,25 +0,0 @@ -export function fmtCPPString(str: string) { - return ( - '"' + - str - .replace(/\\/g, "\\\\") - .replace(/"/g, '\\"') - .replace(/\n/g, "\\n") - .replace(/\r/g, "\\r") - .replace(/\t/g, "\\t") - .replace(/\?/g, "\\?") + // https://stackoverflow.com/questions/1234582 - '"' - ); -} - -export function cap(str: string) { - return str[0].toUpperCase() + str.slice(1); -} - -export function low(str: string) { - if (str.startsWith("JS")) { - return "js" + str.slice(2); - } - - return str[0].toLowerCase() + str.slice(1); -} diff --git a/src/js/builtins/codegen/index.ts b/src/js/builtins/codegen/index.ts deleted file mode 100644 index 564b69689..000000000 --- a/src/js/builtins/codegen/index.ts +++ /dev/null @@ -1,661 +0,0 @@ -import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "fs"; -import path from "path"; -import { sliceSourceCode } from "./builtin-parser"; -import { applyGlobalReplacements, enums, globalsToPrefix } from "./replacements"; -import { cap, fmtCPPString, low } from "./helpers"; -import { spawn } from "bun"; - -async function createStaticHashtables() { - const STATIC_HASH_TABLES = ["src/bun.js/bindings/Process.cpp"]; - console.time("Creating static hash tables..."); - const create_hash_table = path.join(import.meta.dir, "../../../../src/bun.js/scripts/create_hash_table"); - if (!create_hash_table) { - console.warn( - "Could not find create_hash_table executable. Run `bun i` or clone webkit to build static hash tables", - ); - return; - } - for (let cpp of STATIC_HASH_TABLES) { - cpp = path.join(import.meta.dir, "../../../../", cpp); - const { stdout, exited } = spawn({ - cmd: [create_hash_table, cpp], - stdout: "pipe", - stderr: "inherit", - }); - await exited; - let str = await new Response(stdout).text(); - str = str.replaceAll(/^\/\/.*$/gm, ""); - str = str.replaceAll(/^#include.*$/gm, ""); - str = str.replaceAll(`namespace JSC {`, ""); - str = str.replaceAll(`} // namespace JSC`, ""); - str = "// File generated via `make generate-builtins`\n" + str.trim() + "\n"; - await Bun.write(cpp.replace(/\.cpp$/, ".lut.h"), str); - } - console.timeEnd("Creating static hash tables..."); -} - -const staticHashTablePromise = createStaticHashtables(); -console.log("Bundling Bun builtins..."); - -const MINIFY = process.argv.includes("--minify") || process.argv.includes("-m"); -const PARALLEL = process.argv.includes("--parallel") || process.argv.includes("-p"); -const KEEP_TMP = process.argv.includes("--keep-tmp") || process.argv.includes("-k") || true; - -const SRC_DIR = path.join(import.meta.dir, "../"); -const OUT_DIR = path.join(SRC_DIR, "../out"); -const TMP_DIR = path.join(SRC_DIR, "../out/tmp"); - -if (existsSync(TMP_DIR)) rmSync(TMP_DIR, { recursive: true }); -mkdirSync(TMP_DIR); - -const define = { - "process.env.NODE_ENV": "development", - "process.platform": process.platform, - "process.arch": process.arch, -}; - -for (const name in enums) { - const value = enums[name]; - if (typeof value !== "object") throw new Error("Invalid enum object " + name + " defined in " + import.meta.file); - if (typeof value === null) throw new Error("Invalid enum object " + name + " defined in " + import.meta.file); - const keys = Array.isArray(value) ? value : Object.keys(value).filter(k => !k.match(/^[0-9]+$/)); - define[`__intrinsic__${name}IdToLabel`] = "[" + keys.map(k => `"${k}"`).join(", ") + "]"; - define[`__intrinsic__${name}LabelToId`] = "{" + keys.map(k => `"${k}": ${keys.indexOf(k)}`).join(", ") + "}"; -} - -for (const name of globalsToPrefix) { - define[name] = "__intrinsic__" + name; -} - -interface ParsedBuiltin { - name: string; - params: string[]; - directives: Record<string, any>; - source: string; - async: boolean; -} -interface BundledBuiltin { - name: string; - directives: Record<string, any>; - isGetter: boolean; - isConstructor: boolean; - isLinkTimeConstant: boolean; - isNakedConstructor: boolean; - intrinsic: string; - overriddenName: string; - source: string; - params: string[]; - visibility: string; -} - -/** - * Source .ts file --> Array<bundled js function code> - */ -async function processFileSplit(filename: string): Promise<{ functions: BundledBuiltin[]; internal: boolean }> { - const basename = path.basename(filename, ".ts"); - let contents = await Bun.file(filename).text(); - - contents = applyGlobalReplacements(contents); - - // first approach doesnt work perfectly because we actually need to split each function declaration - // and then compile those separately - - const consumeWhitespace = /^\s*/; - const consumeTopLevelContent = /^(\/\*|\/\/|type|import|interface|\$|export (?:async )?function|(?:async )?function)/; - const consumeEndOfType = /;|.(?=export|type|interface|\$|\/\/|\/\*|function)/; - - const functions: ParsedBuiltin[] = []; - let directives: Record<string, any> = {}; - const bundledFunctions: BundledBuiltin[] = []; - let internal = false; - - while (contents.length) { - contents = contents.replace(consumeWhitespace, ""); - if (!contents.length) break; - const match = contents.match(consumeTopLevelContent); - if (!match) { - throw new SyntaxError("Could not process input:\n" + contents.slice(0, contents.indexOf("\n"))); - } - contents = contents.slice(match.index!); - if (match[1] === "import") { - // TODO: we may want to do stuff with these - const i = contents.indexOf(";"); - contents = contents.slice(i + 1); - } else if (match[1] === "/*") { - const i = contents.indexOf("*/") + 2; - internal ||= contents.slice(0, i).includes("@internal"); - contents = contents.slice(i); - } else if (match[1] === "//") { - const i = contents.indexOf("\n") + 1; - internal ||= contents.slice(0, i).includes("@internal"); - contents = contents.slice(i); - } else if (match[1] === "type" || match[1] === "export type") { - const i = contents.search(consumeEndOfType); - contents = contents.slice(i + 1); - } else if (match[1] === "interface") { - contents = sliceSourceCode(contents, false).rest; - } else if (match[1] === "$") { - const directive = contents.match(/^\$([a-zA-Z0-9]+)(?:\s*=\s*([^\n]+?))?\s*;?\n/); - if (!directive) { - throw new SyntaxError("Could not parse directive:\n" + contents.slice(0, contents.indexOf("\n"))); - } - const name = directive[1]; - let value; - try { - value = directive[2] ? JSON.parse(directive[2]) : true; - } catch (error) { - throw new SyntaxError("Could not parse directive value " + directive[2] + " (must be JSON parsable)"); - } - if (name === "constructor") { - throw new SyntaxError("$constructor not implemented"); - } - if (name === "nakedConstructor") { - throw new SyntaxError("$nakedConstructor not implemented"); - } - directives[name] = value; - contents = contents.slice(directive[0].length); - } else if (match[1] === "export function" || match[1] === "export async function") { - const declaration = contents.match( - /^export\s+(async\s+)?function\s+([a-zA-Z0-9]+)\s*\(([^)]*)\)(?:\s*:\s*([^{\n]+))?\s*{?/, - ); - if (!declaration) - throw new SyntaxError("Could not parse function declaration:\n" + contents.slice(0, contents.indexOf("\n"))); - - const async = !!declaration[1]; - const name = declaration[2]; - const paramString = declaration[3]; - const params = - paramString.trim().length === 0 ? [] : paramString.split(",").map(x => x.replace(/:.+$/, "").trim()); - if (params[0] === "this") { - params.shift(); - } - - const { result, rest } = sliceSourceCode(contents.slice(declaration[0].length - 1), true); - functions.push({ - name, - params, - directives, - source: result.trim().slice(1, -1), - async, - }); - contents = rest; - directives = {}; - } else if (match[1] === "function" || match[1] === "async function") { - const fnname = contents.match(/^function ([a-zA-Z0-9]+)\(([^)]*)\)(?:\s*:\s*([^{\n]+))?\s*{?/)![1]; - throw new SyntaxError("All top level functions must be exported: " + fnname); - } else { - throw new Error("TODO: parse " + match[1]); - } - } - - for (const fn of functions) { - const tmpFile = path.join(TMP_DIR, `${basename}.${fn.name}.ts`); - - // not sure if this optimization works properly in jsc builtins - // const useThis = fn.usesThis; - const useThis = true; - - // TODO: we should use format=IIFE so we could bundle imports and extra functions. - await Bun.write( - tmpFile, - `// @ts-nocheck -// GENERATED TEMP FILE - DO NOT EDIT -// Sourced from ${path.relative(TMP_DIR, filename)} - -// do not allow the bundler to rename a symbol to $ -($); - -$$capture_start$$(${fn.async ? "async " : ""}${ - useThis - ? `function(${fn.params.join(",")})` - : `${fn.params.length === 1 ? fn.params[0] : `(${fn.params.join(",")})`}=>` - } {${fn.source}}).$$capture_end$$; -`, - ); - await Bun.sleep(1); - const build = await Bun.build({ - entrypoints: [tmpFile], - define, - minify: true, - }); - if (!build.success) { - throw new AggregateError(build.logs, "Failed bundling builtin function " + fn.name + " from " + basename + ".ts"); - } - if (build.outputs.length !== 1) { - throw new Error("expected one output"); - } - const output = await build.outputs[0].text(); - const captured = output.match(/\$\$capture_start\$\$([\s\S]+)\.\$\$capture_end\$\$/)![1]; - const finalReplacement = - (fn.directives.sloppy ? captured : captured.replace(/function\s*\(.*?\)\s*{/, '$&"use strict";')) - .replace(/^\((async )?function\(/, "($1function (") - .replace(/__intrinsic__lazy\(/g, "globalThis[globalThis.Symbol.for('Bun.lazy')](") - .replace(/__intrinsic__/g, "@") + "\n"; - - bundledFunctions.push({ - name: fn.name, - directives: fn.directives, - source: finalReplacement, - params: fn.params, - visibility: fn.directives.visibility ?? (fn.directives.linkTimeConstant ? "Private" : "Public"), - isGetter: !!fn.directives.getter, - isConstructor: !!fn.directives.constructor, - isLinkTimeConstant: !!fn.directives.linkTimeConstant, - isNakedConstructor: !!fn.directives.nakedConstructor, - intrinsic: fn.directives.intrinsic ?? "NoIntrinsic", - overriddenName: fn.directives.getter - ? `"get ${fn.name}"_s` - : fn.directives.overriddenName - ? `"${fn.directives.overriddenName}"_s` - : "ASCIILiteral()", - }); - } - - return { - functions: bundledFunctions, - internal, - }; -} - -const filesToProcess = readdirSync(SRC_DIR).filter(x => x.endsWith(".ts") && !x.endsWith(".d.ts")); - -const files: Array<{ basename: string; functions: BundledBuiltin[]; internal: boolean }> = []; -async function processFile(x: string) { - const basename = path.basename(x, ".ts"); - try { - files.push({ - basename, - ...(await processFileSplit(path.join(SRC_DIR, x))), - }); - } catch (error) { - console.error("Failed to process file: " + basename + ".ts"); - console.error(error); - process.exit(1); - } -} - -// Bun seems to crash if this is parallelized, :( -if (PARALLEL) { - await Promise.all(filesToProcess.map(processFile)); -} else { - for (const x of filesToProcess) { - await processFile(x); - } -} - -// C++ codegen -let bundledCPP = `// Generated by \`bun src/js/builtins/codegen\` -// Do not edit by hand. -namespace Zig { class GlobalObject; } -#include "root.h" -#include "config.h" -#include "JSDOMGlobalObject.h" -#include "WebCoreJSClientData.h" -#include <JavaScriptCore/JSObjectInlines.h> - -namespace WebCore { - -`; - -for (const { basename, functions } of files) { - bundledCPP += `/* ${basename}.ts */\n`; - const lowerBasename = low(basename); - for (const fn of functions) { - const name = `${lowerBasename}${cap(fn.name)}Code`; - bundledCPP += `// ${fn.name} -const JSC::ConstructAbility s_${name}ConstructAbility = JSC::ConstructAbility::CannotConstruct; -const JSC::ConstructorKind s_${name}ConstructorKind = JSC::ConstructorKind::None; -const JSC::ImplementationVisibility s_${name}ImplementationVisibility = JSC::ImplementationVisibility::${fn.visibility}; -const int s_${name}Length = ${fn.source.length}; -static const JSC::Intrinsic s_${name}Intrinsic = JSC::NoIntrinsic; -const char* const s_${name} = ${fmtCPPString(fn.source)}; - -`; - } - bundledCPP += `#define DEFINE_BUILTIN_GENERATOR(codeName, functionName, overriddenName, argumentCount) \\ -JSC::FunctionExecutable* codeName##Generator(JSC::VM& vm) \\ -{\\ - JSVMClientData* clientData = static_cast<JSVMClientData*>(vm.clientData); \\ - return clientData->builtinFunctions().${lowerBasename}Builtins().codeName##Executable()->link(vm, nullptr, clientData->builtinFunctions().${lowerBasename}Builtins().codeName##Source(), std::nullopt, s_##codeName##Intrinsic); \\ -} -WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_CODE(DEFINE_BUILTIN_GENERATOR) -#undef DEFINE_BUILTIN_GENERATOR - -`; -} - -bundledCPP += ` - -JSBuiltinInternalFunctions::JSBuiltinInternalFunctions(JSC::VM& vm) - : m_vm(vm) -`; - -for (const { basename, internal } of files) { - if (internal) { - bundledCPP += ` , m_${low(basename)}(vm)\n`; - } -} - -bundledCPP += ` -{ - UNUSED_PARAM(vm); -} - -template<typename Visitor> -void JSBuiltinInternalFunctions::visit(Visitor& visitor) -{ -`; -for (const { basename, internal } of files) { - if (internal) bundledCPP += ` m_${low(basename)}.visit(visitor);\n`; -} - -bundledCPP += ` - UNUSED_PARAM(visitor); -} - -template void JSBuiltinInternalFunctions::visit(AbstractSlotVisitor&); -template void JSBuiltinInternalFunctions::visit(SlotVisitor&); - -SUPPRESS_ASAN void JSBuiltinInternalFunctions::initialize(Zig::GlobalObject& globalObject) -{ - UNUSED_PARAM(globalObject); -`; - -for (const { basename, internal } of files) { - if (internal) { - bundledCPP += ` m_${low(basename)}.init(globalObject);\n`; - } -} - -bundledCPP += ` - JSVMClientData& clientData = *static_cast<JSVMClientData*>(m_vm.clientData); - Zig::GlobalObject::GlobalPropertyInfo staticGlobals[] = { -`; - -for (const { basename, internal } of files) { - if (internal) { - bundledCPP += `#define DECLARE_GLOBAL_STATIC(name) \\ - Zig::GlobalObject::GlobalPropertyInfo( \\ - clientData.builtinFunctions().${low(basename)}Builtins().name##PrivateName(), ${low( - basename, - )}().m_##name##Function.get() , JSC::PropertyAttribute::DontDelete | JSC::PropertyAttribute::ReadOnly), - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_FUNCTION_NAME(DECLARE_GLOBAL_STATIC) - #undef DECLARE_GLOBAL_STATIC - `; - } -} - -bundledCPP += ` - }; - globalObject.addStaticGlobals(staticGlobals, std::size(staticGlobals)); - UNUSED_PARAM(clientData); -} - -} // namespace WebCore -`; - -// C++ Header codegen -let bundledHeader = `// Generated by \`bun src/js/builtins/codegen\` -// Do not edit by hand. -#pragma once -namespace Zig { class GlobalObject; } -#include "root.h" -#include <JavaScriptCore/BuiltinUtils.h> -#include <JavaScriptCore/Identifier.h> -#include <JavaScriptCore/JSFunction.h> -#include <JavaScriptCore/UnlinkedFunctionExecutable.h> -#include <JavaScriptCore/VM.h> -#include <JavaScriptCore/WeakInlines.h> - -namespace JSC { -class FunctionExecutable; -} - -namespace WebCore { -`; -for (const { basename, functions, internal } of files) { - bundledHeader += `/* ${basename}.ts */ -`; - const lowerBasename = low(basename); - - for (const fn of functions) { - const name = `${lowerBasename}${cap(fn.name)}Code`; - bundledHeader += `// ${fn.name} -#define WEBCORE_BUILTIN_${basename.toUpperCase()}_${fn.name.toUpperCase()} 1 -extern const char* const s_${name}; -extern const int s_${name}Length; -extern const JSC::ConstructAbility s_${name}ConstructAbility; -extern const JSC::ConstructorKind s_${name}ConstructorKind; -extern const JSC::ImplementationVisibility s_${name}ImplementationVisibility; - -`; - } - bundledHeader += `#define WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_DATA(macro) \\\n`; - for (const fn of functions) { - bundledHeader += ` macro(${fn.name}, ${lowerBasename}${cap(fn.name)}, ${fn.params.length}) \\\n`; - } - bundledHeader += "\n"; - bundledHeader += `#define WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_CODE(macro) \\\n`; - for (const fn of functions) { - const name = `${lowerBasename}${cap(fn.name)}Code`; - bundledHeader += ` macro(${name}, ${fn.name}, ${fn.overriddenName}, s_${name}Length) \\\n`; - } - bundledHeader += "\n"; - bundledHeader += `#define WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_FUNCTION_NAME(macro) \\\n`; - for (const fn of functions) { - bundledHeader += ` macro(${fn.name}) \\\n`; - } - bundledHeader += ` -#define DECLARE_BUILTIN_GENERATOR(codeName, functionName, overriddenName, argumentCount) \\ - JSC::FunctionExecutable* codeName##Generator(JSC::VM&); - -WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_CODE(DECLARE_BUILTIN_GENERATOR) -#undef DECLARE_BUILTIN_GENERATOR - -class ${basename}BuiltinsWrapper : private JSC::WeakHandleOwner { -public: - explicit ${basename}BuiltinsWrapper(JSC::VM& vm) - : m_vm(vm) - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_FUNCTION_NAME(INITIALIZE_BUILTIN_NAMES) -#define INITIALIZE_BUILTIN_SOURCE_MEMBERS(name, functionName, overriddenName, length) , m_##name##Source(JSC::makeSource(StringImpl::createWithoutCopying(s_##name, length), { })) - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_CODE(INITIALIZE_BUILTIN_SOURCE_MEMBERS) -#undef INITIALIZE_BUILTIN_SOURCE_MEMBERS - { - } - -#define EXPOSE_BUILTIN_EXECUTABLES(name, functionName, overriddenName, length) \\ - JSC::UnlinkedFunctionExecutable* name##Executable(); \\ - const JSC::SourceCode& name##Source() const { return m_##name##Source; } - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_CODE(EXPOSE_BUILTIN_EXECUTABLES) -#undef EXPOSE_BUILTIN_EXECUTABLES - - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_FUNCTION_NAME(DECLARE_BUILTIN_IDENTIFIER_ACCESSOR) - - void exportNames(); - -private: - JSC::VM& m_vm; - - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_FUNCTION_NAME(DECLARE_BUILTIN_NAMES) - -#define DECLARE_BUILTIN_SOURCE_MEMBERS(name, functionName, overriddenName, length) \\ - JSC::SourceCode m_##name##Source;\\ - JSC::Weak<JSC::UnlinkedFunctionExecutable> m_##name##Executable; - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_CODE(DECLARE_BUILTIN_SOURCE_MEMBERS) -#undef DECLARE_BUILTIN_SOURCE_MEMBERS - -}; - -#define DEFINE_BUILTIN_EXECUTABLES(name, functionName, overriddenName, length) \\ -inline JSC::UnlinkedFunctionExecutable* ${basename}BuiltinsWrapper::name##Executable() \\ -{\\ - if (!m_##name##Executable) {\\ - JSC::Identifier executableName = functionName##PublicName();\\ - if (overriddenName)\\ - executableName = JSC::Identifier::fromString(m_vm, overriddenName);\\ - m_##name##Executable = JSC::Weak<JSC::UnlinkedFunctionExecutable>(JSC::createBuiltinExecutable(m_vm, m_##name##Source, executableName, s_##name##ImplementationVisibility, s_##name##ConstructorKind, s_##name##ConstructAbility), this, &m_##name##Executable);\\ - }\\ - return m_##name##Executable.get();\\ -} -WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_CODE(DEFINE_BUILTIN_EXECUTABLES) -#undef DEFINE_BUILTIN_EXECUTABLES - -inline void ${basename}BuiltinsWrapper::exportNames() -{ -#define EXPORT_FUNCTION_NAME(name) m_vm.propertyNames->appendExternalName(name##PublicName(), name##PrivateName()); - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_FUNCTION_NAME(EXPORT_FUNCTION_NAME) -#undef EXPORT_FUNCTION_NAME -} -`; - - if (internal) { - bundledHeader += `class ${basename}BuiltinFunctions { -public: - explicit ${basename}BuiltinFunctions(JSC::VM& vm) : m_vm(vm) { } - - void init(JSC::JSGlobalObject&); - template<typename Visitor> void visit(Visitor&); - -public: - JSC::VM& m_vm; - -#define DECLARE_BUILTIN_SOURCE_MEMBERS(functionName) \\ - JSC::WriteBarrier<JSC::JSFunction> m_##functionName##Function; - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_FUNCTION_NAME(DECLARE_BUILTIN_SOURCE_MEMBERS) -#undef DECLARE_BUILTIN_SOURCE_MEMBERS -}; - -inline void ${basename}BuiltinFunctions::init(JSC::JSGlobalObject& globalObject) -{ -#define EXPORT_FUNCTION(codeName, functionName, overriddenName, length) \\ - m_##functionName##Function.set(m_vm, &globalObject, JSC::JSFunction::create(m_vm, codeName##Generator(m_vm), &globalObject)); - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_CODE(EXPORT_FUNCTION) -#undef EXPORT_FUNCTION -} - -template<typename Visitor> -inline void ${basename}BuiltinFunctions::visit(Visitor& visitor) -{ -#define VISIT_FUNCTION(name) visitor.append(m_##name##Function); - WEBCORE_FOREACH_${basename.toUpperCase()}_BUILTIN_FUNCTION_NAME(VISIT_FUNCTION) -#undef VISIT_FUNCTION -} - -template void ${basename}BuiltinFunctions::visit(JSC::AbstractSlotVisitor&); -template void ${basename}BuiltinFunctions::visit(JSC::SlotVisitor&); - `; - } -} -bundledHeader += `class JSBuiltinFunctions { -public: - explicit JSBuiltinFunctions(JSC::VM& vm) - : m_vm(vm) -`; - -for (const { basename } of files) { - bundledHeader += ` , m_${low(basename)}Builtins(m_vm)\n`; -} - -bundledHeader += ` - { -`; - -for (const { basename, internal } of files) { - if (internal) { - bundledHeader += ` m_${low(basename)}Builtins.exportNames();\n`; - } -} - -bundledHeader += ` } -`; - -for (const { basename } of files) { - bundledHeader += ` ${basename}BuiltinsWrapper& ${low(basename)}Builtins() { return m_${low( - basename, - )}Builtins; }\n`; -} - -bundledHeader += ` -private: - JSC::VM& m_vm; -`; - -for (const { basename } of files) { - bundledHeader += ` ${basename}BuiltinsWrapper m_${low(basename)}Builtins;\n`; -} - -bundledHeader += `; -}; - -class JSBuiltinInternalFunctions { -public: - explicit JSBuiltinInternalFunctions(JSC::VM&); - - template<typename Visitor> void visit(Visitor&); - void initialize(Zig::GlobalObject&); -`; - -for (const { basename, internal } of files) { - if (internal) { - bundledHeader += ` ${basename}BuiltinFunctions& ${low(basename)}() { return m_${low(basename)}; }\n`; - } -} - -bundledHeader += ` -private: - JSC::VM& m_vm; -`; - -for (const { basename, internal } of files) { - if (internal) { - bundledHeader += ` ${basename}BuiltinFunctions m_${low(basename)};\n`; - } -} - -bundledHeader += ` -}; - -} // namespace WebCore -`; - -await Bun.write(path.join(OUT_DIR, "WebCoreJSBuiltins.h"), bundledHeader); -await Bun.write(path.join(OUT_DIR, "WebCoreJSBuiltins.cpp"), bundledCPP); - -// Generate TS types -let dts = `// Generated by \`bun src/js/builtins/codegen\` -// Do not edit by hand. -type RemoveThis<F> = F extends (this: infer T, ...args: infer A) => infer R ? (...args: A) => R : F; -`; - -for (const { basename, functions, internal } of files) { - if (internal) { - dts += `\n// ${basename}.ts\n`; - for (const fn of functions) { - dts += `declare const \$${fn.name}: RemoveThis<typeof import("${path.relative( - OUT_DIR, - path.join(SRC_DIR, basename), - )}")[${JSON.stringify(fn.name)}]>;\n`; - } - } -} - -await Bun.write(path.join(OUT_DIR, "WebCoreJSBuiltins.d.ts"), dts); - -const totalJSSize = files.reduce( - (acc, { functions }) => acc + functions.reduce((acc, fn) => acc + fn.source.length, 0), - 0, -); - -if (!KEEP_TMP) { - await rmSync(TMP_DIR, { recursive: true }); -} - -await staticHashTablePromise; - -console.log( - `Embedded JS size: %s bytes (across %s functions, %s files)`, - totalJSSize, - files.reduce((acc, { functions }) => acc + functions.length, 0), - files.length, -); -console.log(`[${performance.now().toFixed(1)}ms]`); diff --git a/src/js/builtins/codegen/replacements.ts b/src/js/builtins/codegen/replacements.ts deleted file mode 100644 index 87157a587..000000000 --- a/src/js/builtins/codegen/replacements.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { LoaderKeys } from "../../../api/schema"; - -// This is a list of extra syntax replacements to do. Kind of like macros -// These are only run on code itself, not string contents or comments. -export const replacements: ReplacementRule[] = [ - { from: /\bthrow new TypeError\b/g, to: "$throwTypeError" }, - { from: /\bthrow new RangeError\b/g, to: "$throwRangeError" }, - { from: /\bthrow new OutOfMemoryError\b/g, to: "$throwOutOfMemoryError" }, - { from: /\bnew TypeError\b/g, to: "$makeTypeError" }, -]; - -// These rules are run on the entire file, including within strings. -export const globalReplacements: ReplacementRule[] = [ - { - from: /\bnotImplementedIssue\(\s*([0-9]+)\s*,\s*((?:"[^"]*"|'[^']+'))\s*\)/g, - to: "new TypeError(`${$2} is not implemented yet. See https://github.com/oven-sh/bun/issues/$1`)", - }, - { - from: /\bnotImplementedIssueFn\(\s*([0-9]+)\s*,\s*((?:"[^"]*"|'[^']+'))\s*\)/g, - to: "() => $throwTypeError(`${$2} is not implemented yet. See https://github.com/oven-sh/bun/issues/$1`)", - }, -]; - -// This is a list of globals we should access using @ notation -// undefined -> __intrinsic__undefined -> @undefined -export const globalsToPrefix = [ - "AbortSignal", - "Array", - "ArrayBuffer", - "Buffer", - "Bun", - "Infinity", - "Loader", - "Promise", - "ReadableByteStreamController", - "ReadableStream", - "ReadableStreamBYOBReader", - "ReadableStreamBYOBRequest", - "ReadableStreamDefaultController", - "ReadableStreamDefaultReader", - "TransformStream", - "TransformStreamDefaultController", - "Uint8Array", - "WritableStream", - "WritableStreamDefaultController", - "WritableStreamDefaultWriter", - "isFinite", - "undefined", -]; - -// These enums map to $<enum>IdToLabel and $<enum>LabelToId -// Make sure to define in ./builtins.d.ts -export const enums = { - Loader: LoaderKeys, - ImportKind: [ - "entry-point", - "import-statement", - "require-call", - "dynamic-import", - "require-resolve", - "import-rule", - "url-token", - "internal", - ], -}; - -// These identifiers have typedef but not present at runtime (converted with replacements) -// If they are present in the bundle after runtime, we warn at the user. -// TODO: implement this check. -export const warnOnIdentifiersNotPresentAtRuntime = [ - // - "OutOfMemoryError", - "notImplementedIssue", - "notImplementedIssueFn", -]; - -export interface ReplacementRule { - from: RegExp; - to: string; - global?: boolean; -} - -/** Applies source code replacements as defined in `replacements` */ -export function applyReplacements(src: string) { - let result = src.replace(/\$([a-zA-Z0-9_]+)\b/gm, `__intrinsic__$1`); - for (const replacement of replacements) { - result = result.replace(replacement.from, replacement.to.replaceAll("$", "__intrinsic__")); - } - return result; -} - -/** Applies source code replacements as defined in `globalReplacements` */ -export function applyGlobalReplacements(src: string) { - let result = src; - for (const replacement of globalReplacements) { - result = result.replace(replacement.from, replacement.to.replaceAll("$", "__intrinsic__")); - } - return result; -} diff --git a/src/js/builtins/tsconfig.json b/src/js/builtins/tsconfig.json index 6cdbe0eef..edaedd2a9 100644 --- a/src/js/builtins/tsconfig.json +++ b/src/js/builtins/tsconfig.json @@ -6,8 +6,9 @@ "include": [ ".", "../private.d.ts", - "builtins.d.ts", + "../builtins.d.ts", "../out/WebCoreJSBuiltins.d.ts", - "../../../packages/bun-types/index.d.ts" + "../../../packages/bun-types/index.d.ts", + "../_codegen/builtin-parser.ts" ] } |