From 6362414d65b69cd01624e84d08eca654fc8cb101 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 7 Apr 2023 20:08:01 -0700 Subject: Bun gets a new bundler (#2312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * alright now just gotta try running it * fix a gajillion compiler errors * even more code * okay i fixed more errors * wip * Update launch.json * Update string_builder.zig * `fast_debug_build_mode` makes debug build 2x faster * Update bundle_v2.zig * more code! * It bundles! * Rename `Bun.Transpiler` to `Bun.Bundler` * `import()` expressions almost work * wip attempt to get import() expr to work * Bundle namespace imports * Attempt to fix the issue with import() unsuccessfully * consider current working directory when resolving relative paths (#2313) * consider current working directory when resolving relative paths fixes #2298 * comment test --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> * support `expect().toThrow(/pattern/)` (#2314) - fix time-zone-dependent test failure * fix missing `Blob` error messages on Linux (#2315) * fix & clean up tests (#2318) - skip flaky tests when running as `root` - use `expect().toThrow()` - clean up temporary files after tests * feat(tty): add some `tty.WriteStream` methods to `process.{stdout, stderr}` (#2320) * feat(stdio): add some `tty.WriteStream` methods * chore(builtins): add process builtin gen'd code * Fix docker install command * `bun test` on macOS in GitHub Actions (#2322) * Fixes #2323 * throw invalid parameter errors in `crypto.scryptSync` (#2331) * throw invalid parameter errors * remove comptime, add empty buffer function * remove error_name comptime * Add reference documentation for bun:test (#2327) * Reorganize tests (#2332) * Fix html-rewriter.test.js * fix the wrong thing being incremented in hmr example (#2334) * Add more test harness * Improve Benchmarking page, small fixes (#2339) * Improve benchmarking page * WIP * Add typescript instructions to hot * Document preload in Plugins. Fix loader in plugin types. * Fix typo * Fix links * run prettier * Document openInEditor * improve `Buffer` compatibility with Node.js (#2341) * improve `Buffer` compatibility with Node.js * use `memmove()` allow `encoding` to be `undefined` * run `bun test` after macOS builds (#2343) * "binary" is an alias of "latin1" Fixes https://github.com/oven-sh/bun/issues/2110 * More spec compliant `Blob.prototype.type` (#2340) * Make `Blob.prototype. type` more spec compliant * Add a few more checks for isNumber() * Fix `make headers` * Safer JSValue.isString() * More tests for blob.slice * Make `Blob.prototype.type` more spec compliant * Add isASCII check * Fix types * Fix failing type test * Update blob.zig * Update blob.zig * Fix .eql check on empty values --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> * Fix bug in test runner * Support `import()` expressions * Implement `require()` * clean up bit_set.zig slightly * Move some things around * misc cleanup * Cleanup some things * Fix a lot of stuff * Fix `module.exports.fn = fn;` in ESM entry point * Fix crash due when printing file * Fix issue with class names * Fix issue with `export default identifier` * Update js_parser.zig * optimization: inline single-property object acceses and arrays * Fix undefined memory in renamed symbols list * Handle call target * wip * Inline it * Fix undefined memory issue when reclaiming blocks in ast * Halt linking on any parse errors * alias * Rename `enable_bundling` to `enable_legacy_bundling` * Workaround anonymous struct literal zig bug * Use slower approach (without bitset) because it doesn't break after 8 symbols * Fix incorrectly-renaming statically defined symbols * Handle more edgecases in our bit_set fork * Reduce number of allocations for `define` * Do not rename unbound symbols * Clean up dot defines a little more * Make the generated names prettier * Workaround runtime symbol missing issue * Fail the build on errors * Support export * from * Support `--outfile` * partially fix renaming * fanicer symbol renaming impl * misc, extremely revertible cleanup * Fix up some bugs with symbol renaming * formatting * Update launch.json * Parse `__PURE__` comments * clean up simd code for pure comments * changes to merge * workaround runtime issue * Fix issue with `export * as` not propagating correctly * Make all top-level declarations `var` when bundling * Fix missing prefix * Fix assigning to stack copy * Fix missing runtime symbol * Fix bug with namespace exports * Dramatically reduce allocations * Update launch.json * Add missing flags * Update js_parser.zig * small cleanup * Make the export name better * Fix unnecessary `var foo = foo` * Implement CommonJS -> ESM conversion * Implement module redirects * Port esbuild bundler tests for new bundler (#2380) * started porting esbuild tests * clean up test names and api before moving on * port tests using a program i wrote * replace todo generated comment * fix generated tests not including some files * work on tests * [github web editor] add define, external, inject, minifySyntax, minifyWhitespace options. * get most of the todo comments out of the way, but expectBundled does not handle most of the cases * continue working on esbuild tests * use test.skip for unsupported tests * Fixups for test runner * Hoist imports & exports * Fix test * Hoist classes * bundler test refining, 51/835 * Fix runtime require * bundler test refining, 81/835 * bundler test refining, 93/835 * Make the test work in any timezone * feat(expect): update toBeInstanceOf (#2396) * feat: update instanceof binding * fix: according to PR comments * Rename `expectObjectTypeCount` to `expectMaxObjectTypeCount` * Fix socket tests with connection errors (#2403) * release pending activity with connection error handler * unref poll_ref * remove trailing comma * Organize Dockerfiles for official status * Remove test Dockerfile * Remove old Docker workflow * Feat(test): add toMatch (#2404) * Fix various fetch/response/request tests (#2416) * fix most fetch tests, skip a few * fastGet, toValueGC, and invalid init * bigint unreachable, range error, log process as process * remove extra fetch_headers * remove js_type parameter, check isObject() * throw invalid mime type error, use enum literal * switch back to promise rejection * RangeError pascal case * Fix several bugs (#2418) * utf16 codepoint with replacement character * Fix test failure with `TextEncoder("ascii')` * Add missing type * Fix Response.prototype.bodyUsed and Request.prototype.bodyUsed * Fix bug with scrypt error not clearing * Update server.zig * oopsie * :nail_care: * docs: Use correct url in the 'Issues' link in README header (#2420) * Fix crash when rendering error page and the server or network is slow * [fetch] Make the default body value `null` when unspecified This is better aligned with the fetch spec * Make node-net tests less flaky * [node:net] Fix issue with `listen` callback firing before it's listening * Always clear timers in node test harness * Fix out of bounds access Repro'd in Buffer tests * Update UWS cc @cirospaciari * Make this test more thorough * Hanging abort test * 0 length body is a null stream * Several bug fixes (#2427) * Fix test * Fix segfault when unexpected type is passed in `expect().toThrow` * Fix issues with request constructor * Don't bother cloning headers when its empty * woops * more tests * fix incorrect test * Make the fetch error messages better * Update response.zig * Fix test that failed on macOS * Fix test * Remove extra hash table lookups * Support running dummy registry directly cc @alexlamsl * Update test * Update test * fixup * Workaround crash in test runner * Fixup test * Fixup test * Update os.test.js --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> * Remove usages of port numbers in tests * Set -O2 and -fno-rtti * Remove -g * Prevent undefined memory access * [bun test] Implement `--rerun-each` flag to run each test N times * Reduce number of module scopes created * add some extra abort checks into streams (#2430) * add some checks to avoid UAF * avoid multiple calls to finalize if endFromJS is called more than once * fix no-op comment * mark as requested_end on abort * remove requested_end from abort * remove unnecessary check (#2432) * Fix bug with scoped aliased dependencies in bun install on macOS * remove `addLog`, remove `--prominent-compile-errors` * Finish the upgrade * Optional chaining flag * Implement same_target_becomes_destructuring optimization * bundler test refining, 109/835 * Reset bindings * Support multiple entry points * Implement `--entry-names` flag * Use a tempdir with a better name * prettier * Log file name * Update js_parser.zig * Mark all bun builtins as external * Make resolve errors actually errors * Update bundler_default.test.ts * Fix `await import(foo)` * WIP react server components * Do more stuff at runtime * :scissors: * Support automatic JSX imports * Use a module cache for now * Update tsconfig.base.json * Fix ThisOutsideFunctionNotRenamed * woopsie * moar cpu * clamp it * fixup * Add a bunch of assertions * Bun uses automatic runtime by default * Parse Import Attributes * Add a note about Valgrind * Update developing.md * Fix up code splitting for React Server Components * Implement client component manifest * Fix crash with --react-server-components and no client components * Backport https://github.com/ziglang/zig/commit/4d31e3c917a05541394c544708f0047cfb53331a * Update launch.json * Fix for latest zig * Workaround bug with ?[]const string Occasionally saw alignment errors in this code Workaround https://github.com/ziglang/zig/issues/15085 related: https://github.com/ziglang/zig/pull/15089 * switch to regular slice * Avoid initializing named_imports and named_exports as undefined * Reduce usages of `undefined` * Add more assertions * --watch wip * Update javascript.zig * Possibly fix the race condition * Faster `do` * bump allocator * Reduce the size of `Symbol` slightly * Alphabetically sort runtime import symbols, for determinism * Prepare for code splitting * handle overlapping stdout * pure * clean up some things * Fix bug with `$$typeof` * Address CommonJS -> ESM hoisting bug * Support `"use server"` in manifest * Implement `"use server"` * Fix importing bun builtins when bundling * Make `commonjs_to_esm` a feature flag, fix some splitting bugs * :scissors: * fixme remove this * Fix crash in longestCommonPath * Chunking! Just need to do import paths now. * Import paths work...now trying to figure out how to make runtime symbols work * add workaround * Replace `bun bun` with `bun build` * Fix crash with dual package hazard * Fix many CommonJS <> ESM interop bugs * Support package.json `"sideEffects"` also skip loading unnecessary package.json data in `bun run` * add a not good --watch implementation * bundler test refining, 140/831 * remove accidentally committed file * do not return status code 1 on successful bundles * bundler test refining, 159/830 * pass exit code to exitOrWatch * clean up help menu -remove two spaces to line up bun build -moved all tags to the end of the text they are colorizing -moved other colors to the start of the text they colorize -removed unneeded tags, keeping only one at the start of the block * importstar is fully ported * wip * you can run code in this branch now * Disable this transform * organize and document bundler tests * Fix double import * Fix sloppy mode function declarations * Disable our CommonJS transform for now * add `assertNotPresent` to make splitting cases easier * Bump! * Update bun.d.ts * use import.meta.require in runtime code * Disable this again * Fix dirname * Fix ESM -> CJS wrapper * :nail_care: --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> Co-authored-by: Alex Lam S.L Co-authored-by: Derrick Farris Co-authored-by: Ashcon Partovi Co-authored-by: Dylan Conway <35280289+dylan-conway@users.noreply.github.com> Co-authored-by: pfg Co-authored-by: Colin McDonnell Co-authored-by: dave caruso Co-authored-by: zhiyuan <32867472+zhiyuang@users.noreply.github.com> Co-authored-by: Dylan Conway Co-authored-by: Kamil Ogórek Co-authored-by: Ciro Spaciari --- src/resolver/package_json.zig | 232 ++++++++++++++++++++++++++---------------- 1 file changed, 143 insertions(+), 89 deletions(-) (limited to 'src/resolver/package_json.zig') diff --git a/src/resolver/package_json.zig b/src/resolver/package_json.zig index 0785e0493..f7ff5f3e4 100644 --- a/src/resolver/package_json.zig +++ b/src/resolver/package_json.zig @@ -37,6 +37,7 @@ const FolderResolver = @import("../install/resolvers/folder_resolver.zig"); const Architecture = @import("../install/npm.zig").Architecture; const OperatingSystem = @import("../install/npm.zig").OperatingSystem; +pub const SideEffectsMap = std.HashMapUnmanaged(bun.StringHashMapUnowned.Key, void, bun.StringHashMapUnowned.Adapter, 80); pub const DependencyMap = struct { map: HashMap = .{}, source_buf: []const u8 = "", @@ -72,6 +73,7 @@ pub const PackageJSON = struct { } const node_modules_path = std.fs.path.sep_str ++ "node_modules" ++ std.fs.path.sep_str; + pub fn nameForImport(this: *const PackageJSON, allocator: std.mem.Allocator) !string { if (strings.indexOf(this.source.path.text, node_modules_path)) |_| { return this.name; @@ -110,6 +112,12 @@ pub const PackageJSON = struct { package_manager_package_id: Install.PackageID = Install.invalid_package_id, dependencies: DependencyMap = .{}, + side_effects: union(enum) { + unspecified: void, + false: void, + map: SideEffectsMap, + } = .{ .unspecified = {} }, + // Present if the "browser" field is present. This field is intended to be // used by bundlers and lets you redirect the paths of certain 3rd-party // modules that don't work in the browser to other modules that shim that @@ -641,107 +649,155 @@ pub const PackageJSON = struct { } } - if (json.asProperty("type")) |type_json| { - if (type_json.expr.asString(r.allocator)) |type_str| { - switch (options.ModuleType.List.get(type_str) orelse options.ModuleType.unknown) { - .cjs => { - package_json.module_type = .cjs; - }, - .esm => { - package_json.module_type = .esm; - }, - .unknown => { - r.log.addRangeWarningFmt( - &json_source, - json_source.rangeOfString(type_json.loc), - r.allocator, - "\"{s}\" is not a valid value for \"type\" field (must be either \"commonjs\" or \"module\")", - .{type_str}, - ) catch unreachable; - }, + // If we're coming from `bun run` + // We do not need to parse all this stuff. + if (comptime !include_scripts) { + if (json.asProperty("type")) |type_json| { + if (type_json.expr.asString(r.allocator)) |type_str| { + switch (options.ModuleType.List.get(type_str) orelse options.ModuleType.unknown) { + .cjs => { + package_json.module_type = .cjs; + }, + .esm => { + package_json.module_type = .esm; + }, + .unknown => { + r.log.addRangeWarningFmt( + &json_source, + json_source.rangeOfString(type_json.loc), + r.allocator, + "\"{s}\" is not a valid value for \"type\" field (must be either \"commonjs\" or \"module\")", + .{type_str}, + ) catch unreachable; + }, + } + } else { + r.log.addWarning(&json_source, type_json.loc, "The value for \"type\" must be a string") catch unreachable; } - } else { - r.log.addWarning(&json_source, type_json.loc, "The value for \"type\" must be a string") catch unreachable; } - } - // Read the "main" fields - for (r.opts.main_fields) |main| { - if (json.asProperty(main)) |main_json| { - const expr: js_ast.Expr = main_json.expr; + // Read the "main" fields + for (r.opts.main_fields) |main| { + if (json.asProperty(main)) |main_json| { + const expr: js_ast.Expr = main_json.expr; - if ((expr.asString(r.allocator))) |str| { - if (str.len > 0) { - package_json.main_fields.put(main, r.allocator.dupe(u8, str) catch unreachable) catch unreachable; + if ((expr.asString(r.allocator))) |str| { + if (str.len > 0) { + package_json.main_fields.put(main, r.allocator.dupe(u8, str) catch unreachable) catch unreachable; + } } } } - } - // Read the "browser" property, but only when targeting the browser - if (r.opts.platform.supportsBrowserField()) { - // We both want the ability to have the option of CJS vs. ESM and the - // option of having node vs. browser. The way to do this is to use the - // object literal form of the "browser" field like this: - // - // "main": "dist/index.node.cjs.js", - // "module": "dist/index.node.esm.js", - // "browser": { - // "./dist/index.node.cjs.js": "./dist/index.browser.cjs.js", - // "./dist/index.node.esm.js": "./dist/index.browser.esm.js" - // }, - // - if (json.asProperty("browser")) |browser_prop| { - switch (browser_prop.expr.data) { - .e_object => |obj| { - // The value is an object - - // Remap all files in the browser field - for (obj.properties.slice()) |*prop| { - var _key_str = (prop.key orelse continue).asString(r.allocator) orelse continue; - const value: js_ast.Expr = prop.value orelse continue; - - // Normalize the path so we can compare against it without getting - // confused by "./". There is no distinction between package paths and - // relative paths for these values because some tools (i.e. Browserify) - // don't make such a distinction. - // - // This leads to weird things like a mapping for "./foo" matching an - // import of "foo", but that's actually not a bug. Or arguably it's a - // bug in Browserify but we have to replicate this bug because packages - // do this in the wild. - const key = r.allocator.dupe(u8, r.fs.normalize(_key_str)) catch unreachable; - - switch (value.data) { - .e_string => |str| { - // If this is a string, it's a replacement package - package_json.browser_map.put(key, str.string(r.allocator) catch unreachable) catch unreachable; - }, - .e_boolean => |boolean| { - if (!boolean.value) { - package_json.browser_map.put(key, "") catch unreachable; - } - }, - else => { - r.log.addWarning(&json_source, value.loc, "Each \"browser\" mapping must be a string or boolean") catch unreachable; - }, + // Read the "browser" property, but only when targeting the browser + if (r.opts.platform.supportsBrowserField()) { + // We both want the ability to have the option of CJS vs. ESM and the + // option of having node vs. browser. The way to do this is to use the + // object literal form of the "browser" field like this: + // + // "main": "dist/index.node.cjs.js", + // "module": "dist/index.node.esm.js", + // "browser": { + // "./dist/index.node.cjs.js": "./dist/index.browser.cjs.js", + // "./dist/index.node.esm.js": "./dist/index.browser.esm.js" + // }, + // + if (json.asProperty("browser")) |browser_prop| { + switch (browser_prop.expr.data) { + .e_object => |obj| { + // The value is an object + + // Remap all files in the browser field + for (obj.properties.slice()) |*prop| { + var _key_str = (prop.key orelse continue).asString(r.allocator) orelse continue; + const value: js_ast.Expr = prop.value orelse continue; + + // Normalize the path so we can compare against it without getting + // confused by "./". There is no distinction between package paths and + // relative paths for these values because some tools (i.e. Browserify) + // don't make such a distinction. + // + // This leads to weird things like a mapping for "./foo" matching an + // import of "foo", but that's actually not a bug. Or arguably it's a + // bug in Browserify but we have to replicate this bug because packages + // do this in the wild. + const key = r.allocator.dupe(u8, r.fs.normalize(_key_str)) catch unreachable; + + switch (value.data) { + .e_string => |str| { + // If this is a string, it's a replacement package + package_json.browser_map.put(key, str.string(r.allocator) catch unreachable) catch unreachable; + }, + .e_boolean => |boolean| { + if (!boolean.value) { + package_json.browser_map.put(key, "") catch unreachable; + } + }, + else => { + r.log.addWarning(&json_source, value.loc, "Each \"browser\" mapping must be a string or boolean") catch unreachable; + }, + } } - } - }, - else => {}, + }, + else => {}, + } } } - } - if (json.asProperty("exports")) |exports_prop| { - if (ExportsMap.parse(r.allocator, &json_source, r.log, exports_prop.expr, exports_prop.loc)) |exports_map| { - package_json.exports = exports_map; + if (json.asProperty("exports")) |exports_prop| { + if (ExportsMap.parse(r.allocator, &json_source, r.log, exports_prop.expr, exports_prop.loc)) |exports_map| { + package_json.exports = exports_map; + } } - } - if (json.asProperty("imports")) |imports_prop| { - if (ExportsMap.parse(r.allocator, &json_source, r.log, imports_prop.expr, imports_prop.loc)) |imports_map| { - package_json.imports = imports_map; + if (json.asProperty("imports")) |imports_prop| { + if (ExportsMap.parse(r.allocator, &json_source, r.log, imports_prop.expr, imports_prop.loc)) |imports_map| { + package_json.imports = imports_map; + } + } + + if (json.get("sideEffects")) |side_effects_field| outer: { + if (side_effects_field.asBool()) |boolean| { + if (!boolean) + package_json.side_effects = .{ .false = {} }; + } else if (side_effects_field.asArray()) |array_| { + var array = array_; + // TODO: switch to only storing hashes + var map = SideEffectsMap{}; + map.ensureTotalCapacity(r.allocator, array.array.items.len) catch unreachable; + while (array.next()) |item| { + if (item.asString(r.allocator)) |name| { + // TODO: support RegExp using JavaScriptCore <> C++ bindings + if (strings.containsChar(name, '*')) { + // https://sourcegraph.com/search?q=context:global+file:package.json+sideEffects%22:+%5B&patternType=standard&sm=1&groupBy=repo + // a lot of these seem to be css files which we don't care about for now anyway + // so we can just skip them in here + if (strings.eqlComptime(std.fs.path.extension(name), ".css")) + continue; + + r.log.addWarning( + &json_source, + item.loc, + "wildcard sideEffects are not supported yet, which means this package will be deoptimized", + ) catch unreachable; + map.deinit(r.allocator); + + package_json.side_effects = .{ .unspecified = {} }; + break :outer; + } + + var joined = [_]string{ + json_source.path.name.dirWithTrailingSlash(), + name, + }; + + _ = map.getOrPutAssumeCapacity( + bun.StringHashMapUnowned.Key.init(r.fs.join(&joined)), + ); + } + } + package_json.side_effects = .{ .map = map }; + } } } @@ -938,8 +994,6 @@ pub const PackageJSON = struct { } } - // TODO: side effects - if (generate_hash) { if (package_json.name.len > 0 and package_json.version.len > 0) { package_json.generateHash(); -- cgit v1.2.3