From 6362414d65b69cd01624e84d08eca654fc8cb101 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 7 Apr 2023 20:08:01 -0700 Subject: Bun gets a new bundler (#2312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * alright now just gotta try running it * fix a gajillion compiler errors * even more code * okay i fixed more errors * wip * Update launch.json * Update string_builder.zig * `fast_debug_build_mode` makes debug build 2x faster * Update bundle_v2.zig * more code! * It bundles! * Rename `Bun.Transpiler` to `Bun.Bundler` * `import()` expressions almost work * wip attempt to get import() expr to work * Bundle namespace imports * Attempt to fix the issue with import() unsuccessfully * consider current working directory when resolving relative paths (#2313) * consider current working directory when resolving relative paths fixes #2298 * comment test --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> * support `expect().toThrow(/pattern/)` (#2314) - fix time-zone-dependent test failure * fix missing `Blob` error messages on Linux (#2315) * fix & clean up tests (#2318) - skip flaky tests when running as `root` - use `expect().toThrow()` - clean up temporary files after tests * feat(tty): add some `tty.WriteStream` methods to `process.{stdout, stderr}` (#2320) * feat(stdio): add some `tty.WriteStream` methods * chore(builtins): add process builtin gen'd code * Fix docker install command * `bun test` on macOS in GitHub Actions (#2322) * Fixes #2323 * throw invalid parameter errors in `crypto.scryptSync` (#2331) * throw invalid parameter errors * remove comptime, add empty buffer function * remove error_name comptime * Add reference documentation for bun:test (#2327) * Reorganize tests (#2332) * Fix html-rewriter.test.js * fix the wrong thing being incremented in hmr example (#2334) * Add more test harness * Improve Benchmarking page, small fixes (#2339) * Improve benchmarking page * WIP * Add typescript instructions to hot * Document preload in Plugins. Fix loader in plugin types. * Fix typo * Fix links * run prettier * Document openInEditor * improve `Buffer` compatibility with Node.js (#2341) * improve `Buffer` compatibility with Node.js * use `memmove()` allow `encoding` to be `undefined` * run `bun test` after macOS builds (#2343) * "binary" is an alias of "latin1" Fixes https://github.com/oven-sh/bun/issues/2110 * More spec compliant `Blob.prototype.type` (#2340) * Make `Blob.prototype. type` more spec compliant * Add a few more checks for isNumber() * Fix `make headers` * Safer JSValue.isString() * More tests for blob.slice * Make `Blob.prototype.type` more spec compliant * Add isASCII check * Fix types * Fix failing type test * Update blob.zig * Update blob.zig * Fix .eql check on empty values --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> * Fix bug in test runner * Support `import()` expressions * Implement `require()` * clean up bit_set.zig slightly * Move some things around * misc cleanup * Cleanup some things * Fix a lot of stuff * Fix `module.exports.fn = fn;` in ESM entry point * Fix crash due when printing file * Fix issue with class names * Fix issue with `export default identifier` * Update js_parser.zig * optimization: inline single-property object acceses and arrays * Fix undefined memory in renamed symbols list * Handle call target * wip * Inline it * Fix undefined memory issue when reclaiming blocks in ast * Halt linking on any parse errors * alias * Rename `enable_bundling` to `enable_legacy_bundling` * Workaround anonymous struct literal zig bug * Use slower approach (without bitset) because it doesn't break after 8 symbols * Fix incorrectly-renaming statically defined symbols * Handle more edgecases in our bit_set fork * Reduce number of allocations for `define` * Do not rename unbound symbols * Clean up dot defines a little more * Make the generated names prettier * Workaround runtime symbol missing issue * Fail the build on errors * Support export * from * Support `--outfile` * partially fix renaming * fanicer symbol renaming impl * misc, extremely revertible cleanup * Fix up some bugs with symbol renaming * formatting * Update launch.json * Parse `__PURE__` comments * clean up simd code for pure comments * changes to merge * workaround runtime issue * Fix issue with `export * as` not propagating correctly * Make all top-level declarations `var` when bundling * Fix missing prefix * Fix assigning to stack copy * Fix missing runtime symbol * Fix bug with namespace exports * Dramatically reduce allocations * Update launch.json * Add missing flags * Update js_parser.zig * small cleanup * Make the export name better * Fix unnecessary `var foo = foo` * Implement CommonJS -> ESM conversion * Implement module redirects * Port esbuild bundler tests for new bundler (#2380) * started porting esbuild tests * clean up test names and api before moving on * port tests using a program i wrote * replace todo generated comment * fix generated tests not including some files * work on tests * [github web editor] add define, external, inject, minifySyntax, minifyWhitespace options. * get most of the todo comments out of the way, but expectBundled does not handle most of the cases * continue working on esbuild tests * use test.skip for unsupported tests * Fixups for test runner * Hoist imports & exports * Fix test * Hoist classes * bundler test refining, 51/835 * Fix runtime require * bundler test refining, 81/835 * bundler test refining, 93/835 * Make the test work in any timezone * feat(expect): update toBeInstanceOf (#2396) * feat: update instanceof binding * fix: according to PR comments * Rename `expectObjectTypeCount` to `expectMaxObjectTypeCount` * Fix socket tests with connection errors (#2403) * release pending activity with connection error handler * unref poll_ref * remove trailing comma * Organize Dockerfiles for official status * Remove test Dockerfile * Remove old Docker workflow * Feat(test): add toMatch (#2404) * Fix various fetch/response/request tests (#2416) * fix most fetch tests, skip a few * fastGet, toValueGC, and invalid init * bigint unreachable, range error, log process as process * remove extra fetch_headers * remove js_type parameter, check isObject() * throw invalid mime type error, use enum literal * switch back to promise rejection * RangeError pascal case * Fix several bugs (#2418) * utf16 codepoint with replacement character * Fix test failure with `TextEncoder("ascii')` * Add missing type * Fix Response.prototype.bodyUsed and Request.prototype.bodyUsed * Fix bug with scrypt error not clearing * Update server.zig * oopsie * :nail_care: * docs: Use correct url in the 'Issues' link in README header (#2420) * Fix crash when rendering error page and the server or network is slow * [fetch] Make the default body value `null` when unspecified This is better aligned with the fetch spec * Make node-net tests less flaky * [node:net] Fix issue with `listen` callback firing before it's listening * Always clear timers in node test harness * Fix out of bounds access Repro'd in Buffer tests * Update UWS cc @cirospaciari * Make this test more thorough * Hanging abort test * 0 length body is a null stream * Several bug fixes (#2427) * Fix test * Fix segfault when unexpected type is passed in `expect().toThrow` * Fix issues with request constructor * Don't bother cloning headers when its empty * woops * more tests * fix incorrect test * Make the fetch error messages better * Update response.zig * Fix test that failed on macOS * Fix test * Remove extra hash table lookups * Support running dummy registry directly cc @alexlamsl * Update test * Update test * fixup * Workaround crash in test runner * Fixup test * Fixup test * Update os.test.js --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> * Remove usages of port numbers in tests * Set -O2 and -fno-rtti * Remove -g * Prevent undefined memory access * [bun test] Implement `--rerun-each` flag to run each test N times * Reduce number of module scopes created * add some extra abort checks into streams (#2430) * add some checks to avoid UAF * avoid multiple calls to finalize if endFromJS is called more than once * fix no-op comment * mark as requested_end on abort * remove requested_end from abort * remove unnecessary check (#2432) * Fix bug with scoped aliased dependencies in bun install on macOS * remove `addLog`, remove `--prominent-compile-errors` * Finish the upgrade * Optional chaining flag * Implement same_target_becomes_destructuring optimization * bundler test refining, 109/835 * Reset bindings * Support multiple entry points * Implement `--entry-names` flag * Use a tempdir with a better name * prettier * Log file name * Update js_parser.zig * Mark all bun builtins as external * Make resolve errors actually errors * Update bundler_default.test.ts * Fix `await import(foo)` * WIP react server components * Do more stuff at runtime * :scissors: * Support automatic JSX imports * Use a module cache for now * Update tsconfig.base.json * Fix ThisOutsideFunctionNotRenamed * woopsie * moar cpu * clamp it * fixup * Add a bunch of assertions * Bun uses automatic runtime by default * Parse Import Attributes * Add a note about Valgrind * Update developing.md * Fix up code splitting for React Server Components * Implement client component manifest * Fix crash with --react-server-components and no client components * Backport https://github.com/ziglang/zig/commit/4d31e3c917a05541394c544708f0047cfb53331a * Update launch.json * Fix for latest zig * Workaround bug with ?[]const string Occasionally saw alignment errors in this code Workaround https://github.com/ziglang/zig/issues/15085 related: https://github.com/ziglang/zig/pull/15089 * switch to regular slice * Avoid initializing named_imports and named_exports as undefined * Reduce usages of `undefined` * Add more assertions * --watch wip * Update javascript.zig * Possibly fix the race condition * Faster `do` * bump allocator * Reduce the size of `Symbol` slightly * Alphabetically sort runtime import symbols, for determinism * Prepare for code splitting * handle overlapping stdout * pure * clean up some things * Fix bug with `$$typeof` * Address CommonJS -> ESM hoisting bug * Support `"use server"` in manifest * Implement `"use server"` * Fix importing bun builtins when bundling * Make `commonjs_to_esm` a feature flag, fix some splitting bugs * :scissors: * fixme remove this * Fix crash in longestCommonPath * Chunking! Just need to do import paths now. * Import paths work...now trying to figure out how to make runtime symbols work * add workaround * Replace `bun bun` with `bun build` * Fix crash with dual package hazard * Fix many CommonJS <> ESM interop bugs * Support package.json `"sideEffects"` also skip loading unnecessary package.json data in `bun run` * add a not good --watch implementation * bundler test refining, 140/831 * remove accidentally committed file * do not return status code 1 on successful bundles * bundler test refining, 159/830 * pass exit code to exitOrWatch * clean up help menu -remove two spaces to line up bun build -moved all tags to the end of the text they are colorizing -moved other colors to the start of the text they colorize -removed unneeded tags, keeping only one at the start of the block * importstar is fully ported * wip * you can run code in this branch now * Disable this transform * organize and document bundler tests * Fix double import * Fix sloppy mode function declarations * Disable our CommonJS transform for now * add `assertNotPresent` to make splitting cases easier * Bump! * Update bun.d.ts * use import.meta.require in runtime code * Disable this again * Fix dirname * Fix ESM -> CJS wrapper * :nail_care: --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> Co-authored-by: Alex Lam S.L Co-authored-by: Derrick Farris Co-authored-by: Ashcon Partovi Co-authored-by: Dylan Conway <35280289+dylan-conway@users.noreply.github.com> Co-authored-by: pfg Co-authored-by: Colin McDonnell Co-authored-by: dave caruso Co-authored-by: zhiyuan <32867472+zhiyuang@users.noreply.github.com> Co-authored-by: Dylan Conway Co-authored-by: Kamil Ogórek Co-authored-by: Ciro Spaciari --- src/cli/build_command.zig | 314 ++++++++++++++++++++++++++++------------------ 1 file changed, 192 insertions(+), 122 deletions(-) (limited to 'src/cli/build_command.zig') diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index 92d2d5b3a..1b8e5c632 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -1,167 +1,237 @@ +const std = @import("std"); +const Command = @import("../cli.zig").Command; const bun = @import("bun"); const string = bun.string; const Output = bun.Output; const Global = bun.Global; const Environment = bun.Environment; -const FeatureFlags = bun.FeatureFlags; const strings = bun.strings; const MutableString = bun.MutableString; const stringZ = bun.stringZ; const default_allocator = bun.default_allocator; const C = bun.C; -const std = @import("std"); const lex = bun.js_lexer; const logger = @import("bun").logger; const options = @import("../options.zig"); const js_parser = bun.js_parser; +const json_parser = bun.JSON; +const js_printer = bun.js_printer; const js_ast = bun.JSAst; const linker = @import("../linker.zig"); -const allocators = @import("../allocators.zig"); const sync = @import("../sync.zig"); const Api = @import("../api/schema.zig").Api; const resolve_path = @import("../resolver/resolve_path.zig"); const configureTransformOptionsForBun = @import("../bun.js/config.zig").configureTransformOptionsForBun; -const Command = @import("../cli.zig").Command; const bundler = bun.bundler; const NodeModuleBundle = @import("../node_module_bundle.zig").NodeModuleBundle; +const GenerateNodeModuleBundle = @import("../bundler/generate_node_modules_bundle.zig"); +const DotEnv = @import("../env_loader.zig"); + const fs = @import("../fs.zig"); -const constStrToU8 = bun.constStrToU8; +const Router = @import("../router.zig"); +const BundleV2 = @import("../bundler/bundle_v2.zig").BundleV2; +var estimated_input_lines_of_code_: usize = undefined; pub const BuildCommand = struct { - pub fn exec(ctx: Command.Context) !void { - var result: options.TransformResult = undefined; - switch (ctx.args.resolve orelse Api.ResolveMode.dev) { - .lazy => { - result = try bundler.Bundler.bundle( - ctx.allocator, - ctx.log, - ctx.args, - ); - }, - else => { - result = try bundler.Bundler.bundle( - ctx.allocator, - ctx.log, - ctx.args, - ); - }, - } - var did_write = false; + pub fn exec( + ctx: Command.Context, + ) !void { + Global.configureAllocator(.{ .long_running = true }); + var allocator = ctx.allocator; + var log = ctx.log; + estimated_input_lines_of_code_ = 0; + + var this_bundler = try bundler.Bundler.init(allocator, log, ctx.args, null, null); + this_bundler.options.entry_names = ctx.bundler_options.entry_names; + this_bundler.resolver.opts.entry_names = ctx.bundler_options.entry_names; + this_bundler.options.output_dir = ctx.bundler_options.outdir; + this_bundler.resolver.opts.output_dir = ctx.bundler_options.outdir; + this_bundler.options.react_server_components = ctx.bundler_options.react_server_components; + this_bundler.resolver.opts.react_server_components = ctx.bundler_options.react_server_components; + this_bundler.options.code_splitting = ctx.bundler_options.code_splitting; + this_bundler.resolver.opts.code_splitting = ctx.bundler_options.code_splitting; + + this_bundler.configureLinker(); + + // This step is optional + // If it fails for any reason, ignore it and continue bundling + // This is partially a workaround for the 'error.MissingRoutesDir' error + this_bundler.configureRouter(true) catch { + this_bundler.options.routes.routes_enabled = false; + this_bundler.options.framework = null; + if (this_bundler.router) |*router| { + router.config.routes_enabled = false; + router.config.single_page_app_routing = false; + router.config.static_dir_enabled = false; + this_bundler.router = null; + } + this_bundler.options.node_modules_bundle = null; + this_bundler.options.node_modules_bundle_pretty_path = ""; + this_bundler.options.node_modules_bundle_url = ""; + }; - defer Output.flush(); - var writer = Output.errorWriter(); - var err_writer = writer; + if (ctx.debug.macros) |macros| { + this_bundler.options.macro_remap = macros; + } - var open_file_limit: usize = fs.FileSystem.RealFS.Limit.handles; - if (ctx.args.write) |write| { - if (write) { - const root_dir = result.root_dir orelse unreachable; + // var env_loader = this_bundler.env; - var all_paths = try ctx.allocator.alloc([]const u8, result.output_files.len); - var max_path_len: usize = 0; - for (result.output_files, 0..) |f, i| { - all_paths[i] = f.input.text; - } + if (ctx.debug.dump_environment_variables) { + this_bundler.dumpEnvironmentVariables(); + return; + } - var from_path = resolve_path.longestCommonPath(all_paths); + if (ctx.debug.dump_limits) { + fs.FileSystem.printLimits(); + Global.exit(0); + return; + } - for (result.output_files) |f| { - max_path_len = std.math.max( - std.math.max(from_path.len, f.input.text.len) + 2 - from_path.len, - max_path_len, - ); + // var generated_server = false; + // if (this_bundler.options.framework) |*framework| { + // if (framework.toAPI(allocator, this_bundler.fs.top_level_dir) catch null) |_server_conf| { + // ServerBundleGeneratorThread.generate( + // log, + // env_loader, + // ctx, + // server_bundle_filepath, + // _server_conf, + // loaded_route_config, + // this_bundler.router, + // ); + // generated_server = true; + + // if (log.msgs.items.len > 0) { + // try log.printForLogLevel(Output.errorWriter()); + // log.* = logger.Log.init(allocator); + // Output.flush(); + // } + // } + // } + + { + + // Always generate the client-only bundle + // we can revisit this decision if people ask + const output_files = BundleV2.generate( + &this_bundler, + allocator, + &estimated_input_lines_of_code_, + ctx.debug.package_bundle_map, + bun.JSC.AnyEventLoop.init(ctx.allocator), + std.crypto.random.int(u64), + ctx.debug.hot_reload == .watch, + ) catch |err| { + if (log.msgs.items.len > 0) { + try log.printForLogLevel(Output.errorWriter()); + } else { + try Output.errorWriter().print("error: {s}", .{@errorName(err)}); } - did_write = true; - - // On posix, file handles automatically close on process exit by the OS - // Closing files shows up in profiling. - // So don't do that unless we actually need to. - // const do_we_need_to_close = !FeatureFlags.store_file_descriptors or (@intCast(usize, root_dir.fd) + open_file_limit) < result.output_files.len; - - var filepath_buf: [bun.MAX_PATH_BYTES]u8 = undefined; - filepath_buf[0] = '.'; - filepath_buf[1] = '/'; - - for (result.output_files) |f| { - var rel_path: []const u8 = undefined; - switch (f.value) { - // easy mode: write the buffer - .buffer => |value| { - rel_path = resolve_path.relative(from_path, f.input.text); - - try root_dir.writeFile(rel_path, value); - }, - .move => |value| { - // const primary = f.input.text[from_path.len..]; - // bun.copy(u8, filepath_buf[2..], primary); - // rel_path = filepath_buf[0 .. primary.len + 2]; - rel_path = value.pathname; - - // try f.moveTo(result.outbase, constStrToU8(rel_path), root_dir.fd); - }, - .copy => |value| { - rel_path = value.pathname; - - try f.copyTo(result.outbase, constStrToU8(rel_path), root_dir.fd); - }, - .noop => {}, - .pending => unreachable, + Output.flush(); + exitOrWatch(1, ctx.debug.hot_reload == .watch); + unreachable; + }; + + { + dump: { + defer Output.flush(); + var writer = Output.errorWriter(); + var output_dir = this_bundler.options.output_dir; + if (ctx.bundler_options.outfile.len > 0 and output_files.items.len == 1 and output_files.items[0].value == .buffer) { + output_dir = std.fs.path.dirname(ctx.bundler_options.outfile) orelse "."; + output_files.items[0].input.text = std.fs.path.basename(ctx.bundler_options.outfile); } - // Print summary - _ = try writer.write("\n"); - const padding_count = 2 + (std.math.max(rel_path.len, max_path_len) - rel_path.len); - try writer.writeByteNTimes(' ', 2); - try writer.writeAll(rel_path); - try writer.writeByteNTimes(' ', padding_count); - const size = @intToFloat(f64, f.size) / 1000.0; - try std.fmt.formatFloatDecimal(size, .{ .precision = 2 }, writer); - try writer.writeAll(" KB\n"); - } - } - } + if (output_dir.len == 0 and output_files.items.len == 1 and output_files.items[0].value == .buffer) { + try writer.writeAll(output_files.items[0].value.buffer); + break :dump; + } - if (Environment.isDebug) { - err_writer.print("\nExpr count: {d}\n", .{js_ast.Expr.icount}) catch {}; - err_writer.print("Stmt count: {d}\n", .{js_ast.Stmt.icount}) catch {}; - err_writer.print("Binding count: {d}\n", .{js_ast.Binding.icount}) catch {}; - err_writer.print("File Descriptors: {d} / {d}\n", .{ - fs.FileSystem.max_fd, - open_file_limit, - }) catch {}; - } + const root_path = output_dir; + const root_dir = try std.fs.cwd().makeOpenPathIterable(root_path, .{}); + var all_paths = try ctx.allocator.alloc([]const u8, output_files.items.len); + var max_path_len: usize = 0; + for (all_paths, output_files.items) |*dest, src| { + dest.* = src.input.text; + } - if (Output.enable_ansi_colors) { - for (result.errors) |err| { - try err.writeFormat(err_writer, true); - _ = try err_writer.write("\n"); - } + var from_path = resolve_path.longestCommonPath(all_paths); - for (result.warnings) |err| { - try err.writeFormat(err_writer, true); - _ = try err_writer.write("\n"); - } - } else { - for (result.errors) |err| { - try err.writeFormat(err_writer, false); - _ = try err_writer.write("\n"); - } + for (output_files.items) |f| { + max_path_len = std.math.max( + std.math.max(from_path.len, f.input.text.len) + 2 - from_path.len, + max_path_len, + ); + } - for (result.warnings) |err| { - try err.writeFormat(err_writer, false); - _ = try err_writer.write("\n"); + // On posix, file handles automatically close on process exit by the OS + // Closing files shows up in profiling. + // So don't do that unless we actually need to. + // const do_we_need_to_close = !FeatureFlags.store_file_descriptors or (@intCast(usize, root_dir.fd) + open_file_limit) < output_files.items.len; + + var filepath_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + filepath_buf[0] = '.'; + filepath_buf[1] = '/'; + + for (output_files.items) |f| { + var rel_path: []const u8 = undefined; + switch (f.value) { + // easy mode: write the buffer + .buffer => |value| { + rel_path = f.input.text; + if (f.input.text.len > from_path.len) { + rel_path = resolve_path.relative(from_path, f.input.text); + if (std.fs.path.dirname(rel_path)) |parent| { + if (parent.len > root_path.len) { + try root_dir.dir.makePath(parent); + } + } + } + try root_dir.dir.writeFile(rel_path, value); + }, + .move => |value| { + // const primary = f.input.text[from_path.len..]; + // bun.copy(u8, filepath_buf[2..], primary); + // rel_path = filepath_buf[0 .. primary.len + 2]; + rel_path = value.pathname; + + // try f.moveTo(result.outbase, constStrToU8(rel_path), root_dir.fd); + }, + .copy => |value| { + rel_path = value.pathname; + + try f.copyTo(root_path, bun.constStrToU8(rel_path), root_dir.dir.fd); + }, + .noop => {}, + .pending => unreachable, + } + + // Print summary + _ = try writer.write("\n"); + const padding_count = 2 + (std.math.max(rel_path.len, max_path_len) - rel_path.len); + try writer.writeByteNTimes(' ', 2); + try writer.writeAll(rel_path); + try writer.writeByteNTimes(' ', padding_count); + const size = @intToFloat(f64, f.size) / 1000.0; + try std.fmt.formatFloatDecimal(size, .{ .precision = 2 }, writer); + try writer.writeAll(" KB\n"); + } + } } - } - - const duration = std.time.nanoTimestamp() - ctx.start_time; - - if (did_write and duration < @as(i128, @as(i128, std.time.ns_per_s) * @as(i128, 2))) { - var elapsed = @divTrunc(duration, @as(i128, std.time.ns_per_ms)); - try err_writer.print("\nCompleted in {d}ms", .{elapsed}); + try log.printForLogLevel(Output.errorWriter()); + exitOrWatch(0, ctx.debug.hot_reload == .watch); } } }; + +fn exitOrWatch(code: u8, watch: bool) void { + if (watch) { + // the watcher thread will exit the process + std.time.sleep(std.math.maxInt(u64) - 1); + } + Global.exit(code); +} -- cgit v1.2.3