diff options
author | 2022-04-16 17:33:16 -0700 | |
---|---|---|
committer | 2022-04-16 17:33:16 -0700 | |
commit | 8d28e72e8a4c080c9d937e3f327d66d120508990 (patch) | |
tree | 38a95c9ca05001b74937f37ef4170304851ff9a5 | |
parent | 0137e5cf94a2cfd510f70d8881f67e8066e0d098 (diff) | |
download | bun-8d28e72e8a4c080c9d937e3f327d66d120508990.tar.gz bun-8d28e72e8a4c080c9d937e3f327d66d120508990.tar.zst bun-8d28e72e8a4c080c9d937e3f327d66d120508990.zip |
Move some code around + delete dead code
-rw-r--r-- | src/bundler.zig | 2105 | ||||
-rw-r--r-- | src/bundler/generate_node_modules_bundle.zig | 1885 | ||||
-rw-r--r-- | src/cli/build_command.zig | 7 | ||||
-rw-r--r-- | src/cli/bun_command.zig | 5 |
4 files changed, 1889 insertions, 2113 deletions
diff --git a/src/bundler.zig b/src/bundler.zig index 957c4af5b..3cf3af2dc 100644 --- a/src/bundler.zig +++ b/src/bundler.zig @@ -476,1835 +476,7 @@ pub const Bundler = struct { Output.flush(); } - pub const GenerateNodeModuleBundle = struct { - const BunQueue = sync.Channel(PendingImports, .Dynamic); - - pub const ThreadPool = struct { - pool: ThreadPoolLib = undefined, - // Hardcode 512 as max number of threads for now. - workers: [512]Worker = undefined, - workers_used: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), - cpu_count: u32 = 0, - started_workers: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), - stopped_workers: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), - completed_count: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), - pending_count: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), - - generator: *GenerateNodeModuleBundle = undefined, - - pub fn start(this: *ThreadPool, generator: *GenerateNodeModuleBundle) !void { - generator.bundler.env.loadProcess(); - this.generator = generator; - - this.cpu_count = @truncate(u32, @divFloor((try std.Thread.getCpuCount()) + 1, 2)); - - if (generator.bundler.env.map.get("GOMAXPROCS")) |max_procs| { - if (std.fmt.parseInt(u32, max_procs, 10)) |cpu_count| { - this.cpu_count = std.math.min(this.cpu_count, cpu_count); - } else |_| {} - } - - this.pool = ThreadPoolLib.init(.{ - .max_threads = this.cpu_count, - }); - this.pool.on_thread_spawn = Worker.onSpawn; - this.pool.threadpool_context = this; - var workers_used: u32 = 0; - while (workers_used < this.cpu_count) : (workers_used += 1) { - try this.workers[workers_used].init(generator); - } - if (workers_used > 0) - this.pool.forceSpawn(); - } - - pub fn wait(this: *ThreadPool, generator: *GenerateNodeModuleBundle) !void { - while (true) { - while (this.generator.queue.tryReadItem() catch null) |queue| { - var iter = queue.iterator(); - var batch = ThreadPoolLib.Batch{}; - var count: u32 = 0; - while (iter.next()) |entry| { - const module_id: u32 = entry.key_ptr.*; - const exists = generator.enqueued_map.getOrPut(module_id) catch unreachable; - if (exists.found_existing) { - continue; - } - batch.push(ThreadPoolLib.Batch.from(&entry.value_ptr.*.task)); - count += 1; - } - _ = this.pending_count.fetchAdd(count, .Monotonic); - this.pool.schedule(batch); - } - - if (this.completed_count.load(.Monotonic) > 0 and this.completed_count.load(.Monotonic) == this.pending_count.load(.Monotonic)) { - break; - } - - std.atomic.spinLoopHint(); - } - - const workers: []const Worker = this.workers[0..this.workers_used.loadUnchecked()]; - for (workers) |worker| { - this.generator.estimated_input_lines_of_code += worker.data.estimated_input_lines_of_code; - try worker.data.log.appendTo(this.generator.log); - } - } - - pub const Task = struct { - result: _resolver.Result, - generator: *GenerateNodeModuleBundle, - }; - - pub const Worker = struct { - thread_id: std.Thread.Id, - thread: std.Thread, - - allocator: std.mem.Allocator, - generator: *GenerateNodeModuleBundle, - data: *WorkerData = undefined, - quit: bool = false, - - has_notify_started: bool = false, - - pub const WorkerData = struct { - shared_buffer: MutableString = undefined, - scan_pass_result: js_parser.ScanPassResult = undefined, - log: *logger.Log, - estimated_input_lines_of_code: usize = 0, - macro_context: js_ast.Macro.MacroContext, - bundler: Bundler = undefined, - - pub fn deinit(this: *WorkerData, allocator: std.mem.Allocator) void { - this.shared_buffer.deinit(); - this.scan_pass_result.named_imports.deinit(); - this.scan_pass_result.import_records.deinit(); - allocator.destroy(this); - } - }; - - pub fn init(worker: *Worker, generator: *GenerateNodeModuleBundle) !void { - worker.generator = generator; - worker.allocator = generator.allocator; - } - - pub fn onSpawn(ctx: ?*anyopaque) ?*anyopaque { - var pool = @ptrCast(*ThreadPool, @alignCast(@alignOf(*ThreadPool), ctx.?)); - - const id = pool.workers_used.fetchAdd(1, .Monotonic); - pool.workers[id].run(); - return &pool.workers[id]; - } - - pub fn notifyStarted(this: *Worker) void { - if (!this.has_notify_started) { - this.has_notify_started = true; - _ = this.generator.pool.started_workers.fetchAdd(1, .Release); - std.Thread.Futex.wake(&this.generator.pool.started_workers, std.math.maxInt(u32)); - } - } - - pub fn run(this: *Worker) void { - Output.Source.configureThread(); - this.thread_id = std.Thread.getCurrentId(); - if (Environment.isDebug) { - Output.prettyln("Thread started.\n", .{}); - } - js_ast.Expr.Data.Store.create(this.generator.allocator); - js_ast.Stmt.Data.Store.create(this.generator.allocator); - this.data = this.generator.allocator.create(WorkerData) catch unreachable; - this.data.* = WorkerData{ - .log = this.generator.allocator.create(logger.Log) catch unreachable, - .estimated_input_lines_of_code = 0, - .macro_context = js_ast.Macro.MacroContext.init(this.generator.bundler), - }; - this.data.log.* = logger.Log.init(this.generator.allocator); - this.data.shared_buffer = MutableString.init(this.generator.allocator, 0) catch unreachable; - this.data.scan_pass_result = js_parser.ScanPassResult.init(this.generator.allocator); - this.data.bundler = this.generator.bundler.*; - var bundler_ptr = &this.data.bundler; - const CacheSet = @import("./cache.zig"); - // no funny business mr. cache - bundler_ptr.resolver.caches = CacheSet.Set.init(this.allocator); - bundler_ptr.linker.resolver = &bundler_ptr.resolver; - bundler_ptr.log = this.data.log; - bundler_ptr.linker.log = this.data.log; - bundler_ptr.linker.resolver.log = this.data.log; - } - - pub const ProcessFileTask = struct { - resolution: _resolver.Result, - task: ThreadPoolLib.Task = .{ .callback = callback }, - - pub fn callback(task: *ThreadPoolLib.Task) void { - var worker = @ptrCast( - *ThreadPool.Worker, - @alignCast( - @alignOf(*ThreadPool.Worker), - ThreadPoolLib.Thread.current.?.ctx.?, - ), - ); - var process: *ProcessFileTask = @fieldParentPtr(ProcessFileTask, "task", task); - - worker.generator.processFile( - worker, - &worker.data.bundler, - process.resolution, - ) catch {}; - _ = worker.generator.pool.completed_count.fetchAdd(1, .Monotonic); - } - }; - }; - }; - write_lock: Lock, - log_lock: Lock = Lock.init(), - module_list: std.ArrayList(Api.JavascriptBundledModule), - package_list: std.ArrayList(Api.JavascriptBundledPackage), - header_string_buffer: MutableString, - - // Just need to know if we've already enqueued this one - package_list_map: std.AutoHashMap(u64, u32), - enqueued_map: std.AutoHashMap(u32, void), - queue: BunQueue, - bundler: *ThisBundler, - - allocator: std.mem.Allocator, - tmpfile: std.fs.File, - log: *logger.Log, - pool: *ThreadPool, - tmpfile_byte_offset: u32 = 0, - code_end_byte_offset: u32 = 0, - has_jsx: bool = false, - estimated_input_lines_of_code: usize = 0, - - work_waiter: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), - list_lock: Lock = Lock.init(), - - dynamic_import_file_size_store: U32Map, - dynamic_import_file_size_store_lock: Lock, - - always_bundled_package_hashes: []u32 = &[_]u32{}, - always_bundled_package_jsons: []*const PackageJSON = &.{}, - always_bundled_booleans: []bool = &.{}, - package_bundle_map: options.BundlePackage.Map = options.BundlePackage.Map{}, - - const U32Map = std.AutoHashMap(u32, u32); - pub const current_version: u32 = 1; - const dist_index_js_string_pointer = Api.StringPointer{ .length = "dist/index.js".len }; - const index_js_string_pointer = Api.StringPointer{ .length = "index.js".len, .offset = "dist/".len }; - - fn upsert(this: *GenerateNodeModuleBundle, module_id: u32, resolve: _resolver.Result) !void { - var dedupe = try this.enqueued_map.getOrPut(module_id); - if (dedupe.found_existing) return; - var task = try this.allocator.create(ThreadPool.Worker.ProcessFileTask); - task.* = ThreadPool.Worker.ProcessFileTask{ - .resolution = resolve, - }; - _ = this.pool.pending_count.fetchAdd(1, .Monotonic); - this.pool.pool.schedule(ThreadPoolLib.Batch.from(&task.task)); - } - - pub fn ensurePathIsAllocated(this: *GenerateNodeModuleBundle, path_: ?*Fs.Path) !void { - var path = path_ orelse return; - - const loader = this.bundler.options.loaders.get(path.name.ext) orelse .file; - if (!loader.isJavaScriptLikeOrJSON()) return; - path.* = try path.dupeAlloc(this.allocator); - } - - pub fn enqueueItem(this: *GenerateNodeModuleBundle, resolve: _resolver.Result) !void { - var result = resolve; - var path = result.path() orelse return; - - const loader = this.bundler.options.loaders.get(path.name.ext) orelse .file; - if (!loader.isJavaScriptLikeOrJSON()) return; - path.* = try path.dupeAlloc(this.allocator); - - if (BundledModuleData.get(this, &result)) |mod| { - try this.upsert(mod.module_id, result); - } else { - try this.upsert(result.hash(this.bundler.fs.top_level_dir, loader), result); - } - } - - // The bun Bundle Format - // All the node_modules your app uses in a single compact file with metadata - // A binary JavaScript bundle format prioritizing generation time and deserialization time - pub const magic_bytes = "#!/usr/bin/env bun\n\n"; - // This makes it possible to do ./path-to-bundle on posix systems so you can see the raw JS contents - // https://en.wikipedia.org/wiki/Magic_number_(programming)#In_files - // Immediately after the magic bytes, the next character is a uint32 followed by a newline - // 0x00000000\n - // That uint32 denotes the byte offset in the file where the code for the bundle ends - // - If the value is 0, that means the file did not finish writing or there are no modules - // - This imposes a maximum bundle size of around 4,294,967,295 bytes. If your JS is more than 4 GB, it won't work. - // The raw JavaScript is encoded as a UTF-8 string starting from the current position + 1 until the above byte offset. - // This uint32 is useful for HTTP servers to separate: - // - Which part of the bundle is the JS code? - // - Which part is the metadata? - // Without needing to do a full pass through the file, or necessarily care about the metadata. - // The metadata is at the bottom of the file instead of the top because the metadata is written after all JS code in the bundle is written. - // The rationale there is: - // 1. We cannot prepend to a file without rewriting the entire file - // 2. The metadata is variable-length and that format will change often. - // 3. We won't have all the metadata until after all JS is finished writing - // If you have 32 MB of JavaScript dependencies, you really want to avoid reading the code in memory. - // - This lets you seek to the specific position in the file. - // - HTTP servers should use sendfile() instead of copying the file to userspace memory. - // So instead, we append metadata to the file after printing each node_module - // When there are no more modules to process, we generate the metadata - // To find the metadata, you look at the byte offset: initial_header[magic_bytes.len..initial_header.len - 1] - // Then, you add that number to initial_header.len - const initial_header = brk: { - var buf = std.mem.zeroes([magic_bytes.len + 5]u8); - std.mem.copy(u8, &buf, magic_bytes); - var remainder = buf[magic_bytes.len..]; - // Write an invalid byte offset to be updated after we finish generating the code - std.mem.writeIntNative(u32, remainder[0 .. remainder.len - 1], 0); - buf[buf.len - 1] = '\n'; - break :brk buf; - }; - const code_start_byte_offset: u32 = initial_header.len; - // The specifics of the metadata is not documented here. You can find it in src/api/schema.peechy. - - pub fn appendHeaderString(generator: *GenerateNodeModuleBundle, str: string) !Api.StringPointer { - // This is so common we might as well just reuse it - // Plus this is one machine word so it's a quick comparison - if (strings.eqlComptime(str, "index.js")) { - return index_js_string_pointer; - } else if (strings.eqlComptime(str, "dist/index.js")) { - return dist_index_js_string_pointer; - } - - var offset = generator.header_string_buffer.list.items.len; - try generator.header_string_buffer.append(str); - return Api.StringPointer{ - .offset = @truncate(u32, offset), - .length = @truncate(u32, str.len), - }; - } - - pub fn generate( - bundler: *ThisBundler, - allocator: std.mem.Allocator, - framework_config: ?Api.LoadedFramework, - route_config: ?Api.LoadedRouteConfig, - destination: [*:0]const u8, - estimated_input_lines_of_code: *usize, - package_bundle_map: options.BundlePackage.Map, - ) !?Api.JavascriptBundleContainer { - _ = try bundler.fs.fs.openTmpDir(); - var tmpname_buf: [64]u8 = undefined; - bundler.resetStore(); - try bundler.configureDefines(); - - const tmpname = try bundler.fs.tmpname( - ".bun", - std.mem.span(&tmpname_buf), - std.hash.Wyhash.hash(@intCast(usize, std.time.milliTimestamp()) % std.math.maxInt(u32), std.mem.span(destination)), - ); - - var tmpfile = Fs.FileSystem.RealFS.Tmpfile{}; - try tmpfile.create(&bundler.fs.fs, tmpname); - - errdefer tmpfile.closeAndDelete(tmpname); - - var generator = try allocator.create(GenerateNodeModuleBundle); - var queue = BunQueue.init(allocator); - defer allocator.destroy(generator); - generator.* = GenerateNodeModuleBundle{ - .module_list = std.ArrayList(Api.JavascriptBundledModule).init(allocator), - .package_list = std.ArrayList(Api.JavascriptBundledPackage).init(allocator), - .header_string_buffer = try MutableString.init(allocator, "dist/index.js".len), - .allocator = allocator, - .enqueued_map = std.AutoHashMap(u32, void).init(allocator), - .queue = queue, - .estimated_input_lines_of_code = 0, - // .resolve_queue = queue, - .bundler = bundler, - .tmpfile = tmpfile.file(), - - .dynamic_import_file_size_store = U32Map.init(allocator), - .dynamic_import_file_size_store_lock = Lock.init(), - .log = bundler.log, - .package_list_map = std.AutoHashMap(u64, u32).init(allocator), - .pool = undefined, - .write_lock = Lock.init(), - .package_bundle_map = package_bundle_map, - }; - // dist/index.js appears more common than /index.js - // but this means we can store both "dist/index.js" and "index.js" in one. - try generator.header_string_buffer.append("dist/index.js"); - try generator.package_list_map.ensureTotalCapacity(128); - var pool = try allocator.create(ThreadPool); - pool.* = ThreadPool{}; - generator.pool = pool; - - var this = generator; - // Always inline the runtime into the bundle - try generator.appendBytes(&initial_header); - - if (bundler.log.level == .verbose) { - bundler.resolver.debug_logs = try DebugLogs.init(allocator); - } - - Analytics.Features.bun_bun = true; - - always_bundled: { - const root_package_json_resolved: _resolver.Result = bundler.resolver.resolve(bundler.fs.top_level_dir, "./package.json", .stmt) catch { - generator.log.addWarning(null, logger.Loc.Empty, "Please run `bun bun` from a directory containing a package.json.") catch unreachable; - break :always_bundled; - }; - const root_package_json = root_package_json_resolved.package_json orelse brk: { - const read_dir = (bundler.resolver.readDirInfo(bundler.fs.top_level_dir) catch unreachable).?; - Analytics.Features.tsconfig = Analytics.Features.tsconfig or read_dir.tsconfig_json != null; - break :brk read_dir.package_json.?; - }; - Analytics.setProjectID(std.fs.path.dirname(root_package_json.source.path.text) orelse "/", root_package_json.name); - if (bundler.macro_context) |macro_ctx| { - Analytics.Features.macros = macro_ctx.remap.count() > 0; - } - - const bundle_keys = package_bundle_map.keys(); - const do_always_bundle = package_bundle_map.values(); - var always_bundle_count: u32 = @truncate(u32, bundle_keys.len); - - if (always_bundle_count != 0) { - Analytics.Features.always_bundle = true; - var always_bundled_package_jsons = bundler.allocator.alloc(*PackageJSON, always_bundle_count) catch unreachable; - var always_bundled_package_hashes = bundler.allocator.alloc(u32, always_bundle_count) catch unreachable; - var always_bundled_booleans = bundler.allocator.alloc(bool, always_bundle_count) catch unreachable; - var i: u16 = 0; - - inner: for (bundle_keys) |name, k| { - std.mem.copy(u8, &tmp_buildfile_buf, name); - std.mem.copy(u8, tmp_buildfile_buf[name.len..], "/package.json"); - const package_json_import = tmp_buildfile_buf[0 .. name.len + "/package.json".len]; - const result = bundler.resolver.resolve(bundler.fs.top_level_dir, package_json_import, .stmt) catch |err| { - generator.log.addErrorFmt(null, logger.Loc.Empty, bundler.allocator, "{s} resolving always bundled module \"{s}\"", .{ @errorName(err), name }) catch unreachable; - continue :inner; - }; - - var package_json: *PackageJSON = result.package_json orelse brk: { - const read_dir = (bundler.resolver.readDirInfo(package_json_import) catch unreachable).?; - if (read_dir.package_json == null) { - generator.log.addWarningFmt(null, logger.Loc.Empty, bundler.allocator, "{s} missing package.json. It will not be bundled", .{name}) catch unreachable; - continue :inner; - } - break :brk read_dir.package_json.?; - }; - - package_json.source.key_path = result.path_pair.primary; - - // if (!strings.contains(result.path_pair.primary.text, package_json.name)) { - // generator.log.addErrorFmt( - // null, - // logger.Loc.Empty, - // bundler.allocator, - // "Bundling \"{s}\" is not supported because the package isn.\n To fix this, move the package's code to a directory containing the name.\n Location: \"{s}\"", - // .{ - // name, - // name, - // result.path_pair.primary.text, - // }, - // ) catch unreachable; - // continue :inner; - // } - - always_bundled_package_jsons[i] = package_json; - always_bundled_package_hashes[i] = package_json.hash; - always_bundled_booleans[i] = do_always_bundle[k] == .always; - i += 1; - } - generator.always_bundled_package_hashes = always_bundled_package_hashes[0..i]; - generator.always_bundled_package_jsons = always_bundled_package_jsons[0..i]; - generator.always_bundled_booleans = always_bundled_booleans[0..i]; - } - } - if (generator.log.errors > 0) return error.BundleFailed; - - this.bundler.macro_context = js_ast.Macro.MacroContext.init(bundler); - - const include_refresh_runtime = - !this.bundler.options.production and - this.bundler.options.jsx.supports_fast_refresh and - bundler.options.platform.isWebLike(); - - if (framework_config != null) { - defer this.bundler.resetStore(); - - try this.bundler.configureFramework(true); - if (bundler.options.framework) |framework| { - Analytics.Features.framework = true; - - if (framework.override_modules.keys.len > 0) { - bundler.options.framework.?.override_modules_hashes = allocator.alloc(u64, framework.override_modules.keys.len) catch unreachable; - for (framework.override_modules.keys) |key, i| { - bundler.options.framework.?.override_modules_hashes[i] = std.hash.Wyhash.hash(0, key); - } - } - } - } else {} - - this.pool.start(this) catch |err| { - Analytics.enqueue(Analytics.EventName.bundle_fail); - return err; - }; - - // The ordering matters here The runtime must appear at the top of - // the file But, we don't know which version of the runtime is - // necessary until we know if they'll use JSX and if they'll use - // react refresh - var _new_jsx_runtime_resolve_result: ?_resolver.Result = null; - var fast_refresh_resolve_result: ?_resolver.Result = null; - // Normally, this is automatic - // However, since we only do the parsing pass, it may not get imported automatically. - if (bundler.options.jsx.parse) { - defer this.bundler.resetStore(); - if (this.bundler.resolver.resolve( - this.bundler.fs.top_level_dir, - this.bundler.options.jsx.import_source, - .require, - )) |new_jsx_runtime| { - _new_jsx_runtime_resolve_result = new_jsx_runtime; - this.ensurePathIsAllocated(_new_jsx_runtime_resolve_result.?.path()) catch unreachable; - Analytics.Features.jsx = true; - } else |_| {} - } - - const include_fast_refresh_in_bundle = Analytics.Features.jsx and - include_refresh_runtime and - !Analytics.Features.fast_refresh and !bundler.options.platform.isServerSide(); - - var refresh_runtime_module_id: u32 = 0; - if (include_refresh_runtime) { - defer this.bundler.resetStore(); - - if (this.bundler.resolver.resolve( - this.bundler.fs.top_level_dir, - this.bundler.options.jsx.refresh_runtime, - .require, - )) |refresh_runtime| { - fast_refresh_resolve_result = refresh_runtime; - this.ensurePathIsAllocated(fast_refresh_resolve_result.?.path()) catch unreachable; - Analytics.Features.fast_refresh = true; - - if (BundledModuleData.get(this, &refresh_runtime)) |mod| { - refresh_runtime_module_id = mod.module_id; - } - } else |_| {} - } - - if (Environment.isDebug) { - switch (bundler.options.platform) { - .node => try generator.appendBytes(runtime.Runtime.sourceContentNode()), - .bun_macro, .bun => try generator.appendBytes(runtime.Runtime.sourceContentBun()), - else => try generator.appendBytes(runtime.Runtime.sourceContent(include_fast_refresh_in_bundle)), - } - - try generator.appendBytes("\n\n"); - } else if (include_fast_refresh_in_bundle) { - try generator.appendBytes(comptime runtime.Runtime.sourceContent(true) ++ "\n\n"); - } else { - try generator.appendBytes( - switch (bundler.options.platform) { - .bun_macro, .bun => comptime @as(string, runtime.Runtime.sourceContentBun() ++ "\n\n"), - .node => comptime @as(string, runtime.Runtime.sourceContentNode() ++ "\n\n"), - else => comptime @as(string, runtime.Runtime.sourceContentWithoutRefresh() ++ "\n\n"), - }, - ); - } - - if (_new_jsx_runtime_resolve_result) |new_jsx_runtime| { - try this.enqueueItem(new_jsx_runtime); - _new_jsx_runtime_resolve_result = null; - } - - if (fast_refresh_resolve_result) |fast_refresh| { - try this.enqueueItem(fast_refresh); - fast_refresh_resolve_result = null; - } - - if (bundler.router) |router| { - defer this.bundler.resetStore(); - Analytics.Features.filesystem_router = true; - - const entry_points = try router.getEntryPoints(); - for (entry_points) |entry_point| { - const resolved = bundler.resolveEntryPoint(entry_point) catch continue; - try this.enqueueItem(resolved); - } - this.bundler.resetStore(); - } else {} - - if (bundler.options.framework) |framework| { - if (bundler.options.platform.isBun()) { - if (framework.server.isEnabled()) { - Analytics.Features.bunjs = true; - const resolved = try bundler.resolver.resolve( - bundler.fs.top_level_dir, - framework.server.path, - .entry_point, - ); - try this.enqueueItem(resolved); - } - } else { - if (framework.client.isEnabled()) { - const resolved = try bundler.resolver.resolve( - bundler.fs.top_level_dir, - framework.client.path, - .entry_point, - ); - try this.enqueueItem(resolved); - } - - if (framework.fallback.isEnabled()) { - const resolved = try bundler.resolver.resolve( - bundler.fs.top_level_dir, - framework.fallback.path, - .entry_point, - ); - try this.enqueueItem(resolved); - } - } - } - - for (bundler.options.entry_points) |entry_point| { - defer this.bundler.resetStore(); - - const entry_point_path = bundler.normalizeEntryPointPath(entry_point); - const resolved = bundler.resolveEntryPoint(entry_point_path) catch continue; - try this.enqueueItem(resolved); - } - - Analytics.enqueue(Analytics.EventName.bundle_start); - this.bundler.resetStore(); - this.pool.wait(this) catch |err| { - Analytics.enqueue(Analytics.EventName.bundle_fail); - return err; - }; - Analytics.enqueue(Analytics.EventName.bundle_success); - - estimated_input_lines_of_code.* = generator.estimated_input_lines_of_code; - - // if (comptime !isRelease) { - // this.queue.checkDuplicatesSlow(); - // } - - if (this.log.errors > 0) { - tmpfile.closeAndDelete(std.mem.span(tmpname)); - // We stop here because if there are errors we don't know if the bundle is valid - // This manifests as a crash when sorting through the module list because we may have added files to the bundle which were never actually finished being added. - return null; - } - - if (include_refresh_runtime and refresh_runtime_module_id > 0) { - var refresh_runtime_injector_buf: [1024]u8 = undefined; - var fixed_buffer = std.io.fixedBufferStream(&refresh_runtime_injector_buf); - var fixed_buffer_writer = fixed_buffer.writer(); - - fixed_buffer_writer.print( - \\if ('window' in globalThis) {{ - \\ (function() {{ - \\ BUN_RUNTIME.__injectFastRefresh(${x}()); - \\ }})(); - \\}} - , - .{refresh_runtime_module_id}, - ) catch unreachable; - try this.tmpfile.writeAll(fixed_buffer.buffer[0..fixed_buffer.pos]); - } - - // Ensure we never overflow - this.code_end_byte_offset = @truncate( - u32, - // Doing this math ourself seems to not necessarily produce correct results - (try this.tmpfile.getPos()), - ); - - var javascript_bundle_container = std.mem.zeroes(Api.JavascriptBundleContainer); - - std.sort.sort( - Api.JavascriptBundledModule, - this.module_list.items, - this, - GenerateNodeModuleBundle.sortJavascriptModuleByPath, - ); - - if (comptime Environment.isDebug) { - const SeenHash = std.AutoHashMap(u64, void); - var map = SeenHash.init(this.allocator); - var ids = SeenHash.init(this.allocator); - try map.ensureTotalCapacity(@truncate(u32, this.module_list.items.len)); - try ids.ensureTotalCapacity(@truncate(u32, this.module_list.items.len)); - - for (this.module_list.items) |a| { - const a_pkg: Api.JavascriptBundledPackage = this.package_list.items[a.package_id]; - const a_name = this.metadataStringPointer(a_pkg.name); - const a_version = this.metadataStringPointer(a_pkg.version); - const a_path = this.metadataStringPointer(a.path); - - std.debug.assert(a_name.len > 0); - std.debug.assert(a_version.len > 0); - std.debug.assert(a_path.len > 0); - var hash_print = std.mem.zeroes([4096]u8); - const hash = std.hash.Wyhash.hash(0, std.fmt.bufPrint(&hash_print, "{s}@{s}/{s}", .{ a_name, a_version, a_path }) catch unreachable); - var result1 = map.getOrPutAssumeCapacity(hash); - std.debug.assert(!result1.found_existing); - - var result2 = ids.getOrPutAssumeCapacity(a.id); - std.debug.assert(!result2.found_existing); - } - } - - var hasher = std.hash.Wyhash.init(0); - - // We want to sort the packages as well as the files - // The modules sort the packages already - // So can just copy it in the below loop. - var sorted_package_list = try allocator.alloc(Api.JavascriptBundledPackage, this.package_list.items.len); - - // At this point, the module_list is sorted. - if (this.module_list.items.len > 0) { - var package_id_i: u32 = 0; - var i: usize = 0; - // Assumption: node_modules are immutable - // Assumption: module files are immutable - // (They're not. But, for our purposes that's okay) - // The etag is: - // - The hash of each module's path in sorted order - // - The hash of each module's code size in sorted order - // - hash(hash(package_name, package_version)) - // If this doesn't prove strong enough, we will do a proper content hash - // But I want to avoid that overhead unless proven necessary. - // There's a good chance we don't even strictly need an etag here. - var bytes: [4]u8 = undefined; - while (i < this.module_list.items.len) { - var current_package_id = this.module_list.items[i].package_id; - this.module_list.items[i].package_id = package_id_i; - var offset = @truncate(u32, i); - - i += 1; - - while (i < this.module_list.items.len and this.module_list.items[i].package_id == current_package_id) : (i += 1) { - this.module_list.items[i].package_id = package_id_i; - // Hash the file path - hasher.update(this.metadataStringPointer(this.module_list.items[i].path)); - // Then the length of the code - std.mem.writeIntNative(u32, &bytes, this.module_list.items[i].code.length); - hasher.update(&bytes); - } - - this.package_list.items[current_package_id].modules_offset = offset; - this.package_list.items[current_package_id].modules_length = @truncate(u32, i) - offset; - - // Hash the hash of the package name - // it's hash(hash(package_name, package_version)) - std.mem.writeIntNative(u32, &bytes, this.package_list.items[current_package_id].hash); - hasher.update(&bytes); - - sorted_package_list[package_id_i] = this.package_list.items[current_package_id]; - package_id_i += 1; - } - } - - var javascript_bundle = std.mem.zeroes(Api.JavascriptBundle); - javascript_bundle.modules = this.module_list.items; - javascript_bundle.packages = sorted_package_list; - javascript_bundle.manifest_string = this.header_string_buffer.list.items; - const etag_u64 = hasher.final(); - // We store the etag as a ascii hex encoded u64 - // This is so we can send the bytes directly in the HTTP server instead of formatting it as hex each time. - javascript_bundle.etag = try std.fmt.allocPrint(allocator, "{x}", .{etag_u64}); - javascript_bundle.generated_at = @truncate(u32, @intCast(u64, std.time.milliTimestamp())); - - const basename = std.fs.path.basename(std.mem.span(destination)); - const extname = std.fs.path.extension(basename); - javascript_bundle.import_from_name = if (bundler.options.platform.isBun()) - "/node_modules.server.bun" - else - try std.fmt.allocPrint( - this.allocator, - "/{s}.{x}.bun", - .{ - basename[0 .. basename.len - extname.len], - etag_u64, - }, - ); - - javascript_bundle_container.bundle_format_version = current_version; - javascript_bundle_container.bundle = javascript_bundle; - javascript_bundle_container.code_length = this.code_end_byte_offset; - javascript_bundle_container.framework = framework_config; - javascript_bundle_container.routes = route_config; - - var start_pos = try this.tmpfile.getPos(); - var tmpwriter = std.io.bufferedWriter(this.tmpfile.writer()); - const SchemaWriter = schema.Writer(@TypeOf(tmpwriter.writer())); - var schema_file_writer = SchemaWriter.init(tmpwriter.writer()); - try javascript_bundle_container.encode(&schema_file_writer); - try tmpwriter.flush(); - - // sanity check - if (Environment.isDebug) { - try this.tmpfile.seekTo(start_pos); - var contents = try allocator.alloc(u8, (try this.tmpfile.getEndPos()) - start_pos); - var read_bytes = try this.tmpfile.read(contents); - var buf = contents[0..read_bytes]; - var reader = schema.Reader.init(buf, allocator); - - var decoder = try Api.JavascriptBundleContainer.decode( - &reader, - ); - std.debug.assert(decoder.code_length.? == javascript_bundle_container.code_length.?); - } - - var code_length_bytes: [4]u8 = undefined; - std.mem.writeIntNative(u32, &code_length_bytes, this.code_end_byte_offset); - _ = try std.os.pwrite(this.tmpfile.handle, &code_length_bytes, magic_bytes.len); - - // Without his mutex, we get a crash at this location: - // try std.os.renameat(tmpdir.fd, tmpname, top_dir.fd, destination); - // ^ - const top_dir = try std.fs.openDirAbsolute(Fs.FileSystem.instance.top_level_dir, .{}); - _ = C.fchmod( - this.tmpfile.handle, - // chmod 777 - 0000010 | 0000100 | 0000001 | 0001000 | 0000040 | 0000004 | 0000002 | 0000400 | 0000200 | 0000020, - ); - try tmpfile.promote(tmpname, top_dir.fd, destination); - // Print any errors at the end - // try this.log.print(Output.errorWriter()); - return javascript_bundle_container; - } - - pub fn metadataStringPointer(this: *GenerateNodeModuleBundle, ptr: Api.StringPointer) string { - return this.header_string_buffer.list.items[ptr.offset .. ptr.offset + ptr.length]; - } - - // Since we trim the prefixes, we must also compare the package name and version - pub fn sortJavascriptModuleByPath(ctx: *GenerateNodeModuleBundle, a: Api.JavascriptBundledModule, b: Api.JavascriptBundledModule) bool { - return switch (std.mem.order( - u8, - ctx.metadataStringPointer( - ctx.package_list.items[a.package_id].name, - ), - ctx.metadataStringPointer( - ctx.package_list.items[b.package_id].name, - ), - )) { - .eq => switch (std.mem.order( - u8, - ctx.metadataStringPointer( - ctx.package_list.items[a.package_id].version, - ), - ctx.metadataStringPointer( - ctx.package_list.items[b.package_id].version, - ), - )) { - .eq => std.mem.order( - u8, - ctx.metadataStringPointer(a.path), - ctx.metadataStringPointer(b.path), - ) == .lt, - .lt => true, - else => false, - }, - .lt => true, - else => false, - }; - } - - // pub fn sortJavascriptPackageByName(ctx: *GenerateNodeModuleBundle, a: Api.JavascriptBundledPackage, b: Api.JavascriptBundledPackage) bool { - // return std.mem.order(u8, ctx.metadataStringPointer(a.name), ctx.metadataStringPointer(b.name)) == .lt; - // } - - pub fn appendBytes(generator: *GenerateNodeModuleBundle, bytes: anytype) !void { - try generator.tmpfile.writeAll(bytes); - generator.tmpfile_byte_offset += @truncate(u32, bytes.len); - } - - const BundledModuleData = struct { - import_path: string, - package_path: string, - package: *const PackageJSON, - module_id: u32, - - pub fn getForceBundle(this: *GenerateNodeModuleBundle, resolve_result: *const _resolver.Result) ?BundledModuleData { - return _get(this, resolve_result, true, false); - } - - pub fn getForceBundleForMain(this: *GenerateNodeModuleBundle, resolve_result: *const _resolver.Result) ?BundledModuleData { - return _get(this, resolve_result, true, true); - } - - threadlocal var normalized_package_path: [512]u8 = undefined; - threadlocal var normalized_package_path2: [512]u8 = undefined; - inline fn _get(this: *GenerateNodeModuleBundle, resolve_result: *const _resolver.Result, comptime force: bool, comptime is_main: bool) ?BundledModuleData { - const path = resolve_result.pathConst() orelse return null; - if (strings.eqlComptime(path.namespace, "node")) { - const _import_path = path.text["/bun-vfs/node_modules/".len..][resolve_result.package_json.?.name.len + 1 ..]; - return BundledModuleData{ - .import_path = _import_path, - .package_path = path.text["/bun-vfs/node_modules/".len..], - .package = resolve_result.package_json.?, - .module_id = resolve_result.package_json.?.hashModule(_import_path), - }; - } - - var import_path = path.text; - var package_path = path.text; - var file_path = path.text; - - if (resolve_result.package_json) |pkg_| { - var pkg: *const PackageJSON = pkg_; - if (this.package_bundle_map.get(pkg.name)) |result| { - if (result == .never) return null; - } - - if (std.mem.indexOfScalar(u32, this.always_bundled_package_hashes, pkg.hash)) |pkg_i| { - pkg = this.always_bundled_package_jsons[pkg_i]; - if (!this.always_bundled_booleans[pkg_i]) return null; - const key_path_source_dir = pkg.source.key_path.sourceDir(); - const default_source_dir = pkg.source.path.sourceDir(); - - if (strings.startsWith(path.text, key_path_source_dir)) { - import_path = path.text[key_path_source_dir.len..]; - } else if (strings.startsWith(path.text, default_source_dir)) { - import_path = path.text[default_source_dir.len..]; - } else if (strings.startsWith(path.pretty, pkg.name)) { - import_path = path.pretty[pkg.name.len + 1 ..]; - } - - var buf_to_use: []u8 = if (is_main) &normalized_package_path2 else &normalized_package_path; - - std.mem.copy(u8, buf_to_use, pkg.name); - buf_to_use[pkg.name.len] = '/'; - std.mem.copy(u8, buf_to_use[pkg.name.len + 1 ..], import_path); - package_path = buf_to_use[0 .. pkg.name.len + import_path.len + 1]; - return BundledModuleData{ - .import_path = import_path, - .package_path = package_path, - .package = pkg, - .module_id = pkg.hashModule(package_path), - }; - } - } - - const root: _resolver.RootPathPair = this.bundler.resolver.rootNodeModulePackageJSON( - resolve_result, - ) orelse return null; - - var base_path = root.base_path; - const package_json = root.package_json; - - if (std.mem.indexOfScalar(u32, this.always_bundled_package_hashes, package_json.hash)) |pkg_i| { - var pkg = this.always_bundled_package_jsons[pkg_i]; - if (!this.always_bundled_booleans[pkg_i]) return null; - const key_path_source_dir = pkg.source.key_path.sourceDir(); - const default_source_dir = pkg.source.path.sourceDir(); - - if (strings.startsWith(path.text, key_path_source_dir)) { - import_path = path.text[key_path_source_dir.len..]; - } else if (strings.startsWith(path.text, default_source_dir)) { - import_path = path.text[default_source_dir.len..]; - } else if (strings.startsWith(path.pretty, pkg.name)) { - import_path = path.pretty[pkg.name.len + 1 ..]; - } - - var buf_to_use: []u8 = if (is_main) &normalized_package_path2 else &normalized_package_path; - - std.mem.copy(u8, buf_to_use, pkg.name); - buf_to_use[pkg.name.len] = '/'; - std.mem.copy(u8, buf_to_use[pkg.name.len + 1 ..], import_path); - package_path = buf_to_use[0 .. pkg.name.len + import_path.len + 1]; - return BundledModuleData{ - .import_path = import_path, - .package_path = package_path, - .package = pkg, - .module_id = pkg.hashModule(package_path), - }; - } - - // Easymode: the file path doesn't need to be remapped. - if (strings.startsWith(file_path, base_path)) { - import_path = std.mem.trimLeft(u8, path.text[base_path.len..], "/"); - package_path = std.mem.trim(u8, path.text[base_path.len - package_json.name.len - 1 ..], "/"); - std.debug.assert(import_path.len > 0); - return BundledModuleData{ - .import_path = import_path, - .package_path = package_path, - .package = package_json, - .module_id = package_json.hashModule(package_path), - }; - } - - if (std.mem.lastIndexOf(u8, file_path, package_json.name)) |i| { - package_path = file_path[i..]; - import_path = package_path[package_json.name.len + 1 ..]; - std.debug.assert(import_path.len > 0); - return BundledModuleData{ - .import_path = import_path, - .package_path = package_path, - .package = package_json, - .module_id = package_json.hashModule(package_path), - }; - } - - if (comptime force) { - if (std.mem.indexOfScalar(u32, this.always_bundled_package_hashes, root.package_json.hash)) |pkg_json_i| { - const pkg_json = this.always_bundled_package_jsons[pkg_json_i]; - - base_path = pkg_json.source.key_path.sourceDir(); - - if (strings.startsWith(file_path, base_path)) { - import_path = std.mem.trimLeft(u8, path.text[base_path.len..], "/"); - package_path = std.mem.trim(u8, path.text[base_path.len - package_json.name.len - 1 ..], "/"); - std.debug.assert(import_path.len > 0); - return BundledModuleData{ - .import_path = import_path, - .package_path = package_path, - .package = package_json, - .module_id = package_json.hashModule(package_path), - }; - } - - if (std.mem.lastIndexOf(u8, file_path, package_json.name)) |i| { - package_path = file_path[i..]; - import_path = package_path[package_json.name.len + 1 ..]; - std.debug.assert(import_path.len > 0); - return BundledModuleData{ - .import_path = import_path, - .package_path = package_path, - .package = package_json, - .module_id = package_json.hashModule(package_path), - }; - } - - // This is our last resort. - // The package is supposed to be bundled - // package.json says its' in packages/@foo/bar/index.js - // The file really is in packages/bar/index.js - // But we need the import path to contain the package path for bundling to work - // so we fake it - // we say "" - if (package_json.name[0] == '@') { - if (std.mem.indexOfScalar(u8, package_json.name, '/')) |at| { - const package_subpath = package_json.name[at + 1 ..]; - if (std.mem.lastIndexOf(u8, file_path, package_subpath)) |i| { - package_path = this.bundler.fs.dirname_store.print("{s}/{s}", .{ package_json.name, file_path[i + package_subpath.len ..] }) catch unreachable; - import_path = package_path[package_json.name.len + 1 ..]; - return BundledModuleData{ - .import_path = import_path, - .package_path = package_path, - .package = package_json, - .module_id = package_json.hashModule(package_path), - }; - } - } - } - } - } - - return null; - } - - pub fn get(this: *GenerateNodeModuleBundle, resolve_result: *const _resolver.Result) ?BundledModuleData { - return _get(this, resolve_result, false, false); - } - }; - - fn writeEmptyModule(this: *GenerateNodeModuleBundle, package_relative_path: string, module_id: u32) !u32 { - this.write_lock.lock(); - defer this.write_lock.unlock(); - var code_offset = @truncate(u32, try this.tmpfile.getPos()); - var writer = this.tmpfile.writer(); - var buffered = std.io.bufferedWriter(writer); - - var bufwriter = buffered.writer(); - try bufwriter.writeAll("// "); - try bufwriter.writeAll(package_relative_path); - try bufwriter.writeAll(" (disabled/empty)\nexport var $"); - std.fmt.formatInt(module_id, 16, .lower, .{}, bufwriter) catch unreachable; - try bufwriter.writeAll(" = () => { var obj = {}; Object.defineProperty(obj, 'default', { value: obj, enumerable: false, configurable: true }, obj); return obj; }; \n"); - try buffered.flush(); - this.tmpfile_byte_offset = @truncate(u32, try this.tmpfile.getPos()); - return code_offset; - } - - fn processImportRecord(_: *GenerateNodeModuleBundle, _: ImportRecord) !void {} - var json_ast_symbols = [_]js_ast.Symbol{ - js_ast.Symbol{ .original_name = "$$m" }, - js_ast.Symbol{ .original_name = "exports" }, - js_ast.Symbol{ .original_name = "module" }, - js_ast.Symbol{ .original_name = "CONGRATS_YOU_FOUND_A_BUG" }, - js_ast.Symbol{ .original_name = "$$bun_runtime_json_parse" }, - }; - const json_parse_string = "parse"; - var json_ast_symbols_list = std.mem.span(&json_ast_symbols); - threadlocal var override_file_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; - - pub fn appendToModuleList( - this: *GenerateNodeModuleBundle, - package: *const PackageJSON, - module_id: u32, - code_offset: u32, - package_relative_path: string, - ) !void { - this.list_lock.lock(); - defer this.list_lock.unlock(); - - const code_length = @atomicLoad(u32, &this.tmpfile_byte_offset, .SeqCst) - code_offset; - - if (comptime Environment.isDebug) { - std.debug.assert(code_length > 0); - std.debug.assert(package.hash != 0); - std.debug.assert(package.version.len > 0); - std.debug.assert(package.name.len > 0); - std.debug.assert(module_id > 0); - } - - var package_get_or_put_entry = try this.package_list_map.getOrPut(package.hash); - - if (!package_get_or_put_entry.found_existing) { - package_get_or_put_entry.value_ptr.* = @truncate(u32, this.package_list.items.len); - try this.package_list.append( - Api.JavascriptBundledPackage{ - .name = try this.appendHeaderString(package.name), - .version = try this.appendHeaderString(package.version), - .hash = package.hash, - }, - ); - this.has_jsx = this.has_jsx or strings.eql(package.name, this.bundler.options.jsx.package_name); - } - - var path_extname_length = @truncate(u8, std.fs.path.extension(package_relative_path).len); - try this.module_list.append( - Api.JavascriptBundledModule{ - .path = try this.appendHeaderString( - package_relative_path, - ), - .path_extname_length = path_extname_length, - .package_id = package_get_or_put_entry.value_ptr.*, - .id = module_id, - .code = Api.StringPointer{ - .length = @truncate(u32, code_length), - .offset = @truncate(u32, code_offset), - }, - }, - ); - } - threadlocal var json_e_string: js_ast.E.String = undefined; - threadlocal var json_e_call: js_ast.E.Call = undefined; - threadlocal var json_e_identifier: js_ast.E.Identifier = undefined; - threadlocal var json_call_args: [1]js_ast.Expr = undefined; - const PendingImports = std.AutoArrayHashMap(u32, ThreadPool.Worker.ProcessFileTask); - pub fn processFile(this: *GenerateNodeModuleBundle, worker: *ThreadPool.Worker, bundler: *Bundler, _resolve: _resolver.Result) !void { - const resolve = _resolve; - if (resolve.is_external) return; - - var shared_buffer = &worker.data.shared_buffer; - var scan_pass_result = &worker.data.scan_pass_result; - var file_path = (resolve.pathConst() orelse unreachable).*; - - var add_to_bundle = brk: { - if (resolve.package_json) |package_json| { - if (this.package_bundle_map.get(package_json.name)) |result| { - break :brk result == .always; - } - } - - break :brk resolve.isLikelyNodeModule(); - }; - - const source_dir = file_path.sourceDir(); - const loader = bundler.options.loader(file_path.name.ext); - const platform = bundler.options.platform; - - defer scan_pass_result.reset(); - defer shared_buffer.reset(); - defer this.bundler.resetStore(); - var log = worker.data.log; - var queue = PendingImports.init(worker.allocator); - var __module_data: ?BundledModuleData = null; - if (add_to_bundle) { - __module_data = BundledModuleData.getForceBundleForMain(this, &resolve); - } - // If we're in a node_module, build that almost normally - if (add_to_bundle and __module_data != null) { - const module_data = __module_data.?; - - var code_offset: u32 = 0; - - const module_id = module_data.module_id; - const package = module_data.package; - const package_relative_path = module_data.import_path; - - file_path.pretty = module_data.package_path; - - const entry: CacheEntry = brk: { - if (this.bundler.options.framework) |framework| { - if (framework.override_modules_hashes.len > 0) { - const package_relative_path_hash = std.hash.Wyhash.hash(0, module_data.package_path); - if (std.mem.indexOfScalar( - u64, - framework.override_modules_hashes, - package_relative_path_hash, - )) |index| { - const relative_path = [_]string{ - framework.resolved_dir, - framework.override_modules.values[index], - }; - var override_path = this.bundler.fs.absBuf( - &relative_path, - &override_file_path_buf, - ); - override_file_path_buf[override_path.len] = 0; - var override_pathZ = override_file_path_buf[0..override_path.len :0]; - break :brk try bundler.resolver.caches.fs.readFileShared( - bundler.fs, - override_pathZ, - 0, - null, - shared_buffer, - ); - } - } - } - - if (!strings.eqlComptime(file_path.namespace, "node")) - break :brk try bundler.resolver.caches.fs.readFileShared( - bundler.fs, - file_path.textZ(), - resolve.dirname_fd, - if (resolve.file_fd != 0) resolve.file_fd else null, - shared_buffer, - ); - - break :brk CacheEntry{ - .contents = NodeFallbackModules.contentsFromPath(file_path.text) orelse "", - }; - }; - - var approximate_newline_count: usize = 0; - defer worker.data.estimated_input_lines_of_code += approximate_newline_count; - - // Handle empty files - // We can't just ignore them. Sometimes code will try to import it. Often because of TypeScript types. - // So we just say it's an empty object. Empty object mimicks what "browser": false does as well. - // TODO: optimize this so that all the exports for these are done in one line instead of writing repeatedly - if (entry.contents.len == 0 or (entry.contents.len < 33 and strings.trim(entry.contents, " \n\r").len == 0)) { - code_offset = try this.writeEmptyModule(module_data.package_path, module_id); - } else { - var ast: js_ast.Ast = undefined; - - const source = logger.Source.initRecycledFile( - Fs.File{ - .path = file_path, - .contents = entry.contents, - }, - bundler.allocator, - ) catch return null; - - switch (loader) { - .jsx, - .tsx, - .js, - .ts, - => { - var jsx = _resolve.jsx; - jsx.parse = loader.isJSX(); - - var opts = js_parser.Parser.Options.init(jsx, loader); - opts.transform_require_to_import = false; - opts.enable_bundling = true; - opts.warn_about_unbundled_modules = false; - opts.macro_context = &worker.data.macro_context; - opts.features.auto_import_jsx = jsx.parse; - opts.features.trim_unused_imports = this.bundler.options.trim_unused_imports orelse loader.isTypeScript(); - opts.tree_shaking = this.bundler.options.tree_shaking; - ast = (bundler.resolver.caches.js.parse( - bundler.allocator, - opts, - bundler.options.define, - log, - &source, - ) catch null) orelse return; - - approximate_newline_count = ast.approximate_newline_count; - if (ast.import_records.len > 0) { - for (ast.import_records) |*import_record| { - - // Don't resolve the runtime - if (import_record.is_internal or import_record.is_unused) { - continue; - } - - if (bundler.resolver.resolve(source_dir, import_record.path.text, import_record.kind)) |*_resolved_import| { - if (_resolved_import.is_external) { - continue; - } - var path = _resolved_import.path() orelse { - import_record.path.is_disabled = true; - import_record.is_bundled = true; - continue; - }; - - const loader_ = bundler.options.loader(path.name.ext); - - if (!loader_.isJavaScriptLikeOrJSON()) { - import_record.path.is_disabled = true; - import_record.is_bundled = true; - continue; - } - - // if (_resolved_import.package_json == null) |pkg_json| { - // _resolved_import.package_json = if (pkg_json.hash == resolve.package_json.?.hash) - // resolve.package_json - // else - // _resolved_import.package_json; - // } - - const resolved_import: *const _resolver.Result = _resolved_import; - - const _module_data = BundledModuleData.getForceBundle(this, resolved_import) orelse { - // if a macro imports code that cannot be bundled - // we just silently disable it - // because...we need some kind of hook to say "don't bundle this" - import_record.path.is_disabled = true; - import_record.is_bundled = false; - - continue; - }; - import_record.module_id = _module_data.module_id; - std.debug.assert(import_record.module_id != 0); - import_record.is_bundled = true; - - path.* = try path.dupeAlloc(this.allocator); - - import_record.path = path.*; - _ = queue.getOrPutValue( - _module_data.module_id, - .{ - .resolution = _resolved_import.*, - }, - ) catch unreachable; - } else |err| { - if (comptime Environment.isDebug) { - if (!import_record.handles_import_errors) { - Output.prettyErrorln("\n<r><red>{s}<r> resolving \"{s}\" from \"{s}\"", .{ - @errorName(err), - import_record.path.text, - file_path.text, - }); - } - } - - // Disable failing packages from being printed. - // This may cause broken code to write. - // However, doing this means we tell them all the resolve errors - // Rather than just the first one. - import_record.path.is_disabled = true; - - switch (err) { - error.ModuleNotFound => { - const addError = logger.Log.addResolveErrorWithTextDupeMaybeWarn; - - if (!import_record.handles_import_errors) { - if (isPackagePath(import_record.path.text)) { - if (platform.isWebLike() and options.ExternalModules.isNodeBuiltin(import_record.path.text)) { - try addError( - log, - &source, - import_record.range, - this.allocator, - "Could not resolve Node.js builtin: \"{s}\".", - .{import_record.path.text}, - import_record.kind, - platform.isBun(), - ); - } else { - try addError( - log, - &source, - import_record.range, - this.allocator, - "Could not resolve: \"{s}\". Maybe you need to \"bun install\"?", - .{import_record.path.text}, - import_record.kind, - platform.isBun(), - ); - } - } else if (!platform.isBun()) { - try addError( - log, - &source, - import_record.range, - this.allocator, - "Could not resolve: \"{s}\"", - .{ - import_record.path.text, - }, - import_record.kind, - platform.isBun(), - ); - } - } - }, - // assume other errors are already in the log - else => {}, - } - } - } - } - }, - .json => { - // parse the JSON _only_ to catch errors at build time. - const json_parse_result = json_parser.ParseJSONForBundling(&source, worker.data.log, worker.allocator) catch return; - - if (json_parse_result.tag != .empty) { - const expr = brk: { - // If it's an ascii string, we just print it out with a big old JSON.parse() - if (json_parse_result.tag == .ascii) { - json_e_string = js_ast.E.String{ .utf8 = source.contents, .prefer_template = true }; - var json_string_expr = js_ast.Expr{ .data = .{ .e_string = &json_e_string }, .loc = logger.Loc{ .start = 0 } }; - json_call_args[0] = json_string_expr; - json_e_identifier = js_ast.E.Identifier{ .ref = Ref.atIndex(json_ast_symbols_list.len - 1) }; - - json_e_call = js_ast.E.Call{ - .target = js_ast.Expr{ .data = .{ .e_identifier = json_e_identifier }, .loc = logger.Loc{ .start = 0 } }, - .args = js_ast.ExprNodeList.init(std.mem.span(&json_call_args)), - }; - break :brk js_ast.Expr{ .data = .{ .e_call = &json_e_call }, .loc = logger.Loc{ .start = 0 } }; - // If we're going to have to convert it to a UTF16, just make it an object actually - } else { - break :brk json_parse_result.expr; - } - }; - - var stmt = js_ast.Stmt.alloc(js_ast.S.ExportDefault, js_ast.S.ExportDefault{ - .value = js_ast.StmtOrExpr{ .expr = expr }, - .default_name = js_ast.LocRef{ - .loc = logger.Loc{}, - .ref = Ref.None, - }, - }, logger.Loc{ .start = 0 }); - var stmts = worker.allocator.alloc(js_ast.Stmt, 1) catch unreachable; - stmts[0] = stmt; - var parts = worker.allocator.alloc(js_ast.Part, 1) catch unreachable; - parts[0] = js_ast.Part{ .stmts = stmts }; - ast = js_ast.Ast.initTest(parts); - - ast.runtime_imports = runtime.Runtime.Imports{}; - ast.runtime_imports.@"$$m" = .{ .ref = Ref.atIndex(0), .primary = Ref.None, .backup = Ref.None }; - ast.runtime_imports.__export = .{ .ref = Ref.atIndex(1), .primary = Ref.None, .backup = Ref.None }; - ast.symbols = json_ast_symbols_list; - ast.module_ref = Ref.atIndex(2); - ast.exports_ref = ast.runtime_imports.__export.?.ref; - ast.bundle_export_ref = Ref.atIndex(3); - } else { - var parts = &[_]js_ast.Part{}; - ast = js_ast.Ast.initTest(parts); - } - }, - else => { - return; - }, - } - - switch (ast.parts.len) { - // It can be empty after parsing too - // A file like this is an example: - // "//# sourceMappingURL=validator.js.map" - 0 => { - code_offset = try this.writeEmptyModule(module_data.package_path, module_id); - }, - else => { - const register_ref = ast.runtime_imports.@"$$m".?.ref; - const E = js_ast.E; - const Expr = js_ast.Expr; - const Stmt = js_ast.Stmt; - - var prepend_part: js_ast.Part = undefined; - var needs_prepend_part = false; - if (ast.parts.len > 1) { - for (ast.parts) |part| { - if (part.tag != .none and part.stmts.len > 0) { - prepend_part = part; - needs_prepend_part = true; - break; - } - } - } - - var package_path = js_ast.E.String{ .utf8 = module_data.package_path }; - - var target_identifier = E.Identifier{ .ref = register_ref }; - var cjs_args: [2]js_ast.G.Arg = undefined; - var module_binding = js_ast.B.Identifier{ .ref = ast.module_ref.? }; - var exports_binding = js_ast.B.Identifier{ .ref = ast.exports_ref.? }; - - var part = &ast.parts[ast.parts.len - 1]; - - var new_stmts: [1]Stmt = undefined; - var register_args: [1]Expr = undefined; - var closure = E.Arrow{ - .args = &cjs_args, - .body = .{ - .loc = logger.Loc.Empty, - .stmts = part.stmts, - }, - }; - - cjs_args[0] = js_ast.G.Arg{ - .binding = js_ast.Binding{ - .loc = logger.Loc.Empty, - .data = .{ .b_identifier = &module_binding }, - }, - }; - cjs_args[1] = js_ast.G.Arg{ - .binding = js_ast.Binding{ - .loc = logger.Loc.Empty, - .data = .{ .b_identifier = &exports_binding }, - }, - }; - - var properties: [1]js_ast.G.Property = undefined; - var e_object = E.Object{ - .properties = js_ast.G.Property.List.init(&properties), - }; - const module_path_str = js_ast.Expr{ .data = .{ .e_string = &package_path }, .loc = logger.Loc.Empty }; - properties[0] = js_ast.G.Property{ - .key = module_path_str, - .value = Expr{ .loc = logger.Loc.Empty, .data = .{ .e_arrow = &closure } }, - }; - - // if (!ast.uses_module_ref) { - // var symbol = &ast.symbols[ast.module_ref.?.innerIndex()]; - // symbol.original_name = "_$$"; - // } - - // $$m(12345, "react", "index.js", function(module, exports) { - - // }) - var accessor = js_ast.E.Index{ .index = module_path_str, .target = js_ast.Expr{ - .data = .{ .e_object = &e_object }, - .loc = logger.Loc.Empty, - } }; - register_args[0] = Expr{ .loc = logger.Loc.Empty, .data = .{ .e_index = &accessor } }; - - var call_register = E.Call{ - .target = Expr{ - .data = .{ .e_identifier = target_identifier }, - .loc = logger.Loc{ .start = 0 }, - }, - .args = js_ast.ExprNodeList.init(®ister_args), - }; - var register_expr = Expr{ .loc = call_register.target.loc, .data = .{ .e_call = &call_register } }; - var decls: [1]js_ast.G.Decl = undefined; - var bundle_export_binding = js_ast.B.Identifier{ .ref = ast.runtime_imports.@"$$m".?.ref }; - var binding = js_ast.Binding{ - .loc = register_expr.loc, - .data = .{ .b_identifier = &bundle_export_binding }, - }; - decls[0] = js_ast.G.Decl{ - .value = register_expr, - .binding = binding, - }; - var export_var = js_ast.S.Local{ - .decls = &decls, - .is_export = true, - }; - new_stmts[0] = Stmt{ .loc = register_expr.loc, .data = .{ .s_local = &export_var } }; - part.stmts = &new_stmts; - - var writer = js_printer.NewFileWriter(this.tmpfile); - var symbols: [][]js_ast.Symbol = &([_][]js_ast.Symbol{ast.symbols}); - - // It should only have one part. - ast.parts = ast.parts[ast.parts.len - 1 ..]; - const write_result = - // The difference between these two is `ascii_only` - // We try our best to print UTF-8. - // However, JavaScriptCore does not accept UTF-8. - // It accepts either latin1 characters as unsigned char - // or UTF-16 chars as uint16 - // We don't want to add a UTF decoding pass to the .bun files - // because it's potentially 15 MB of code. - // If we store it as UTF-16 directly, then that 15 MB of code becomes 30 MB of code! - // lots of code! - // - - if (!bundler.options.platform.isBun()) - try js_printer.printCommonJSThreaded( - @TypeOf(writer), - writer, - ast, - js_ast.Symbol.Map.initList(symbols), - &source, - false, - js_printer.Options{ - .to_module_ref = Ref.RuntimeRef, - .bundle_export_ref = ast.runtime_imports.@"$$m".?.ref, - .source_path = file_path, - .externals = ast.externals, - .indent = 0, - .require_ref = ast.require_ref, - .module_hash = module_id, - .runtime_imports = ast.runtime_imports, - .prepend_part_value = &prepend_part, - .prepend_part_key = if (needs_prepend_part) closure.body.stmts.ptr else null, - }, - Linker, - &bundler.linker, - &this.write_lock, - std.fs.File, - this.tmpfile, - std.fs.File.getPos, - &this.tmpfile_byte_offset, - ) - else - try js_printer.printCommonJSThreaded( - @TypeOf(writer), - writer, - ast, - js_ast.Symbol.Map.initList(symbols), - &source, - true, - js_printer.Options{ - .to_module_ref = Ref.RuntimeRef, - .bundle_export_ref = ast.runtime_imports.@"$$m".?.ref, - .source_path = file_path, - .externals = ast.externals, - .indent = 0, - .require_ref = ast.require_ref, - .module_hash = module_id, - .runtime_imports = ast.runtime_imports, - .prepend_part_value = &prepend_part, - .prepend_part_key = if (needs_prepend_part) closure.body.stmts.ptr else null, - }, - Linker, - &bundler.linker, - &this.write_lock, - std.fs.File, - this.tmpfile, - std.fs.File.getPos, - &this.tmpfile_byte_offset, - ); - - code_offset = write_result.off; - }, - } - } - - if (comptime Environment.isDebug) { - Output.prettyln("{s}@{s}/{s} - {d}:{d} \n", .{ package.name, package.version, package_relative_path, package.hash, module_id }); - Output.flush(); - std.debug.assert(package_relative_path.len > 0); - } - - try this.appendToModuleList( - package, - module_id, - code_offset, - package_relative_path, - ); - } else { - // If it's app code, scan but do not fully parse. - switch (loader) { - .jsx, - .tsx, - .js, - .ts, - => { - const entry = bundler.resolver.caches.fs.readFileShared( - bundler.fs, - file_path.textZ(), - resolve.dirname_fd, - if (resolve.file_fd != 0) resolve.file_fd else null, - shared_buffer, - ) catch return; - if (entry.contents.len == 0 or (entry.contents.len < 33 and strings.trim(entry.contents, " \n\r").len == 0)) return; - - const source = logger.Source.initRecycledFile(Fs.File{ .path = file_path, .contents = entry.contents }, bundler.allocator) catch return null; - - var jsx = bundler.options.jsx; - - jsx.parse = loader.isJSX(); - var opts = js_parser.Parser.Options.init(jsx, loader); - opts.macro_context = &worker.data.macro_context; - opts.features.trim_unused_imports = bundler.options.trim_unused_imports orelse loader.isTypeScript(); - - try bundler.resolver.caches.js.scan( - bundler.allocator, - scan_pass_result, - opts, - bundler.options.define, - log, - &source, - ); - worker.data.estimated_input_lines_of_code += scan_pass_result.approximate_newline_count; - - { - for (scan_pass_result.import_records.items) |*import_record| { - if (import_record.is_internal or import_record.is_unused) { - continue; - } - - if (bundler.resolver.resolve(source_dir, import_record.path.text, import_record.kind)) |*_resolved_import| { - if (_resolved_import.is_external) { - continue; - } - - var path = _resolved_import.path() orelse continue; - - const loader_ = this.bundler.options.loader(path.name.ext); - if (!loader_.isJavaScriptLikeOrJSON()) continue; - - path.* = try path.dupeAlloc(this.allocator); - - if (BundledModuleData.get(this, _resolved_import)) |mod| { - if (comptime !FeatureFlags.bundle_dynamic_import) { - if (import_record.kind == .dynamic) - continue; - } else { - // When app code dynamically imports a large file - // Don't bundle it. Leave it as a separate file. - // The main value from bundling in development is to minimize tiny, waterfall http requests - // If you're importing > 100 KB file dynamically, developer is probably explicitly trying to do that. - // There's a tradeoff between "I want to minimize page load time" - if (import_record.kind == .dynamic) { - this.dynamic_import_file_size_store_lock.lock(); - defer this.dynamic_import_file_size_store_lock.unlock(); - var dynamic_import_file_size = this.dynamic_import_file_size_store.getOrPut(mod.module_id) catch unreachable; - if (!dynamic_import_file_size.found_existing) { - var fd = _resolved_import.file_fd; - var can_close = false; - if (fd == 0) { - dynamic_import_file_size.value_ptr.* = 0; - fd = (std.fs.openFileAbsolute(path.textZ(), .{}) catch |err| { - this.log.addRangeWarningFmt( - &source, - import_record.range, - worker.allocator, - "{s} opening file: \"{s}\"", - .{ @errorName(err), path.text }, - ) catch unreachable; - continue; - }).handle; - can_close = true; - Fs.FileSystem.setMaxFd(fd); - } - - defer { - if (can_close and bundler.fs.fs.needToCloseFiles()) { - var _file = std.fs.File{ .handle = fd }; - _file.close(); - _resolved_import.file_fd = 0; - } else if (FeatureFlags.store_file_descriptors) { - _resolved_import.file_fd = fd; - } - } - - var file = std.fs.File{ .handle = fd }; - var stat = file.stat() catch |err| { - this.log.addRangeWarningFmt( - &source, - import_record.range, - worker.allocator, - "{s} stat'ing file: \"{s}\"", - .{ @errorName(err), path.text }, - ) catch unreachable; - dynamic_import_file_size.value_ptr.* = 0; - continue; - }; - - dynamic_import_file_size.value_ptr.* = @truncate(u32, stat.size); - } - - if (dynamic_import_file_size.value_ptr.* > 1024 * 100) - continue; - } - } - - std.debug.assert(mod.module_id != 0); - _ = queue.getOrPutValue( - mod.module_id, - .{ - .resolution = _resolved_import.*, - }, - ) catch unreachable; - } else { - _ = queue.getOrPutValue( - _resolved_import.hash( - this.bundler.fs.top_level_dir, - loader_, - ), - .{ - .resolution = _resolved_import.*, - }, - ) catch unreachable; - } - } else |err| { - switch (err) { - error.ModuleNotFound => { - if (!import_record.handles_import_errors) { - const addError = logger.Log.addResolveErrorWithTextDupeMaybeWarn; - if (isPackagePath(import_record.path.text)) { - if (platform.isWebLike() and options.ExternalModules.isNodeBuiltin(import_record.path.text)) { - try addError( - log, - &source, - import_record.range, - this.allocator, - "Could not resolve Node.js builtin: \"{s}\".", - .{import_record.path.text}, - import_record.kind, - platform.isBun(), - ); - } else { - try addError( - log, - &source, - import_record.range, - this.allocator, - "Could not resolve: \"{s}\". Maybe you need to \"bun install\"?", - .{import_record.path.text}, - import_record.kind, - platform.isBun(), - ); - } - } else if (!platform.isBun()) { - try addError( - log, - &source, - import_record.range, - this.allocator, - "Could not resolve: \"{s}\"", - .{ - import_record.path.text, - }, - import_record.kind, - platform.isBun(), - ); - } - } - }, - // assume other errors are already in the log - else => {}, - } - } - } - } - }, - else => {}, - } - } - - if (queue.count() > 0) { - try this.queue.writeItem(queue); - } - } - }; + pub const GenerateNodeModulesBundle = @import("./bundler/generate_node_modules_bundle.zig"); pub const BuildResolveResultPair = struct { written: usize, @@ -3387,281 +1559,6 @@ pub const Bundler = struct { } }; -pub const Transformer = struct { - opts: Api.TransformOptions, - log: *logger.Log, - allocator: std.mem.Allocator, - platform: options.Platform = undefined, - out_extensions: std.StringHashMap(string) = undefined, - output_path: string, - cwd: string, - define: *Define, - - pub fn transform( - allocator: std.mem.Allocator, - log: *logger.Log, - opts: Api.TransformOptions, - ) !options.TransformResult { - js_ast.Expr.Data.Store.create(allocator); - js_ast.Stmt.Data.Store.create(allocator); - const platform = options.Platform.from(opts.platform); - - var define = try options.definesFromTransformOptions( - allocator, - log, - opts.define, - false, - platform, - null, - null, - ); - - const cwd = if (opts.absolute_working_dir) |workdir| try std.fs.realpathAlloc(allocator, workdir) else try std.process.getCwdAlloc(allocator); - - const output_dir_parts = [_]string{ try std.process.getCwdAlloc(allocator), opts.output_dir orelse "out" }; - const output_dir = try std.fs.path.join(allocator, &output_dir_parts); - var output_files = try std.ArrayList(options.OutputFile).initCapacity(allocator, opts.entry_points.len); - const out_extensions = platform.outExtensions(allocator); - - var loader_map = try options.loadersFromTransformOptions(allocator, opts.loaders); - var use_default_loaders = loader_map.count() == 0; - - var jsx = if (opts.jsx) |_jsx| try options.JSX.Pragma.fromApi(_jsx, allocator) else options.JSX.Pragma{}; - - var ulimit: usize = Fs.FileSystem.RealFS.adjustUlimit() catch unreachable; - var care_about_closing_files = !(FeatureFlags.store_file_descriptors and opts.entry_points.len * 2 < ulimit); - - var transformer = Transformer{ - .log = log, - .allocator = allocator, - .opts = opts, - .cwd = cwd, - .platform = platform, - .out_extensions = out_extensions, - .define = define, - .output_path = output_dir, - }; - - const write_to_output_dir = opts.entry_points.len > 1 or opts.output_dir != null; - - var output_dir_handle: ?std.fs.Dir = null; - if (write_to_output_dir) { - output_dir_handle = try options.openOutputDir(output_dir); - } - - if (write_to_output_dir) { - for (opts.entry_points) |entry_point, i| { - try transformer.processEntryPoint( - entry_point, - i, - &output_files, - output_dir_handle, - .disk, - care_about_closing_files, - use_default_loaders, - loader_map, - &jsx, - ); - } - } else { - for (opts.entry_points) |entry_point, i| { - try transformer.processEntryPoint( - entry_point, - i, - &output_files, - output_dir_handle, - .stdout, - care_about_closing_files, - use_default_loaders, - loader_map, - &jsx, - ); - } - } - - return try options.TransformResult.init(output_dir, output_files.toOwnedSlice(), log, allocator); - } - - pub fn processEntryPoint( - transformer: *Transformer, - entry_point: string, - _: usize, - output_files: *std.ArrayList(options.OutputFile), - _output_dir: ?std.fs.Dir, - comptime write_destination_type: options.WriteDestination, - care_about_closing_files: bool, - use_default_loaders: bool, - loader_map: std.StringHashMap(options.Loader), - jsx: *options.JSX.Pragma, - ) !void { - var allocator = transformer.allocator; - var log = transformer.log; - - var _log = logger.Log.init(allocator); - const absolutePath = resolve_path.joinAbs(transformer.cwd, .auto, entry_point); - - const file = try std.fs.openFileAbsolute(absolutePath, std.fs.File.OpenFlags{ .mode = .read_only }); - defer { - if (care_about_closing_files) { - file.close(); - } - } - - const stat = try file.stat(); - - const code = try file.readToEndAlloc(allocator, stat.size); - defer { - if (_log.msgs.items.len == 0) { - allocator.free(code); - } - _log.appendTo(log) catch {}; - } - const _file = Fs.File{ .path = Fs.Path.init(entry_point), .contents = code }; - var source = try logger.Source.initFile(_file, allocator); - var loader: options.Loader = undefined; - if (use_default_loaders) { - loader = options.defaultLoaders.get(std.fs.path.extension(absolutePath)) orelse return; - } else { - loader = options.Loader.forFileName( - entry_point, - loader_map, - ) orelse return; - } - - var _source = &source; - - var output_file = options.OutputFile{ - .input = _file.path, - .loader = loader, - .value = undefined, - }; - - var file_to_write: std.fs.File = undefined; - var output_path: Fs.Path = undefined; - - switch (write_destination_type) { - .stdout => { - file_to_write = std.io.getStdOut(); - output_path = Fs.Path.init("stdout"); - }, - .disk => { - const output_dir = _output_dir orelse unreachable; - output_path = Fs.Path.init(try allocator.dupe(u8, resolve_path.relative(transformer.cwd, entry_point))); - file_to_write = try output_dir.createFile(entry_point, .{}); - }, - } - - switch (loader) { - .jsx, .js, .ts, .tsx => { - jsx.parse = loader.isJSX(); - var file_op = options.OutputFile.FileOperation.fromFile(file_to_write.handle, output_path.pretty); - - var parser_opts = js_parser.Parser.Options.init(jsx.*, loader); - file_op.is_tmpdir = false; - output_file.value = .{ .move = file_op }; - - if (_output_dir) |output_dir| { - file_op.dir = output_dir.fd; - } - - file_op.fd = file.handle; - var parser = try js_parser.Parser.init(parser_opts, log, _source, transformer.define, allocator); - parser_opts.can_import_from_bundle = false; - const result = try parser.parse(); - - const ast = result.ast; - var symbols: [][]js_ast.Symbol = &([_][]js_ast.Symbol{ast.symbols}); - - output_file.size = try js_printer.printAst( - js_printer.FileWriter, - js_printer.NewFileWriter(file_to_write), - ast, - js_ast.Symbol.Map.initList(symbols), - _source, - false, - js_printer.Options{ - .to_module_ref = Ref.RuntimeRef, - .externals = ast.externals, - .transform_imports = false, - .runtime_imports = ast.runtime_imports, - }, - ?*anyopaque, - null, - false, - ); - }, - else => { - unreachable; - }, - } - - js_ast.Expr.Data.Store.reset(); - js_ast.Stmt.Data.Store.reset(); - try output_files.append(output_file); - } - - pub fn _transform( - allocator: std.mem.Allocator, - log: *logger.Log, - opts: js_parser.Parser.Options, - loader: options.Loader, - define: *const Define, - source: *const logger.Source, - comptime Writer: type, - writer: Writer, - ) !usize { - var ast: js_ast.Ast = undefined; - - switch (loader) { - .json => { - var expr = try json_parser.ParseJSON(source, log, allocator); - var stmt = js_ast.Stmt.alloc(js_ast.S.ExportDefault{ - .value = js_ast.StmtOrExpr{ .expr = expr }, - .default_name = js_ast.LocRef{ - .loc = logger.Loc{}, - }, - }, logger.Loc{ .start = 0 }); - var stmts = try allocator.alloc(js_ast.Stmt, 1); - stmts[0] = stmt; - var parts = try allocator.alloc(js_ast.Part, 1); - parts[0] = js_ast.Part{ .stmts = stmts }; - - ast = js_ast.Ast.initTest(parts); - }, - - .jsx, .tsx, .ts, .js => { - var parser = try js_parser.Parser.init(opts, log, source, define, allocator); - var res = try parser.parse(); - ast = res.ast; - - if (FeatureFlags.print_ast) { - try ast.toJSON(allocator, std.io.getStdErr().writer()); - } - }, - else => { - Global.panic("Unsupported loader: {s} for path: {s}", .{ loader, source.path.text }); - }, - } - - var symbols: [][]js_ast.Symbol = &([_][]js_ast.Symbol{ast.symbols}); - - return try js_printer.printAst( - Writer, - writer, - ast, - js_ast.Symbol.Map.initList(symbols), - source, - false, - js_printer.Options{ - .to_module_ref = ast.module_ref orelse js_ast.Ref.None, - .transform_imports = false, - .runtime_imports = ast.runtime_imports, - }, - null, - ); - } -}; - pub const ServeResult = struct { file: options.OutputFile, mime_type: MimeType, diff --git a/src/bundler/generate_node_modules_bundle.zig b/src/bundler/generate_node_modules_bundle.zig new file mode 100644 index 000000000..32fba8664 --- /dev/null +++ b/src/bundler/generate_node_modules_bundle.zig @@ -0,0 +1,1885 @@ +const Bundler = @import("../bundler.zig").Bundler; +const GenerateNodeModulesBundle = @This(); +const bun = @import("../global.zig"); +const string = bun.string; +const Output = bun.Output; +const Global = bun.Global; +const Environment = bun.Environment; +const strings = bun.strings; +const MutableString = bun.MutableString; +const stringZ = bun.stringZ; +const default_allocator = bun.default_allocator; +const StoredFileDescriptorType = bun.StoredFileDescriptorType; +const FeatureFlags = bun.FeatureFlags; +const C = bun.C; +const std = @import("std"); +const lex = @import("../js_lexer.zig"); +const logger = @import("../logger.zig"); +const options = @import("../options.zig"); +const js_parser = @import("../js_parser.zig"); +const json_parser = @import("../json_parser.zig"); +const js_printer = @import("../js_printer.zig"); +const js_ast = @import("../js_ast.zig"); +const linker = @import("../linker.zig"); +const Ref = @import("../ast/base.zig").Ref; +const Define = @import("../defines.zig").Define; +const DebugOptions = @import("../cli.zig").Command.DebugOptions; +const ThreadPoolLib = @import("../thread_pool.zig"); + +const panicky = @import("../panic_handler.zig"); +const Fs = @import("../fs.zig"); +const schema = @import("../api/schema.zig"); +const Api = schema.Api; +const _resolver = @import("../resolver/resolver.zig"); +const sync = @import("../sync.zig"); +const ImportRecord = @import("../import_record.zig").ImportRecord; +const allocators = @import("../allocators.zig"); +const MimeType = @import("../http/mime_type.zig"); +const resolve_path = @import("../resolver/resolve_path.zig"); +const runtime = @import("../runtime.zig"); +const Timer = @import("../timer.zig"); +const PackageJSON = @import("../resolver/package_json.zig").PackageJSON; +const MacroRemap = @import("../resolver/package_json.zig").MacroMap; +const DebugLogs = _resolver.DebugLogs; +const NodeModuleBundle = @import("../node_module_bundle.zig").NodeModuleBundle; +const Router = @import("../router.zig"); +const isPackagePath = _resolver.isPackagePath; +const Lock = @import("../lock.zig").Lock; +const NodeFallbackModules = @import("../node_fallbacks.zig"); +const CacheEntry = @import("../cache.zig").FsCacheEntry; +const Analytics = @import("../analytics/analytics_thread.zig"); +const URL = @import("../url.zig").URL; +const Report = @import("../report.zig"); +const Linker = linker.Linker; +const Resolver = _resolver.Resolver; +const TOML = @import("../toml/toml_parser.zig").TOML; + +const EntryPoints = @import("./entry_points.zig"); +const BunQueue = sync.Channel(PendingImports, .Dynamic); +const GenerateNodeModuleBundle = @This(); +const ThisBundler = @import("../bundler.zig").Bundler; +pub const ThreadPool = struct { + pool: ThreadPoolLib = undefined, + // Hardcode 512 as max number of threads for now. + workers: [512]Worker = undefined, + workers_used: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), + cpu_count: u32 = 0, + started_workers: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), + stopped_workers: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), + completed_count: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), + pending_count: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), + + generator: *GenerateNodeModuleBundle = undefined, + + pub fn start(this: *ThreadPool, generator: *GenerateNodeModuleBundle) !void { + generator.bundler.env.loadProcess(); + this.generator = generator; + + this.cpu_count = @truncate(u32, @divFloor((try std.Thread.getCpuCount()) + 1, 2)); + + if (generator.bundler.env.map.get("GOMAXPROCS")) |max_procs| { + if (std.fmt.parseInt(u32, max_procs, 10)) |cpu_count| { + this.cpu_count = std.math.min(this.cpu_count, cpu_count); + } else |_| {} + } + + this.pool = ThreadPoolLib.init(.{ + .max_threads = this.cpu_count, + }); + this.pool.on_thread_spawn = Worker.onSpawn; + this.pool.threadpool_context = this; + var workers_used: u32 = 0; + while (workers_used < this.cpu_count) : (workers_used += 1) { + try this.workers[workers_used].init(generator); + } + if (workers_used > 0) + this.pool.forceSpawn(); + } + + pub fn wait(this: *ThreadPool, generator: *GenerateNodeModuleBundle) !void { + while (true) { + while (this.generator.queue.tryReadItem() catch null) |queue| { + var iter = queue.iterator(); + var batch = ThreadPoolLib.Batch{}; + var count: u32 = 0; + while (iter.next()) |entry| { + const module_id: u32 = entry.key_ptr.*; + const exists = generator.enqueued_map.getOrPut(module_id) catch unreachable; + if (exists.found_existing) { + continue; + } + batch.push(ThreadPoolLib.Batch.from(&entry.value_ptr.*.task)); + count += 1; + } + _ = this.pending_count.fetchAdd(count, .Monotonic); + this.pool.schedule(batch); + } + + if (this.completed_count.load(.Monotonic) > 0 and this.completed_count.load(.Monotonic) == this.pending_count.load(.Monotonic)) { + break; + } + + std.atomic.spinLoopHint(); + } + + const workers: []const Worker = this.workers[0..this.workers_used.loadUnchecked()]; + for (workers) |worker| { + this.generator.estimated_input_lines_of_code += worker.data.estimated_input_lines_of_code; + try worker.data.log.appendTo(this.generator.log); + } + } + + pub const Task = struct { + result: _resolver.Result, + generator: *GenerateNodeModuleBundle, + }; + + pub const Worker = struct { + thread_id: std.Thread.Id, + thread: std.Thread, + + allocator: std.mem.Allocator, + generator: *GenerateNodeModuleBundle, + data: *WorkerData = undefined, + quit: bool = false, + + has_notify_started: bool = false, + + pub const WorkerData = struct { + shared_buffer: MutableString = undefined, + scan_pass_result: js_parser.ScanPassResult = undefined, + log: *logger.Log, + estimated_input_lines_of_code: usize = 0, + macro_context: js_ast.Macro.MacroContext, + bundler: Bundler = undefined, + + pub fn deinit(this: *WorkerData, allocator: std.mem.Allocator) void { + this.shared_buffer.deinit(); + this.scan_pass_result.named_imports.deinit(); + this.scan_pass_result.import_records.deinit(); + allocator.destroy(this); + } + }; + + pub fn init(worker: *Worker, generator: *GenerateNodeModuleBundle) !void { + worker.generator = generator; + worker.allocator = generator.allocator; + } + + pub fn onSpawn(ctx: ?*anyopaque) ?*anyopaque { + var pool = @ptrCast(*ThreadPool, @alignCast(@alignOf(*ThreadPool), ctx.?)); + + const id = pool.workers_used.fetchAdd(1, .Monotonic); + pool.workers[id].run(); + return &pool.workers[id]; + } + + pub fn notifyStarted(this: *Worker) void { + if (!this.has_notify_started) { + this.has_notify_started = true; + _ = this.generator.pool.started_workers.fetchAdd(1, .Release); + std.Thread.Futex.wake(&this.generator.pool.started_workers, std.math.maxInt(u32)); + } + } + + pub fn run(this: *Worker) void { + Output.Source.configureThread(); + this.thread_id = std.Thread.getCurrentId(); + if (Environment.isDebug) { + Output.prettyln("Thread started.\n", .{}); + } + js_ast.Expr.Data.Store.create(this.generator.allocator); + js_ast.Stmt.Data.Store.create(this.generator.allocator); + this.data = this.generator.allocator.create(WorkerData) catch unreachable; + this.data.* = WorkerData{ + .log = this.generator.allocator.create(logger.Log) catch unreachable, + .estimated_input_lines_of_code = 0, + .macro_context = js_ast.Macro.MacroContext.init(this.generator.bundler), + }; + this.data.log.* = logger.Log.init(this.generator.allocator); + this.data.shared_buffer = MutableString.init(this.generator.allocator, 0) catch unreachable; + this.data.scan_pass_result = js_parser.ScanPassResult.init(this.generator.allocator); + this.data.bundler = this.generator.bundler.*; + var bundler_ptr = &this.data.bundler; + const CacheSet = @import("../cache.zig"); + // no funny business mr. cache + bundler_ptr.resolver.caches = CacheSet.Set.init(this.allocator); + bundler_ptr.linker.resolver = &bundler_ptr.resolver; + bundler_ptr.log = this.data.log; + bundler_ptr.linker.log = this.data.log; + bundler_ptr.linker.resolver.log = this.data.log; + } + + pub const ProcessFileTask = struct { + resolution: _resolver.Result, + task: ThreadPoolLib.Task = .{ .callback = callback }, + + pub fn callback(task: *ThreadPoolLib.Task) void { + var worker = @ptrCast( + *ThreadPool.Worker, + @alignCast( + @alignOf(*ThreadPool.Worker), + ThreadPoolLib.Thread.current.?.ctx.?, + ), + ); + var process: *ProcessFileTask = @fieldParentPtr(ProcessFileTask, "task", task); + + worker.generator.processFile( + worker, + &worker.data.bundler, + process.resolution, + ) catch {}; + _ = worker.generator.pool.completed_count.fetchAdd(1, .Monotonic); + } + }; + }; +}; +write_lock: Lock, +log_lock: Lock = Lock.init(), +module_list: std.ArrayList(Api.JavascriptBundledModule), +package_list: std.ArrayList(Api.JavascriptBundledPackage), +header_string_buffer: MutableString, + +// Just need to know if we've already enqueued this one +package_list_map: std.AutoHashMap(u64, u32), +enqueued_map: std.AutoHashMap(u32, void), +queue: BunQueue, +bundler: *ThisBundler, + +allocator: std.mem.Allocator, +tmpfile: std.fs.File, +log: *logger.Log, +pool: *ThreadPool, +tmpfile_byte_offset: u32 = 0, +code_end_byte_offset: u32 = 0, +has_jsx: bool = false, +estimated_input_lines_of_code: usize = 0, + +work_waiter: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0), +list_lock: Lock = Lock.init(), + +dynamic_import_file_size_store: U32Map, +dynamic_import_file_size_store_lock: Lock, + +always_bundled_package_hashes: []u32 = &[_]u32{}, +always_bundled_package_jsons: []*const PackageJSON = &.{}, +always_bundled_booleans: []bool = &.{}, +package_bundle_map: options.BundlePackage.Map = options.BundlePackage.Map{}, + +const U32Map = std.AutoHashMap(u32, u32); +pub const current_version: u32 = 1; +const dist_index_js_string_pointer = Api.StringPointer{ .length = "dist/index.js".len }; +const index_js_string_pointer = Api.StringPointer{ .length = "index.js".len, .offset = "dist/".len }; + +fn upsert(this: *GenerateNodeModuleBundle, module_id: u32, resolve: _resolver.Result) !void { + var dedupe = try this.enqueued_map.getOrPut(module_id); + if (dedupe.found_existing) return; + var task = try this.allocator.create(ThreadPool.Worker.ProcessFileTask); + task.* = ThreadPool.Worker.ProcessFileTask{ + .resolution = resolve, + }; + _ = this.pool.pending_count.fetchAdd(1, .Monotonic); + this.pool.pool.schedule(ThreadPoolLib.Batch.from(&task.task)); +} + +pub fn ensurePathIsAllocated(this: *GenerateNodeModuleBundle, path_: ?*Fs.Path) !void { + var path = path_ orelse return; + + const loader = this.bundler.options.loaders.get(path.name.ext) orelse .file; + if (!loader.isJavaScriptLikeOrJSON()) return; + path.* = try path.dupeAlloc(this.allocator); +} + +pub fn enqueueItem(this: *GenerateNodeModuleBundle, resolve: _resolver.Result) !void { + var result = resolve; + var path = result.path() orelse return; + + const loader = this.bundler.options.loaders.get(path.name.ext) orelse .file; + if (!loader.isJavaScriptLikeOrJSON()) return; + path.* = try path.dupeAlloc(this.allocator); + + if (BundledModuleData.get(this, &result)) |mod| { + try this.upsert(mod.module_id, result); + } else { + try this.upsert(result.hash(this.bundler.fs.top_level_dir, loader), result); + } +} + +// The bun Bundle Format +// All the node_modules your app uses in a single compact file with metadata +// A binary JavaScript bundle format prioritizing generation time and deserialization time +pub const magic_bytes = "#!/usr/bin/env bun\n\n"; +// This makes it possible to do ./path-to-bundle on posix systems so you can see the raw JS contents +// https://en.wikipedia.org/wiki/Magic_number_(programming)#In_files +// Immediately after the magic bytes, the next character is a uint32 followed by a newline +// 0x00000000\n +// That uint32 denotes the byte offset in the file where the code for the bundle ends +// - If the value is 0, that means the file did not finish writing or there are no modules +// - This imposes a maximum bundle size of around 4,294,967,295 bytes. If your JS is more than 4 GB, it won't work. +// The raw JavaScript is encoded as a UTF-8 string starting from the current position + 1 until the above byte offset. +// This uint32 is useful for HTTP servers to separate: +// - Which part of the bundle is the JS code? +// - Which part is the metadata? +// Without needing to do a full pass through the file, or necessarily care about the metadata. +// The metadata is at the bottom of the file instead of the top because the metadata is written after all JS code in the bundle is written. +// The rationale there is: +// 1. We cannot prepend to a file without rewriting the entire file +// 2. The metadata is variable-length and that format will change often. +// 3. We won't have all the metadata until after all JS is finished writing +// If you have 32 MB of JavaScript dependencies, you really want to avoid reading the code in memory. +// - This lets you seek to the specific position in the file. +// - HTTP servers should use sendfile() instead of copying the file to userspace memory. +// So instead, we append metadata to the file after printing each node_module +// When there are no more modules to process, we generate the metadata +// To find the metadata, you look at the byte offset: initial_header[magic_bytes.len..initial_header.len - 1] +// Then, you add that number to initial_header.len +const initial_header = brk: { + var buf = std.mem.zeroes([magic_bytes.len + 5]u8); + std.mem.copy(u8, &buf, magic_bytes); + var remainder = buf[magic_bytes.len..]; + // Write an invalid byte offset to be updated after we finish generating the code + std.mem.writeIntNative(u32, remainder[0 .. remainder.len - 1], 0); + buf[buf.len - 1] = '\n'; + break :brk buf; +}; +const code_start_byte_offset: u32 = initial_header.len; +// The specifics of the metadata is not documented here. You can find it in src/api/schema.peechy. + +pub fn appendHeaderString(generator: *GenerateNodeModuleBundle, str: string) !Api.StringPointer { + // This is so common we might as well just reuse it + // Plus this is one machine word so it's a quick comparison + if (strings.eqlComptime(str, "index.js")) { + return index_js_string_pointer; + } else if (strings.eqlComptime(str, "dist/index.js")) { + return dist_index_js_string_pointer; + } + + var offset = generator.header_string_buffer.list.items.len; + try generator.header_string_buffer.append(str); + return Api.StringPointer{ + .offset = @truncate(u32, offset), + .length = @truncate(u32, str.len), + }; +} + +pub fn generate( + bundler: *ThisBundler, + allocator: std.mem.Allocator, + framework_config: ?Api.LoadedFramework, + route_config: ?Api.LoadedRouteConfig, + destination: [*:0]const u8, + estimated_input_lines_of_code: *usize, + package_bundle_map: options.BundlePackage.Map, +) !?Api.JavascriptBundleContainer { + _ = try bundler.fs.fs.openTmpDir(); + var tmpname_buf: [64]u8 = undefined; + bundler.resetStore(); + try bundler.configureDefines(); + + const tmpname = try bundler.fs.tmpname( + ".bun", + std.mem.span(&tmpname_buf), + std.hash.Wyhash.hash(@intCast(usize, std.time.milliTimestamp()) % std.math.maxInt(u32), std.mem.span(destination)), + ); + + var tmpfile = Fs.FileSystem.RealFS.Tmpfile{}; + try tmpfile.create(&bundler.fs.fs, tmpname); + + errdefer tmpfile.closeAndDelete(tmpname); + + var generator = try allocator.create(GenerateNodeModuleBundle); + var queue = BunQueue.init(allocator); + defer allocator.destroy(generator); + generator.* = GenerateNodeModuleBundle{ + .module_list = std.ArrayList(Api.JavascriptBundledModule).init(allocator), + .package_list = std.ArrayList(Api.JavascriptBundledPackage).init(allocator), + .header_string_buffer = try MutableString.init(allocator, "dist/index.js".len), + .allocator = allocator, + .enqueued_map = std.AutoHashMap(u32, void).init(allocator), + .queue = queue, + .estimated_input_lines_of_code = 0, + // .resolve_queue = queue, + .bundler = bundler, + .tmpfile = tmpfile.file(), + + .dynamic_import_file_size_store = U32Map.init(allocator), + .dynamic_import_file_size_store_lock = Lock.init(), + .log = bundler.log, + .package_list_map = std.AutoHashMap(u64, u32).init(allocator), + .pool = undefined, + .write_lock = Lock.init(), + .package_bundle_map = package_bundle_map, + }; + // dist/index.js appears more common than /index.js + // but this means we can store both "dist/index.js" and "index.js" in one. + try generator.header_string_buffer.append("dist/index.js"); + try generator.package_list_map.ensureTotalCapacity(128); + var pool = try allocator.create(ThreadPool); + pool.* = ThreadPool{}; + generator.pool = pool; + + var this = generator; + // Always inline the runtime into the bundle + try generator.appendBytes(&initial_header); + + if (bundler.log.level == .verbose) { + bundler.resolver.debug_logs = try DebugLogs.init(allocator); + } + + Analytics.Features.bun_bun = true; + + always_bundled: { + const root_package_json_resolved: _resolver.Result = bundler.resolver.resolve(bundler.fs.top_level_dir, "./package.json", .stmt) catch { + generator.log.addWarning(null, logger.Loc.Empty, "Please run `bun bun` from a directory containing a package.json.") catch unreachable; + break :always_bundled; + }; + const root_package_json = root_package_json_resolved.package_json orelse brk: { + const read_dir = (bundler.resolver.readDirInfo(bundler.fs.top_level_dir) catch unreachable).?; + Analytics.Features.tsconfig = Analytics.Features.tsconfig or read_dir.tsconfig_json != null; + break :brk read_dir.package_json.?; + }; + Analytics.setProjectID(std.fs.path.dirname(root_package_json.source.path.text) orelse "/", root_package_json.name); + if (bundler.macro_context) |macro_ctx| { + Analytics.Features.macros = macro_ctx.remap.count() > 0; + } + + const bundle_keys = package_bundle_map.keys(); + const do_always_bundle = package_bundle_map.values(); + var always_bundle_count: u32 = @truncate(u32, bundle_keys.len); + + if (always_bundle_count != 0) { + Analytics.Features.always_bundle = true; + var always_bundled_package_jsons = bundler.allocator.alloc(*PackageJSON, always_bundle_count) catch unreachable; + var always_bundled_package_hashes = bundler.allocator.alloc(u32, always_bundle_count) catch unreachable; + var always_bundled_booleans = bundler.allocator.alloc(bool, always_bundle_count) catch unreachable; + var i: u16 = 0; + + inner: for (bundle_keys) |name, k| { + std.mem.copy(u8, &Bundler.tmp_buildfile_buf, name); + std.mem.copy(u8, Bundler.tmp_buildfile_buf[name.len..], "/package.json"); + const package_json_import = Bundler.tmp_buildfile_buf[0 .. name.len + "/package.json".len]; + const result = bundler.resolver.resolve(bundler.fs.top_level_dir, package_json_import, .stmt) catch |err| { + generator.log.addErrorFmt(null, logger.Loc.Empty, bundler.allocator, "{s} resolving always bundled module \"{s}\"", .{ @errorName(err), name }) catch unreachable; + continue :inner; + }; + + var package_json: *PackageJSON = result.package_json orelse brk: { + const read_dir = (bundler.resolver.readDirInfo(package_json_import) catch unreachable).?; + if (read_dir.package_json == null) { + generator.log.addWarningFmt(null, logger.Loc.Empty, bundler.allocator, "{s} missing package.json. It will not be bundled", .{name}) catch unreachable; + continue :inner; + } + break :brk read_dir.package_json.?; + }; + + package_json.source.key_path = result.path_pair.primary; + + // if (!strings.contains(result.path_pair.primary.text, package_json.name)) { + // generator.log.addErrorFmt( + // null, + // logger.Loc.Empty, + // bundler.allocator, + // "Bundling \"{s}\" is not supported because the package isn.\n To fix this, move the package's code to a directory containing the name.\n Location: \"{s}\"", + // .{ + // name, + // name, + // result.path_pair.primary.text, + // }, + // ) catch unreachable; + // continue :inner; + // } + + always_bundled_package_jsons[i] = package_json; + always_bundled_package_hashes[i] = package_json.hash; + always_bundled_booleans[i] = do_always_bundle[k] == .always; + i += 1; + } + generator.always_bundled_package_hashes = always_bundled_package_hashes[0..i]; + generator.always_bundled_package_jsons = always_bundled_package_jsons[0..i]; + generator.always_bundled_booleans = always_bundled_booleans[0..i]; + } + } + if (generator.log.errors > 0) return error.BundleFailed; + + this.bundler.macro_context = js_ast.Macro.MacroContext.init(bundler); + + const include_refresh_runtime = + !this.bundler.options.production and + this.bundler.options.jsx.supports_fast_refresh and + bundler.options.platform.isWebLike(); + + if (framework_config != null) { + defer this.bundler.resetStore(); + + try this.bundler.configureFramework(true); + if (bundler.options.framework) |framework| { + Analytics.Features.framework = true; + + if (framework.override_modules.keys.len > 0) { + bundler.options.framework.?.override_modules_hashes = allocator.alloc(u64, framework.override_modules.keys.len) catch unreachable; + for (framework.override_modules.keys) |key, i| { + bundler.options.framework.?.override_modules_hashes[i] = std.hash.Wyhash.hash(0, key); + } + } + } + } else {} + + this.pool.start(this) catch |err| { + Analytics.enqueue(Analytics.EventName.bundle_fail); + return err; + }; + + // The ordering matters here The runtime must appear at the top of + // the file But, we don't know which version of the runtime is + // necessary until we know if they'll use JSX and if they'll use + // react refresh + var _new_jsx_runtime_resolve_result: ?_resolver.Result = null; + var fast_refresh_resolve_result: ?_resolver.Result = null; + // Normally, this is automatic + // However, since we only do the parsing pass, it may not get imported automatically. + if (bundler.options.jsx.parse) { + defer this.bundler.resetStore(); + if (this.bundler.resolver.resolve( + this.bundler.fs.top_level_dir, + this.bundler.options.jsx.import_source, + .require, + )) |new_jsx_runtime| { + _new_jsx_runtime_resolve_result = new_jsx_runtime; + this.ensurePathIsAllocated(_new_jsx_runtime_resolve_result.?.path()) catch unreachable; + Analytics.Features.jsx = true; + } else |_| {} + } + + const include_fast_refresh_in_bundle = Analytics.Features.jsx and + include_refresh_runtime and + !Analytics.Features.fast_refresh and !bundler.options.platform.isServerSide(); + + var refresh_runtime_module_id: u32 = 0; + if (include_refresh_runtime) { + defer this.bundler.resetStore(); + + if (this.bundler.resolver.resolve( + this.bundler.fs.top_level_dir, + this.bundler.options.jsx.refresh_runtime, + .require, + )) |refresh_runtime| { + fast_refresh_resolve_result = refresh_runtime; + this.ensurePathIsAllocated(fast_refresh_resolve_result.?.path()) catch unreachable; + Analytics.Features.fast_refresh = true; + + if (BundledModuleData.get(this, &refresh_runtime)) |mod| { + refresh_runtime_module_id = mod.module_id; + } + } else |_| {} + } + + if (Environment.isDebug) { + switch (bundler.options.platform) { + .node => try generator.appendBytes(runtime.Runtime.sourceContentNode()), + .bun_macro, .bun => try generator.appendBytes(runtime.Runtime.sourceContentBun()), + else => try generator.appendBytes(runtime.Runtime.sourceContent(include_fast_refresh_in_bundle)), + } + + try generator.appendBytes("\n\n"); + } else if (include_fast_refresh_in_bundle) { + try generator.appendBytes(comptime runtime.Runtime.sourceContent(true) ++ "\n\n"); + } else { + try generator.appendBytes( + switch (bundler.options.platform) { + .bun_macro, .bun => comptime @as(string, runtime.Runtime.sourceContentBun() ++ "\n\n"), + .node => comptime @as(string, runtime.Runtime.sourceContentNode() ++ "\n\n"), + else => comptime @as(string, runtime.Runtime.sourceContentWithoutRefresh() ++ "\n\n"), + }, + ); + } + + if (_new_jsx_runtime_resolve_result) |new_jsx_runtime| { + try this.enqueueItem(new_jsx_runtime); + _new_jsx_runtime_resolve_result = null; + } + + if (fast_refresh_resolve_result) |fast_refresh| { + try this.enqueueItem(fast_refresh); + fast_refresh_resolve_result = null; + } + + if (bundler.router) |router| { + defer this.bundler.resetStore(); + Analytics.Features.filesystem_router = true; + + const entry_points = try router.getEntryPoints(); + for (entry_points) |entry_point| { + const resolved = bundler.resolveEntryPoint(entry_point) catch continue; + try this.enqueueItem(resolved); + } + this.bundler.resetStore(); + } else {} + + if (bundler.options.framework) |framework| { + if (bundler.options.platform.isBun()) { + if (framework.server.isEnabled()) { + Analytics.Features.bunjs = true; + const resolved = try bundler.resolver.resolve( + bundler.fs.top_level_dir, + framework.server.path, + .entry_point, + ); + try this.enqueueItem(resolved); + } + } else { + if (framework.client.isEnabled()) { + const resolved = try bundler.resolver.resolve( + bundler.fs.top_level_dir, + framework.client.path, + .entry_point, + ); + try this.enqueueItem(resolved); + } + + if (framework.fallback.isEnabled()) { + const resolved = try bundler.resolver.resolve( + bundler.fs.top_level_dir, + framework.fallback.path, + .entry_point, + ); + try this.enqueueItem(resolved); + } + } + } + + for (bundler.options.entry_points) |entry_point| { + defer this.bundler.resetStore(); + + const entry_point_path = bundler.normalizeEntryPointPath(entry_point); + const resolved = bundler.resolveEntryPoint(entry_point_path) catch continue; + try this.enqueueItem(resolved); + } + + Analytics.enqueue(Analytics.EventName.bundle_start); + this.bundler.resetStore(); + this.pool.wait(this) catch |err| { + Analytics.enqueue(Analytics.EventName.bundle_fail); + return err; + }; + Analytics.enqueue(Analytics.EventName.bundle_success); + + estimated_input_lines_of_code.* = generator.estimated_input_lines_of_code; + + // if (comptime !isRelease) { + // this.queue.checkDuplicatesSlow(); + // } + + if (this.log.errors > 0) { + tmpfile.closeAndDelete(std.mem.span(tmpname)); + // We stop here because if there are errors we don't know if the bundle is valid + // This manifests as a crash when sorting through the module list because we may have added files to the bundle which were never actually finished being added. + return null; + } + + if (include_refresh_runtime and refresh_runtime_module_id > 0) { + var refresh_runtime_injector_buf: [1024]u8 = undefined; + var fixed_buffer = std.io.fixedBufferStream(&refresh_runtime_injector_buf); + var fixed_buffer_writer = fixed_buffer.writer(); + + fixed_buffer_writer.print( + \\if ('window' in globalThis) {{ + \\ (function() {{ + \\ BUN_RUNTIME.__injectFastRefresh(${x}()); + \\ }})(); + \\}} + , + .{refresh_runtime_module_id}, + ) catch unreachable; + try this.tmpfile.writeAll(fixed_buffer.buffer[0..fixed_buffer.pos]); + } + + // Ensure we never overflow + this.code_end_byte_offset = @truncate( + u32, + // Doing this math ourself seems to not necessarily produce correct results + (try this.tmpfile.getPos()), + ); + + var javascript_bundle_container = std.mem.zeroes(Api.JavascriptBundleContainer); + + std.sort.sort( + Api.JavascriptBundledModule, + this.module_list.items, + this, + GenerateNodeModuleBundle.sortJavascriptModuleByPath, + ); + + if (comptime Environment.isDebug) { + const SeenHash = std.AutoHashMap(u64, void); + var map = SeenHash.init(this.allocator); + var ids = SeenHash.init(this.allocator); + try map.ensureTotalCapacity(@truncate(u32, this.module_list.items.len)); + try ids.ensureTotalCapacity(@truncate(u32, this.module_list.items.len)); + + for (this.module_list.items) |a| { + const a_pkg: Api.JavascriptBundledPackage = this.package_list.items[a.package_id]; + const a_name = this.metadataStringPointer(a_pkg.name); + const a_version = this.metadataStringPointer(a_pkg.version); + const a_path = this.metadataStringPointer(a.path); + + std.debug.assert(a_name.len > 0); + std.debug.assert(a_version.len > 0); + std.debug.assert(a_path.len > 0); + var hash_print = std.mem.zeroes([4096]u8); + const hash = std.hash.Wyhash.hash(0, std.fmt.bufPrint(&hash_print, "{s}@{s}/{s}", .{ a_name, a_version, a_path }) catch unreachable); + var result1 = map.getOrPutAssumeCapacity(hash); + std.debug.assert(!result1.found_existing); + + var result2 = ids.getOrPutAssumeCapacity(a.id); + std.debug.assert(!result2.found_existing); + } + } + + var hasher = std.hash.Wyhash.init(0); + + // We want to sort the packages as well as the files + // The modules sort the packages already + // So can just copy it in the below loop. + var sorted_package_list = try allocator.alloc(Api.JavascriptBundledPackage, this.package_list.items.len); + + // At this point, the module_list is sorted. + if (this.module_list.items.len > 0) { + var package_id_i: u32 = 0; + var i: usize = 0; + // Assumption: node_modules are immutable + // Assumption: module files are immutable + // (They're not. But, for our purposes that's okay) + // The etag is: + // - The hash of each module's path in sorted order + // - The hash of each module's code size in sorted order + // - hash(hash(package_name, package_version)) + // If this doesn't prove strong enough, we will do a proper content hash + // But I want to avoid that overhead unless proven necessary. + // There's a good chance we don't even strictly need an etag here. + var bytes: [4]u8 = undefined; + while (i < this.module_list.items.len) { + var current_package_id = this.module_list.items[i].package_id; + this.module_list.items[i].package_id = package_id_i; + var offset = @truncate(u32, i); + + i += 1; + + while (i < this.module_list.items.len and this.module_list.items[i].package_id == current_package_id) : (i += 1) { + this.module_list.items[i].package_id = package_id_i; + // Hash the file path + hasher.update(this.metadataStringPointer(this.module_list.items[i].path)); + // Then the length of the code + std.mem.writeIntNative(u32, &bytes, this.module_list.items[i].code.length); + hasher.update(&bytes); + } + + this.package_list.items[current_package_id].modules_offset = offset; + this.package_list.items[current_package_id].modules_length = @truncate(u32, i) - offset; + + // Hash the hash of the package name + // it's hash(hash(package_name, package_version)) + std.mem.writeIntNative(u32, &bytes, this.package_list.items[current_package_id].hash); + hasher.update(&bytes); + + sorted_package_list[package_id_i] = this.package_list.items[current_package_id]; + package_id_i += 1; + } + } + + var javascript_bundle = std.mem.zeroes(Api.JavascriptBundle); + javascript_bundle.modules = this.module_list.items; + javascript_bundle.packages = sorted_package_list; + javascript_bundle.manifest_string = this.header_string_buffer.list.items; + const etag_u64 = hasher.final(); + // We store the etag as a ascii hex encoded u64 + // This is so we can send the bytes directly in the HTTP server instead of formatting it as hex each time. + javascript_bundle.etag = try std.fmt.allocPrint(allocator, "{x}", .{etag_u64}); + javascript_bundle.generated_at = @truncate(u32, @intCast(u64, std.time.milliTimestamp())); + + const basename = std.fs.path.basename(std.mem.span(destination)); + const extname = std.fs.path.extension(basename); + javascript_bundle.import_from_name = if (bundler.options.platform.isBun()) + "/node_modules.server.bun" + else + try std.fmt.allocPrint( + this.allocator, + "/{s}.{x}.bun", + .{ + basename[0 .. basename.len - extname.len], + etag_u64, + }, + ); + + javascript_bundle_container.bundle_format_version = current_version; + javascript_bundle_container.bundle = javascript_bundle; + javascript_bundle_container.code_length = this.code_end_byte_offset; + javascript_bundle_container.framework = framework_config; + javascript_bundle_container.routes = route_config; + + var start_pos = try this.tmpfile.getPos(); + var tmpwriter = std.io.bufferedWriter(this.tmpfile.writer()); + const SchemaWriter = schema.Writer(@TypeOf(tmpwriter.writer())); + var schema_file_writer = SchemaWriter.init(tmpwriter.writer()); + try javascript_bundle_container.encode(&schema_file_writer); + try tmpwriter.flush(); + + // sanity check + if (Environment.isDebug) { + try this.tmpfile.seekTo(start_pos); + var contents = try allocator.alloc(u8, (try this.tmpfile.getEndPos()) - start_pos); + var read_bytes = try this.tmpfile.read(contents); + var buf = contents[0..read_bytes]; + var reader = schema.Reader.init(buf, allocator); + + var decoder = try Api.JavascriptBundleContainer.decode( + &reader, + ); + std.debug.assert(decoder.code_length.? == javascript_bundle_container.code_length.?); + } + + var code_length_bytes: [4]u8 = undefined; + std.mem.writeIntNative(u32, &code_length_bytes, this.code_end_byte_offset); + _ = try std.os.pwrite(this.tmpfile.handle, &code_length_bytes, magic_bytes.len); + + // Without his mutex, we get a crash at this location: + // try std.os.renameat(tmpdir.fd, tmpname, top_dir.fd, destination); + // ^ + const top_dir = try std.fs.openDirAbsolute(Fs.FileSystem.instance.top_level_dir, .{}); + _ = C.fchmod( + this.tmpfile.handle, + // chmod 777 + 0000010 | 0000100 | 0000001 | 0001000 | 0000040 | 0000004 | 0000002 | 0000400 | 0000200 | 0000020, + ); + try tmpfile.promote(tmpname, top_dir.fd, destination); + // Print any errors at the end + // try this.log.print(Output.errorWriter()); + return javascript_bundle_container; +} + +pub fn metadataStringPointer(this: *GenerateNodeModuleBundle, ptr: Api.StringPointer) string { + return this.header_string_buffer.list.items[ptr.offset .. ptr.offset + ptr.length]; +} + +// Since we trim the prefixes, we must also compare the package name and version +pub fn sortJavascriptModuleByPath(ctx: *GenerateNodeModuleBundle, a: Api.JavascriptBundledModule, b: Api.JavascriptBundledModule) bool { + return switch (std.mem.order( + u8, + ctx.metadataStringPointer( + ctx.package_list.items[a.package_id].name, + ), + ctx.metadataStringPointer( + ctx.package_list.items[b.package_id].name, + ), + )) { + .eq => switch (std.mem.order( + u8, + ctx.metadataStringPointer( + ctx.package_list.items[a.package_id].version, + ), + ctx.metadataStringPointer( + ctx.package_list.items[b.package_id].version, + ), + )) { + .eq => std.mem.order( + u8, + ctx.metadataStringPointer(a.path), + ctx.metadataStringPointer(b.path), + ) == .lt, + .lt => true, + else => false, + }, + .lt => true, + else => false, + }; +} + +// pub fn sortJavascriptPackageByName(ctx: *GenerateNodeModuleBundle, a: Api.JavascriptBundledPackage, b: Api.JavascriptBundledPackage) bool { +// return std.mem.order(u8, ctx.metadataStringPointer(a.name), ctx.metadataStringPointer(b.name)) == .lt; +// } + +pub fn appendBytes(generator: *GenerateNodeModuleBundle, bytes: anytype) !void { + try generator.tmpfile.writeAll(bytes); + generator.tmpfile_byte_offset += @truncate(u32, bytes.len); +} + +const BundledModuleData = struct { + import_path: string, + package_path: string, + package: *const PackageJSON, + module_id: u32, + + pub fn getForceBundle(this: *GenerateNodeModuleBundle, resolve_result: *const _resolver.Result) ?BundledModuleData { + return _get(this, resolve_result, true, false); + } + + pub fn getForceBundleForMain(this: *GenerateNodeModuleBundle, resolve_result: *const _resolver.Result) ?BundledModuleData { + return _get(this, resolve_result, true, true); + } + + threadlocal var normalized_package_path: [512]u8 = undefined; + threadlocal var normalized_package_path2: [512]u8 = undefined; + inline fn _get(this: *GenerateNodeModuleBundle, resolve_result: *const _resolver.Result, comptime force: bool, comptime is_main: bool) ?BundledModuleData { + const path = resolve_result.pathConst() orelse return null; + if (strings.eqlComptime(path.namespace, "node")) { + const _import_path = path.text["/bun-vfs/node_modules/".len..][resolve_result.package_json.?.name.len + 1 ..]; + return BundledModuleData{ + .import_path = _import_path, + .package_path = path.text["/bun-vfs/node_modules/".len..], + .package = resolve_result.package_json.?, + .module_id = resolve_result.package_json.?.hashModule(_import_path), + }; + } + + var import_path = path.text; + var package_path = path.text; + var file_path = path.text; + + if (resolve_result.package_json) |pkg_| { + var pkg: *const PackageJSON = pkg_; + if (this.package_bundle_map.get(pkg.name)) |result| { + if (result == .never) return null; + } + + if (std.mem.indexOfScalar(u32, this.always_bundled_package_hashes, pkg.hash)) |pkg_i| { + pkg = this.always_bundled_package_jsons[pkg_i]; + if (!this.always_bundled_booleans[pkg_i]) return null; + const key_path_source_dir = pkg.source.key_path.sourceDir(); + const default_source_dir = pkg.source.path.sourceDir(); + + if (strings.startsWith(path.text, key_path_source_dir)) { + import_path = path.text[key_path_source_dir.len..]; + } else if (strings.startsWith(path.text, default_source_dir)) { + import_path = path.text[default_source_dir.len..]; + } else if (strings.startsWith(path.pretty, pkg.name)) { + import_path = path.pretty[pkg.name.len + 1 ..]; + } + + var buf_to_use: []u8 = if (is_main) &normalized_package_path2 else &normalized_package_path; + + std.mem.copy(u8, buf_to_use, pkg.name); + buf_to_use[pkg.name.len] = '/'; + std.mem.copy(u8, buf_to_use[pkg.name.len + 1 ..], import_path); + package_path = buf_to_use[0 .. pkg.name.len + import_path.len + 1]; + return BundledModuleData{ + .import_path = import_path, + .package_path = package_path, + .package = pkg, + .module_id = pkg.hashModule(package_path), + }; + } + } + + const root: _resolver.RootPathPair = this.bundler.resolver.rootNodeModulePackageJSON( + resolve_result, + ) orelse return null; + + var base_path = root.base_path; + const package_json = root.package_json; + + if (std.mem.indexOfScalar(u32, this.always_bundled_package_hashes, package_json.hash)) |pkg_i| { + var pkg = this.always_bundled_package_jsons[pkg_i]; + if (!this.always_bundled_booleans[pkg_i]) return null; + const key_path_source_dir = pkg.source.key_path.sourceDir(); + const default_source_dir = pkg.source.path.sourceDir(); + + if (strings.startsWith(path.text, key_path_source_dir)) { + import_path = path.text[key_path_source_dir.len..]; + } else if (strings.startsWith(path.text, default_source_dir)) { + import_path = path.text[default_source_dir.len..]; + } else if (strings.startsWith(path.pretty, pkg.name)) { + import_path = path.pretty[pkg.name.len + 1 ..]; + } + + var buf_to_use: []u8 = if (is_main) &normalized_package_path2 else &normalized_package_path; + + std.mem.copy(u8, buf_to_use, pkg.name); + buf_to_use[pkg.name.len] = '/'; + std.mem.copy(u8, buf_to_use[pkg.name.len + 1 ..], import_path); + package_path = buf_to_use[0 .. pkg.name.len + import_path.len + 1]; + return BundledModuleData{ + .import_path = import_path, + .package_path = package_path, + .package = pkg, + .module_id = pkg.hashModule(package_path), + }; + } + + // Easymode: the file path doesn't need to be remapped. + if (strings.startsWith(file_path, base_path)) { + import_path = std.mem.trimLeft(u8, path.text[base_path.len..], "/"); + package_path = std.mem.trim(u8, path.text[base_path.len - package_json.name.len - 1 ..], "/"); + std.debug.assert(import_path.len > 0); + return BundledModuleData{ + .import_path = import_path, + .package_path = package_path, + .package = package_json, + .module_id = package_json.hashModule(package_path), + }; + } + + if (std.mem.lastIndexOf(u8, file_path, package_json.name)) |i| { + package_path = file_path[i..]; + import_path = package_path[package_json.name.len + 1 ..]; + std.debug.assert(import_path.len > 0); + return BundledModuleData{ + .import_path = import_path, + .package_path = package_path, + .package = package_json, + .module_id = package_json.hashModule(package_path), + }; + } + + if (comptime force) { + if (std.mem.indexOfScalar(u32, this.always_bundled_package_hashes, root.package_json.hash)) |pkg_json_i| { + const pkg_json = this.always_bundled_package_jsons[pkg_json_i]; + + base_path = pkg_json.source.key_path.sourceDir(); + + if (strings.startsWith(file_path, base_path)) { + import_path = std.mem.trimLeft(u8, path.text[base_path.len..], "/"); + package_path = std.mem.trim(u8, path.text[base_path.len - package_json.name.len - 1 ..], "/"); + std.debug.assert(import_path.len > 0); + return BundledModuleData{ + .import_path = import_path, + .package_path = package_path, + .package = package_json, + .module_id = package_json.hashModule(package_path), + }; + } + + if (std.mem.lastIndexOf(u8, file_path, package_json.name)) |i| { + package_path = file_path[i..]; + import_path = package_path[package_json.name.len + 1 ..]; + std.debug.assert(import_path.len > 0); + return BundledModuleData{ + .import_path = import_path, + .package_path = package_path, + .package = package_json, + .module_id = package_json.hashModule(package_path), + }; + } + + // This is our last resort. + // The package is supposed to be bundled + // package.json says its' in packages/@foo/bar/index.js + // The file really is in packages/bar/index.js + // But we need the import path to contain the package path for bundling to work + // so we fake it + // we say "" + if (package_json.name[0] == '@') { + if (std.mem.indexOfScalar(u8, package_json.name, '/')) |at| { + const package_subpath = package_json.name[at + 1 ..]; + if (std.mem.lastIndexOf(u8, file_path, package_subpath)) |i| { + package_path = this.bundler.fs.dirname_store.print("{s}/{s}", .{ package_json.name, file_path[i + package_subpath.len ..] }) catch unreachable; + import_path = package_path[package_json.name.len + 1 ..]; + return BundledModuleData{ + .import_path = import_path, + .package_path = package_path, + .package = package_json, + .module_id = package_json.hashModule(package_path), + }; + } + } + } + } + } + + return null; + } + + pub fn get(this: *GenerateNodeModuleBundle, resolve_result: *const _resolver.Result) ?BundledModuleData { + return _get(this, resolve_result, false, false); + } +}; + +fn writeEmptyModule(this: *GenerateNodeModuleBundle, package_relative_path: string, module_id: u32) !u32 { + this.write_lock.lock(); + defer this.write_lock.unlock(); + var code_offset = @truncate(u32, try this.tmpfile.getPos()); + var writer = this.tmpfile.writer(); + var buffered = std.io.bufferedWriter(writer); + + var bufwriter = buffered.writer(); + try bufwriter.writeAll("// "); + try bufwriter.writeAll(package_relative_path); + try bufwriter.writeAll(" (disabled/empty)\nexport var $"); + std.fmt.formatInt(module_id, 16, .lower, .{}, bufwriter) catch unreachable; + try bufwriter.writeAll(" = () => { var obj = {}; Object.defineProperty(obj, 'default', { value: obj, enumerable: false, configurable: true }, obj); return obj; }; \n"); + try buffered.flush(); + this.tmpfile_byte_offset = @truncate(u32, try this.tmpfile.getPos()); + return code_offset; +} + +fn processImportRecord(_: *GenerateNodeModuleBundle, _: ImportRecord) !void {} +var json_ast_symbols = [_]js_ast.Symbol{ + js_ast.Symbol{ .original_name = "$$m" }, + js_ast.Symbol{ .original_name = "exports" }, + js_ast.Symbol{ .original_name = "module" }, + js_ast.Symbol{ .original_name = "CONGRATS_YOU_FOUND_A_BUG" }, + js_ast.Symbol{ .original_name = "$$bun_runtime_json_parse" }, +}; +const json_parse_string = "parse"; +var json_ast_symbols_list = std.mem.span(&json_ast_symbols); +threadlocal var override_file_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + +pub fn appendToModuleList( + this: *GenerateNodeModuleBundle, + package: *const PackageJSON, + module_id: u32, + code_offset: u32, + package_relative_path: string, +) !void { + this.list_lock.lock(); + defer this.list_lock.unlock(); + + const code_length = @atomicLoad(u32, &this.tmpfile_byte_offset, .SeqCst) - code_offset; + + if (comptime Environment.isDebug) { + std.debug.assert(code_length > 0); + std.debug.assert(package.hash != 0); + std.debug.assert(package.version.len > 0); + std.debug.assert(package.name.len > 0); + std.debug.assert(module_id > 0); + } + + var package_get_or_put_entry = try this.package_list_map.getOrPut(package.hash); + + if (!package_get_or_put_entry.found_existing) { + package_get_or_put_entry.value_ptr.* = @truncate(u32, this.package_list.items.len); + try this.package_list.append( + Api.JavascriptBundledPackage{ + .name = try this.appendHeaderString(package.name), + .version = try this.appendHeaderString(package.version), + .hash = package.hash, + }, + ); + this.has_jsx = this.has_jsx or strings.eql(package.name, this.bundler.options.jsx.package_name); + } + + var path_extname_length = @truncate(u8, std.fs.path.extension(package_relative_path).len); + try this.module_list.append( + Api.JavascriptBundledModule{ + .path = try this.appendHeaderString( + package_relative_path, + ), + .path_extname_length = path_extname_length, + .package_id = package_get_or_put_entry.value_ptr.*, + .id = module_id, + .code = Api.StringPointer{ + .length = @truncate(u32, code_length), + .offset = @truncate(u32, code_offset), + }, + }, + ); +} +threadlocal var json_e_string: js_ast.E.String = undefined; +threadlocal var json_e_call: js_ast.E.Call = undefined; +threadlocal var json_e_identifier: js_ast.E.Identifier = undefined; +threadlocal var json_call_args: [1]js_ast.Expr = undefined; +const PendingImports = std.AutoArrayHashMap(u32, ThreadPool.Worker.ProcessFileTask); +pub fn processFile(this: *GenerateNodeModuleBundle, worker: *ThreadPool.Worker, bundler: *Bundler, _resolve: _resolver.Result) !void { + const resolve = _resolve; + if (resolve.is_external) return; + + var shared_buffer = &worker.data.shared_buffer; + var scan_pass_result = &worker.data.scan_pass_result; + var file_path = (resolve.pathConst() orelse unreachable).*; + + var add_to_bundle = brk: { + if (resolve.package_json) |package_json| { + if (this.package_bundle_map.get(package_json.name)) |result| { + break :brk result == .always; + } + } + + break :brk resolve.isLikelyNodeModule(); + }; + + const source_dir = file_path.sourceDir(); + const loader = bundler.options.loader(file_path.name.ext); + const platform = bundler.options.platform; + + defer scan_pass_result.reset(); + defer shared_buffer.reset(); + defer this.bundler.resetStore(); + var log = worker.data.log; + var queue = PendingImports.init(worker.allocator); + var __module_data: ?BundledModuleData = null; + if (add_to_bundle) { + __module_data = BundledModuleData.getForceBundleForMain(this, &resolve); + } + // If we're in a node_module, build that almost normally + if (add_to_bundle and __module_data != null) { + const module_data = __module_data.?; + + var code_offset: u32 = 0; + + const module_id = module_data.module_id; + const package = module_data.package; + const package_relative_path = module_data.import_path; + + file_path.pretty = module_data.package_path; + + const entry: CacheEntry = brk: { + if (this.bundler.options.framework) |framework| { + if (framework.override_modules_hashes.len > 0) { + const package_relative_path_hash = std.hash.Wyhash.hash(0, module_data.package_path); + if (std.mem.indexOfScalar( + u64, + framework.override_modules_hashes, + package_relative_path_hash, + )) |index| { + const relative_path = [_]string{ + framework.resolved_dir, + framework.override_modules.values[index], + }; + var override_path = this.bundler.fs.absBuf( + &relative_path, + &override_file_path_buf, + ); + override_file_path_buf[override_path.len] = 0; + var override_pathZ = override_file_path_buf[0..override_path.len :0]; + break :brk try bundler.resolver.caches.fs.readFileShared( + bundler.fs, + override_pathZ, + 0, + null, + shared_buffer, + ); + } + } + } + + if (!strings.eqlComptime(file_path.namespace, "node")) + break :brk try bundler.resolver.caches.fs.readFileShared( + bundler.fs, + file_path.textZ(), + resolve.dirname_fd, + if (resolve.file_fd != 0) resolve.file_fd else null, + shared_buffer, + ); + + break :brk CacheEntry{ + .contents = NodeFallbackModules.contentsFromPath(file_path.text) orelse "", + }; + }; + + var approximate_newline_count: usize = 0; + defer worker.data.estimated_input_lines_of_code += approximate_newline_count; + + // Handle empty files + // We can't just ignore them. Sometimes code will try to import it. Often because of TypeScript types. + // So we just say it's an empty object. Empty object mimicks what "browser": false does as well. + // TODO: optimize this so that all the exports for these are done in one line instead of writing repeatedly + if (entry.contents.len == 0 or (entry.contents.len < 33 and strings.trim(entry.contents, " \n\r").len == 0)) { + code_offset = try this.writeEmptyModule(module_data.package_path, module_id); + } else { + var ast: js_ast.Ast = undefined; + + const source = logger.Source.initRecycledFile( + Fs.File{ + .path = file_path, + .contents = entry.contents, + }, + bundler.allocator, + ) catch return null; + + switch (loader) { + .jsx, + .tsx, + .js, + .ts, + => { + var jsx = _resolve.jsx; + jsx.parse = loader.isJSX(); + + var opts = js_parser.Parser.Options.init(jsx, loader); + opts.transform_require_to_import = false; + opts.enable_bundling = true; + opts.warn_about_unbundled_modules = false; + opts.macro_context = &worker.data.macro_context; + opts.features.auto_import_jsx = jsx.parse; + opts.features.trim_unused_imports = this.bundler.options.trim_unused_imports orelse loader.isTypeScript(); + opts.tree_shaking = this.bundler.options.tree_shaking; + ast = (bundler.resolver.caches.js.parse( + bundler.allocator, + opts, + bundler.options.define, + log, + &source, + ) catch null) orelse return; + + approximate_newline_count = ast.approximate_newline_count; + if (ast.import_records.len > 0) { + for (ast.import_records) |*import_record| { + + // Don't resolve the runtime + if (import_record.is_internal or import_record.is_unused) { + continue; + } + + if (bundler.resolver.resolve(source_dir, import_record.path.text, import_record.kind)) |*_resolved_import| { + if (_resolved_import.is_external) { + continue; + } + var path = _resolved_import.path() orelse { + import_record.path.is_disabled = true; + import_record.is_bundled = true; + continue; + }; + + const loader_ = bundler.options.loader(path.name.ext); + + if (!loader_.isJavaScriptLikeOrJSON()) { + import_record.path.is_disabled = true; + import_record.is_bundled = true; + continue; + } + + // if (_resolved_import.package_json == null) |pkg_json| { + // _resolved_import.package_json = if (pkg_json.hash == resolve.package_json.?.hash) + // resolve.package_json + // else + // _resolved_import.package_json; + // } + + const resolved_import: *const _resolver.Result = _resolved_import; + + const _module_data = BundledModuleData.getForceBundle(this, resolved_import) orelse { + // if a macro imports code that cannot be bundled + // we just silently disable it + // because...we need some kind of hook to say "don't bundle this" + import_record.path.is_disabled = true; + import_record.is_bundled = false; + + continue; + }; + import_record.module_id = _module_data.module_id; + std.debug.assert(import_record.module_id != 0); + import_record.is_bundled = true; + + path.* = try path.dupeAlloc(this.allocator); + + import_record.path = path.*; + _ = queue.getOrPutValue( + _module_data.module_id, + .{ + .resolution = _resolved_import.*, + }, + ) catch unreachable; + } else |err| { + if (comptime Environment.isDebug) { + if (!import_record.handles_import_errors) { + Output.prettyErrorln("\n<r><red>{s}<r> resolving \"{s}\" from \"{s}\"", .{ + @errorName(err), + import_record.path.text, + file_path.text, + }); + } + } + + // Disable failing packages from being printed. + // This may cause broken code to write. + // However, doing this means we tell them all the resolve errors + // Rather than just the first one. + import_record.path.is_disabled = true; + + switch (err) { + error.ModuleNotFound => { + const addError = logger.Log.addResolveErrorWithTextDupeMaybeWarn; + + if (!import_record.handles_import_errors) { + if (isPackagePath(import_record.path.text)) { + if (platform.isWebLike() and options.ExternalModules.isNodeBuiltin(import_record.path.text)) { + try addError( + log, + &source, + import_record.range, + this.allocator, + "Could not resolve Node.js builtin: \"{s}\".", + .{import_record.path.text}, + import_record.kind, + platform.isBun(), + ); + } else { + try addError( + log, + &source, + import_record.range, + this.allocator, + "Could not resolve: \"{s}\". Maybe you need to \"bun install\"?", + .{import_record.path.text}, + import_record.kind, + platform.isBun(), + ); + } + } else if (!platform.isBun()) { + try addError( + log, + &source, + import_record.range, + this.allocator, + "Could not resolve: \"{s}\"", + .{ + import_record.path.text, + }, + import_record.kind, + platform.isBun(), + ); + } + } + }, + // assume other errors are already in the log + else => {}, + } + } + } + } + }, + .json => { + // parse the JSON _only_ to catch errors at build time. + const json_parse_result = json_parser.ParseJSONForBundling(&source, worker.data.log, worker.allocator) catch return; + + if (json_parse_result.tag != .empty) { + const expr = brk: { + // If it's an ascii string, we just print it out with a big old JSON.parse() + if (json_parse_result.tag == .ascii) { + json_e_string = js_ast.E.String{ .utf8 = source.contents, .prefer_template = true }; + var json_string_expr = js_ast.Expr{ .data = .{ .e_string = &json_e_string }, .loc = logger.Loc{ .start = 0 } }; + json_call_args[0] = json_string_expr; + json_e_identifier = js_ast.E.Identifier{ .ref = Ref.atIndex(json_ast_symbols_list.len - 1) }; + + json_e_call = js_ast.E.Call{ + .target = js_ast.Expr{ .data = .{ .e_identifier = json_e_identifier }, .loc = logger.Loc{ .start = 0 } }, + .args = js_ast.ExprNodeList.init(std.mem.span(&json_call_args)), + }; + break :brk js_ast.Expr{ .data = .{ .e_call = &json_e_call }, .loc = logger.Loc{ .start = 0 } }; + // If we're going to have to convert it to a UTF16, just make it an object actually + } else { + break :brk json_parse_result.expr; + } + }; + + var stmt = js_ast.Stmt.alloc(js_ast.S.ExportDefault, js_ast.S.ExportDefault{ + .value = js_ast.StmtOrExpr{ .expr = expr }, + .default_name = js_ast.LocRef{ + .loc = logger.Loc{}, + .ref = Ref.None, + }, + }, logger.Loc{ .start = 0 }); + var stmts = worker.allocator.alloc(js_ast.Stmt, 1) catch unreachable; + stmts[0] = stmt; + var parts = worker.allocator.alloc(js_ast.Part, 1) catch unreachable; + parts[0] = js_ast.Part{ .stmts = stmts }; + ast = js_ast.Ast.initTest(parts); + + ast.runtime_imports = runtime.Runtime.Imports{}; + ast.runtime_imports.@"$$m" = .{ .ref = Ref.atIndex(0), .primary = Ref.None, .backup = Ref.None }; + ast.runtime_imports.__export = .{ .ref = Ref.atIndex(1), .primary = Ref.None, .backup = Ref.None }; + ast.symbols = json_ast_symbols_list; + ast.module_ref = Ref.atIndex(2); + ast.exports_ref = ast.runtime_imports.__export.?.ref; + ast.bundle_export_ref = Ref.atIndex(3); + } else { + var parts = &[_]js_ast.Part{}; + ast = js_ast.Ast.initTest(parts); + } + }, + else => { + return; + }, + } + + switch (ast.parts.len) { + // It can be empty after parsing too + // A file like this is an example: + // "//# sourceMappingURL=validator.js.map" + 0 => { + code_offset = try this.writeEmptyModule(module_data.package_path, module_id); + }, + else => { + const register_ref = ast.runtime_imports.@"$$m".?.ref; + const E = js_ast.E; + const Expr = js_ast.Expr; + const Stmt = js_ast.Stmt; + + var prepend_part: js_ast.Part = undefined; + var needs_prepend_part = false; + if (ast.parts.len > 1) { + for (ast.parts) |part| { + if (part.tag != .none and part.stmts.len > 0) { + prepend_part = part; + needs_prepend_part = true; + break; + } + } + } + + var package_path = js_ast.E.String{ .utf8 = module_data.package_path }; + + var target_identifier = E.Identifier{ .ref = register_ref }; + var cjs_args: [2]js_ast.G.Arg = undefined; + var module_binding = js_ast.B.Identifier{ .ref = ast.module_ref.? }; + var exports_binding = js_ast.B.Identifier{ .ref = ast.exports_ref.? }; + + var part = &ast.parts[ast.parts.len - 1]; + + var new_stmts: [1]Stmt = undefined; + var register_args: [1]Expr = undefined; + var closure = E.Arrow{ + .args = &cjs_args, + .body = .{ + .loc = logger.Loc.Empty, + .stmts = part.stmts, + }, + }; + + cjs_args[0] = js_ast.G.Arg{ + .binding = js_ast.Binding{ + .loc = logger.Loc.Empty, + .data = .{ .b_identifier = &module_binding }, + }, + }; + cjs_args[1] = js_ast.G.Arg{ + .binding = js_ast.Binding{ + .loc = logger.Loc.Empty, + .data = .{ .b_identifier = &exports_binding }, + }, + }; + + var properties: [1]js_ast.G.Property = undefined; + var e_object = E.Object{ + .properties = js_ast.G.Property.List.init(&properties), + }; + const module_path_str = js_ast.Expr{ .data = .{ .e_string = &package_path }, .loc = logger.Loc.Empty }; + properties[0] = js_ast.G.Property{ + .key = module_path_str, + .value = Expr{ .loc = logger.Loc.Empty, .data = .{ .e_arrow = &closure } }, + }; + + // if (!ast.uses_module_ref) { + // var symbol = &ast.symbols[ast.module_ref.?.innerIndex()]; + // symbol.original_name = "_$$"; + // } + + // $$m(12345, "react", "index.js", function(module, exports) { + + // }) + var accessor = js_ast.E.Index{ .index = module_path_str, .target = js_ast.Expr{ + .data = .{ .e_object = &e_object }, + .loc = logger.Loc.Empty, + } }; + register_args[0] = Expr{ .loc = logger.Loc.Empty, .data = .{ .e_index = &accessor } }; + + var call_register = E.Call{ + .target = Expr{ + .data = .{ .e_identifier = target_identifier }, + .loc = logger.Loc{ .start = 0 }, + }, + .args = js_ast.ExprNodeList.init(®ister_args), + }; + var register_expr = Expr{ .loc = call_register.target.loc, .data = .{ .e_call = &call_register } }; + var decls: [1]js_ast.G.Decl = undefined; + var bundle_export_binding = js_ast.B.Identifier{ .ref = ast.runtime_imports.@"$$m".?.ref }; + var binding = js_ast.Binding{ + .loc = register_expr.loc, + .data = .{ .b_identifier = &bundle_export_binding }, + }; + decls[0] = js_ast.G.Decl{ + .value = register_expr, + .binding = binding, + }; + var export_var = js_ast.S.Local{ + .decls = &decls, + .is_export = true, + }; + new_stmts[0] = Stmt{ .loc = register_expr.loc, .data = .{ .s_local = &export_var } }; + part.stmts = &new_stmts; + + var writer = js_printer.NewFileWriter(this.tmpfile); + var symbols: [][]js_ast.Symbol = &([_][]js_ast.Symbol{ast.symbols}); + + // It should only have one part. + ast.parts = ast.parts[ast.parts.len - 1 ..]; + const write_result = + // The difference between these two is `ascii_only` + // We try our best to print UTF-8. + // However, JavaScriptCore does not accept UTF-8. + // It accepts either latin1 characters as unsigned char + // or UTF-16 chars as uint16 + // We don't want to add a UTF decoding pass to the .bun files + // because it's potentially 15 MB of code. + // If we store it as UTF-16 directly, then that 15 MB of code becomes 30 MB of code! + // lots of code! + // + + if (!bundler.options.platform.isBun()) + try js_printer.printCommonJSThreaded( + @TypeOf(writer), + writer, + ast, + js_ast.Symbol.Map.initList(symbols), + &source, + false, + js_printer.Options{ + .to_module_ref = Ref.RuntimeRef, + .bundle_export_ref = ast.runtime_imports.@"$$m".?.ref, + .source_path = file_path, + .externals = ast.externals, + .indent = 0, + .require_ref = ast.require_ref, + .module_hash = module_id, + .runtime_imports = ast.runtime_imports, + .prepend_part_value = &prepend_part, + .prepend_part_key = if (needs_prepend_part) closure.body.stmts.ptr else null, + }, + Linker, + &bundler.linker, + &this.write_lock, + std.fs.File, + this.tmpfile, + std.fs.File.getPos, + &this.tmpfile_byte_offset, + ) + else + try js_printer.printCommonJSThreaded( + @TypeOf(writer), + writer, + ast, + js_ast.Symbol.Map.initList(symbols), + &source, + true, + js_printer.Options{ + .to_module_ref = Ref.RuntimeRef, + .bundle_export_ref = ast.runtime_imports.@"$$m".?.ref, + .source_path = file_path, + .externals = ast.externals, + .indent = 0, + .require_ref = ast.require_ref, + .module_hash = module_id, + .runtime_imports = ast.runtime_imports, + .prepend_part_value = &prepend_part, + .prepend_part_key = if (needs_prepend_part) closure.body.stmts.ptr else null, + }, + Linker, + &bundler.linker, + &this.write_lock, + std.fs.File, + this.tmpfile, + std.fs.File.getPos, + &this.tmpfile_byte_offset, + ); + + code_offset = write_result.off; + }, + } + } + + if (comptime Environment.isDebug) { + Output.prettyln("{s}@{s}/{s} - {d}:{d} \n", .{ package.name, package.version, package_relative_path, package.hash, module_id }); + Output.flush(); + std.debug.assert(package_relative_path.len > 0); + } + + try this.appendToModuleList( + package, + module_id, + code_offset, + package_relative_path, + ); + } else { + // If it's app code, scan but do not fully parse. + switch (loader) { + .jsx, + .tsx, + .js, + .ts, + => { + const entry = bundler.resolver.caches.fs.readFileShared( + bundler.fs, + file_path.textZ(), + resolve.dirname_fd, + if (resolve.file_fd != 0) resolve.file_fd else null, + shared_buffer, + ) catch return; + if (entry.contents.len == 0 or (entry.contents.len < 33 and strings.trim(entry.contents, " \n\r").len == 0)) return; + + const source = logger.Source.initRecycledFile(Fs.File{ .path = file_path, .contents = entry.contents }, bundler.allocator) catch return null; + + var jsx = bundler.options.jsx; + + jsx.parse = loader.isJSX(); + var opts = js_parser.Parser.Options.init(jsx, loader); + opts.macro_context = &worker.data.macro_context; + opts.features.trim_unused_imports = bundler.options.trim_unused_imports orelse loader.isTypeScript(); + + try bundler.resolver.caches.js.scan( + bundler.allocator, + scan_pass_result, + opts, + bundler.options.define, + log, + &source, + ); + worker.data.estimated_input_lines_of_code += scan_pass_result.approximate_newline_count; + + { + for (scan_pass_result.import_records.items) |*import_record| { + if (import_record.is_internal or import_record.is_unused) { + continue; + } + + if (bundler.resolver.resolve(source_dir, import_record.path.text, import_record.kind)) |*_resolved_import| { + if (_resolved_import.is_external) { + continue; + } + + var path = _resolved_import.path() orelse continue; + + const loader_ = this.bundler.options.loader(path.name.ext); + if (!loader_.isJavaScriptLikeOrJSON()) continue; + + path.* = try path.dupeAlloc(this.allocator); + + if (BundledModuleData.get(this, _resolved_import)) |mod| { + if (comptime !FeatureFlags.bundle_dynamic_import) { + if (import_record.kind == .dynamic) + continue; + } else { + // When app code dynamically imports a large file + // Don't bundle it. Leave it as a separate file. + // The main value from bundling in development is to minimize tiny, waterfall http requests + // If you're importing > 100 KB file dynamically, developer is probably explicitly trying to do that. + // There's a tradeoff between "I want to minimize page load time" + if (import_record.kind == .dynamic) { + this.dynamic_import_file_size_store_lock.lock(); + defer this.dynamic_import_file_size_store_lock.unlock(); + var dynamic_import_file_size = this.dynamic_import_file_size_store.getOrPut(mod.module_id) catch unreachable; + if (!dynamic_import_file_size.found_existing) { + var fd = _resolved_import.file_fd; + var can_close = false; + if (fd == 0) { + dynamic_import_file_size.value_ptr.* = 0; + fd = (std.fs.openFileAbsolute(path.textZ(), .{}) catch |err| { + this.log.addRangeWarningFmt( + &source, + import_record.range, + worker.allocator, + "{s} opening file: \"{s}\"", + .{ @errorName(err), path.text }, + ) catch unreachable; + continue; + }).handle; + can_close = true; + Fs.FileSystem.setMaxFd(fd); + } + + defer { + if (can_close and bundler.fs.fs.needToCloseFiles()) { + var _file = std.fs.File{ .handle = fd }; + _file.close(); + _resolved_import.file_fd = 0; + } else if (FeatureFlags.store_file_descriptors) { + _resolved_import.file_fd = fd; + } + } + + var file = std.fs.File{ .handle = fd }; + var stat = file.stat() catch |err| { + this.log.addRangeWarningFmt( + &source, + import_record.range, + worker.allocator, + "{s} stat'ing file: \"{s}\"", + .{ @errorName(err), path.text }, + ) catch unreachable; + dynamic_import_file_size.value_ptr.* = 0; + continue; + }; + + dynamic_import_file_size.value_ptr.* = @truncate(u32, stat.size); + } + + if (dynamic_import_file_size.value_ptr.* > 1024 * 100) + continue; + } + } + + std.debug.assert(mod.module_id != 0); + _ = queue.getOrPutValue( + mod.module_id, + .{ + .resolution = _resolved_import.*, + }, + ) catch unreachable; + } else { + _ = queue.getOrPutValue( + _resolved_import.hash( + this.bundler.fs.top_level_dir, + loader_, + ), + .{ + .resolution = _resolved_import.*, + }, + ) catch unreachable; + } + } else |err| { + switch (err) { + error.ModuleNotFound => { + if (!import_record.handles_import_errors) { + const addError = logger.Log.addResolveErrorWithTextDupeMaybeWarn; + if (isPackagePath(import_record.path.text)) { + if (platform.isWebLike() and options.ExternalModules.isNodeBuiltin(import_record.path.text)) { + try addError( + log, + &source, + import_record.range, + this.allocator, + "Could not resolve Node.js builtin: \"{s}\".", + .{import_record.path.text}, + import_record.kind, + platform.isBun(), + ); + } else { + try addError( + log, + &source, + import_record.range, + this.allocator, + "Could not resolve: \"{s}\". Maybe you need to \"bun install\"?", + .{import_record.path.text}, + import_record.kind, + platform.isBun(), + ); + } + } else if (!platform.isBun()) { + try addError( + log, + &source, + import_record.range, + this.allocator, + "Could not resolve: \"{s}\"", + .{ + import_record.path.text, + }, + import_record.kind, + platform.isBun(), + ); + } + } + }, + // assume other errors are already in the log + else => {}, + } + } + } + } + }, + else => {}, + } + } + + if (queue.count() > 0) { + try this.queue.writeItem(queue); + } +} diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index b8be52118..eb2bb3141 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -34,13 +34,6 @@ pub const BuildCommand = struct { pub fn exec(ctx: Command.Context) !void { var result: options.TransformResult = undefined; switch (ctx.args.resolve orelse Api.ResolveMode.dev) { - Api.ResolveMode.disable => { - result = try bundler.Transformer.transform( - ctx.allocator, - ctx.log, - ctx.args, - ); - }, .lazy => { result = try bundler.Bundler.bundle( ctx.allocator, diff --git a/src/cli/bun_command.zig b/src/cli/bun_command.zig index d9fbaea75..8305379d7 100644 --- a/src/cli/bun_command.zig +++ b/src/cli/bun_command.zig @@ -27,6 +27,7 @@ const resolve_path = @import("../resolver/resolve_path.zig"); const configureTransformOptionsForBun = @import("../javascript/jsc/config.zig").configureTransformOptionsForBun; const bundler = @import("../bundler.zig"); const NodeModuleBundle = @import("../node_module_bundle.zig").NodeModuleBundle; +const GenerateNodeModuleBundle = @import("../bundler/generate_node_modules_bundle.zig"); const DotEnv = @import("../env_loader.zig"); const fs = @import("../fs.zig"); @@ -65,7 +66,7 @@ const ServerBundleGeneratorThread = struct { } var estimated_input_lines_of_code: usize = 0; - _ = try bundler.Bundler.GenerateNodeModuleBundle.generate( + _ = try GenerateNodeModuleBundle.generate( &server_bundler, allocator_, server_conf, @@ -177,7 +178,7 @@ pub const BunCommand = struct { // Always generate the client-only bundle // we can revisit this decision if people ask - var node_modules_ = try bundler.Bundler.GenerateNodeModuleBundle.generate( + var node_modules_ = try GenerateNodeModuleBundle.generate( &this_bundler, allocator, loaded_framework, |