aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGravatar Jarred Sumner <jarred@jarredsumner.com> 2023-04-22 19:44:23 -0700
committerGravatar GitHub <noreply@github.com> 2023-04-22 19:44:23 -0700
commit4b24bb464c5a54c728bee8c9b4aa30a60c3fb7d4 (patch)
tree7983dd2d5ba18f9a6111be2e7e6d17f69d170bba
parent7d6b5f5358617f92ace6f3103dd835ddff73f92a (diff)
downloadbun-4b24bb464c5a54c728bee8c9b4aa30a60c3fb7d4.tar.gz
bun-4b24bb464c5a54c728bee8c9b4aa30a60c3fb7d4.tar.zst
bun-4b24bb464c5a54c728bee8c9b4aa30a60c3fb7d4.zip
Make `Bun.build` more reliable (#2718)
* One possible implementation to make `Bun.build` work better * Pass allocator in * Make our temporary buffers a little safer * rename * Fix memory corruption in symbol table * Add support for deferred idle events in ThreadPool * Free more memory * Use a global allocator FS cache * more `inline` * Make duping keys optional in StringMap * Close file handles more often * Update router.zig * wip possibly delete this commit * Fix memory issues and reduce memory usage * > 0.8 * Switch to AsyncIO.Waker and fix memory leak in JSBundleCompletionTask * We don't need to clone this actually * Fix error * Format * Fixup * Fixup --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>
-rw-r--r--src/bit_set.zig8
-rw-r--r--src/bun.js/api/JSBundler.zig4
-rw-r--r--src/bun.zig33
-rw-r--r--src/bundler.zig5
-rw-r--r--src/bundler/bundle_v2.zig520
-rw-r--r--src/cache.zig21
-rw-r--r--src/cli/build_command.zig4
-rw-r--r--src/cli/create_command.zig4
-rw-r--r--src/cli/init_command.zig2
-rw-r--r--src/cli/upgrade_command.zig2
-rw-r--r--src/fs.zig56
-rw-r--r--src/http.zig8
-rw-r--r--src/install/install.zig8
-rw-r--r--src/install/lockfile.zig4
-rw-r--r--src/js_ast.zig63
-rw-r--r--src/js_parser.zig6
-rw-r--r--src/json_parser.zig10
-rw-r--r--src/options.zig26
-rw-r--r--src/resolver/dir_info.zig4
-rw-r--r--src/resolver/package_json.zig90
-rw-r--r--src/resolver/resolver.zig243
-rw-r--r--src/router.zig4
-rw-r--r--src/thread_pool.zig58
-rw-r--r--src/work_pool.zig2
24 files changed, 761 insertions, 424 deletions
diff --git a/src/bit_set.zig b/src/bit_set.zig
index deee255bc..5f18eb32b 100644
--- a/src/bit_set.zig
+++ b/src/bit_set.zig
@@ -402,7 +402,7 @@ pub fn ArrayBitSet(comptime MaskIntType: type, comptime size: usize) type {
/// Returns true if the bit at the specified index
/// is present in the set, false otherwise.
- pub fn isSet(self: *const Self, index: usize) bool {
+ pub inline fn isSet(self: *const Self, index: usize) bool {
if (comptime Environment.allow_assert) std.debug.assert(index < bit_length);
if (num_masks == 0) return false; // doesn't compile in this case
return (self.masks[maskIndex(index)] & maskBit(index)) != 0;
@@ -646,13 +646,13 @@ pub fn ArrayBitSet(comptime MaskIntType: type, comptime size: usize) type {
return BitSetIterator(MaskInt, options);
}
- fn maskBit(index: usize) MaskInt {
+ inline fn maskBit(index: usize) MaskInt {
return @as(MaskInt, 1) << @truncate(ShiftInt, index);
}
- fn maskIndex(index: usize) usize {
+ inline fn maskIndex(index: usize) usize {
return index >> @bitSizeOf(ShiftInt);
}
- fn boolMaskBit(index: usize, value: bool) MaskInt {
+ inline fn boolMaskBit(index: usize, value: bool) MaskInt {
return @as(MaskInt, @boolToInt(value)) << @intCast(ShiftInt, index);
}
};
diff --git a/src/bun.js/api/JSBundler.zig b/src/bun.js/api/JSBundler.zig
index 1149880fe..4bb5cad3d 100644
--- a/src/bun.js/api/JSBundler.zig
+++ b/src/bun.js/api/JSBundler.zig
@@ -54,7 +54,7 @@ pub const JSBundler = struct {
target: options.Platform = options.Platform.browser,
entry_points: bun.StringSet = bun.StringSet.init(bun.default_allocator),
hot: bool = false,
- define: bun.StringMap = bun.StringMap.init(bun.default_allocator),
+ define: bun.StringMap = bun.StringMap.init(bun.default_allocator, true),
dir: OwnedString = OwnedString.initEmpty(bun.default_allocator),
outdir: OwnedString = OwnedString.initEmpty(bun.default_allocator),
serve: Serve = .{},
@@ -89,7 +89,7 @@ pub const JSBundler = struct {
var this = Config{
.entry_points = bun.StringSet.init(allocator),
.external = bun.StringSet.init(allocator),
- .define = bun.StringMap.init(allocator),
+ .define = bun.StringMap.init(allocator, true),
.dir = OwnedString.initEmpty(allocator),
.label = OwnedString.initEmpty(allocator),
.outdir = OwnedString.initEmpty(allocator),
diff --git a/src/bun.zig b/src/bun.zig
index c6fce74de..e6349fee9 100644
--- a/src/bun.zig
+++ b/src/bun.zig
@@ -20,6 +20,10 @@ else
pub const huge_allocator_threshold: comptime_int = @import("./memory_allocator.zig").huge_threshold;
+/// We cannot use a threadlocal memory allocator for FileSystem-related things
+/// FileSystem is a singleton.
+pub const fs_allocator = default_allocator;
+
pub const C = @import("c.zig");
pub const FeatureFlags = @import("feature_flags.zig");
@@ -1280,12 +1284,14 @@ pub const Schema = @import("./api/schema.zig");
pub const StringMap = struct {
map: Map,
+ dupe_keys: bool = false,
pub const Map = StringArrayHashMap(string);
- pub fn init(allocator: std.mem.Allocator) StringMap {
+ pub fn init(allocator: std.mem.Allocator, dupe_keys: bool) StringMap {
return StringMap{
.map = Map.init(allocator),
+ .dupe_keys = dupe_keys,
};
}
@@ -1311,7 +1317,8 @@ pub const StringMap = struct {
pub fn insert(self: *StringMap, key: []const u8, value: []const u8) !void {
var entry = try self.map.getOrPut(key);
if (!entry.found_existing) {
- entry.key_ptr.* = try self.map.allocator.dupe(u8, key);
+ if (self.dupe_keys)
+ entry.key_ptr.* = try self.map.allocator.dupe(u8, key);
} else {
self.map.allocator.free(entry.value_ptr.*);
}
@@ -1319,13 +1326,20 @@ pub const StringMap = struct {
entry.value_ptr.* = try self.map.allocator.dupe(u8, value);
}
+ pub const put = insert;
+ pub fn get(self: *const StringMap, key: []const u8) ?[]const u8 {
+ return self.map.get(key);
+ }
+
pub fn deinit(self: *StringMap) void {
for (self.map.values()) |value| {
self.map.allocator.free(value);
}
- for (self.map.keys()) |key| {
- self.map.allocator.free(key);
+ if (self.dupe_keys) {
+ for (self.map.keys()) |key| {
+ self.map.allocator.free(key);
+ }
}
self.map.deinit();
@@ -1334,3 +1348,14 @@ pub const StringMap = struct {
pub const DotEnv = @import("./env_loader.zig");
pub const BundleV2 = @import("./bundler/bundle_v2.zig").BundleV2;
+
+pub const Lock = @import("./lock.zig").Lock;
+pub const UnboundedQueue = @import("./bun.js/unbounded_queue.zig").UnboundedQueue;
+
+pub fn threadlocalAllocator() std.mem.Allocator {
+ if (comptime use_mimalloc) {
+ return MimallocArena.getThreadlocalDefault();
+ }
+
+ return default_allocator;
+}
diff --git a/src/bundler.zig b/src/bundler.zig
index f2da1a960..9030aaf8c 100644
--- a/src/bundler.zig
+++ b/src/bundler.zig
@@ -423,8 +423,7 @@ pub const Bundler = struct {
) !Bundler {
js_ast.Expr.Data.Store.create(allocator);
js_ast.Stmt.Data.Store.create(allocator);
- var fs = try Fs.FileSystem.init1(
- allocator,
+ var fs = try Fs.FileSystem.init(
opts.absolute_working_dir,
);
const bundle_options = try options.BundleOptions.fromApi(
@@ -1779,7 +1778,7 @@ pub const Bundler = struct {
if (bundler.linker.any_needs_runtime) {
try bundler.output_files.append(
- options.OutputFile.initBuf(runtime.Runtime.sourceContent(false), Linker.runtime_source_path, .js),
+ options.OutputFile.initBuf(runtime.Runtime.sourceContent(false), bun.default_allocator, Linker.runtime_source_path, .js),
);
}
diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig
index 140229dc1..edfbc50eb 100644
--- a/src/bundler/bundle_v2.zig
+++ b/src/bundler/bundle_v2.zig
@@ -85,8 +85,8 @@ const BitSet = bun.bit_set.DynamicBitSetUnmanaged;
pub const ThreadPool = struct {
pool: *ThreadPoolLib = undefined,
// Hardcode 512 as max number of threads for now.
- workers: [512]Worker = undefined,
- workers_used: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0),
+ workers_assignments: std.AutoArrayHashMap(std.Thread.Id, *Worker) = std.AutoArrayHashMap(std.Thread.Id, *Worker).init(bun.default_allocator),
+ workers_assignments_lock: bun.Lock = bun.Lock.init(),
cpu_count: u32 = 0,
started_workers: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0),
stopped_workers: std.atomic.Atomic(u32) = std.atomic.Atomic(u32).init(0),
@@ -108,10 +108,12 @@ pub const ThreadPool = struct {
if (v2.bundler.env.map.get("GOMAXPROCS")) |max_procs| {
if (std.fmt.parseInt(u32, max_procs, 10)) |cpu_count| {
- this.cpu_count = @max(cpu_count, 2);
+ this.cpu_count = cpu_count;
} else |_| {}
}
- this.cpu_count = @min(this.cpu_count, @truncate(u32, this.workers.len - 1));
+
+ this.cpu_count = @max(@min(this.cpu_count, @truncate(u32, 128 - 1)), 2);
+
if (existing_thread_pool) |pool| {
this.pool = pool;
} else {
@@ -120,102 +122,120 @@ pub const ThreadPool = struct {
.max_threads = this.cpu_count,
});
}
- this.pool.on_thread_spawn = Worker.onSpawn;
- this.pool.threadpool_context = this;
- var workers_used: u32 = 0;
- while (workers_used < this.cpu_count) : (workers_used += 1) {
- try this.workers[workers_used].init(v2);
- }
+ this.pool.setThreadContext(this);
- if (workers_used > 0)
- this.pool.forceSpawn();
debug("allocated {d} workers", .{this.cpu_count});
}
+ pub fn getWorker(this: *ThreadPool, id: std.Thread.Id) *Worker {
+ var worker: *Worker = undefined;
+ {
+ this.workers_assignments_lock.lock();
+ defer this.workers_assignments_lock.unlock();
+ var entry = this.workers_assignments.getOrPut(id) catch unreachable;
+ if (entry.found_existing) {
+ return entry.value_ptr.*;
+ }
+
+ worker = bun.default_allocator.create(Worker) catch unreachable;
+ entry.value_ptr.* = worker;
+ }
+
+ worker.* = .{
+ .ctx = this.v2,
+ .allocator = undefined,
+ .thread = ThreadPoolLib.Thread.current,
+ };
+ worker.init(this.v2);
+
+ return worker;
+ }
+
pub const Worker = struct {
- thread_id: std.Thread.Id,
- thread: std.Thread,
heap: ThreadlocalArena = ThreadlocalArena{},
allocator: std.mem.Allocator,
ctx: *BundleV2,
- data: *WorkerData = undefined,
+ data: WorkerData = undefined,
quit: bool = false,
ast_memory_allocator: js_ast.ASTMemoryAllocator = undefined,
- has_notify_started: bool = false,
has_created: bool = false,
+ thread: ?*ThreadPoolLib.Thread = null,
+
+ deinit_task: ThreadPoolLib.Task = .{ .callback = deinitCallback },
+
+ pub fn deinitCallback(task: *ThreadPoolLib.Task) void {
+ var this = @fieldParentPtr(Worker, "deinit_task", task);
+ this.deinit();
+ }
+
+ pub fn deinitSoon(this: *Worker) void {
+ if (this.thread) |thread| {
+ thread.pushIdleTask(&this.deinit_task);
+ }
+ }
pub fn deinit(this: *Worker) void {
- this.data.deinit(this.allocator);
- this.heap.deinit();
+ if (this.has_created) {
+ this.heap.deinit();
+ }
+
+ bun.default_allocator.destroy(this);
}
- pub fn get() *Worker {
- var worker = @ptrCast(
- *ThreadPool.Worker,
- @alignCast(
- @alignOf(*ThreadPool.Worker),
- ThreadPoolLib.Thread.current.?.ctx.?,
- ),
- );
+ pub fn get(ctx: *BundleV2) *Worker {
+ var worker = ctx.graph.pool.getWorker(std.Thread.getCurrentId());
if (!worker.has_created) {
- worker.create();
+ worker.create(ctx);
+ }
+
+ worker.ast_memory_allocator.push();
+
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ worker.heap.gc(true);
}
return worker;
}
+ pub fn unget(this: *Worker) void {
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.heap.gc(true);
+ }
+
+ this.ast_memory_allocator.pop();
+ }
+
pub const WorkerData = struct {
log: *Logger.Log,
estimated_input_lines_of_code: usize = 0,
macro_context: js_ast.Macro.MacroContext,
bundler: Bundler = undefined,
-
- pub fn deinit(this: *WorkerData, allocator: std.mem.Allocator) void {
- allocator.destroy(this);
- }
};
- pub fn init(worker: *Worker, v2: *BundleV2) !void {
+ pub fn init(worker: *Worker, v2: *BundleV2) void {
worker.ctx = v2;
}
- pub fn onSpawn(ctx: ?*anyopaque) ?*anyopaque {
- var pool = @ptrCast(*ThreadPool, @alignCast(@alignOf(*ThreadPool), ctx.?));
-
- const id = pool.workers_used.fetchAdd(1, .Monotonic);
- pool.workers[id].run();
- return &pool.workers[id];
- }
-
- pub fn notifyStarted(this: *Worker) void {
- if (!this.has_notify_started) {
- this.has_notify_started = true;
- _ = this.v2.pool.started_workers.fetchAdd(1, .Release);
- std.Thread.Futex.wake(&this.v2.pool.started_workers, std.math.maxInt(u32));
- }
- }
-
- fn create(this: *Worker) void {
+ fn create(this: *Worker, ctx: *BundleV2) void {
this.has_created = true;
Output.Source.configureThread();
- this.thread_id = std.Thread.getCurrentId();
this.heap = ThreadlocalArena.init() catch unreachable;
this.allocator = this.heap.allocator();
var allocator = this.allocator;
this.ast_memory_allocator = .{ .allocator = this.allocator };
- this.ast_memory_allocator.push();
+ this.ast_memory_allocator.reset();
- this.data = allocator.create(WorkerData) catch unreachable;
- this.data.* = WorkerData{
+ this.data = WorkerData{
.log = allocator.create(Logger.Log) catch unreachable,
.estimated_input_lines_of_code = 0,
.macro_context = undefined,
};
this.data.log.* = Logger.Log.init(allocator);
- this.data.bundler = this.ctx.bundler.*;
+ this.ctx = ctx;
+ this.data.bundler = ctx.bundler.*;
this.data.bundler.setLog(this.data.log);
this.data.bundler.setAllocator(allocator);
this.data.bundler.linker.resolver = &this.data.bundler.resolver;
@@ -227,9 +247,9 @@ pub const ThreadPool = struct {
this.data.bundler.resolver.caches = CacheSet.Set.init(this.allocator);
}
- pub fn run(this: *Worker) void {
+ pub fn run(this: *Worker, ctx: *BundleV2) void {
if (!this.has_created) {
- this.create();
+ this.create(ctx);
}
// no funny business mr. cache
@@ -374,7 +394,7 @@ pub const BundleV2 = struct {
.side_effects = resolve.primary_side_effects_data,
});
var task = try this.graph.allocator.create(ParseTask);
- task.* = ParseTask.init(&result, source_index);
+ task.* = ParseTask.init(&result, source_index, this);
task.loader = loader;
task.jsx = this.bundler.options.jsx;
task.task.node.next = null;
@@ -389,6 +409,7 @@ pub const BundleV2 = struct {
event_loop: EventLoop,
enable_reloading: bool,
thread_pool: ?*ThreadPoolLib,
+ heap: ?ThreadlocalArena,
) !*BundleV2 {
var generator = try allocator.create(BundleV2);
bundler.options.mark_builtins_as_external = bundler.options.platform.isBun() or bundler.options.platform == .node;
@@ -401,7 +422,7 @@ pub const BundleV2 = struct {
.server_bundler = bundler,
.graph = .{
.pool = undefined,
- .heap = try ThreadlocalArena.init(),
+ .heap = heap orelse try ThreadlocalArena.init(),
.allocator = undefined,
},
.linker = .{
@@ -458,7 +479,10 @@ pub const BundleV2 = struct {
this.graph.path_to_source_index_map.put(this.graph.allocator, bun.hash("bun:wrap"), Index.runtime.get()) catch unreachable;
var runtime_parse_task = try this.graph.allocator.create(ParseTask);
runtime_parse_task.* = ParseTask.runtime;
- runtime_parse_task.task.node.next = null;
+ runtime_parse_task.ctx = this;
+ runtime_parse_task.task = .{
+ .callback = &ParseTask.callback,
+ };
runtime_parse_task.tree_shaking = true;
runtime_parse_task.loader = .js;
this.graph.parse_pending += 1;
@@ -488,7 +512,6 @@ pub const BundleV2 = struct {
try this.graph.input_files.ensureUnusedCapacity(this.graph.allocator, user_entry_points.len);
try this.graph.path_to_source_index_map.ensureUnusedCapacity(this.graph.allocator, @truncate(u32, user_entry_points.len));
- defer this.bundler.resetStore();
for (user_entry_points) |entry_point| {
const resolved = this.bundler.resolveEntryPoint(entry_point) catch continue;
if (try this.enqueueItem(null, &batch, resolved)) |source_index| {
@@ -505,10 +528,12 @@ pub const BundleV2 = struct {
this.linker.graph.allocator = this.bundler.allocator;
this.linker.graph.ast = try this.graph.ast.clone(this.linker.allocator);
var ast = this.linker.graph.ast.slice();
- for (ast.items(.module_scope)) |*new_module_scope| {
- for (new_module_scope.children.slice()) |new_child| {
- new_child.parent = new_module_scope;
+ for (ast.items(.module_scope)) |*module_scope| {
+ for (module_scope.children.slice()) |child| {
+ child.parent = module_scope;
}
+
+ module_scope.generated = try module_scope.generated.clone(this.linker.allocator);
}
}
@@ -519,7 +544,7 @@ pub const BundleV2 = struct {
unique_key: u64,
enable_reloading: bool,
) !std.ArrayList(options.OutputFile) {
- var this = try BundleV2.init(bundler, allocator, event_loop, enable_reloading, null);
+ var this = try BundleV2.init(bundler, allocator, event_loop, enable_reloading, null, null);
if (this.bundler.log.msgs.items.len > 0) {
return error.BuildFailed;
@@ -562,7 +587,7 @@ pub const BundleV2 = struct {
.jsc_event_loop = event_loop,
.promise = JSC.JSPromise.Strong.init(globalThis),
.globalThis = globalThis,
- .ref = JSC.Ref.init(),
+ .ref = JSC.PollRef.init(),
.env = globalThis.bunVM().bundler.env,
.log = Logger.Log.init(bun.default_allocator),
.task = JSBundleCompletionTask.TaskCompletion.init(completion),
@@ -572,8 +597,21 @@ pub const BundleV2 = struct {
// conditions from creating two
_ = JSC.WorkPool.get();
- var thread = try std.Thread.spawn(.{}, generateInNewThreadWrap, .{completion});
- thread.detach();
+ if (!BundleThread.created) {
+ BundleThread.created = true;
+
+ var instance = bun.default_allocator.create(BundleThread) catch unreachable;
+ instance.queue = .{};
+ instance.waker = bun.AsyncIO.Waker.init(bun.default_allocator) catch @panic("Failed to create waker");
+ instance.queue.push(completion);
+ BundleThread.instance = instance;
+
+ var thread = try std.Thread.spawn(.{}, generateInNewThreadWrap, .{instance});
+ thread.detach();
+ } else {
+ BundleThread.instance.queue.push(completion);
+ BundleThread.instance.waker.wake() catch {};
+ }
completion.ref.ref(globalThis.bunVM());
@@ -590,12 +628,14 @@ pub const BundleV2 = struct {
task: bun.JSC.AnyTask,
globalThis: *JSC.JSGlobalObject,
promise: JSC.JSPromise.Strong,
- ref: JSC.Ref = JSC.Ref.init(),
+ ref: JSC.PollRef = JSC.PollRef.init(),
env: *bun.DotEnv.Loader,
log: Logger.Log,
result: Result = .{ .pending = {} },
+ next: ?*JSBundleCompletionTask = null,
+
pub const Result = union(enum) {
pending: void,
err: anyerror,
@@ -609,6 +649,7 @@ pub const BundleV2 = struct {
defer {
this.config.deinit(bun.default_allocator);
+ bun.default_allocator.destroy(this);
}
this.ref.unref(globalThis.bunVM());
@@ -641,6 +682,7 @@ pub const BundleV2 = struct {
JSC.ZigString.static("path"),
JSC.ZigString.fromUTF8(output_file.input.text).toValueGC(globalThis),
);
+ defer bun.default_allocator.free(output_file.input.text);
obj.put(
globalThis,
@@ -669,29 +711,56 @@ pub const BundleV2 = struct {
};
pub fn generateInNewThreadWrap(
- completion: *JSBundleCompletionTask,
+ instance: *BundleThread,
) void {
Output.Source.configureNamedThread("Bundler");
- generateInNewThread(completion) catch |err| {
- completion.result = .{ .err = err };
- var concurrent_task = bun.default_allocator.create(JSC.ConcurrentTask) catch unreachable;
- concurrent_task.* = JSC.ConcurrentTask{
- .auto_delete = true,
- .task = completion.task.task(),
- .next = null,
- };
- completion.jsc_event_loop.enqueueTaskConcurrent(concurrent_task);
- };
+ var any = false;
+
+ while (true) {
+ while (instance.queue.pop()) |completion| {
+ generateInNewThread(completion) catch |err| {
+ completion.result = .{ .err = err };
+ var concurrent_task = bun.default_allocator.create(JSC.ConcurrentTask) catch unreachable;
+ concurrent_task.* = JSC.ConcurrentTask{
+ .auto_delete = true,
+ .task = completion.task.task(),
+ .next = null,
+ };
+ completion.jsc_event_loop.enqueueTaskConcurrent(concurrent_task);
+ };
+ any = true;
+ }
+
+ if (any) {
+ bun.Mimalloc.mi_collect(false);
+ }
+ _ = instance.waker.wait() catch 0;
+ }
}
+ pub const BundleThread = struct {
+ waker: bun.AsyncIO.Waker,
+ queue: bun.UnboundedQueue(JSBundleCompletionTask, .next) = .{},
+ pub var created = false;
+ pub var instance: *BundleThread = undefined;
+ };
+
fn generateInNewThread(
completion: *JSBundleCompletionTask,
) !void {
- const allocator = bun.default_allocator;
+ var heap = try ThreadlocalArena.init();
+ defer heap.deinit();
+
+ const allocator = heap.allocator();
+ var ast_memory_allocator = try allocator.create(js_ast.ASTMemoryAllocator);
+ ast_memory_allocator.* = .{
+ .allocator = allocator,
+ };
+ ast_memory_allocator.reset();
+ ast_memory_allocator.push();
const config = &completion.config;
var bundler = try allocator.create(bun.Bundler);
- errdefer allocator.destroy(bundler);
bundler.* = try bun.Bundler.init(
allocator,
@@ -723,14 +792,16 @@ pub const BundleV2 = struct {
bundler.resolver.opts = bundler.options;
- var event_loop = try allocator.create(JSC.AnyEventLoop);
- defer allocator.destroy(event_loop);
+ var this = try BundleV2.init(bundler, allocator, JSC.AnyEventLoop.init(allocator), false, JSC.WorkPool.get(), heap);
- // Ensure uWS::Loop is initialized
- _ = bun.uws.Loop.get().?;
+ defer {
+ if (this.graph.pool.pool.threadpool_context == @ptrCast(?*anyopaque, this.graph.pool)) {
+ this.graph.pool.pool.threadpool_context = null;
+ }
- var this = try BundleV2.init(bundler, allocator, JSC.AnyEventLoop.init(allocator), false, JSC.WorkPool.get());
- defer this.deinit();
+ ast_memory_allocator.pop();
+ this.deinit();
+ }
completion.result = .{
.value = .{
@@ -748,10 +819,18 @@ pub const BundleV2 = struct {
}
pub fn deinit(this: *BundleV2) void {
- for (this.graph.pool.workers[0..this.graph.pool.workers_used.loadUnchecked()]) |*worker| {
- worker.deinit();
+ if (this.graph.pool.workers_assignments.count() > 0) {
+ {
+ this.graph.pool.workers_assignments_lock.lock();
+ defer this.graph.pool.workers_assignments_lock.unlock();
+ for (this.graph.pool.workers_assignments.values()) |worker| {
+ worker.deinitSoon();
+ }
+ this.graph.pool.workers_assignments.deinit();
+ }
+
+ this.graph.pool.pool.wakeForIdleEvents();
}
- this.graph.heap.deinit();
}
pub fn runFromJSInNewThread(this: *BundleV2, config: *const bun.JSC.API.JSBundler.Config) !std.ArrayList(options.OutputFile) {
@@ -759,17 +838,32 @@ pub const BundleV2 = struct {
return error.BuildFailed;
}
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.graph.heap.gc(true);
+ bun.Mimalloc.mi_collect(true);
+ }
+
this.graph.pool.pool.schedule(try this.enqueueEntryPoints(config.entry_points.keys()));
// We must wait for all the parse tasks to complete, even if there are errors.
this.waitForParse();
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.graph.heap.gc(true);
+ bun.Mimalloc.mi_collect(true);
+ }
+
if (this.bundler.log.errors > 0) {
return error.BuildFailed;
}
try this.cloneAST();
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.graph.heap.gc(true);
+ bun.Mimalloc.mi_collect(true);
+ }
+
var chunks = try this.linker.link(
this,
this.graph.entry_points.items,
@@ -786,6 +880,8 @@ pub const BundleV2 = struct {
}
pub fn onParseTaskComplete(parse_result: *ParseTask.Result, this: *BundleV2) void {
+ defer bun.default_allocator.destroy(parse_result);
+
var graph = &this.graph;
var batch = ThreadPoolLib.Batch{};
var diff: isize = -1;
@@ -850,6 +946,7 @@ pub const BundleV2 = struct {
}
var iter = result.resolve_queue.iterator();
+ defer result.resolve_queue.deinit();
while (iter.next()) |entry| {
const hash = entry.key_ptr.*;
@@ -868,21 +965,25 @@ pub const BundleV2 = struct {
}
if (!existing.found_existing) {
+ var new_task = value;
var new_input_file = Graph.InputFile{
- .source = Logger.Source.initEmptyFile(entry.value_ptr.path.text),
+ .source = Logger.Source.initEmptyFile(new_task.path.text),
.side_effects = value.side_effects,
};
new_input_file.source.index = Index.source(graph.input_files.len);
- new_input_file.source.path = entry.value_ptr.path;
+ new_input_file.source.path = new_task.path;
new_input_file.source.key_path = new_input_file.source.path;
// graph.source_index_map.put(graph.allocator, new_input_file.source.index.get, new_input_file.source) catch unreachable;
existing.value_ptr.* = new_input_file.source.index.get();
- entry.value_ptr.source_index = new_input_file.source.index;
+ new_task.source_index = new_input_file.source.index;
+ new_task.ctx = this;
graph.input_files.append(graph.allocator, new_input_file) catch unreachable;
graph.ast.append(graph.allocator, js_ast.Ast.empty) catch unreachable;
- batch.push(ThreadPoolLib.Batch.from(&entry.value_ptr.task));
+ batch.push(ThreadPoolLib.Batch.from(&new_task.task));
diff += 1;
+ } else {
+ bun.default_allocator.destroy(value);
}
}
@@ -958,13 +1059,15 @@ const ParseTask = struct {
tree_shaking: bool = false,
known_platform: ?options.Platform = null,
module_type: options.ModuleType = .unknown,
+ ctx: *BundleV2,
const debug = Output.scoped(.ParseTask, false);
- pub const ResolveQueue = std.AutoArrayHashMap(u64, ParseTask);
+ pub const ResolveQueue = std.AutoArrayHashMap(u64, *ParseTask);
- pub fn init(resolve_result: *const _resolver.Result, source_index: ?Index) ParseTask {
+ pub fn init(resolve_result: *const _resolver.Result, source_index: ?Index, ctx: *BundleV2) ParseTask {
return .{
+ .ctx = ctx,
.path = resolve_result.path_pair.primary,
.contents_or_fd = .{
.fd = .{
@@ -980,6 +1083,7 @@ const ParseTask = struct {
}
pub const runtime = ParseTask{
+ .ctx = undefined,
.path = Fs.Path.initWithNamespace("runtime", "bun:runtime"),
.side_effects = _resolver.SideEffects.no_side_effects__pure_data,
.jsx = options.JSX.Pragma{
@@ -1108,7 +1212,7 @@ const ParseTask = struct {
step: *ParseTask.Result.Error.Step,
log: *Logger.Log,
) anyerror!?Result.Success {
- var allocator = this.allocator;
+ const allocator = this.allocator;
var data = this.data;
var bundler = &data.bundler;
@@ -1138,7 +1242,8 @@ const ParseTask = struct {
override_file_path_buf[override_path.len] = 0;
var override_pathZ = override_file_path_buf[0..override_path.len :0];
debug("{s} -> {s}", .{ file_path.text, override_path });
- break :brk try resolver.caches.fs.readFile(
+ break :brk try resolver.caches.fs.readFileWithAllocator(
+ allocator,
bundler.fs,
override_pathZ,
0,
@@ -1154,7 +1259,8 @@ const ParseTask = struct {
.contents = NodeFallbackModules.contentsFromPath(file_path.text) orelse "",
};
- break :brk resolver.caches.fs.readFile(
+ break :brk resolver.caches.fs.readFileWithAllocator(
+ allocator,
bundler.fs,
file_path.text,
task.contents_or_fd.fd.dir,
@@ -1205,7 +1311,7 @@ const ParseTask = struct {
const is_empty = entry.contents.len == 0 or (entry.contents.len < 33 and strings.trim(entry.contents, " \n\r").len == 0);
- const use_directive = if (!is_empty and this.ctx.bundler.options.react_server_components)
+ const use_directive = if (!is_empty and bundler.options.react_server_components)
UseDirective.parse(entry.contents)
else
.none;
@@ -1348,7 +1454,7 @@ const ParseTask = struct {
var resolve_entry = try resolve_queue.getOrPut(wyhash(0, path.text));
if (resolve_entry.found_existing) {
- import_record.path = resolve_entry.value_ptr.path;
+ import_record.path = resolve_entry.value_ptr.*.path;
continue;
}
@@ -1375,21 +1481,25 @@ const ParseTask = struct {
import_record.path = path.*;
debug("created ParseTask: {s}", .{path.text});
- resolve_entry.value_ptr.* = ParseTask.init(&resolve_result, null);
- resolve_entry.value_ptr.secondary_path_for_commonjs_interop = secondary_path_to_copy;
+ var resolve_task = bun.default_allocator.create(ParseTask) catch @panic("Ran out of memory");
+ resolve_task.* = ParseTask.init(&resolve_result, null, this.ctx);
+
+ resolve_task.secondary_path_for_commonjs_interop = secondary_path_to_copy;
if (use_directive != .none) {
- resolve_entry.value_ptr.known_platform = platform;
+ resolve_task.known_platform = platform;
} else if (task.known_platform) |known_platform| {
- resolve_entry.value_ptr.known_platform = known_platform;
+ resolve_task.known_platform = known_platform;
}
- resolve_entry.value_ptr.jsx.development = task.jsx.development;
+ resolve_task.jsx.development = task.jsx.development;
- if (resolve_entry.value_ptr.loader == null) {
- resolve_entry.value_ptr.loader = path.loader(&bundler.options.loaders);
- resolve_entry.value_ptr.tree_shaking = task.tree_shaking;
+ if (resolve_task.loader == null) {
+ resolve_task.loader = path.loader(&bundler.options.loaders);
+ resolve_task.tree_shaking = task.tree_shaking;
}
+
+ resolve_entry.value_ptr.* = resolve_task;
} else |err| {
// Disable failing packages from being printed.
// This may cause broken code to write.
@@ -1453,10 +1563,6 @@ const ParseTask = struct {
return err;
}
- // Allow the AST to outlive this call
- _ = js_ast.Expr.Data.Store.toOwnedSlice();
- _ = js_ast.Stmt.Data.Store.toOwnedSlice();
-
// never a react client component if RSC is not enabled.
std.debug.assert(use_directive == .none or bundler.options.react_server_components);
@@ -1481,21 +1587,12 @@ const ParseTask = struct {
}
fn run(this: *ParseTask) void {
- var worker = @ptrCast(
- *ThreadPool.Worker,
- @alignCast(
- @alignOf(*ThreadPool.Worker),
- ThreadPoolLib.Thread.current.?.ctx.?,
- ),
- );
+ var worker = ThreadPool.Worker.get(this.ctx);
+ defer worker.unget();
var step: ParseTask.Result.Error.Step = .pending;
var log = Logger.Log.init(worker.allocator);
std.debug.assert(this.source_index.isValid()); // forgot to set source_index
- defer {
- if (comptime FeatureFlags.help_catch_memory_issues) {
- worker.heap.gc(false);
- }
- }
+
var result = bun.default_allocator.create(Result) catch unreachable;
result.* = .{
.value = brk: {
@@ -1992,12 +2089,23 @@ const LinkerGraph = struct {
var parts_list = g.ast.items(.parts)[source_index].slice();
var part: *js_ast.Part = &parts_list[part_index];
var uses = part.symbol_uses;
+ var needs_reindex = false;
+ if (uses.capacity() < uses.count() + 1 and !uses.contains(ref)) {
+ var symbol_uses = js_ast.Part.SymbolUseMap{};
+ try symbol_uses.ensureTotalCapacity(g.allocator, uses.count() + 1);
+ symbol_uses.entries.len = uses.keys().len;
+ bun.copy(std.meta.Child(@TypeOf(symbol_uses.keys())), symbol_uses.keys(), uses.keys());
+ bun.copy(std.meta.Child(@TypeOf(symbol_uses.values())), symbol_uses.values(), uses.values());
+ needs_reindex = true;
+ uses = symbol_uses;
+ }
var entry = uses.getOrPut(g.allocator, ref) catch unreachable;
if (entry.found_existing) {
entry.value_ptr.count_estimate += use_count;
} else {
entry.value_ptr.* = .{ .count_estimate = use_count };
}
+ if (needs_reindex) uses.reIndex(g.allocator) catch unreachable;
part.symbol_uses = uses;
const exports_ref = g.ast.items(.exports_ref)[source_index];
@@ -2420,6 +2528,10 @@ const LinkerContext = struct {
reachable,
);
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.checkForMemoryCorruption();
+ }
+
try this.scanImportsAndExports();
// Stop now if there were errors
@@ -2427,17 +2539,39 @@ const LinkerContext = struct {
return &[_]Chunk{};
}
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.checkForMemoryCorruption();
+ }
+
try this.treeShakingAndCodeSplitting();
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.checkForMemoryCorruption();
+ }
+
const chunks = try this.computeChunks(unique_key);
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.checkForMemoryCorruption();
+ }
+
try this.computeCrossChunkDependencies(chunks);
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.checkForMemoryCorruption();
+ }
+
this.graph.symbols.followAll();
return chunks;
}
+ fn checkForMemoryCorruption(this: *LinkerContext) void {
+ // For this to work, you need mimalloc's debug build enabled.
+ // make mimalloc-debug
+ this.parse_graph.heap.gc(true);
+ }
+
pub noinline fn computeChunks(
this: *LinkerContext,
unique_key: u64,
@@ -3196,6 +3330,10 @@ const LinkerContext = struct {
}
}
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.checkForMemoryCorruption();
+ }
+
// Step 4: Match imports with exports. This must be done after we process all
// export stars because imports can bind to export star re-exports.
{
@@ -3271,11 +3409,17 @@ const LinkerContext = struct {
// Note: `do` will wait for all to finish before moving forward
try this.parse_graph.pool.pool.do(this.allocator, &this.wait_group, this, doStep5, this.graph.reachable_files);
}
+
+ if (comptime FeatureFlags.help_catch_memory_issues) {
+ this.checkForMemoryCorruption();
+ }
+
// Step 6: Bind imports to exports. This adds non-local dependencies on the
// parts that declare the export to all parts that use the import. Also
// generate wrapper parts for wrapped files.
{
const bufPrint = std.fmt.bufPrint;
+ _ = bufPrint;
var parts_list: []js_ast.Part.List = this.graph.ast.items(.parts);
var wrapper_refs = this.graph.ast.items(.wrapper_ref);
// const needs_export_symbol_from_runtime: []const bool = this.graph.meta.items(.needs_export_symbol_from_runtime);
@@ -3301,15 +3445,8 @@ const LinkerContext = struct {
const source: *const Logger.Source = &this.parse_graph.input_files.items(.source)[source_index];
const exports_ref = exports_refs[id];
- var exports_symbol: ?*js_ast.Symbol = if (exports_ref.isValid())
- this.graph.symbols.get(exports_ref)
- else
- null;
+
const module_ref = module_refs[id];
- var module_symbol: ?*js_ast.Symbol = if (module_ref.isValid())
- this.graph.symbols.get(module_ref)
- else
- null;
// TODO: see if counting and batching into a single large allocation instead of per-file improves perf
const string_buffer_len: usize = brk: {
@@ -3338,9 +3475,13 @@ const LinkerContext = struct {
};
var string_buffer = this.allocator.alloc(u8, string_buffer_len) catch unreachable;
- var buf = string_buffer;
+ var builder = bun.StringBuilder{
+ .len = 0,
+ .cap = string_buffer.len,
+ .ptr = string_buffer.ptr,
+ };
- defer std.debug.assert(buf.len == 0); // ensure we used all of it
+ defer std.debug.assert(builder.len == builder.cap); // ensure we used all of it
// Pre-generate symbols for re-exports CommonJS symbols in case they
// are necessary later. This is done now because the symbols map cannot be
@@ -3349,8 +3490,7 @@ const LinkerContext = struct {
var copies = this.allocator.alloc(Ref, aliases.len) catch unreachable;
for (aliases, copies) |alias, *copy| {
- const original_name = bufPrint(buf, "export_{}", .{strings.fmtIdentifier(alias)}) catch unreachable;
- buf = buf[original_name.len..];
+ const original_name = builder.fmt("export_{}", .{strings.fmtIdentifier(alias)});
copy.* = this.graph.generateNewSymbol(source_index, .other, original_name);
}
this.graph.meta.items(.cjs_export_copies)[id] = copies;
@@ -3358,15 +3498,13 @@ const LinkerContext = struct {
// Use "init_*" for ESM wrappers instead of "require_*"
if (wrap == .esm) {
- const original_name = bufPrint(
- buf,
+ const original_name = builder.fmt(
"init_{}",
.{
source.fmtIdentifier(),
},
- ) catch unreachable;
+ );
- buf = buf[original_name.len..];
this.graph.symbols.get(wrapper_refs[id]).?.original_name = original_name;
}
@@ -3376,10 +3514,20 @@ const LinkerContext = struct {
// aesthetics and is not about correctness. This is done here because by
// this point, we know the CommonJS status will not change further.
if (wrap != .cjs and export_kind != .cjs) {
- const exports_name = bufPrint(buf, "exports_{any}", .{source.fmtIdentifier()}) catch unreachable;
- buf = buf[exports_name.len..];
- const module_name = bufPrint(buf, "module_{any}", .{source.fmtIdentifier()}) catch unreachable;
- buf = buf[module_name.len..];
+ const exports_name = builder.fmt("exports_{}", .{source.fmtIdentifier()});
+ const module_name = builder.fmt("module_{}", .{source.fmtIdentifier()});
+
+ // Note: it's possible for the symbols table to be resized
+ // so we cannot call .get() above this scope.
+ var exports_symbol: ?*js_ast.Symbol = if (exports_ref.isValid())
+ this.graph.symbols.get(exports_ref)
+ else
+ null;
+ var module_symbol: ?*js_ast.Symbol = if (module_ref.isValid())
+ this.graph.symbols.get(module_ref)
+ else
+ null;
+
if (exports_symbol != null)
exports_symbol.?.original_name = exports_name;
if (module_symbol != null)
@@ -3419,10 +3567,13 @@ const LinkerContext = struct {
var part: *js_ast.Part = &parts[part_index];
const parts_declaring_symbol: []u32 = this.graph.topLevelSymbolToParts(import_id, import.data.import_ref);
- part.dependencies.ensureUnusedCapacity(
- this.allocator,
- parts_declaring_symbol.len + @as(usize, import.re_exports.len),
- ) catch unreachable;
+ const total_len = parts_declaring_symbol.len + @as(usize, import.re_exports.len) + @as(usize, part.dependencies.len);
+ if (part.dependencies.cap < total_len) {
+ var list = std.ArrayList(Dependency).init(this.allocator);
+ list.ensureUnusedCapacity(total_len) catch unreachable;
+ list.appendSliceAssumeCapacity(part.dependencies.slice());
+ part.dependencies.update(list);
+ }
// Depend on the file containing the imported symbol
for (parts_declaring_symbol) |resolved_part_index| {
@@ -3975,23 +4126,11 @@ const LinkerContext = struct {
const id = source_index;
if (id > c.graph.meta.len) return;
- var worker: *ThreadPool.Worker = @ptrCast(
- *ThreadPool.Worker,
- @alignCast(
- @alignOf(*ThreadPool.Worker),
- ThreadPoolLib.Thread.current.?.ctx.?,
- ),
- );
+ var worker: *ThreadPool.Worker = ThreadPool.Worker.get(@fieldParentPtr(BundleV2, "linker", c));
+ defer worker.unget();
+
// we must use this allocator here
const allocator_ = worker.allocator;
- if (comptime FeatureFlags.help_catch_memory_issues) {
- worker.heap.gc(false);
- }
- defer {
- if (comptime FeatureFlags.help_catch_memory_issues) {
- worker.heap.gc(false);
- }
- }
var resolved_exports: *ResolvedExports = &c.graph.meta.items(.resolved_exports)[id];
@@ -4924,11 +5063,8 @@ const LinkerContext = struct {
fn generateChunkJS_(ctx: GenerateChunkCtx, chunk: *Chunk, chunk_index: usize) !void {
_ = chunk_index;
defer ctx.wg.finish();
- var worker = ThreadPool.Worker.get();
-
- if (comptime FeatureFlags.help_catch_memory_issues) {
- worker.heap.gc(false);
- }
+ var worker = ThreadPool.Worker.get(@fieldParentPtr(BundleV2, "linker", ctx.c));
+ defer worker.unget();
const allocator = worker.allocator;
const c = ctx.c;
@@ -4942,9 +5078,6 @@ const LinkerContext = struct {
const toESMRef = c.graph.symbols.follow(runtime_members.get("__toESM").?.ref);
const runtimeRequireRef = c.graph.symbols.follow(runtime_members.get("__require").?.ref);
- js_ast.Expr.Data.Store.create(bun.default_allocator);
- js_ast.Stmt.Data.Store.create(bun.default_allocator);
-
var r = try c.renameSymbolsInChunk(allocator, chunk, repr.files_in_chunk_order);
defer r.deinit();
const part_ranges = repr.parts_in_chunk_in_order;
@@ -4998,7 +5131,7 @@ const LinkerContext = struct {
// the final chunk owns the memory buffer
compile_results.appendAssumeCapacity(.{
.javascript = .{
- .result = result,
+ .result = result.clone(allocator) catch unreachable,
.source_index = part_range.source_index.get(),
},
});
@@ -5116,7 +5249,7 @@ const LinkerContext = struct {
if (cross_chunk_prefix.len > 0) {
newline_before_comment = true;
line_offset.advance(cross_chunk_prefix);
- j.push(cross_chunk_prefix);
+ j.append(cross_chunk_prefix, 0, bun.default_allocator);
}
// Concatenate the generated JavaScript chunks together
@@ -5183,10 +5316,10 @@ const LinkerContext = struct {
if (is_runtime) {
line_offset.advance(compile_result.code());
- j.append(compile_result.code(), 0, allocator);
+ j.append(compile_result.code(), 0, bun.default_allocator);
} else {
line_offset.advance(compile_result.code());
- j.append(compile_result.code(), 0, allocator);
+ j.append(compile_result.code(), 0, bun.default_allocator);
// TODO: sourcemap
}
@@ -5200,7 +5333,7 @@ const LinkerContext = struct {
// Stick the entry point tail at the end of the file. Deliberately don't
// include any source mapping information for this because it's automatically
// generated and doesn't correspond to a location in the input file.
- j.push(tail_code);
+ j.append(tail_code, 0, bun.default_allocator);
}
// Put the cross-chunk suffix inside the IIFE
@@ -5210,7 +5343,7 @@ const LinkerContext = struct {
line_offset.advance("\n");
}
- j.push(cross_chunk_suffix);
+ j.append(cross_chunk_suffix, 0, bun.default_allocator);
}
if (c.options.output_format == .iife) {
@@ -5232,6 +5365,7 @@ const LinkerContext = struct {
chunk.intermediate_output = c.breakOutputIntoPieces(
allocator,
&j,
+
cross_chunk_prefix.len > 0 or
cross_chunk_suffix.len > 0,
@truncate(u32, ctx.chunks.len),
@@ -6444,9 +6578,6 @@ const LinkerContext = struct {
// referencing everything by array makes the code a lot more annoying :(
const ast: js_ast.Ast = c.graph.ast.get(part_range.source_index.get());
- js_ast.Expr.Data.Store.reset();
- js_ast.Stmt.Data.Store.reset();
-
var needs_wrapper = false;
const namespace_export_part_index = js_ast.namespace_export_part_index;
@@ -6628,8 +6759,6 @@ const LinkerContext = struct {
mergeAdjacentLocalStmts(&stmts.all_stmts, temp_allocator);
}
- // TODO: mergeAdjacentLocalStmts
-
var out_stmts: []js_ast.Stmt = stmts.all_stmts.items;
// Optionally wrap all statements in a closure
@@ -7104,7 +7233,7 @@ const LinkerContext = struct {
.export_names = export_names.items,
.contents = bytes.items,
};
- var byte_buffer = std.ArrayList(u8).initCapacity(c.allocator, bytes.items.len) catch unreachable;
+ var byte_buffer = std.ArrayList(u8).initCapacity(bun.default_allocator, bytes.items.len) catch unreachable;
var byte_buffer_writer = byte_buffer.writer();
const SchemaWriter = schema.Writer(@TypeOf(&byte_buffer_writer));
var writer = SchemaWriter.init(&byte_buffer_writer);
@@ -7113,15 +7242,18 @@ const LinkerContext = struct {
} else &.{};
// Generate the final output files by joining file pieces together
- var output_files = std.ArrayList(options.OutputFile).initCapacity(c.allocator, chunks.len + @as(
+ var output_files = std.ArrayList(options.OutputFile).initCapacity(bun.default_allocator, chunks.len + @as(
usize,
@boolToInt(react_client_components_manifest.len > 0),
)) catch unreachable;
output_files.items.len = chunks.len;
for (chunks, output_files.items) |*chunk, *output_file| {
+ const buffer = chunk.intermediate_output.code(chunk, chunks) catch @panic("Failed to allocate memory for output file");
output_file.* = options.OutputFile.initBuf(
- chunk.intermediate_output.code(c.allocator, chunk, chunks) catch @panic("Failed to allocate memory for output file"),
- chunk.final_rel_path,
+ buffer,
+ Chunk.IntermediateOutput.allocatorForSize(buffer.len),
+ // clone for main thread
+ bun.default_allocator.dupe(u8, chunk.final_rel_path) catch unreachable,
// TODO: remove this field
.js,
);
@@ -7130,6 +7262,7 @@ const LinkerContext = struct {
if (react_client_components_manifest.len > 0) {
output_files.appendAssumeCapacity(options.OutputFile.initBuf(
react_client_components_manifest,
+ bun.default_allocator,
"./components-manifest.blob",
.file,
));
@@ -8487,7 +8620,14 @@ pub const Chunk = struct {
empty: void,
- pub fn code(this: IntermediateOutput, allocator: std.mem.Allocator, chunk: *Chunk, chunks: []Chunk) ![]const u8 {
+ pub fn allocatorForSize(size: usize) std.mem.Allocator {
+ if (size >= 512 * 1024)
+ return std.heap.page_allocator
+ else
+ return bun.default_allocator;
+ }
+
+ pub fn code(this: IntermediateOutput, chunk: *Chunk, chunks: []Chunk) ![]const u8 {
switch (this) {
.pieces => |*pieces| {
var count: usize = 0;
@@ -8505,7 +8645,7 @@ pub const Chunk = struct {
}
}
- var total_buf = try allocator.alloc(u8, count);
+ var total_buf = try allocatorForSize(count).alloc(u8, count);
var remain = total_buf;
for (pieces.slice()) |piece| {
@@ -8539,7 +8679,7 @@ pub const Chunk = struct {
.joiner => |joiner_| {
// TODO: make this safe
var joiny = joiner_;
- return joiny.done(allocator);
+ return joiny.done(allocatorForSize(joiny.len));
},
.empty => return "",
}
diff --git a/src/cache.zig b/src/cache.zig
index d77c5c6b2..88cf19025 100644
--- a/src/cache.zig
+++ b/src/cache.zig
@@ -143,6 +143,18 @@ pub const Fs = struct {
comptime use_shared_buffer: bool,
_file_handle: ?StoredFileDescriptorType,
) !Entry {
+ return c.readFileWithAllocator(bun.fs_allocator, _fs, path, dirname_fd, use_shared_buffer, _file_handle);
+ }
+
+ pub fn readFileWithAllocator(
+ c: *Fs,
+ allocator: std.mem.Allocator,
+ _fs: *fs.FileSystem,
+ path: string,
+ dirname_fd: StoredFileDescriptorType,
+ comptime use_shared_buffer: bool,
+ _file_handle: ?StoredFileDescriptorType,
+ ) !Entry {
var rfs = _fs.fs;
var file_handle: std.fs.File = if (_file_handle) |__file| std.fs.File{ .handle = __file } else undefined;
@@ -167,21 +179,22 @@ pub const Fs = struct {
}
}
+ const will_close = rfs.needToCloseFiles() and _file_handle == null;
defer {
- if (rfs.needToCloseFiles() and _file_handle == null) {
+ if (will_close) {
file_handle.close();
}
}
const file = if (c.stream)
- rfs.readFileWithHandle(path, null, file_handle, use_shared_buffer, c.sharedBuffer(), true) catch |err| {
+ rfs.readFileWithHandleAndAllocator(allocator, path, null, file_handle, use_shared_buffer, c.sharedBuffer(), true) catch |err| {
if (Environment.isDebug) {
Output.printError("{s}: readFile error -- {s}", .{ path, @errorName(err) });
}
return err;
}
else
- rfs.readFileWithHandle(path, null, file_handle, use_shared_buffer, c.sharedBuffer(), false) catch |err| {
+ rfs.readFileWithHandleAndAllocator(allocator, path, null, file_handle, use_shared_buffer, c.sharedBuffer(), false) catch |err| {
if (Environment.isDebug) {
Output.printError("{s}: readFile error -- {s}", .{ path, @errorName(err) });
}
@@ -190,7 +203,7 @@ pub const Fs = struct {
return Entry{
.contents = file.contents,
- .fd = if (FeatureFlags.store_file_descriptors) file_handle.handle else 0,
+ .fd = if (FeatureFlags.store_file_descriptors and !will_close) file_handle.handle else 0,
};
}
};
diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig
index 33ab05632..d6ca3236d 100644
--- a/src/cli/build_command.zig
+++ b/src/cli/build_command.zig
@@ -158,7 +158,7 @@ pub const BuildCommand = struct {
if (output_dir.len == 0 and ctx.bundler_options.outfile.len == 0) {
// if --transform is passed, it won't have an output dir
if (output_files[0].value == .buffer)
- try writer.writeAll(output_files[0].value.buffer);
+ try writer.writeAll(output_files[0].value.buffer.bytes);
break :dump;
}
@@ -202,7 +202,7 @@ pub const BuildCommand = struct {
}
}
}
- try root_dir.dir.writeFile(rel_path, value);
+ try root_dir.dir.writeFile(rel_path, value.bytes);
},
.move => |value| {
const primary = f.input.text[from_path.len..];
diff --git a/src/cli/create_command.zig b/src/cli/create_command.zig
index cdfe10156..60261a446 100644
--- a/src/cli/create_command.zig
+++ b/src/cli/create_command.zig
@@ -260,7 +260,7 @@ pub const CreateCommand = struct {
return try CreateListExamplesCommand.exec(ctx);
}
- var filesystem = try fs.FileSystem.init1(ctx.allocator, null);
+ var filesystem = try fs.FileSystem.init(null);
var env_loader: DotEnv.Loader = brk: {
var map = try ctx.allocator.create(DotEnv.Map);
map.* = DotEnv.Map.init(ctx.allocator);
@@ -2098,7 +2098,7 @@ pub const Example = struct {
pub const CreateListExamplesCommand = struct {
pub fn exec(ctx: Command.Context) !void {
- var filesystem = try fs.FileSystem.init1(ctx.allocator, null);
+ var filesystem = try fs.FileSystem.init(null);
var env_loader: DotEnv.Loader = brk: {
var map = try ctx.allocator.create(DotEnv.Map);
map.* = DotEnv.Map.init(ctx.allocator);
diff --git a/src/cli/init_command.zig b/src/cli/init_command.zig
index 1b95d47fa..bc8867368 100644
--- a/src/cli/init_command.zig
+++ b/src/cli/init_command.zig
@@ -99,7 +99,7 @@ pub const InitCommand = struct {
};
pub fn exec(alloc: std.mem.Allocator, argv: [][*:0]u8) !void {
- var fs = try Fs.FileSystem.init1(alloc, null);
+ var fs = try Fs.FileSystem.init(null);
const pathname = Fs.PathName.init(fs.topLevelDirWithoutTrailingSlash());
const destination_dir = std.fs.cwd();
diff --git a/src/cli/upgrade_command.zig b/src/cli/upgrade_command.zig
index b23914c28..665508833 100644
--- a/src/cli/upgrade_command.zig
+++ b/src/cli/upgrade_command.zig
@@ -382,7 +382,7 @@ pub const UpgradeCommand = struct {
fn _exec(ctx: Command.Context) !void {
try HTTP.HTTPThread.init();
- var filesystem = try fs.FileSystem.init1(ctx.allocator, null);
+ var filesystem = try fs.FileSystem.init(null);
var env_loader: DotEnv.Loader = brk: {
var map = try ctx.allocator.create(DotEnv.Map);
map.* = DotEnv.Map.init(ctx.allocator);
diff --git a/src/fs.zig b/src/fs.zig
index 2a0c45283..e7c0df19e 100644
--- a/src/fs.zig
+++ b/src/fs.zig
@@ -78,7 +78,6 @@ pub const BytecodeCacheFetcher = struct {
};
pub const FileSystem = struct {
- allocator: std.mem.Allocator,
top_level_dir: string = "/",
// used on subsequent updates
@@ -144,18 +143,17 @@ pub const FileSystem = struct {
ENOTDIR,
};
- pub fn init1(
- allocator: std.mem.Allocator,
+ pub fn init(
top_level_dir: ?string,
) !*FileSystem {
- return init1WithForce(allocator, top_level_dir, false);
+ return initWithForce(top_level_dir, false);
}
- pub fn init1WithForce(
- allocator: std.mem.Allocator,
+ pub fn initWithForce(
top_level_dir: ?string,
comptime force: bool,
) !*FileSystem {
+ const allocator = bun.fs_allocator;
var _top_level_dir = top_level_dir orelse (if (Environment.isBrowser) "/project/" else try std.process.getCwdAlloc(allocator));
// Ensure there's a trailing separator in the top level directory
@@ -172,10 +170,8 @@ pub const FileSystem = struct {
if (!instance_loaded or force) {
instance = FileSystem{
- .allocator = allocator,
.top_level_dir = _top_level_dir,
.fs = Implementation.init(
- allocator,
_top_level_dir,
),
// must always use default_allocator since the other allocators may not be threadsafe when an element resizes
@@ -282,7 +278,7 @@ pub const FileSystem = struct {
const query = strings.copyLowercaseIfNeeded(_query, &scratch_lookup_buffer);
const result = entry.data.get(query) orelse return null;
const basename = result.base();
- if (!strings.eql(basename, _query)) {
+ if (!strings.eqlLong(basename, _query, true)) {
return Entry.Lookup{ .entry = result, .diff_case = Entry.Lookup.DifferentCase{
.dir = entry.dir,
.query = _query,
@@ -367,11 +363,11 @@ pub const FileSystem = struct {
abs_path: PathString = PathString.empty,
- pub inline fn base(this: *const Entry) string {
+ pub inline fn base(this: *Entry) string {
return this.base_.slice();
}
- pub inline fn base_lowercase(this: *const Entry) string {
+ pub inline fn base_lowercase(this: *Entry) string {
return this.base_lowercase_.slice();
}
@@ -538,7 +534,6 @@ pub const FileSystem = struct {
pub const RealFS = struct {
entries_mutex: Mutex = Mutex.init(),
entries: *EntriesOption.Map,
- allocator: std.mem.Allocator,
cwd: string,
parent_fs: *FileSystem = undefined,
file_limit: usize = 32,
@@ -688,19 +683,17 @@ pub const FileSystem = struct {
var _entries_option_map: *EntriesOption.Map = undefined;
var _entries_option_map_loaded: bool = false;
pub fn init(
- allocator: std.mem.Allocator,
cwd: string,
) RealFS {
const file_limit = adjustUlimit() catch unreachable;
if (!_entries_option_map_loaded) {
- _entries_option_map = EntriesOption.Map.init(allocator);
+ _entries_option_map = EntriesOption.Map.init(bun.fs_allocator);
_entries_option_map_loaded = true;
}
return RealFS{
.entries = _entries_option_map,
- .allocator = allocator,
.cwd = cwd,
.file_limit = file_limit,
.file_quota = file_limit,
@@ -796,7 +789,7 @@ pub const FileSystem = struct {
}
pub const EntriesOption = union(Tag) {
- entries: DirEntry,
+ entries: *DirEntry,
err: DirEntry.Err,
pub const Tag = enum {
@@ -824,9 +817,10 @@ pub const FileSystem = struct {
comptime Iterator: type,
iterator: Iterator,
) !DirEntry {
+ _ = fs;
var iter = (std.fs.IterableDir{ .dir = handle }).iterate();
var dir = DirEntry.init(_dir);
- const allocator = fs.allocator;
+ const allocator = bun.fs_allocator;
errdefer dir.deinit(allocator);
if (FeatureFlags.store_file_descriptors) {
@@ -909,8 +903,10 @@ pub const FileSystem = struct {
};
if (comptime FeatureFlags.enable_entry_cache) {
+ var entries_ptr = bun.fs_allocator.create(DirEntry) catch unreachable;
+ entries_ptr.* = entries;
const result = EntriesOption{
- .entries = entries,
+ .entries = entries_ptr,
};
var out = try fs.entries.put(&cache_result.?, result);
@@ -934,6 +930,28 @@ pub const FileSystem = struct {
shared_buffer: *MutableString,
comptime stream: bool,
) !File {
+ return readFileWithHandleAndAllocator(
+ fs,
+ bun.fs_allocator,
+ path,
+ _size,
+ file,
+ use_shared_buffer,
+ shared_buffer,
+ stream,
+ );
+ }
+
+ pub fn readFileWithHandleAndAllocator(
+ fs: *RealFS,
+ allocator: std.mem.Allocator,
+ path: string,
+ _size: ?usize,
+ file: std.fs.File,
+ comptime use_shared_buffer: bool,
+ shared_buffer: *MutableString,
+ comptime stream: bool,
+ ) !File {
FileSystem.setMaxFd(file.handle);
// Skip the extra file.stat() call when possible
@@ -1004,7 +1022,7 @@ pub const FileSystem = struct {
}
} else {
// We use pread to ensure if the file handle was open, it doesn't seek from the last position
- var buf = try fs.allocator.alloc(u8, size);
+ var buf = try allocator.alloc(u8, size);
const read_count = file.preadAll(buf, 0) catch |err| {
fs.readFileError(path, err);
return err;
diff --git a/src/http.zig b/src/http.zig
index 08ecbda92..cd5e386a1 100644
--- a/src/http.zig
+++ b/src/http.zig
@@ -2654,19 +2654,19 @@ pub const RequestContext = struct {
}
if (FeatureFlags.strong_etags_for_built_files) {
- const did_send = ctx.writeETag(buffer) catch false;
+ const did_send = ctx.writeETag(buffer.bytes) catch false;
if (did_send) return;
}
- if (buffer.len == 0) {
+ if (buffer.bytes.len == 0) {
return try ctx.sendNoContent();
}
defer ctx.done();
try ctx.writeStatus(200);
- try ctx.prepareToSendBody(buffer.len, false);
+ try ctx.prepareToSendBody(buffer.bytes.len, false);
if (!send_body) return;
- _ = try ctx.writeSocket(buffer, SOCKET_FLAGS);
+ _ = try ctx.writeSocket(buffer.bytes, SOCKET_FLAGS);
},
}
}
diff --git a/src/install/install.zig b/src/install/install.zig
index d6ecd2f46..ddad15dd9 100644
--- a/src/install/install.zig
+++ b/src/install/install.zig
@@ -4976,7 +4976,7 @@ pub const PackageManager = struct {
try global_dir.dir.setAsCwd();
}
- var fs = try Fs.FileSystem.init1(ctx.allocator, null);
+ var fs = try Fs.FileSystem.init(null);
var original_cwd = std.mem.trimRight(u8, fs.top_level_dir, "/");
bun.copy(u8, &cwd_buf, original_cwd);
@@ -5041,7 +5041,7 @@ pub const PackageManager = struct {
};
env.loadProcess();
- try env.load(&fs.fs, &entries_option.entries, false);
+ try env.load(&fs.fs, entries_option.entries, false);
if (env.map.get("BUN_INSTALL_VERBOSE") != null) {
PackageManager.verbose_install = true;
@@ -5068,7 +5068,7 @@ pub const PackageManager = struct {
.network_task_fifo = NetworkQueue.init(),
.allocator = ctx.allocator,
.log = ctx.log,
- .root_dir = &entries_option.entries,
+ .root_dir = entries_option.entries,
.env = env,
.cpu_count = cpu_count,
.thread_pool = ThreadPool.init(.{
@@ -5143,7 +5143,7 @@ pub const PackageManager = struct {
.network_task_fifo = NetworkQueue.init(),
.allocator = allocator,
.log = log,
- .root_dir = &root_dir.entries,
+ .root_dir = root_dir.entries,
.env = env,
.cpu_count = cpu_count,
.thread_pool = ThreadPool.init(.{
diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig
index 86ad1ac54..ea1c4fcf8 100644
--- a/src/install/lockfile.zig
+++ b/src/install/lockfile.zig
@@ -854,7 +854,7 @@ pub const Printer = struct {
if (lockfile_path.len > 0 and lockfile_path[0] == std.fs.path.sep)
std.os.chdir(std.fs.path.dirname(lockfile_path) orelse "/") catch {};
- _ = try FileSystem.init1(allocator, null);
+ _ = try FileSystem.init(null);
var lockfile = try allocator.create(Lockfile);
@@ -918,7 +918,7 @@ pub const Printer = struct {
};
env_loader.loadProcess();
- try env_loader.load(&fs.fs, &entries_option.entries, false);
+ try env_loader.load(&fs.fs, entries_option.entries, false);
var log = logger.Log.init(allocator);
try options.load(
allocator,
diff --git a/src/js_ast.zig b/src/js_ast.zig
index dbd0fb010..bccf4e0d8 100644
--- a/src/js_ast.zig
+++ b/src/js_ast.zig
@@ -2130,6 +2130,20 @@ pub const E = struct {
};
}
+ pub fn cloneSliceIfNecessary(str: *const String, allocator: std.mem.Allocator) !bun.string {
+ if (Expr.Data.Store.memory_allocator) |mem| {
+ if (mem == GlobalStoreHandle.global_store_ast) {
+ return str.string(allocator);
+ }
+ }
+
+ if (str.isUTF8()) {
+ return allocator.dupe(u8, str.string(allocator) catch unreachable);
+ }
+
+ return str.string(allocator);
+ }
+
pub fn javascriptLength(s: *const String) u32 {
if (s.rope_len > 0) {
// We only support ascii ropes for now
@@ -2226,6 +2240,14 @@ pub const E = struct {
}
}
+ pub fn stringCloned(s: *const String, allocator: std.mem.Allocator) !bun.string {
+ if (s.isUTF8()) {
+ return try allocator.dupe(u8, s.data);
+ } else {
+ return strings.toUTF8Alloc(allocator, s.slice16());
+ }
+ }
+
pub fn hash(s: *const String) u64 {
if (s.isBlank()) return 0;
@@ -9542,18 +9564,12 @@ pub const ASTMemoryAllocator = struct {
allocator: std.mem.Allocator,
previous: ?*ASTMemoryAllocator = null,
- pub fn push(this: *ASTMemoryAllocator) void {
- if (Stmt.Data.Store.memory_allocator == this) {
- return;
- }
+ pub fn reset(this: *ASTMemoryAllocator) void {
this.stack_allocator.fallback_allocator = this.allocator;
this.bump_allocator = this.stack_allocator.get();
- var prev = Stmt.Data.Store.memory_allocator;
- if (this.previous) |other| {
- other.previous = prev;
- } else {
- this.previous = prev;
- }
+ }
+
+ pub fn push(this: *ASTMemoryAllocator) void {
Stmt.Data.Store.memory_allocator = this;
Expr.Data.Store.memory_allocator = this;
}
@@ -9646,6 +9662,32 @@ pub const UseDirective = enum {
}
};
+pub const GlobalStoreHandle = struct {
+ prev_memory_allocator: ?*ASTMemoryAllocator = null,
+
+ var global_store_ast: ?*ASTMemoryAllocator = null;
+ var global_store_threadsafe: std.heap.ThreadSafeAllocator = undefined;
+
+ pub fn get() ?*ASTMemoryAllocator {
+ if (global_store_ast == null) {
+ var global = bun.default_allocator.create(ASTMemoryAllocator) catch unreachable;
+ global.allocator = bun.default_allocator;
+ global.bump_allocator = bun.default_allocator;
+ global_store_ast = global;
+ }
+
+ var prev = Stmt.Data.Store.memory_allocator;
+ Stmt.Data.Store.memory_allocator = global_store_ast;
+ Expr.Data.Store.memory_allocator = global_store_ast;
+ return prev;
+ }
+
+ pub fn unget(handle: ?*ASTMemoryAllocator) void {
+ Stmt.Data.Store.memory_allocator = handle;
+ Expr.Data.Store.memory_allocator = handle;
+ }
+};
+
// test "Binding.init" {
// var binding = Binding.alloc(
// std.heap.page_allocator,
@@ -9834,3 +9876,4 @@ pub const UseDirective = enum {
// Stmt | 192
// STry | 384
// -- ESBuild bit sizes
+
diff --git a/src/js_parser.zig b/src/js_parser.zig
index cf78826ea..1a0e582f1 100644
--- a/src/js_parser.zig
+++ b/src/js_parser.zig
@@ -2904,6 +2904,10 @@ pub const Parser = struct {
var before = ListManaged(js_ast.Part).init(p.allocator);
var after = ListManaged(js_ast.Part).init(p.allocator);
var parts = ListManaged(js_ast.Part).init(p.allocator);
+ defer {
+ after.deinit();
+ before.deinit();
+ }
if (p.options.bundle) {
// allocate an empty part for the bundle
@@ -4156,8 +4160,6 @@ pub const Parser = struct {
parts_slice = _parts;
} else {
- after.deinit();
- before.deinit();
parts_slice = parts.items;
}
diff --git a/src/json_parser.zig b/src/json_parser.zig
index fa9af406f..913a16bd6 100644
--- a/src/json_parser.zig
+++ b/src/json_parser.zig
@@ -141,8 +141,13 @@ fn JSONLikeParser_(
lexer: Lexer,
log: *logger.Log,
allocator: std.mem.Allocator,
+ list_allocator: std.mem.Allocator,
pub fn init(allocator: std.mem.Allocator, source_: logger.Source, log: *logger.Log) !Parser {
+ return initWithListAllocator(allocator, allocator, source_, log);
+ }
+
+ pub fn initWithListAllocator(allocator: std.mem.Allocator, list_allocator: std.mem.Allocator, source_: logger.Source, log: *logger.Log) !Parser {
Expr.Data.Store.assert();
Stmt.Data.Store.assert();
@@ -150,6 +155,7 @@ fn JSONLikeParser_(
.lexer = try Lexer.init(log, source_, allocator),
.allocator = allocator,
.log = log,
+ .list_allocator = list_allocator,
};
}
@@ -202,7 +208,7 @@ fn JSONLikeParser_(
.t_open_bracket => {
try p.lexer.next();
var is_single_line = !p.lexer.has_newline_before;
- var exprs = std.ArrayList(Expr).init(p.allocator);
+ var exprs = std.ArrayList(Expr).init(p.list_allocator);
while (p.lexer.token != .t_close_bracket) {
if (exprs.items.len > 0) {
@@ -235,7 +241,7 @@ fn JSONLikeParser_(
.t_open_brace => {
try p.lexer.next();
var is_single_line = !p.lexer.has_newline_before;
- var properties = std.ArrayList(G.Property).init(p.allocator);
+ var properties = std.ArrayList(G.Property).init(p.list_allocator);
const DuplicateNodeType = comptime if (opts.json_warn_duplicate_keys) *HashMapPool.LinkedList.Node else void;
const HashMapType = comptime if (opts.json_warn_duplicate_keys) HashMapPool.HashMap else void;
diff --git a/src/options.zig b/src/options.zig
index 59b814894..26485eb70 100644
--- a/src/options.zig
+++ b/src/options.zig
@@ -1974,6 +1974,7 @@ pub const OutputFile = struct {
var blob = globalObject.allocator().create(JSC.WebCore.Blob) catch unreachable;
blob.* = JSC.WebCore.Blob.initWithStore(file_blob, globalObject);
blob.allocator = globalObject.allocator();
+ blob.globalThis = globalObject;
blob.content_type = loader.toMimeType().value;
return blob.toJS(globalObject);
@@ -1996,7 +1997,11 @@ pub const OutputFile = struct {
};
pub const Value = union(Kind) {
- buffer: []const u8,
+ buffer: struct {
+ allocator: std.mem.Allocator,
+ bytes: []const u8,
+ },
+
move: FileOperation,
copy: FileOperation,
noop: u0,
@@ -2029,12 +2034,17 @@ pub const OutputFile = struct {
return res;
}
- pub fn initBuf(buf: []const u8, pathname: string, loader: Loader) OutputFile {
+ pub fn initBuf(buf: []const u8, allocator: std.mem.Allocator, pathname: string, loader: Loader) OutputFile {
return .{
.loader = loader,
.input = Fs.Path.init(pathname),
.size = buf.len,
- .value = .{ .buffer = buf },
+ .value = .{
+ .buffer = .{
+ .bytes = buf,
+ .allocator = allocator,
+ },
+ },
};
}
@@ -2077,12 +2087,14 @@ pub const OutputFile = struct {
.move => this.value.move.toJS(globalObject, this.loader),
.copy => this.value.copy.toJS(globalObject, this.loader),
.buffer => |buffer| brk: {
- var blob = globalObject.allocator().create(JSC.WebCore.Blob) catch unreachable;
- blob.* = JSC.WebCore.Blob.init(@constCast(buffer), bun.default_allocator, globalObject);
+ var blob = bun.default_allocator.create(JSC.WebCore.Blob) catch unreachable;
+ blob.* = JSC.WebCore.Blob.init(@constCast(buffer.bytes), buffer.allocator, globalObject);
blob.store.?.mime_type = this.loader.toMimeType();
blob.content_type = blob.store.?.mime_type.value;
- blob.allocator = globalObject.allocator();
- break :brk blob.toJS(globalObject);
+ blob.allocator = bun.default_allocator;
+ const blob_jsvalue = blob.toJS(globalObject);
+ blob_jsvalue.ensureStillAlive();
+ break :brk blob_jsvalue;
},
};
}
diff --git a/src/resolver/dir_info.zig b/src/resolver/dir_info.zig
index 3ae75d0e7..66d589cfa 100644
--- a/src/resolver/dir_info.zig
+++ b/src/resolver/dir_info.zig
@@ -85,7 +85,7 @@ pub fn getEntries(dirinfo: *const DirInfo) ?*Fs.FileSystem.DirEntry {
var entries_ptr = Fs.FileSystem.instance.fs.entries.atIndex(dirinfo.entries) orelse return null;
switch (entries_ptr.*) {
.entries => {
- return &entries_ptr.entries;
+ return entries_ptr.entries;
},
.err => {
return null;
@@ -97,7 +97,7 @@ pub fn getEntriesConst(dirinfo: *const DirInfo) ?*const Fs.FileSystem.DirEntry {
const entries_ptr = Fs.FileSystem.instance.fs.entries.atIndex(dirinfo.entries) orelse return null;
switch (entries_ptr.*) {
.entries => {
- return &entries_ptr.entries;
+ return entries_ptr.entries;
},
.err => {
return null;
diff --git a/src/resolver/package_json.zig b/src/resolver/package_json.zig
index 4b55faa88..ce9695008 100644
--- a/src/resolver/package_json.zig
+++ b/src/resolver/package_json.zig
@@ -22,8 +22,8 @@ const js_lexer = bun.js_lexer;
const resolve_path = @import("./resolve_path.zig");
// Assume they're not going to have hundreds of main fields or browser map
// so use an array-backed hash table instead of bucketed
-const MainFieldMap = bun.StringArrayHashMap(string);
-pub const BrowserMap = bun.StringArrayHashMap(string);
+const MainFieldMap = bun.StringMap;
+pub const BrowserMap = bun.StringMap;
pub const MacroImportReplacementMap = bun.StringArrayHashMap(string);
pub const MacroMap = bun.StringArrayHashMapUnmanaged(MacroImportReplacementMap);
@@ -588,7 +588,12 @@ pub const PackageJSON = struct {
const package_json_path_ = r.fs.abs(&parts);
const package_json_path = r.fs.dirname_store.append(@TypeOf(package_json_path_), package_json_path_) catch unreachable;
- const entry = r.caches.fs.readFile(
+ // DirInfo cache is reused globally
+ // So we cannot free these
+ const allocator = bun.fs_allocator;
+
+ const entry = r.caches.fs.readFileWithAllocator(
+ allocator,
r.fs,
package_json_path,
dirname_fd,
@@ -596,12 +601,18 @@ pub const PackageJSON = struct {
null,
) catch |err| {
if (err != error.IsDir) {
- r.log.addErrorFmt(null, logger.Loc.Empty, r.allocator, "Cannot read file \"{s}\": {s}", .{ r.prettyPath(fs.Path.init(input_path)), @errorName(err) }) catch unreachable;
+ r.log.addErrorFmt(null, logger.Loc.Empty, allocator, "Cannot read file \"{s}\": {s}", .{ r.prettyPath(fs.Path.init(input_path)), @errorName(err) }) catch unreachable;
}
return null;
};
+ defer {
+ if (entry.fd != 0) {
+ _ = bun.JSC.Node.Syscall.close(entry.fd);
+ }
+ }
+
if (r.debug_logs) |*debug| {
debug.addNoteFmt("The file \"{s}\" exists", .{package_json_path});
}
@@ -611,21 +622,28 @@ pub const PackageJSON = struct {
var json_source = logger.Source.initPathString(key_path.text, entry.contents);
json_source.path.pretty = r.prettyPath(json_source.path);
- const json: js_ast.Expr = (r.caches.json.parseJSON(r.log, json_source, r.allocator) catch |err| {
+ const json: js_ast.Expr = (r.caches.json.parseJSON(r.log, json_source, allocator) catch |err| {
if (Environment.isDebug) {
Output.printError("{s}: JSON parse error: {s}", .{ package_json_path, @errorName(err) });
}
return null;
} orelse return null);
+ if (json.data != .e_object) {
+ // Invalid package.json in node_modules is noisy.
+ // Let's just ignore it.
+ allocator.free(entry.contents);
+ return null;
+ }
+
var package_json = PackageJSON{
.name = "",
.version = "",
.hash = 0xDEADBEEF,
.source = json_source,
.module_type = .unknown,
- .browser_map = BrowserMap.init(r.allocator),
- .main_fields = MainFieldMap.init(r.allocator),
+ .browser_map = BrowserMap.init(allocator, false),
+ .main_fields = MainFieldMap.init(allocator, false),
};
// Note: we tried rewriting this to be fewer loops over all the properties (asProperty loops over each)
@@ -634,17 +652,17 @@ pub const PackageJSON = struct {
// Feels like a codegen issue.
// or that looping over every property doesn't really matter because most package.jsons are < 20 properties
if (json.asProperty("version")) |version_json| {
- if (version_json.expr.asString(r.allocator)) |version_str| {
+ if (version_json.expr.asString(allocator)) |version_str| {
if (version_str.len > 0) {
- package_json.version = r.allocator.dupe(u8, version_str) catch unreachable;
+ package_json.version = allocator.dupe(u8, version_str) catch unreachable;
}
}
}
if (json.asProperty("name")) |version_json| {
- if (version_json.expr.asString(r.allocator)) |version_str| {
+ if (version_json.expr.asString(allocator)) |version_str| {
if (version_str.len > 0) {
- package_json.name = r.allocator.dupe(u8, version_str) catch unreachable;
+ package_json.name = allocator.dupe(u8, version_str) catch unreachable;
}
}
}
@@ -653,7 +671,7 @@ pub const PackageJSON = struct {
// We do not need to parse all this stuff.
if (comptime !include_scripts) {
if (json.asProperty("type")) |type_json| {
- if (type_json.expr.asString(r.allocator)) |type_str| {
+ if (type_json.expr.asString(allocator)) |type_str| {
switch (options.ModuleType.List.get(type_str) orelse options.ModuleType.unknown) {
.cjs => {
package_json.module_type = .cjs;
@@ -665,7 +683,7 @@ pub const PackageJSON = struct {
r.log.addRangeWarningFmt(
&json_source,
json_source.rangeOfString(type_json.loc),
- r.allocator,
+ allocator,
"\"{s}\" is not a valid value for \"type\" field (must be either \"commonjs\" or \"module\")",
.{type_str},
) catch unreachable;
@@ -681,9 +699,9 @@ pub const PackageJSON = struct {
if (json.asProperty(main)) |main_json| {
const expr: js_ast.Expr = main_json.expr;
- if ((expr.asString(r.allocator))) |str| {
+ if ((expr.asString(allocator))) |str| {
if (str.len > 0) {
- package_json.main_fields.put(main, r.allocator.dupe(u8, str) catch unreachable) catch unreachable;
+ package_json.main_fields.put(main, str) catch unreachable;
}
}
}
@@ -709,7 +727,7 @@ pub const PackageJSON = struct {
// Remap all files in the browser field
for (obj.properties.slice()) |*prop| {
- var _key_str = (prop.key orelse continue).asString(r.allocator) orelse continue;
+ var _key_str = (prop.key orelse continue).asString(allocator) orelse continue;
const value: js_ast.Expr = prop.value orelse continue;
// Normalize the path so we can compare against it without getting
@@ -721,12 +739,12 @@ pub const PackageJSON = struct {
// import of "foo", but that's actually not a bug. Or arguably it's a
// bug in Browserify but we have to replicate this bug because packages
// do this in the wild.
- const key = r.allocator.dupe(u8, r.fs.normalize(_key_str)) catch unreachable;
+ const key = allocator.dupe(u8, r.fs.normalize(_key_str)) catch unreachable;
switch (value.data) {
.e_string => |str| {
// If this is a string, it's a replacement package
- package_json.browser_map.put(key, str.string(r.allocator) catch unreachable) catch unreachable;
+ package_json.browser_map.put(key, str.string(allocator) catch unreachable) catch unreachable;
},
.e_boolean => |boolean| {
if (!boolean.value) {
@@ -745,13 +763,13 @@ pub const PackageJSON = struct {
}
if (json.asProperty("exports")) |exports_prop| {
- if (ExportsMap.parse(r.allocator, &json_source, r.log, exports_prop.expr, exports_prop.loc)) |exports_map| {
+ if (ExportsMap.parse(bun.default_allocator, &json_source, r.log, exports_prop.expr, exports_prop.loc)) |exports_map| {
package_json.exports = exports_map;
}
}
if (json.asProperty("imports")) |imports_prop| {
- if (ExportsMap.parse(r.allocator, &json_source, r.log, imports_prop.expr, imports_prop.loc)) |imports_map| {
+ if (ExportsMap.parse(bun.default_allocator, &json_source, r.log, imports_prop.expr, imports_prop.loc)) |imports_map| {
package_json.imports = imports_map;
}
}
@@ -764,9 +782,9 @@ pub const PackageJSON = struct {
var array = array_;
// TODO: switch to only storing hashes
var map = SideEffectsMap{};
- map.ensureTotalCapacity(r.allocator, array.array.items.len) catch unreachable;
+ map.ensureTotalCapacity(allocator, array.array.items.len) catch unreachable;
while (array.next()) |item| {
- if (item.asString(r.allocator)) |name| {
+ if (item.asString(allocator)) |name| {
// TODO: support RegExp using JavaScriptCore <> C++ bindings
if (strings.containsChar(name, '*')) {
// https://sourcegraph.com/search?q=context:global+file:package.json+sideEffects%22:+%5B&patternType=standard&sm=1&groupBy=repo
@@ -780,7 +798,7 @@ pub const PackageJSON = struct {
item.loc,
"wildcard sideEffects are not supported yet, which means this package will be deoptimized",
) catch unreachable;
- map.deinit(r.allocator);
+ map.deinit(allocator);
package_json.side_effects = .{ .unspecified = {} };
break :outer;
@@ -815,7 +833,7 @@ pub const PackageJSON = struct {
if (tag == .npm) {
const sliced = Semver.SlicedString.init(package_json.version, package_json.version);
- if (Dependency.parseWithTag(r.allocator, String.init(package_json.name, package_json.name), package_json.version, .npm, &sliced, r.log)) |dependency_version| {
+ if (Dependency.parseWithTag(allocator, String.init(package_json.name, package_json.name), package_json.version, .npm, &sliced, r.log)) |dependency_version| {
if (dependency_version.value.npm.version.isExact()) {
if (pm.lockfile.resolve(package_json.name, dependency_version)) |resolved| {
package_json.package_manager_package_id = resolved;
@@ -915,7 +933,7 @@ pub const PackageJSON = struct {
.b_buf = json_source.contents,
};
package_json.dependencies.map.ensureTotalCapacityContext(
- r.allocator,
+ allocator,
total_dependency_count,
ctx,
) catch unreachable;
@@ -926,14 +944,14 @@ pub const PackageJSON = struct {
var group_obj = group_json.data.e_object;
for (group_obj.properties.slice()) |*prop| {
const name_prop = prop.key orelse continue;
- const name_str = name_prop.asString(r.allocator) orelse continue;
+ const name_str = name_prop.asString(allocator) orelse continue;
const name = String.init(name_str, name_str);
const version_value = prop.value orelse continue;
- const version_str = version_value.asString(r.allocator) orelse continue;
+ const version_str = version_value.asString(allocator) orelse continue;
const sliced_str = Semver.SlicedString.init(version_str, version_str);
if (Dependency.parse(
- r.allocator,
+ allocator,
name,
version_str,
&sliced_str,
@@ -968,26 +986,26 @@ pub const PackageJSON = struct {
var count: usize = 0;
for (scripts_obj.properties.slice()) |prop| {
- const key = prop.key.?.asString(r.allocator) orelse continue;
- const value = prop.value.?.asString(r.allocator) orelse continue;
+ const key = prop.key.?.asString(allocator) orelse continue;
+ const value = prop.value.?.asString(allocator) orelse continue;
count += @as(usize, @boolToInt(key.len > 0 and value.len > 0));
}
if (count == 0) break :read_scripts;
- var scripts = ScriptsMap.init(r.allocator);
+ var scripts = ScriptsMap.init(allocator);
scripts.ensureUnusedCapacity(count) catch break :read_scripts;
for (scripts_obj.properties.slice()) |prop| {
- const key = prop.key.?.asString(r.allocator) orelse continue;
- const value = prop.value.?.asString(r.allocator) orelse continue;
+ const key = prop.key.?.asString(allocator) orelse continue;
+ const value = prop.value.?.asString(allocator) orelse continue;
if (!(key.len > 0 and value.len > 0)) continue;
scripts.putAssumeCapacity(key, value);
}
- package_json.scripts = r.allocator.create(ScriptsMap) catch unreachable;
+ package_json.scripts = allocator.create(ScriptsMap) catch unreachable;
package_json.scripts.?.* = scripts;
}
}
@@ -1048,7 +1066,7 @@ pub const ExportsMap = struct {
.e_string => |str| {
return Entry{
.data = .{
- .string = str.string(this.allocator) catch unreachable,
+ .string = str.slice(this.allocator),
},
.first_token = this.source.rangeOfString(expr.loc),
};
@@ -1079,7 +1097,7 @@ pub const ExportsMap = struct {
first_token.loc = expr.loc;
first_token.len = 1;
for (e_obj.properties.slice(), 0..) |prop, i| {
- const key: string = prop.key.?.data.e_string.string(this.allocator) catch unreachable;
+ const key: string = prop.key.?.data.e_string.slice(this.allocator);
const key_range: logger.Range = this.source.rangeOfString(prop.key.?.loc);
// If exports is an Object with both a key starting with "." and a key
diff --git a/src/resolver/resolver.zig b/src/resolver/resolver.zig
index 4845de213..0e9bd3a22 100644
--- a/src/resolver/resolver.zig
+++ b/src/resolver/resolver.zig
@@ -60,13 +60,46 @@ pub const SideEffectsData = struct {
is_side_effects_array_in_json: bool = false,
};
-pub const TemporaryBuffer = struct {
- pub threadlocal var ExtensionPathBuf: [512]u8 = undefined;
- pub threadlocal var TSConfigMatchStarBuf: [512]u8 = undefined;
- pub threadlocal var TSConfigMatchPathBuf: [512]u8 = undefined;
- pub threadlocal var TSConfigMatchFullBuf: [bun.MAX_PATH_BYTES]u8 = undefined;
- pub threadlocal var TSConfigMatchFullBuf2: [bun.MAX_PATH_BYTES]u8 = undefined;
-};
+/// A temporary threadlocal buffer with a lifetime more than the current
+/// function call.
+const bufs = struct {
+ // Experimenting with making this one struct instead of a bunch of different
+ // threadlocal vars yielded no performance improvement on macOS when
+ // bundling 10 copies of Three.js. It may be worthwhile for more complicated
+ // packages but we lack a decent module resolution benchmark right now.
+ // Potentially revisit after https://github.com/oven-sh/bun/issues/2716
+ threadlocal var extension_path: [512]u8 = undefined;
+ threadlocal var tsconfig_match_full_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var tsconfig_match_full_buf2: [bun.MAX_PATH_BYTES]u8 = undefined;
+
+ threadlocal var esm_subpath: [512]u8 = undefined;
+ threadlocal var esm_absolute_package_path: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var esm_absolute_package_path_joined: [bun.MAX_PATH_BYTES]u8 = undefined;
+
+ threadlocal var dir_entry_paths_to_resolve: [256]DirEntryResolveQueueItem = undefined;
+ threadlocal var open_dirs: [256]std.fs.IterableDir = undefined;
+ threadlocal var resolve_without_remapping: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var index: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var dir_info_uncached_filename: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var node_bin_path: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var dir_info_uncached_path: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var tsconfig_base_url: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var relative_abs_path: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var load_as_file_or_directory_via_tsconfig_base_path: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var node_modules_check: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var field_abs_path: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var tsconfig_path_abs: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var check_browser_map: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var remap_path: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var load_as_file: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var remap_path_trailing_slash: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var path_in_global_disk_cache: [bun.MAX_PATH_BYTES]u8 = undefined;
+ threadlocal var abs_to_rel: [bun.MAX_PATH_BYTES]u8 = undefined;
+
+ pub inline fn bufs(comptime field: std.meta.DeclEnum(@This())) *@TypeOf(@field(@This(), @tagName(field))) {
+ return &@field(@This(), @tagName(field));
+ }
+}.bufs;
pub const PathPair = struct {
primary: Path,
@@ -262,26 +295,6 @@ pub const DirEntryResolveQueueItem = struct {
fd: StoredFileDescriptorType = 0,
};
-threadlocal var _dir_entry_paths_to_resolve: [256]DirEntryResolveQueueItem = undefined;
-threadlocal var _open_dirs: [256]std.fs.IterableDir = undefined;
-threadlocal var resolve_without_remapping_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var index_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var dir_info_uncached_filename_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var node_bin_path: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var dir_info_uncached_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var tsconfig_base_url_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var relative_abs_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var load_as_file_or_directory_via_tsconfig_base_path: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var node_modules_check_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var field_abs_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var tsconfig_path_abs_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var check_browser_map_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var remap_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var load_as_file_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var remap_path_trailing_slash: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var tsconfig_paths_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-threadlocal var path_in_global_disk_cache_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-
pub const DebugLogs = struct {
what: string = "",
indent: MutableString,
@@ -547,7 +560,7 @@ pub const Resolver = struct {
return ThisResolver{
.allocator = allocator,
- .dir_cache = DirInfo.HashMap.init(allocator),
+ .dir_cache = DirInfo.HashMap.init(bun.default_allocator),
.mutex = &resolver_Mutex,
.caches = CacheSet.init(allocator),
.opts = opts,
@@ -1093,7 +1106,7 @@ pub const Resolver = struct {
if (check_relative) {
const parts = [_]string{ source_dir, import_path };
- const abs_path = r.fs.absBuf(&parts, &relative_abs_path_buf);
+ const abs_path = r.fs.absBuf(&parts, bufs(.relative_abs_path));
if (r.opts.external.abs_paths.count() > 0 and r.opts.external.abs_paths.contains(abs_path)) {
// If the string literal in the source text is an absolute path and has
@@ -1438,9 +1451,6 @@ pub const Resolver = struct {
r.dir_cache.remove(path);
}
- threadlocal var esm_subpath_buf: [512]u8 = undefined;
- threadlocal var esm_absolute_package_path: [bun.MAX_PATH_BYTES]u8 = undefined;
- threadlocal var esm_absolute_package_path_joined: [bun.MAX_PATH_BYTES]u8 = undefined;
pub fn loadNodeModules(
r: *ThisResolver,
import_path: string,
@@ -1475,7 +1485,7 @@ pub const Resolver = struct {
if (tsconfig.hasBaseURL()) {
const base = tsconfig.base_url;
const paths = [_]string{ base, import_path };
- const abs = r.fs.absBuf(&paths, &load_as_file_or_directory_via_tsconfig_base_path);
+ const abs = r.fs.absBuf(&paths, bufs(.load_as_file_or_directory_via_tsconfig_base_path));
if (r.loadAsFileOrDirectory(abs, kind)) |res| {
return .{ .success = res };
@@ -1493,7 +1503,7 @@ pub const Resolver = struct {
return r.loadPackageImports(import_path, dir_info_package_json.?, kind, global_cache);
}
- const esm_ = ESModule.Package.parse(import_path, &esm_subpath_buf);
+ const esm_ = ESModule.Package.parse(import_path, bufs(.esm_subpath));
var source_dir_info = dir_info;
var any_node_modules_folder = false;
@@ -1506,7 +1516,7 @@ pub const Resolver = struct {
if (dir_info.hasNodeModules()) {
any_node_modules_folder = true;
var _paths = [_]string{ dir_info.abs_path, "node_modules", import_path };
- const abs_path = r.fs.absBuf(&_paths, &node_modules_check_buf);
+ const abs_path = r.fs.absBuf(&_paths, bufs(.node_modules_check));
if (r.debug_logs) |*debug| {
debug.addNoteFmt("Checking for a package in the directory \"{s}\"", .{abs_path});
}
@@ -1514,7 +1524,7 @@ pub const Resolver = struct {
if (esm_) |esm| {
const abs_package_path = brk: {
var parts = [_]string{ dir_info.abs_path, "node_modules", esm.name };
- break :brk r.fs.absBuf(&parts, &esm_absolute_package_path);
+ break :brk r.fs.absBuf(&parts, bufs(.esm_absolute_package_path));
};
if (r.dirInfoCached(abs_package_path) catch null) |pkg_dir_info| {
@@ -1717,7 +1727,7 @@ pub const Resolver = struct {
}
};
- const dir_path_for_resolution = manager.pathForResolution(resolved_package_id, resolution, &path_in_global_disk_cache_buf) catch |err| {
+ const dir_path_for_resolution = manager.pathForResolution(resolved_package_id, resolution, bufs(.path_in_global_disk_cache)) catch |err| {
// if it's missing, we need to install it
if (err == error.FileNotFound) {
switch (manager.getPreinstallState(resolved_package_id, manager.lockfile)) {
@@ -1854,7 +1864,7 @@ pub const Resolver = struct {
}
var _paths = [_]string{ pkg_dir_info.abs_path, esm.subpath };
- const abs_path = r.fs.absBuf(&_paths, &node_modules_check_buf);
+ const abs_path = r.fs.absBuf(&_paths, bufs(.node_modules_check));
if (r.debug_logs) |*debug| {
debug.addNoteFmt("Checking for a package in the directory \"{s}\"", .{abs_path});
}
@@ -1911,21 +1921,24 @@ pub const Resolver = struct {
}
if (needs_iter) {
- const allocator = r.fs.allocator;
- dir_entries_option = rfs.entries.put(&cached_dir_entry_result, .{
- .entries = Fs.FileSystem.DirEntry.init(
- Fs.FileSystem.DirnameStore.instance.append(string, dir_path) catch unreachable,
- ),
- }) catch unreachable;
+ const allocator = bun.fs_allocator;
+ var dir_entries_ptr = allocator.create(Fs.FileSystem.DirEntry) catch unreachable;
+ dir_entries_ptr.* = Fs.FileSystem.DirEntry.init(
+ Fs.FileSystem.DirnameStore.instance.append(string, dir_path) catch unreachable,
+ );
if (FeatureFlags.store_file_descriptors) {
Fs.FileSystem.setMaxFd(open_dir.dir.fd);
- dir_entries_option.entries.fd = open_dir.dir.fd;
+ dir_entries_ptr.fd = open_dir.dir.fd;
}
var dir_iterator = open_dir.iterate();
while (dir_iterator.next() catch null) |_value| {
- dir_entries_option.entries.addEntry(_value, allocator, void, {}) catch unreachable;
+ dir_entries_ptr.addEntry(_value, allocator, void, {}) catch unreachable;
}
+
+ dir_entries_option = rfs.entries.put(&cached_dir_entry_result, .{
+ .entries = dir_entries_ptr,
+ }) catch unreachable;
}
// We must initialize it as empty so that the result index is correct.
@@ -2068,7 +2081,7 @@ pub const Resolver = struct {
abs_package_path,
strings.withoutLeadingSlash(esm_resolution.path),
};
- break :brk r.fs.absBuf(&parts, &esm_absolute_package_path_joined);
+ break :brk r.fs.absBuf(&parts, bufs(.esm_absolute_package_path_joined));
};
var missing_suffix: string = undefined;
@@ -2094,9 +2107,9 @@ pub const Resolver = struct {
// Try to have a friendly error message if people forget the extension
if (ends_with_star) {
- bun.copy(u8, &load_as_file_buf, base);
+ bun.copy(u8, bufs(.load_as_file), base);
for (extension_order) |ext| {
- var file_name = load_as_file_buf[0 .. base.len + ext.len];
+ var file_name = bufs(.load_as_file)[0 .. base.len + ext.len];
bun.copy(u8, file_name[base.len..], ext);
if (entries.get(file_name) != null) {
if (r.debug_logs) |*debug| {
@@ -2121,9 +2134,9 @@ pub const Resolver = struct {
if (r.dirInfoCached(abs_esm_path) catch null) |dir_info| {
if (dir_info.getEntries()) |dir_entries| {
const index = "index";
- bun.copy(u8, &load_as_file_buf, index);
+ bun.copy(u8, bufs(.load_as_file), index);
for (extension_order) |ext| {
- var file_name = load_as_file_buf[0 .. index.len + ext.len];
+ var file_name = bufs(.load_as_file)[0 .. index.len + ext.len];
bun.copy(u8, file_name[index.len..], ext);
const index_query = dir_entries.get(file_name);
if (index_query != null and index_query.?.entry.kind(&r.fs.fs) == .file) {
@@ -2191,7 +2204,7 @@ pub const Resolver = struct {
return r.loadNodeModules(import_path, kind, source_dir_info, global_cache, false);
} else {
const paths = [_]string{ source_dir_info.abs_path, import_path };
- var resolved = r.fs.absBuf(&paths, &resolve_without_remapping_buf);
+ var resolved = r.fs.absBuf(&paths, bufs(.resolve_without_remapping));
if (r.loadAsFileOrDirectory(resolved, kind)) |result| {
return .{ .success = result };
}
@@ -2226,14 +2239,14 @@ pub const Resolver = struct {
// this might leak
if (!std.fs.path.isAbsolute(result.base_url)) {
const paths = [_]string{ file_dir, result.base_url };
- result.base_url = r.fs.dirname_store.append(string, r.fs.absBuf(&paths, &tsconfig_base_url_buf)) catch unreachable;
+ result.base_url = r.fs.dirname_store.append(string, r.fs.absBuf(&paths, bufs(.tsconfig_base_url))) catch unreachable;
}
}
if (result.paths.count() > 0 and (result.base_url_for_paths.len == 0 or !std.fs.path.isAbsolute(result.base_url_for_paths))) {
// this might leak
const paths = [_]string{ file_dir, result.base_url };
- result.base_url_for_paths = r.fs.dirname_store.append(string, r.fs.absBuf(&paths, &tsconfig_base_url_buf)) catch unreachable;
+ result.base_url_for_paths = r.fs.dirname_store.append(string, r.fs.absBuf(&paths, bufs(.tsconfig_base_url))) catch unreachable;
}
return result;
@@ -2279,7 +2292,7 @@ pub const Resolver = struct {
) orelse return null;
}
- var _pkg = try r.allocator.create(PackageJSON);
+ var _pkg = try bun.default_allocator.create(PackageJSON);
_pkg.* = pkg;
return _pkg;
}
@@ -2325,11 +2338,13 @@ pub const Resolver = struct {
return r.dir_cache.atIndex(top_result.index);
}
+ var dir_info_uncached_path_buf = bufs(.dir_info_uncached_path);
+
var i: i32 = 1;
- bun.copy(u8, &dir_info_uncached_path_buf, _path);
+ bun.copy(u8, dir_info_uncached_path_buf, _path);
var path = dir_info_uncached_path_buf[0.._path.len];
- _dir_entry_paths_to_resolve[0] = (DirEntryResolveQueueItem{ .result = top_result, .unsafe_path = path, .safe_path = "" });
+ bufs(.dir_entry_paths_to_resolve)[0] = (DirEntryResolveQueueItem{ .result = top_result, .unsafe_path = path, .safe_path = "" });
var top = Dirname.dirname(path);
var top_parent: allocators.Result = allocators.Result{
@@ -2354,15 +2369,15 @@ pub const Resolver = struct {
top_parent = result;
break;
}
- _dir_entry_paths_to_resolve[@intCast(usize, i)] = DirEntryResolveQueueItem{
+ bufs(.dir_entry_paths_to_resolve)[@intCast(usize, i)] = DirEntryResolveQueueItem{
.unsafe_path = top,
.result = result,
.fd = 0,
};
if (rfs.entries.get(top)) |top_entry| {
- _dir_entry_paths_to_resolve[@intCast(usize, i)].safe_path = top_entry.entries.dir;
- _dir_entry_paths_to_resolve[@intCast(usize, i)].fd = top_entry.entries.fd;
+ bufs(.dir_entry_paths_to_resolve)[@intCast(usize, i)].safe_path = top_entry.entries.dir;
+ bufs(.dir_entry_paths_to_resolve)[@intCast(usize, i)].fd = top_entry.entries.fd;
}
i += 1;
}
@@ -2372,21 +2387,21 @@ pub const Resolver = struct {
if (result.status != .unknown) {
top_parent = result;
} else {
- _dir_entry_paths_to_resolve[@intCast(usize, i)] = DirEntryResolveQueueItem{
+ bufs(.dir_entry_paths_to_resolve)[@intCast(usize, i)] = DirEntryResolveQueueItem{
.unsafe_path = root_path,
.result = result,
.fd = 0,
};
if (rfs.entries.get(top)) |top_entry| {
- _dir_entry_paths_to_resolve[@intCast(usize, i)].safe_path = top_entry.entries.dir;
- _dir_entry_paths_to_resolve[@intCast(usize, i)].fd = top_entry.entries.fd;
+ bufs(.dir_entry_paths_to_resolve)[@intCast(usize, i)].safe_path = top_entry.entries.dir;
+ bufs(.dir_entry_paths_to_resolve)[@intCast(usize, i)].fd = top_entry.entries.fd;
}
i += 1;
}
}
- var queue_slice: []DirEntryResolveQueueItem = _dir_entry_paths_to_resolve[0..@intCast(usize, i)];
+ var queue_slice: []DirEntryResolveQueueItem = bufs(.dir_entry_paths_to_resolve)[0..@intCast(usize, i)];
if (Environment.allow_assert) std.debug.assert(queue_slice.len > 0);
var open_dir_count: usize = 0;
@@ -2395,7 +2410,7 @@ pub const Resolver = struct {
// Anything
if (open_dir_count > 0 and r.fs.fs.needToCloseFiles()) {
- var open_dirs: []std.fs.IterableDir = _open_dirs[0..open_dir_count];
+ var open_dirs: []std.fs.IterableDir = bufs(.open_dirs)[0..open_dir_count];
for (open_dirs) |*open_dir| {
open_dir.dir.close();
}
@@ -2487,7 +2502,7 @@ pub const Resolver = struct {
if (queue_top.fd == 0) {
Fs.FileSystem.setMaxFd(open_dir.dir.fd);
// these objects mostly just wrap the file descriptor, so it's fine to keep it.
- _open_dirs[open_dir_count] = open_dir;
+ bufs(.open_dirs)[open_dir_count] = open_dir;
open_dir_count += 1;
}
@@ -2529,19 +2544,21 @@ pub const Resolver = struct {
}
if (needs_iter) {
- const allocator = r.fs.allocator;
- dir_entries_option = try rfs.entries.put(&cached_dir_entry_result, .{
- .entries = Fs.FileSystem.DirEntry.init(dir_path),
- });
-
+ const allocator = bun.fs_allocator;
+ var entries_ptr = allocator.create(Fs.FileSystem.DirEntry) catch unreachable;
+ entries_ptr.* = Fs.FileSystem.DirEntry.init(dir_path);
if (FeatureFlags.store_file_descriptors) {
Fs.FileSystem.setMaxFd(open_dir.dir.fd);
- dir_entries_option.entries.fd = open_dir.dir.fd;
+ entries_ptr.fd = open_dir.dir.fd;
}
var dir_iterator = open_dir.iterate();
while (try dir_iterator.next()) |_value| {
- dir_entries_option.entries.addEntry(_value, allocator, void, {}) catch unreachable;
+ entries_ptr.addEntry(_value, allocator, void, {}) catch unreachable;
}
+
+ dir_entries_option = try rfs.entries.put(&cached_dir_entry_result, .{
+ .entries = entries_ptr,
+ });
}
// We must initialize it as empty so that the result index is correct.
@@ -2610,7 +2627,7 @@ pub const Resolver = struct {
if (!std.fs.path.isAbsolute(absolute_original_path)) {
const parts = [_]string{ abs_base_url, original_path };
- absolute_original_path = r.fs.absBuf(&parts, &tsconfig_path_abs_buf);
+ absolute_original_path = r.fs.absBuf(&parts, bufs(.tsconfig_path_abs));
}
if (r.loadAsFileOrDirectory(absolute_original_path, kind)) |res| {
@@ -2672,14 +2689,14 @@ pub const Resolver = struct {
// 1. Normalize the base path
// so that "/Users/foo/project/", "../components/*" => "/Users/foo/components/""
- var prefix = r.fs.absBuf(&prefix_parts, &TemporaryBuffer.TSConfigMatchFullBuf2);
+ var prefix = r.fs.absBuf(&prefix_parts, bufs(.tsconfig_match_full_buf2));
// 2. Join the new base path with the matched result
// so that "/Users/foo/components/", "/foo/bar" => /Users/foo/components/foo/bar
var parts = [_]string{ prefix, std.mem.trimLeft(u8, matched_text, "/"), std.mem.trimLeft(u8, longest_match.suffix, "/") };
var absolute_original_path = r.fs.absBuf(
&parts,
- &TemporaryBuffer.TSConfigMatchFullBuf,
+ bufs(.tsconfig_match_full_buf),
);
if (r.loadAsFileOrDirectory(absolute_original_path, kind)) |res| {
@@ -2741,8 +2758,6 @@ pub const Resolver = struct {
extension_order: []const string,
map: BrowserMap,
- pub threadlocal var abs_to_rel_buf: [bun.MAX_PATH_BYTES]u8 = undefined;
-
pub const Kind = enum { PackagePath, AbsolutePath };
pub fn checkPath(
@@ -2759,12 +2774,14 @@ pub const Resolver = struct {
return true;
}
- bun.copy(u8, &TemporaryBuffer.ExtensionPathBuf, cleaned);
+ var ext_buf = bufs(.extension_path);
+
+ bun.copy(u8, ext_buf, cleaned);
// If that failed, try adding implicit extensions
for (this.extension_order) |ext| {
- bun.copy(u8, TemporaryBuffer.ExtensionPathBuf[cleaned.len..], ext);
- const new_path = TemporaryBuffer.ExtensionPathBuf[0 .. cleaned.len + ext.len];
+ bun.copy(u8, ext_buf[cleaned.len..], ext);
+ const new_path = ext_buf[0 .. cleaned.len + ext.len];
// if (r.debug_logs) |*debug| {
// debug.addNoteFmt("Checking for \"{s}\" ", .{new_path});
// }
@@ -2781,7 +2798,7 @@ pub const Resolver = struct {
var index_path: string = "";
{
var parts = [_]string{ std.mem.trimRight(u8, path_to_check, std.fs.path.sep_str), std.fs.path.sep_str ++ "index" };
- index_path = ResolvePath.joinStringBuf(&tsconfig_base_url_buf, &parts, .auto);
+ index_path = ResolvePath.joinStringBuf(bufs(.tsconfig_base_url), &parts, .auto);
}
if (map.get(index_path)) |_remapped| {
@@ -2790,11 +2807,11 @@ pub const Resolver = struct {
return true;
}
- bun.copy(u8, &TemporaryBuffer.ExtensionPathBuf, index_path);
+ bun.copy(u8, ext_buf, index_path);
for (this.extension_order) |ext| {
- bun.copy(u8, TemporaryBuffer.ExtensionPathBuf[index_path.len..], ext);
- const new_path = TemporaryBuffer.ExtensionPathBuf[0 .. index_path.len + ext.len];
+ bun.copy(u8, ext_buf[index_path.len..], ext);
+ const new_path = ext_buf[0 .. index_path.len + ext.len];
// if (r.debug_logs) |*debug| {
// debug.addNoteFmt("Checking for \"{s}\" ", .{new_path});
// }
@@ -2839,7 +2856,7 @@ pub const Resolver = struct {
}
// Normalize the path so we can compare against it without getting confused by "./"
- var cleaned = r.fs.normalizeBuf(&check_browser_map_buf, input_path);
+ var cleaned = r.fs.normalizeBuf(bufs(.check_browser_map), input_path);
if (cleaned.len == 1 and cleaned[0] == '.') {
// No bundler supports remapping ".", so we don't either
@@ -2860,11 +2877,12 @@ pub const Resolver = struct {
// First try the import path as a package path
if (isPackagePath(checker.input_path)) {
+ var abs_to_rel = bufs(.abs_to_rel);
switch (comptime kind) {
.AbsolutePath => {
- BrowserMapPath.abs_to_rel_buf[0..2].* = "./".*;
- bun.copy(u8, BrowserMapPath.abs_to_rel_buf[2..], checker.input_path);
- if (checker.checkPath(BrowserMapPath.abs_to_rel_buf[0 .. checker.input_path.len + 2])) {
+ abs_to_rel[0..2].* = "./".*;
+ bun.copy(u8, abs_to_rel[2..], checker.input_path);
+ if (checker.checkPath(abs_to_rel[0 .. checker.input_path.len + 2])) {
return checker.remapped;
}
},
@@ -2882,10 +2900,10 @@ pub const Resolver = struct {
};
if (isInSamePackage) {
- BrowserMapPath.abs_to_rel_buf[0..2].* = "./".*;
- bun.copy(u8, BrowserMapPath.abs_to_rel_buf[2..], checker.input_path);
+ abs_to_rel[0..2].* = "./".*;
+ bun.copy(u8, abs_to_rel[2..], checker.input_path);
- if (checker.checkPath(BrowserMapPath.abs_to_rel_buf[0 .. checker.input_path.len + 2])) {
+ if (checker.checkPath(abs_to_rel[0 .. checker.input_path.len + 2])) {
return checker.remapped;
}
}
@@ -2939,7 +2957,7 @@ pub const Resolver = struct {
}
}
const _paths = [_]string{ path, field_rel_path };
- const field_abs_path = r.fs.absBuf(&_paths, &field_abs_path_buf);
+ const field_abs_path = r.fs.absBuf(&_paths, bufs(.field_abs_path));
// Is this a file?
if (r.loadAsFile(field_abs_path, extension_order)) |result| {
@@ -2972,7 +2990,9 @@ pub const Resolver = struct {
var rfs = &r.fs.fs;
// Try the "index" file with extensions
for (extension_order) |ext| {
- var base = TemporaryBuffer.ExtensionPathBuf[0 .. "index".len + ext.len];
+ var ext_buf = bufs(.extension_path);
+
+ var base = ext_buf[0 .. "index".len + ext.len];
base[0.."index".len].* = "index".*;
bun.copy(u8, base["index".len..], ext);
@@ -2982,7 +3002,7 @@ pub const Resolver = struct {
const out_buf = brk: {
if (lookup.entry.abs_path.isEmpty()) {
const parts = [_]string{ dir_info.abs_path, base };
- const out_buf_ = r.fs.absBuf(&parts, &index_buf);
+ const out_buf_ = r.fs.absBuf(&parts, bufs(.index));
lookup.entry.abs_path =
PathString.init(r.fs.dirname_store.append(@TypeOf(out_buf_), out_buf_) catch unreachable);
}
@@ -3024,10 +3044,11 @@ pub const Resolver = struct {
// In order for our path handling logic to be correct, it must end with a trailing slash.
var path = path_;
if (!strings.endsWithChar(path_, std.fs.path.sep)) {
- bun.copy(u8, &remap_path_trailing_slash, path);
- remap_path_trailing_slash[path.len] = std.fs.path.sep;
- remap_path_trailing_slash[path.len + 1] = 0;
- path = remap_path_trailing_slash[0 .. path.len + 1];
+ var path_buf = bufs(.remap_path_trailing_slash);
+ bun.copy(u8, path_buf, path);
+ path_buf[path.len] = std.fs.path.sep;
+ path_buf[path.len + 1] = 0;
+ path = path_buf[0 .. path.len + 1];
}
if (r.care_about_browser_field) {
@@ -3044,7 +3065,7 @@ pub const Resolver = struct {
// Is the path disabled?
if (remap.len == 0) {
const paths = [_]string{ path, field_rel_path };
- const new_path = r.fs.absBuf(&paths, &remap_path_buf);
+ const new_path = r.fs.absBuf(&paths, bufs(.remap_path));
var _path = Path.init(new_path);
_path.is_disabled = true;
return MatchResult{
@@ -3056,7 +3077,7 @@ pub const Resolver = struct {
}
const new_paths = [_]string{ path, remap };
- const remapped_abs = r.fs.absBuf(&new_paths, &remap_path_buf);
+ const remapped_abs = r.fs.absBuf(&new_paths, bufs(.remap_path));
// Is this a file
if (r.loadAsFile(remapped_abs, extension_order)) |file_result| {
@@ -3283,7 +3304,7 @@ pub const Resolver = struct {
const abs_path = brk: {
if (query.entry.abs_path.isEmpty()) {
const abs_path_parts = [_]string{ query.entry.dir, query.entry.base() };
- query.entry.abs_path = PathString.init(r.fs.dirname_store.append(string, r.fs.absBuf(&abs_path_parts, &load_as_file_buf)) catch unreachable);
+ query.entry.abs_path = PathString.init(r.fs.dirname_store.append(string, r.fs.absBuf(&abs_path_parts, bufs(.load_as_file))) catch unreachable);
}
break :brk query.entry.abs_path.slice();
@@ -3299,9 +3320,9 @@ pub const Resolver = struct {
}
// Try the path with extensions
- bun.copy(u8, &load_as_file_buf, path);
+ bun.copy(u8, bufs(.load_as_file), path);
for (extension_order) |ext| {
- var buffer = load_as_file_buf[0 .. path.len + ext.len];
+ var buffer = bufs(.load_as_file)[0 .. path.len + ext.len];
bun.copy(u8, buffer[path.len..], ext);
const file_name = buffer[path.len - base.len .. buffer.len];
@@ -3352,7 +3373,7 @@ pub const Resolver = struct {
const ext = base[last_dot..base.len];
if (strings.eqlComptime(ext, ".js") or strings.eqlComptime(ext, ".jsx")) {
const segment = base[0..last_dot];
- var tail = load_as_file_buf[path.len - base.len ..];
+ var tail = bufs(.load_as_file)[path.len - base.len ..];
bun.copy(u8, tail, segment);
const exts = .{ ".ts", ".tsx" };
@@ -3460,7 +3481,7 @@ pub const Resolver = struct {
const this_dir = std.fs.Dir{ .fd = fd };
var file = this_dir.openDirZ("node_modules/.bin", .{}, true) catch break :append_bin_dir;
defer file.close();
- var bin_path = bun.getFdPath(file.fd, &node_bin_path) catch break :append_bin_dir;
+ var bin_path = bun.getFdPath(file.fd, bufs(.node_bin_path)) catch break :append_bin_dir;
bin_folders_lock.lock();
defer bin_folders_lock.unlock();
@@ -3485,7 +3506,7 @@ pub const Resolver = struct {
const this_dir = std.fs.Dir{ .fd = fd };
var file = this_dir.openDirZ(".bin", .{}, false) catch break :append_bin_dir;
defer file.close();
- var bin_path = bun.getFdPath(file.fd, &node_bin_path) catch break :append_bin_dir;
+ var bin_path = bun.getFdPath(file.fd, bufs(.node_bin_path)) catch break :append_bin_dir;
bin_folders_lock.lock();
defer bin_folders_lock.unlock();
@@ -3540,7 +3561,7 @@ pub const Resolver = struct {
} else if (parent.?.abs_real_path.len > 0) {
// this might leak a little i'm not sure
const parts = [_]string{ parent.?.abs_real_path, base };
- symlink = r.fs.dirname_store.append(string, r.fs.absBuf(&parts, &dir_info_uncached_filename_buf)) catch unreachable;
+ symlink = r.fs.dirname_store.append(string, r.fs.absBuf(&parts, bufs(.dir_info_uncached_filename))) catch unreachable;
if (r.debug_logs) |*logs| {
logs.addNote(std.fmt.allocPrint(r.allocator, "Resolved symlink \"{s}\" to \"{s}\"", .{ path, symlink }) catch unreachable);
@@ -3592,7 +3613,7 @@ pub const Resolver = struct {
if (entry.kind(rfs) == .file) {
const parts = [_]string{ path, "tsconfig.json" };
- tsconfig_path = r.fs.absBuf(&parts, &dir_info_uncached_filename_buf);
+ tsconfig_path = r.fs.absBuf(&parts, bufs(.dir_info_uncached_filename));
}
}
if (tsconfig_path == null) {
@@ -3600,7 +3621,7 @@ pub const Resolver = struct {
const entry = lookup.entry;
if (entry.kind(rfs) == .file) {
const parts = [_]string{ path, "jsconfig.json" };
- tsconfig_path = r.fs.absBuf(&parts, &dir_info_uncached_filename_buf);
+ tsconfig_path = r.fs.absBuf(&parts, bufs(.dir_info_uncached_filename));
}
}
}
@@ -3629,7 +3650,7 @@ pub const Resolver = struct {
while (current.extends.len > 0) {
var ts_dir_name = Dirname.dirname(current.abs_path);
// not sure why this needs cwd but we'll just pass in the dir of the tsconfig...
- var abs_path = ResolvePath.joinAbsStringBuf(ts_dir_name, &tsconfig_path_abs_buf, &[_]string{ ts_dir_name, current.extends }, .auto);
+ var abs_path = ResolvePath.joinAbsStringBuf(ts_dir_name, bufs(.tsconfig_path_abs), &[_]string{ ts_dir_name, current.extends }, .auto);
var parent_config_maybe = try r.parseTSConfig(abs_path, 0);
if (parent_config_maybe) |parent_config| {
try parent_configs.append(parent_config);
diff --git a/src/router.zig b/src/router.zig
index 926e8ea7e..5b4c56722 100644
--- a/src/router.zig
+++ b/src/router.zig
@@ -941,7 +941,7 @@ pub const Test = struct {
const JSAst = bun.JSAst;
JSAst.Expr.Data.Store.create(default_allocator);
JSAst.Stmt.Data.Store.create(default_allocator);
- var fs = try FileSystem.init1(default_allocator, null);
+ var fs = try FileSystem.init(null);
var top_level_dir = fs.top_level_dir;
var pages_parts = [_]string{ top_level_dir, "pages" };
@@ -998,7 +998,7 @@ pub const Test = struct {
const JSAst = bun.JSAst;
JSAst.Expr.Data.Store.create(default_allocator);
JSAst.Stmt.Data.Store.create(default_allocator);
- var fs = try FileSystem.init1WithForce(default_allocator, null, true);
+ var fs = try FileSystem.initWithForce(null, true);
var top_level_dir = fs.top_level_dir;
var pages_parts = [_]string{ top_level_dir, "pages" };
diff --git a/src/thread_pool.zig b/src/thread_pool.zig
index f287dd866..9b6951cbb 100644
--- a/src/thread_pool.zig
+++ b/src/thread_pool.zig
@@ -63,6 +63,11 @@ pub fn init(config: Config) ThreadPool {
};
}
+pub fn wakeForIdleEvents(this: *ThreadPool) void {
+ // Wake all the threads to check for idle events.
+ this.idle_event.wake(Event.NOTIFIED, std.math.maxInt(u32));
+}
+
/// Wait for a thread to call shutdown() on the thread pool and kill the worker threads.
pub fn deinit(self: *ThreadPool) void {
self.join();
@@ -483,8 +488,6 @@ noinline fn notifySlow(self: *ThreadPool, is_waking: bool) void {
}
}
-// sleep_on_idle seems to impact `bun install` performance negatively
-// so we can just not sleep for that
noinline fn wait(self: *ThreadPool, _is_waking: bool) error{Shutdown}!bool {
var is_idle = false;
var is_waking = _is_waking;
@@ -530,6 +533,10 @@ noinline fn wait(self: *ThreadPool, _is_waking: bool) error{Shutdown}!bool {
continue;
});
} else {
+ if (Thread.current) |current| {
+ current.drainIdleEvents();
+ }
+
self.idle_event.wait();
sync = @bitCast(Sync, self.sync.load(.Monotonic));
}
@@ -574,6 +581,17 @@ fn register(noalias self: *ThreadPool, noalias thread: *Thread) void {
}
}
+pub fn setThreadContext(noalias pool: *ThreadPool, ctx: ?*anyopaque) void {
+ pool.threadpool_context = ctx;
+
+ var thread = pool.threads.load(.Monotonic) orelse return;
+ thread.ctx = pool.threadpool_context;
+ while (thread.next) |next| {
+ next.ctx = pool.threadpool_context;
+ thread = next;
+ }
+}
+
fn unregister(noalias self: *ThreadPool, noalias maybe_thread: ?*Thread) void {
// Un-spawn one thread, either due to a failed OS thread spawning or the thread is exiting.
const one_spawned = @bitCast(u32, Sync{ .spawned = 1 });
@@ -621,25 +639,35 @@ pub const Thread = struct {
target: ?*Thread = null,
join_event: Event = .{},
run_queue: Node.Queue = .{},
+ idle_queue: Node.Queue = .{},
run_buffer: Node.Buffer = .{},
ctx: ?*anyopaque = null,
pub threadlocal var current: ?*Thread = null;
+ pub fn pushIdleTask(self: *Thread, task: *Task) void {
+ const list = Node.List{
+ .head = &task.node,
+ .tail = &task.node,
+ };
+ self.idle_queue.push(list);
+ }
+
/// Thread entry point which runs a worker for the ThreadPool
fn run(thread_pool: *ThreadPool) void {
- Output.Source.configureThread();
+ Output.Source.configureNamedThread("Bun Pool");
- var self = Thread{};
- current = &self;
+ var self_ = Thread{};
+ var self = &self_;
+ current = self;
if (thread_pool.on_thread_spawn) |spawn| {
current.?.ctx = spawn(thread_pool.threadpool_context);
}
- thread_pool.register(&self);
+ thread_pool.register(self);
- defer thread_pool.unregister(&self);
+ defer thread_pool.unregister(self);
var is_waking = false;
while (true) {
@@ -653,13 +681,25 @@ pub const Thread = struct {
const task = @fieldParentPtr(Task, "node", result.node);
(task.callback)(task);
}
+
Output.flush();
+
+ self.drainIdleEvents();
+ }
+ }
+
+ pub fn drainIdleEvents(noalias self: *Thread) void {
+ var consumer = self.idle_queue.tryAcquireConsumer() catch return;
+ defer self.idle_queue.releaseConsumer(consumer);
+ while (self.idle_queue.pop(&consumer)) |node| {
+ const task = @fieldParentPtr(Task, "node", node);
+ (task.callback)(task);
}
}
/// Try to dequeue a Node/Task from the ThreadPool.
/// Spurious reports of dequeue() returning empty are allowed.
- fn pop(noalias self: *Thread, noalias thread_pool: *ThreadPool) ?Node.Buffer.Stole {
+ pub fn pop(noalias self: *Thread, noalias thread_pool: *ThreadPool) ?Node.Buffer.Stole {
// Check our local buffer first
if (self.run_buffer.pop()) |node| {
return Node.Buffer.Stole{
@@ -714,7 +754,7 @@ const Event = struct {
const EMPTY = 0;
const WAITING = 1;
- const NOTIFIED = 2;
+ pub const NOTIFIED = 2;
const SHUTDOWN = 3;
/// Wait for and consume a notification
diff --git a/src/work_pool.zig b/src/work_pool.zig
index f23becdfc..cb59c0143 100644
--- a/src/work_pool.zig
+++ b/src/work_pool.zig
@@ -13,7 +13,7 @@ pub fn NewWorkPool(comptime max_threads: ?usize) type {
@setCold(true);
pool = ThreadPool.init(.{
- .max_threads = max_threads orelse @floatToInt(u32, @floor(@intToFloat(f32, @max(std.Thread.getCpuCount() catch 0, 2)) * 0.8)),
+ .max_threads = max_threads orelse @max(@truncate(u32, std.Thread.getCpuCount() catch 0), 2),
.stack_size = 2 * 1024 * 1024,
});
return &pool;