diff options
author | 2023-04-27 03:45:49 -0700 | |
---|---|---|
committer | 2023-04-27 03:45:49 -0700 | |
commit | df59fe28431837b28e11ee3031667b595f14403e (patch) | |
tree | e3c8282e6682e8f705849e6a9fdb56872387f07c | |
parent | 990f53f98619061b170e265d4f3e06146f39135f (diff) | |
download | bun-df59fe28431837b28e11ee3031667b595f14403e.tar.gz bun-df59fe28431837b28e11ee3031667b595f14403e.tar.zst bun-df59fe28431837b28e11ee3031667b595f14403e.zip |
Implement `outdir` in `Bun.build`
Diffstat (limited to '')
-rw-r--r-- | src/bun.js/base.zig | 8 | ||||
-rw-r--r-- | src/bun.js/webcore/blob.zig | 2 | ||||
-rw-r--r-- | src/bun.zig | 2 | ||||
-rw-r--r-- | src/bundler/bundle_v2.zig | 282 | ||||
-rw-r--r-- | src/cli/build_command.zig | 8 | ||||
-rw-r--r-- | src/http.zig | 2 | ||||
-rw-r--r-- | src/max_heap_allocator.zig | 51 | ||||
-rw-r--r-- | src/options.zig | 33 |
8 files changed, 362 insertions, 26 deletions
diff --git a/src/bun.js/base.zig b/src/bun.js/base.zig index cbd80db87..da9501541 100644 --- a/src/bun.js/base.zig +++ b/src/bun.js/base.zig @@ -1729,10 +1729,10 @@ pub const JSStringList = std.ArrayList(js.JSStringRef); pub const ArrayBuffer = extern struct { ptr: [*]u8 = undefined, - offset: u32, - len: u32, - byte_len: u32, - typed_array_type: JSC.JSValue.JSType, + offset: u32 = 0, + len: u32 = 0, + byte_len: u32 = 0, + typed_array_type: JSC.JSValue.JSType = .Cell, value: JSC.JSValue = JSC.JSValue.zero, shared: bool = false, diff --git a/src/bun.js/webcore/blob.zig b/src/bun.js/webcore/blob.zig index 4ac48f25f..c5e07b13c 100644 --- a/src/bun.js/webcore/blob.zig +++ b/src/bun.js/webcore/blob.zig @@ -2117,7 +2117,7 @@ pub const Blob = struct { } if (os.S.ISREG(stat.mode) and - this.max_length > std.mem.page_size and + this.max_length > bun.C.preallocate_length and this.max_length != Blob.max_size) { bun.C.preallocate_file(this.destination_fd, 0, this.max_length) catch {}; diff --git a/src/bun.zig b/src/bun.zig index 336757ded..d640b4a77 100644 --- a/src/bun.zig +++ b/src/bun.zig @@ -1360,3 +1360,5 @@ pub fn threadlocalAllocator() std.mem.Allocator { return default_allocator; } + +pub const MaxHeapAllocator = @import("./max_heap_allocator.zig").MaxHeapAllocator; diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index 6bb19587a..3f0c08a65 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -7861,30 +7861,271 @@ const LinkerContext = struct { )) catch unreachable; output_files.items.len = chunks.len; + const root_path = c.resolver.opts.output_dir; + + if (root_path.len > 0) { + try c.writeOutputFilesToDisk(root_path, chunks, react_client_components_manifest, &output_files); + } else { + // In-memory build + for (chunks, output_files.items) |*chunk, *output_file| { + const buffer = chunk.intermediate_output.code( + null, + c.parse_graph, + c.resolver.opts.public_path, + chunk, + chunks, + ) catch @panic("Failed to allocate memory for output file"); + output_file.* = options.OutputFile.initBuf( + buffer, + Chunk.IntermediateOutput.allocatorForSize(buffer.len), + // clone for main thread + bun.default_allocator.dupe(u8, chunk.final_rel_path) catch unreachable, + // TODO: remove this field + .js, + ); + } + + if (react_client_components_manifest.len > 0) { + output_files.appendAssumeCapacity(options.OutputFile.initBuf( + react_client_components_manifest, + bun.default_allocator, + components_manifest_path, + .file, + )); + } + + output_files.appendSliceAssumeCapacity(c.parse_graph.additional_output_files.items); + } + + return output_files; + } + + fn writeOutputFilesToDisk( + c: *LinkerContext, + root_path: string, + chunks: []Chunk, + react_client_components_manifest: []const u8, + output_files: *std.ArrayList(options.OutputFile), + ) !void { + var root_dir = std.fs.cwd().makeOpenPathIterable(root_path, .{}) catch |err| { + c.log.addErrorFmt(null, Logger.Loc.Empty, bun.default_allocator, "{s} opening outdir {}", .{ + @errorName(err), + bun.fmt.quote(root_path), + }) catch unreachable; + return err; + }; + defer root_dir.close(); + const from_path: []const u8 = brk: { + var all_paths = c.allocator.alloc( + []const u8, + chunks.len + + @as( + usize, + @boolToInt( + react_client_components_manifest.len > 0, + ), + ) + + c.parse_graph.additional_output_files.items.len, + ) catch unreachable; + defer c.allocator.free(all_paths); + + var remaining_paths = all_paths; + + for (all_paths[0..chunks.len], chunks) |*dest, src| { + dest.* = src.final_rel_path; + } + remaining_paths = remaining_paths[chunks.len..]; + + if (react_client_components_manifest.len > 0) { + remaining_paths[0] = components_manifest_path; + remaining_paths = remaining_paths[1..]; + } + + for (remaining_paths, c.parse_graph.additional_output_files.items) |*dest, output_file| { + dest.* = output_file.input.text; + } + + remaining_paths = remaining_paths[c.parse_graph.additional_output_files.items.len..]; + + std.debug.assert(remaining_paths.len == 0); + + break :brk resolve_path.longestCommonPath(all_paths); + }; + + // Optimization: when writing to disk, we can re-use the memory + var max_heap_allocator: bun.MaxHeapAllocator = undefined; + const code_allocator = max_heap_allocator.init(bun.default_allocator); + defer max_heap_allocator.deinit(); + + var pathbuf: [bun.MAX_PATH_BYTES]u8 = undefined; + for (chunks, output_files.items) |*chunk, *output_file| { - const buffer = chunk.intermediate_output.code(c.parse_graph, c.resolver.opts.public_path, chunk, chunks) catch @panic("Failed to allocate memory for output file"); - output_file.* = options.OutputFile.initBuf( - buffer, - Chunk.IntermediateOutput.allocatorForSize(buffer.len), - // clone for main thread - bun.default_allocator.dupe(u8, chunk.final_rel_path) catch unreachable, - // TODO: remove this field - .js, - ); + defer max_heap_allocator.reset(); + + var rel_path = chunk.final_rel_path; + if (rel_path.len > from_path.len) { + rel_path = resolve_path.relative(from_path, rel_path); + if (std.fs.path.dirname(rel_path)) |parent| { + if (parent.len > root_path.len) { + root_dir.dir.makePath(parent) catch |err| { + c.log.addErrorFmt(null, Logger.Loc.Empty, bun.default_allocator, "{s} creating outdir {} while saving chunk {}", .{ + @errorName(err), + bun.fmt.quote(parent), + bun.fmt.quote(chunk.final_rel_path), + }) catch unreachable; + return err; + }; + } + } + } + + const buffer = chunk.intermediate_output.code( + code_allocator, + c.parse_graph, + c.resolver.opts.public_path, + chunk, + chunks, + ) catch @panic("Failed to allocate memory for output chunk"); + + switch (JSC.Node.NodeFS.writeFileWithPathBuffer( + &pathbuf, + JSC.Node.Arguments.WriteFile{ + .data = JSC.Node.StringOrBuffer{ + .buffer = JSC.Buffer{ + .buffer = .{ + .ptr = @constCast(buffer.ptr), + // TODO: handle > 4 GB files + .len = @truncate(u32, buffer.len), + .byte_len = @truncate(u32, buffer.len), + }, + }, + }, + .encoding = .buffer, + .dirfd = @intCast(bun.FileDescriptor, root_dir.dir.fd), + .file = .{ + .path = JSC.Node.PathLike{ + .string = JSC.PathString.init(rel_path), + }, + }, + }, + )) { + .err => |err| { + c.log.addErrorFmt(null, Logger.Loc.Empty, bun.default_allocator, "{} writing chunk {}", .{ + bun.fmt.quote(err.toSystemError().message.slice()), + bun.fmt.quote(chunk.final_rel_path), + }) catch unreachable; + return error.WriteFailed; + }, + .result => {}, + } + + output_file.* = options.OutputFile{ + .input = Fs.Path.init(chunk.final_rel_path), + .loader = .js, + .size = @truncate(u32, buffer.len), + .value = .{ + .saved = .{}, + }, + }; } if (react_client_components_manifest.len > 0) { - output_files.appendAssumeCapacity(options.OutputFile.initBuf( - react_client_components_manifest, - bun.default_allocator, - "./components-manifest.blob", - .file, - )); + switch (JSC.Node.NodeFS.writeFileWithPathBuffer( + &pathbuf, + JSC.Node.Arguments.WriteFile{ + .data = JSC.Node.StringOrBuffer{ + .buffer = JSC.Buffer{ + .buffer = .{ + .ptr = @constCast(react_client_components_manifest.ptr), + // TODO: handle > 4 GB files + .len = @truncate(u32, react_client_components_manifest.len), + .byte_len = @truncate(u32, react_client_components_manifest.len), + }, + }, + }, + .encoding = .buffer, + .dirfd = @intCast(bun.FileDescriptor, root_dir.dir.fd), + .file = .{ + .path = JSC.Node.PathLike{ + .string = JSC.PathString.init(components_manifest_path), + }, + }, + }, + )) { + .err => |err| { + c.log.addErrorFmt(null, Logger.Loc.Empty, bun.default_allocator, "{} writing chunk {}", .{ + bun.fmt.quote(err.toSystemError().message.slice()), + bun.fmt.quote(components_manifest_path), + }) catch unreachable; + return error.WriteFailed; + }, + .result => {}, + } + + output_files.appendAssumeCapacity(options.OutputFile{ + .input = Fs.Path.init(components_manifest_path), + .loader = .file, + .size = @truncate(u32, react_client_components_manifest.len), + .value = .{ + .saved = .{}, + }, + }); } - output_files.appendSliceAssumeCapacity(c.parse_graph.additional_output_files.items); + { + const offset = output_files.items.len; + output_files.items.len += c.parse_graph.additional_output_files.items.len; + + for (c.parse_graph.additional_output_files.items, output_files.items[offset..][0..c.parse_graph.additional_output_files.items.len]) |*src, *dest| { + const bytes = src.value.buffer.bytes; + src.value.buffer.bytes.len = 0; - return output_files; + defer { + src.value.buffer.allocator.free(bytes); + } + + switch (JSC.Node.NodeFS.writeFileWithPathBuffer( + &pathbuf, + JSC.Node.Arguments.WriteFile{ + .data = JSC.Node.StringOrBuffer{ + .buffer = JSC.Buffer{ + .buffer = .{ + .ptr = @constCast(bytes.ptr), + // TODO: handle > 4 GB files + .len = @truncate(u32, bytes.len), + .byte_len = @truncate(u32, bytes.len), + }, + }, + }, + .encoding = .buffer, + .dirfd = @intCast(bun.FileDescriptor, root_dir.dir.fd), + .file = .{ + .path = JSC.Node.PathLike{ + .string = JSC.PathString.init(src.input.text), + }, + }, + }, + )) { + .err => |err| { + c.log.addErrorFmt(null, Logger.Loc.Empty, bun.default_allocator, "{} writing chunk {}", .{ + bun.fmt.quote(err.toSystemError().message.slice()), + bun.fmt.quote(src.input.text), + }) catch unreachable; + return error.WriteFailed; + }, + .result => {}, + } + + dest.* = .{ + .input = src.input, + .loader = src.loader, + .size = @truncate(u32, bytes.len), + .value = .{ + .saved = .{}, + }, + }; + } + } } // Sort cross-chunk exports by chunk name for determinism @@ -9245,6 +9486,7 @@ pub const Chunk = struct { pub fn code( this: IntermediateOutput, + allocator_to_use: ?std.mem.Allocator, graph: *const Graph, import_prefix: []const u8, chunk: *Chunk, @@ -9285,7 +9527,7 @@ pub const Chunk = struct { } } - var total_buf = try allocatorForSize(count).alloc(u8, count); + var total_buf = try (allocator_to_use orelse allocatorForSize(count)).alloc(u8, count); var remain = total_buf; for (pieces.slice()) |piece| { @@ -9335,7 +9577,7 @@ pub const Chunk = struct { .joiner => |joiner_| { // TODO: make this safe var joiny = joiner_; - return joiny.done(allocatorForSize(joiny.len)); + return joiny.done((allocator_to_use orelse allocatorForSize(joiny.len))); }, .empty => return "", } @@ -9555,3 +9797,5 @@ fn cheapPrefixNormalizer(prefix: []const u8, suffix: []const u8) [2]string { return .{ prefix, suffix }; } + +const components_manifest_path = "./components-manifest.blob"; diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index d6ca3236d..196d43f23 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -191,6 +191,14 @@ pub const BuildCommand = struct { for (output_files) |f| { var rel_path: []const u8 = undefined; switch (f.value) { + // Nothing to do in this case + .saved => { + rel_path = f.input.text; + if (f.input.text.len > from_path.len) { + rel_path = resolve_path.relative(from_path, f.input.text); + } + }, + // easy mode: write the buffer .buffer => |value| { rel_path = f.input.text; diff --git a/src/http.zig b/src/http.zig index cd5e386a1..cf58e5941 100644 --- a/src/http.zig +++ b/src/http.zig @@ -2243,6 +2243,8 @@ pub const RequestContext = struct { const send_body = ctx.method.hasBody(); switch (result.file.value) { + .saved => {}, + .pending => |resolve_result| { const path = resolve_result.pathConst() orelse { try ctx.sendNoContent(); diff --git a/src/max_heap_allocator.zig b/src/max_heap_allocator.zig new file mode 100644 index 000000000..ea616c9ca --- /dev/null +++ b/src/max_heap_allocator.zig @@ -0,0 +1,51 @@ +const bun = @import("root").bun; +const std = @import("std"); + +/// Single allocation only. +/// +pub const MaxHeapAllocator = struct { + array_list: std.ArrayList(u8), + + fn alloc(ptr: *anyopaque, len: usize, _: u8, _: usize) ?[*]u8 { + var this = bun.cast(*MaxHeapAllocator, ptr); + this.array_list.items.len = 0; + this.array_list.ensureTotalCapacity(len) catch return null; + this.array_list.items.len = len; + return this.array_list.items.ptr; + } + + fn resize(_: *anyopaque, buf: []u8, _: u8, new_len: usize, _: usize) bool { + _ = new_len; + _ = buf; + @panic("not implemented"); + } + + fn free( + _: *anyopaque, + _: []u8, + _: u8, + _: usize, + ) void {} + + pub fn reset(this: *MaxHeapAllocator) void { + this.array_list.items.len = 0; + } + + pub fn deinit(this: *MaxHeapAllocator) void { + this.array_list.deinit(); + } + + const vtable = std.mem.Allocator.VTable{ + .alloc = &alloc, + .free = &free, + .resize = &resize, + }; + pub fn init(this: *MaxHeapAllocator, allocator: std.mem.Allocator) std.mem.Allocator { + this.array_list = std.ArrayList(u8).init(allocator); + + return std.mem.Allocator{ + .ptr = this, + .vtable = &vtable, + }; + } +}; diff --git a/src/options.zig b/src/options.zig index 38cf09819..a1aeba88e 100644 --- a/src/options.zig +++ b/src/options.zig @@ -2014,14 +2014,42 @@ pub const OutputFile = struct { allocator: std.mem.Allocator, bytes: []const u8, }, - + saved: SavedFile, move: FileOperation, copy: FileOperation, noop: u0, pending: resolver.Result, }; - pub const Kind = enum { move, copy, noop, buffer, pending }; + pub const SavedFile = struct { + pub fn toJS( + globalThis: *JSC.JSGlobalObject, + path: []const u8, + byte_size: usize, + ) JSC.JSValue { + const mime_type = globalThis.bunVM().mimeType(path); + const store = JSC.WebCore.Blob.Store.initFile( + JSC.Node.PathOrFileDescriptor{ + .path = JSC.Node.PathLike{ + .string = JSC.PathString.init(path), + }, + }, + mime_type, + bun.default_allocator, + ) catch unreachable; + + var blob = bun.default_allocator.create(JSC.WebCore.Blob) catch unreachable; + blob.* = JSC.WebCore.Blob.initWithStore(store, globalThis); + if (mime_type) |mime| { + blob.content_type = mime.value; + } + blob.size = @truncate(JSC.WebCore.Blob.SizeType, byte_size); + blob.allocator = bun.default_allocator; + return blob.toJS(globalThis); + } + }; + + pub const Kind = enum { move, copy, noop, buffer, pending, saved }; pub fn initPending(loader: Loader, pending: resolver.Result) OutputFile { return .{ @@ -2099,6 +2127,7 @@ pub const OutputFile = struct { .noop => JSC.JSValue.undefined, .move => this.value.move.toJS(globalObject, this.loader), .copy => this.value.copy.toJS(globalObject, this.loader), + .saved => SavedFile.toJS(globalObject, this.input.text, this.size), .buffer => |buffer| brk: { var blob = bun.default_allocator.create(JSC.WebCore.Blob) catch unreachable; blob.* = JSC.WebCore.Blob.init(@constCast(buffer.bytes), buffer.allocator, globalObject); |