diff options
author | 2022-03-07 00:33:49 -0800 | |
---|---|---|
committer | 2022-03-07 00:33:49 -0800 | |
commit | d43a6455359062b2c67f914d2f56d7d1ce3044a7 (patch) | |
tree | 456b6a9359f10cc2aae5f53fe18a8dd223969929 | |
parent | 50d5b872f9a008be02b3c36a7fa34cabef5718f6 (diff) | |
download | bun-d43a6455359062b2c67f914d2f56d7d1ce3044a7.tar.gz bun-d43a6455359062b2c67f914d2f56d7d1ce3044a7.tar.zst bun-d43a6455359062b2c67f914d2f56d7d1ce3044a7.zip |
source maps optimizations
-rw-r--r-- | src/api/schema.d.ts | 11 | ||||
-rw-r--r-- | src/api/schema.js | 29 | ||||
-rw-r--r-- | src/api/schema.peechy | 6 | ||||
-rw-r--r-- | src/api/schema.zig | 25 | ||||
-rw-r--r-- | src/bunfig.zig | 5 | ||||
-rw-r--r-- | src/cli.zig | 65 | ||||
-rw-r--r-- | src/cli/build_command.zig | 7 | ||||
-rw-r--r-- | src/fs.zig | 6 | ||||
-rw-r--r-- | src/http.zig | 23 | ||||
-rw-r--r-- | src/sourcemap/sourcemap.zig | 255 | ||||
-rw-r--r-- | src/string_immutable.zig | 111 |
11 files changed, 458 insertions, 85 deletions
diff --git a/src/api/schema.d.ts b/src/api/schema.d.ts index 0a7ed3b9b..3e8474787 100644 --- a/src/api/schema.d.ts +++ b/src/api/schema.d.ts @@ -194,6 +194,16 @@ export const DotEnvBehaviorKeys = { 3: "load_all", load_all: "load_all", }; +export enum SourceMapMode { + inline_into_file = 1, + external = 2, +} +export const SourceMapModeKeys = { + 1: "inline_into_file", + inline_into_file: "inline_into_file", + 2: "external", + external: "external", +}; export enum ImportKind { entry_point = 1, stmt = 2, @@ -520,6 +530,7 @@ export interface TransformOptions { disable_hmr?: boolean; port?: uint16; logLevel?: MessageLevel; + source_map?: SourceMapMode; } export interface FileHandle { diff --git a/src/api/schema.js b/src/api/schema.js index b89ed4a08..4dfa2e245 100644 --- a/src/api/schema.js +++ b/src/api/schema.js @@ -1718,6 +1718,10 @@ function decodeTransformOptions(bb) { result["logLevel"] = MessageLevel[bb.readVarUint()]; break; + case 27: + result["source_map"] = SourceMapMode[bb.readByte()]; + break; + default: throw new Error("Attempted to parse invalid message"); } @@ -1925,8 +1929,31 @@ function encodeTransformOptions(message, bb) { ); bb.writeVarUint(encoded); } + + var value = message["source_map"]; + if (value != null) { + bb.writeByte(27); + var encoded = SourceMapMode[value]; + if (encoded === void 0) + throw new Error( + "Invalid value " + JSON.stringify(value) + ' for enum "SourceMapMode"' + ); + bb.writeByte(encoded); + } bb.writeByte(0); } +const SourceMapMode = { + 1: 1, + 2: 2, + inline_into_file: 1, + external: 2, +}; +const SourceMapModeKeys = { + 1: "inline_into_file", + 2: "external", + inline_into_file: "inline_into_file", + external: "external", +}; function decodeFileHandle(bb) { var result = {}; @@ -3272,6 +3299,8 @@ export { decodeRouteConfig }; export { encodeRouteConfig }; export { decodeTransformOptions }; export { encodeTransformOptions }; +export { SourceMapMode }; +export { SourceMapModeKeys }; export { decodeFileHandle }; export { encodeFileHandle }; export { decodeTransform }; diff --git a/src/api/schema.peechy b/src/api/schema.peechy index cf954cc4c..9fe2fdc7a 100644 --- a/src/api/schema.peechy +++ b/src/api/schema.peechy @@ -345,6 +345,12 @@ message TransformOptions { uint16 port = 25; MessageLevel logLevel = 26; + SourceMapMode source_map = 27; +} + +smol SourceMapMode { + inline_into_file = 1; + external = 2; } struct FileHandle { diff --git a/src/api/schema.zig b/src/api/schema.zig index 5f4532522..2374103eb 100644 --- a/src/api/schema.zig +++ b/src/api/schema.zig @@ -1744,6 +1744,9 @@ pub const Api = struct { /// logLevel log_level: ?MessageLevel = null, + /// source_map + source_map: ?SourceMapMode = null, + pub fn decode(reader: anytype) anyerror!TransformOptions { var this = std.mem.zeroes(TransformOptions); @@ -1831,6 +1834,9 @@ pub const Api = struct { 26 => { this.log_level = try reader.readValue(MessageLevel); }, + 27 => { + this.source_map = try reader.readValue(SourceMapMode); + }, else => { return error.InvalidMessage; }, @@ -1944,10 +1950,29 @@ pub const Api = struct { try writer.writeFieldID(26); try writer.writeEnum(log_level); } + if (this.source_map) |source_map| { + try writer.writeFieldID(27); + try writer.writeEnum(source_map); + } try writer.endMessage(); } }; + pub const SourceMapMode = enum(u8) { + _none, + /// inline_into_file + inline_into_file, + + /// external + external, + + _, + + pub fn jsonStringify(self: *const @This(), opts: anytype, o: anytype) !void { + return try std.json.stringify(@tagName(self), opts, o); + } + }; + pub const FileHandle = struct { /// path path: []const u8, diff --git a/src/bunfig.zig b/src/bunfig.zig index 2088525b5..a9227755e 100644 --- a/src/bunfig.zig +++ b/src/bunfig.zig @@ -349,6 +349,11 @@ pub const Bunfig = struct { try this.expect(file, .e_string); this.bunfig.node_modules_bundle_path = try file.data.e_string.string(allocator); } + + if (bun.get("outdir")) |dir| { + try this.expect(dir, .e_string); + this.bunfig.output_dir = try dir.data.e_string.string(allocator); + } } if (comptime cmd == .BunCommand) { diff --git a/src/cli.zig b/src/cli.zig index dcef720b0..263308b98 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -196,7 +196,15 @@ pub const Arguments = struct { clap.parseParam("--disable-bun.js Disable bun.js from loading in the dev server") catch unreachable, }; - const params = public_params ++ debug_params; + pub const params = public_params ++ debug_params; + + const build_only_params = [_]ParamType{ + clap.parseParam("--sourcemap <STR>? Build with sourcemaps - 'inline', 'external', or 'none'") catch unreachable, + clap.parseParam("--outdir <STR> Default to \"dist\" if multiple files") catch unreachable, + }; + + const build_params_public = public_params ++ build_only_params; + pub const build_params = build_params_public ++ debug_params; fn printVersionAndExit() noreturn { @setCold(true); @@ -296,19 +304,25 @@ pub const Arguments = struct { try loadConfigPath(allocator, auto_loaded, config_path, ctx, comptime cmd); } - pub fn loadConfigWithCmdArgs(allocator: std.mem.Allocator, args: clap.Args(clap.Help, ¶ms), ctx: *Command.Context, comptime cmd: Command.Tag) !void { + fn loadConfigWithCmdArgs( + comptime cmd: Command.Tag, + allocator: std.mem.Allocator, + args: clap.Args(clap.Help, cmd.params()), + ctx: *Command.Context, + ) !void { return try loadConfig(allocator, args.option("--config"), ctx, comptime cmd); } pub fn parse(allocator: std.mem.Allocator, ctx: *Command.Context, comptime cmd: Command.Tag) !Api.TransformOptions { var diag = clap.Diagnostic{}; + const params_to_use = comptime cmd.params(); - var args = clap.parse(clap.Help, ¶ms, .{ + var args = clap.parse(clap.Help, params_to_use, .{ .diagnostic = &diag, .allocator = allocator, }) catch |err| { // Report useful error and exit - clap.help(Output.errorWriter(), ¶ms) catch {}; + clap.help(Output.errorWriter(), params_to_use) catch {}; Output.errorWriter().writeAll("\n") catch {}; diag.report(Output.errorWriter(), err) catch {}; Global.exit(1); @@ -329,7 +343,7 @@ pub const Arguments = struct { ctx.args.absolute_working_dir = cwd; if (comptime Command.Tag.loads_config.get(cmd)) { - try loadConfigWithCmdArgs(allocator, args, ctx, cmd); + try loadConfigWithCmdArgs(cmd, allocator, args, ctx); } var opts: Api.TransformOptions = ctx.args; @@ -389,7 +403,8 @@ pub const Arguments = struct { const print_help = args.flag("--help"); if (print_help) { - clap.help(Output.writer(), std.mem.span(params[0..public_params.len])) catch {}; + const params_len = if (cmd == .BuildCommand) build_params_public.len else public_params.len; + clap.help(Output.writer(), std.mem.span(params_to_use[0..params_len])) catch {}; Output.prettyln("\n-------\n\n", .{}); Output.flush(); HelpCommand.printWithReason(.explicit); @@ -405,6 +420,27 @@ pub const Arguments = struct { var output_dir: ?string = null; const production = false; + if (cmd == .BuildCommand) { + if (args.option("--outdir")) |outdir| { + if (outdir.len > 0) { + output_dir = outdir; + } + } + + if (args.option("--sourcemap")) |setting| { + if (setting.len == 0 or strings.eqlComptime(setting, "inline")) { + opts.source_map = Api.SourceMapMode.inline_into_file; + } else if (strings.eqlComptime(setting, "none")) { + opts.source_map = Api.SourceMapMode._none; + } else if (strings.eqlComptime(setting, "external")) { + opts.source_map = Api.SourceMapMode.external; + } else { + Output.prettyErrorln("<r><red>error<r>: Invalid sourcemap setting: \"{s}\"", .{setting}); + Global.crash(); + } + } + } + if (opts.entry_points.len == 0) { var entry_points = args.positionals(); @@ -439,12 +475,12 @@ pub const Arguments = struct { entry_points = entry_points[1..]; } - var write = entry_points.len > 1 or output_dir != null; - if (write and output_dir == null) { - var _paths = [_]string{ cwd, "out" }; - output_dir = try std.fs.path.resolve(allocator, &_paths); + opts.write = entry_points.len > 1 or + output_dir != null or + @enumToInt(opts.source_map orelse Api.SourceMapMode._none) > 0; + if ((opts.write orelse false) and (output_dir orelse "").len == 0) { + output_dir = "out"; } - opts.write = write; }, .RunCommand => { if (entry_points.len > 0 and (strings.eqlComptime( @@ -1180,6 +1216,13 @@ pub const Command = struct { PackageManagerCommand, TestCommand, + pub fn params(comptime cmd: Tag) []const Arguments.ParamType { + return &comptime switch (cmd) { + Command.Tag.BuildCommand => Arguments.build_params, + else => Arguments.params, + }; + } + pub fn readGlobalConfig(this: Tag) bool { return switch (this) { .PackageManagerCommand, .InstallCommand, .AddCommand, .RemoveCommand => true, diff --git a/src/cli/build_command.zig b/src/cli/build_command.zig index 86376fdd9..3542af5e4 100644 --- a/src/cli/build_command.zig +++ b/src/cli/build_command.zig @@ -59,16 +59,13 @@ pub const BuildCommand = struct { var did_write = false; defer Output.flush(); - var writer = Output.writer(); + var writer = Output.errorWriter(); var err_writer = writer; - var open_file_limit: usize = 32; + var open_file_limit: usize = fs.FileSystem.RealFS.Limit.handles; if (ctx.args.write) |write| { if (write) { const root_dir = result.root_dir orelse unreachable; - if (std.os.getrlimit(.NOFILE)) |limit| { - open_file_limit = limit.cur; - } else |_| {} var all_paths = try ctx.allocator.alloc([]const u8, result.output_files.len); var max_path_len: usize = 0; diff --git a/src/fs.zig b/src/fs.zig index 57f3855c2..604a106fc 100644 --- a/src/fs.zig +++ b/src/fs.zig @@ -102,6 +102,12 @@ pub const FileSystem = struct { return tmpdir_handle.?; } + pub fn getFdPath(this: *const FileSystem, fd: FileDescriptorType) ![]const u8 { + var buf: [_global.MAX_PATH_BYTES]u8 = undefined; + var dir = try std.os.getFdPath(fd, &buf); + return try this.dirname_store.append([]u8, dir); + } + pub fn tmpname(_: *const FileSystem, extname: string, buf: []u8, hash: u64) ![*:0]u8 { // PRNG was...not so random return try std.fmt.bufPrintZ(buf, ".{x}{s}", .{ @truncate(u64, @intCast(u128, hash) * @intCast(u128, std.time.nanoTimestamp())), extname }); diff --git a/src/http.zig b/src/http.zig index 5f6236915..1016af017 100644 --- a/src/http.zig +++ b/src/http.zig @@ -2192,12 +2192,11 @@ pub const RequestContext = struct { rctx: *RequestContext, _loader: Options.Loader, buffer: MutableString = undefined, - threadlocal var buffer: MutableString = undefined; - threadlocal var has_loaded_buffer: bool = false; + threadlocal var buffer: ?*MutableString = null; pub fn reserveNext(this: *SocketPrinterInternal, count: u32) anyerror![*]u8 { try this.buffer.growIfNeeded(count); - return @ptrCast([*]u8, &this.buffer.list.items.ptr[buffer.list.items.len]); + return @ptrCast([*]u8, &this.buffer.list.items.ptr[this.buffer.list.items.len]); } pub fn advanceBy(this: *SocketPrinterInternal, count: u32) void { @@ -2207,17 +2206,17 @@ pub const RequestContext = struct { } pub fn init(rctx: *RequestContext, _loader: Options.Loader) SocketPrinterInternal { - if (!has_loaded_buffer) { - buffer = MutableString.init(default_allocator, 0) catch unreachable; - has_loaded_buffer = true; + if (buffer == null) { + buffer = default_allocator.create(MutableString) catch unreachable; + buffer.?.* = MutableString.init2048(default_allocator) catch unreachable; } - buffer.reset(); + buffer.?.reset(); return SocketPrinterInternal{ .rctx = rctx, ._loader = _loader, - .buffer = buffer, + .buffer = buffer.?.*, }; } pub fn writeByte(this: *SocketPrinterInternal, byte: u8) anyerror!usize { @@ -2247,12 +2246,15 @@ pub const RequestContext = struct { const SourceMapHandler = JSPrinter.SourceMapHandler.For(SocketPrinterInternal, onSourceMapChunk); pub fn onSourceMapChunk(this: *SocketPrinterInternal, chunk: SourceMap.Chunk, source: logger.Source) anyerror!void { + defer { + SocketPrinterInternal.buffer.?.* = this.buffer; + } if (this.rctx.has_called_done) return; this.buffer.reset(); this.buffer = try chunk.printSourceMapContents(source, this.buffer, false); defer { this.buffer.reset(); - buffer = this.buffer; + SocketPrinterInternal.buffer.?.* = this.buffer; } const buf = this.buffer.toOwnedSliceLeaky(); if (buf.len == 0) { @@ -2272,11 +2274,12 @@ pub const RequestContext = struct { pub fn done( chunky: *SocketPrinterInternal, ) anyerror!void { + SocketPrinterInternal.buffer.?.* = chunky.buffer; if (chunky.rctx.has_called_done) return; const buf = chunky.buffer.toOwnedSliceLeaky(); defer { chunky.buffer.reset(); - buffer = chunky.buffer; + SocketPrinterInternal.buffer.?.* = chunky.buffer; } if (chunky.rctx.header("Open-In-Editor") != null) { diff --git a/src/sourcemap/sourcemap.zig b/src/sourcemap/sourcemap.zig index ccd5561c8..cf96afa4e 100644 --- a/src/sourcemap/sourcemap.zig +++ b/src/sourcemap/sourcemap.zig @@ -10,6 +10,10 @@ const Logger = @import("../logger.zig"); const strings = @import("../string_immutable.zig"); const MutableString = @import("../string_mutable.zig").MutableString; const base64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +const Joiner = @import("../string_joiner.zig"); +const JSPrinter = @import("../js_printer.zig"); +const URL = @import("../query_string_map.zig").URL; +const FileSystem = @import("../fs.zig").FileSystem; const base64_lut: [std.math.maxInt(u8)]u8 = brk: { @setEvalBranchQuota(9999); var bytes = [_]u8{255} ** std.math.maxInt(u8); @@ -46,36 +50,48 @@ pub const SourceMapState = struct { sources: [][]const u8 = &[_][]u8{}, sources_content: [][]SourceContent, -mapping: std.ArrayListUnmanaged(Mapping) = .{}, +mapping: Mapping.List = .{}, allocator: std.mem.Allocator, -pub const Mapping = extern struct { - contents: [5]i32 = undefined, +pub const Mapping = struct { + generated: LineColumnOffset, + original: LineColumnOffset, + source_index: i32, + + pub const List = std.MultiArrayList(Mapping); pub inline fn generatedLine(mapping: Mapping) i32 { - return mapping.contents[0]; + return mapping.generated.lines; } pub inline fn generatedColumn(mapping: Mapping) i32 { - return mapping.contents[1]; + return mapping.generated.columns; } pub inline fn sourceIndex(mapping: Mapping) i32 { - return mapping.contents[2]; + return mapping.source_index; } pub inline fn originalLine(mapping: Mapping) i32 { - return mapping.contents[3]; + return mapping.original.lines; } pub inline fn originalColumn(mapping: Mapping) i32 { - return mapping.contents[4]; + return mapping.original.columns; } }; pub const LineColumnOffset = struct { lines: i32 = 0, columns: i32 = 0, + + pub fn cmp(_: void, a: LineColumnOffset, b: LineColumnOffset) std.math.Order { + if (a.lines != b.lines) { + return std.math.order(a.lines, b.lines); + } + + return std.math.order(a.columns, b.columns); + } }; pub const SourceContent = struct { @@ -87,10 +103,83 @@ pub fn find( this: *const SourceMap, line: i32, column: i32, -) ?*const Mapping { +) ?Mapping { _ = this; _ = line; _ = column; + + const generated = this.mapping.items(.generated); + + if (std.sort.binarySearch(LineColumnOffset, LineColumnOffset{ .lines = line, .columns = column }, generated, void{}, LineColumnOffset.cmp)) |i| { + return this.mapping.get(i); + } + + return null; +} + +pub const SourceMapPieces = struct { + prefix: std.ArrayList(u8), + mappings: std.ArrayList(u8), + suffix: std.ArrayList(u8), +}; + +// -- comment from esbuild -- +// Source map chunks are computed in parallel for speed. Each chunk is relative +// to the zero state instead of being relative to the end state of the previous +// chunk, since it's impossible to know the end state of the previous chunk in +// a parallel computation. +// +// After all chunks are computed, they are joined together in a second pass. +// This rewrites the first mapping in each chunk to be relative to the end +// state of the previous chunk. +pub fn appendSourceMapChunk(j: *Joiner, prev_end_state_: SourceMapState, start_state_: SourceMapState, source_map_: MutableString) !void { + var prev_end_state = prev_end_state_; + var start_state = start_state_; + // Handle line breaks in between this mapping and the previous one + if (start_state.generated_line > 0) { + j.append(try strings.repeatingAlloc(source_map_.allocator, @intCast(usize, start_state.generated_line), ';'), 0, source_map_.allocator); + prev_end_state.generated_column = 0; + } + + var source_map = source_map_.list.items; + if (strings.indexOfNotChar(source_map, ';')) |semicolons| { + j.append(source_map[0..semicolons], 0, null); + source_map = source_map[semicolons..]; + prev_end_state.generated_column = 0; + start_state.generated_column = 0; + } + + // Strip off the first mapping from the buffer. The first mapping should be + // for the start of the original file (the printer always generates one for + // the start of the file). + var i: usize = 0; + const generated_column_ = decodeVLQ(source_map, 0); + i = generated_column_.start; + const source_index_ = decodeVLQ(source_map, i); + i = source_index_.start; + const original_line_ = decodeVLQ(source_map, i); + i = original_line_.start; + const original_column_ = decodeVLQ(source_map, i); + i = original_column_.start; + + source_map = source_map[i..]; + + // Rewrite the first mapping to be relative to the end state of the previous + // chunk. We now know what the end state is because we're in the second pass + // where all chunks have already been generated. + start_state.source_index += source_index_.value; + start_state.generated_column += generated_column_.value; + start_state.original_line += original_line_.value; + start_state.original_column += original_column_.value; + + j.append( + appendMappingToBuffer(MutableString.initEmpty(source_map.allocator), j.lastByte(), prev_end_state, start_state).list.items, + 0, + source_map.allocator, + ); + + // Then append everything after that without modification. + j.append(source_map_.list.items, @truncate(u32, @ptrToInt(source_map.ptr) - @ptrToInt(source_map_.list.items.ptr)), source_map_.allocator); } // A single base 64 digit can contain 6 bits of data. For the base 64 variable @@ -203,7 +292,7 @@ pub const LineOffsetTable = struct { pub fn findLine(list: List, loc: Logger.Loc) i32 { const byte_offsets_to_start_of_line = list.items(.byte_offset_to_start_of_line); - var original_line: i32 = 0; + var original_line: u32 = 0; if (loc.start <= -1) { return 0; } @@ -217,7 +306,7 @@ pub const LineOffsetTable = struct { const step = count / 2; i = original_line + step; if (byte_offsets_to_start_of_line[i] <= loc_start) { - original_line = @intCast(i32, i + 1); + original_line = i + 1; count = count - step - 1; } else { count = step; @@ -225,7 +314,7 @@ pub const LineOffsetTable = struct { } } - return original_line - 1; + return @intCast(i32, original_line) - 1; } pub fn generate(allocator: std.mem.Allocator, contents: []const u8, approximate_line_count: i32) List { @@ -247,9 +336,9 @@ pub const LineOffsetTable = struct { var remaining = contents; while (remaining.len > 0) { - // TODO: SIMD const len_ = strings.wtf8ByteSequenceLength(remaining[0]); const c = strings.decodeWTF8RuneT(remaining.ptr[0..4], len_, i32, 0); + const cp_len = @as(usize, len_); if (column == 0) { line_byte_offset = @truncate( @@ -284,10 +373,26 @@ pub const LineOffsetTable = struct { u32, @ptrToInt(remaining.ptr) - @ptrToInt(contents.ptr), )) - line_byte_offset; - try columns_for_non_ascii.ensureUnusedCapacity((line_bytes_so_far - column_byte_offset) + 1); + columns_for_non_ascii.ensureUnusedCapacity((line_bytes_so_far - column_byte_offset) + 1) catch unreachable; while (column_byte_offset <= line_bytes_so_far) : (column_byte_offset += 1) { columns_for_non_ascii.appendAssumeCapacity(column); } + } else { + switch (c) { + (@maximum('\r', '\n') + 1)...127 => { + // skip ahead to the next newline or non-ascii character + if (strings.indexOfNewlineOrNonASCIICheckStart(remaining, @as(u32, len_), false)) |j| { + column += @intCast(i32, j); + remaining = remaining[j..]; + continue; + } else { + // if there are no more lines, we are done! + column += @intCast(i32, remaining.len); + remaining = remaining[remaining.len..]; + } + }, + else => {}, + } } switch (c) { @@ -299,16 +404,16 @@ pub const LineOffsetTable = struct { continue; } var columns_list = columns_for_non_ascii; - if (columns_for_non_ascii.items.len > 0 and stack_fallback.fixed_buffer_allocator.ownsSlice(columns_for_non_ascii.items)) { - columns_for_non_ascii.items = try allocator.dupe(i32, columns_for_non_ascii.toOwnedSlice()); + if (columns_for_non_ascii.items.len > 0 and stack_fallback.fixed_buffer_allocator.ownsSlice(std.mem.sliceAsBytes(columns_for_non_ascii.items))) { + columns_for_non_ascii.items = allocator.dupe(i32, columns_for_non_ascii.toOwnedSlice()) catch unreachable; columns_for_non_ascii.capacity = columns_for_non_ascii.items.len; } - try list.append(allocator, .{ + list.append(allocator, .{ .byte_offset_to_start_of_line = line_byte_offset, .byte_offset_to_first_non_ascii = byte_offset_to_first_non_ascii, .columns_for_non_ascii = BabyList(i32).fromList(columns_list), - }); + }) catch unreachable; column = 0; byte_offset_to_first_non_ascii = 0; column_byte_offset = 0; @@ -323,17 +428,17 @@ pub const LineOffsetTable = struct { }, } - remaining = remaining[len_..]; + remaining = remaining[cp_len..]; } // Mark the start of the next line if (column == 0) { - line_byte_offset = @intCast(i32, contents.len); + line_byte_offset = @intCast(u32, contents.len); } if (columns_for_non_ascii.items.len > 0) { const line_bytes_so_far = @intCast(u32, contents.len) - line_byte_offset; - try columns_for_non_ascii.ensureUnusedCapacity((line_bytes_so_far - column_byte_offset) + 1); + columns_for_non_ascii.ensureUnusedCapacity((line_bytes_so_far - column_byte_offset) + 1) catch unreachable; while (column_byte_offset <= line_bytes_so_far) : (column_byte_offset += 1) { columns_for_non_ascii.appendAssumeCapacity(column); } @@ -341,22 +446,37 @@ pub const LineOffsetTable = struct { { var columns_list = columns_for_non_ascii; - if (columns_for_non_ascii.items.len > 0 and stack_fallback.fixed_buffer_allocator.ownsSlice(columns_for_non_ascii.items)) { - columns_for_non_ascii.items = try allocator.dupe(i32, columns_for_non_ascii.toOwnedSlice()); + if (columns_for_non_ascii.items.len > 0 and stack_fallback.fixed_buffer_allocator.ownsSlice(std.mem.sliceAsBytes(columns_for_non_ascii.items))) { + columns_for_non_ascii.items = allocator.dupe(i32, columns_for_non_ascii.toOwnedSlice()) catch unreachable; columns_for_non_ascii.capacity = columns_for_non_ascii.items.len; } - try list.append(allocator, .{ + list.append(allocator, .{ .byte_offset_to_start_of_line = line_byte_offset, .byte_offset_to_first_non_ascii = byte_offset_to_first_non_ascii, .columns_for_non_ascii = BabyList(i32).fromList(columns_list), - }); + }) catch unreachable; } return list; } }; +pub fn appendSourceMappingURLRemote( + origin: URL, + source: Logger.Source, + asset_prefix_path: []const u8, + comptime Writer: type, + writer: Writer, +) !void { + try writer.writeAll("\n//# sourceMappingURL="); + try writer.writeAll(strings.withoutTrailingSlash(origin.href)); + if (asset_prefix_path.len > 0) + try writer.writeAll(asset_prefix_path); + try writer.writeAll(source.path.pretty); + try writer.writeAll(".map"); +} + pub fn appendMappingToBuffer(buffer_: MutableString, last_byte: u8, prev_state: SourceMapState, current_state: SourceMapState) MutableString { var buffer = buffer_; const needs_comma = last_byte != 0 and last_byte != ';' and last_byte != '"'; @@ -383,9 +503,9 @@ pub fn appendMappingToBuffer(buffer_: MutableString, last_byte: u8, prev_state: if (needs_comma) { buffer.appendCharAssumeCapacity(','); } - - inline for (vlq) |*item| { - buffer.appendAssumeCapacity(item.bytes[0..item.len]); + comptime var i: usize = 0; + inline while (i < vlq.len) : (i += 1) { + buffer.appendAssumeCapacity(vlq[i].bytes[0..vlq[i].len]); } return buffer; @@ -406,6 +526,42 @@ pub const Chunk = struct { /// ignore empty chunks should_ignore: bool = true, + pub fn printSourceMapContents( + chunk: Chunk, + source: Logger.Source, + mutable: MutableString, + comptime ascii_only: bool, + ) !MutableString { + var output = mutable; + + // attempt to pre-allocate + + var filename_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; + var filename = source.path.text; + if (strings.hasPrefix(source.path.text, FileSystem.instance.top_level_dir)) { + filename = filename[FileSystem.instance.top_level_dir.len - 1 ..]; + } else if (filename.len > 0 and filename[0] != '/') { + filename_buf[0] = '/'; + @memcpy(filename_buf[1..], filename.ptr, filename.len); + filename = filename_buf[0 .. filename.len + 1]; + } + + output.growIfNeeded( + filename.len + 2 + source.contents.len + chunk.buffer.list.items.len + 32 + 39 + 29 + 22 + 20, + ) catch unreachable; + try output.append("{\n \"version\":3,\n \"sources\": ["); + + output = try JSPrinter.quoteForJSON(filename, output, ascii_only); + + try output.append("],\n \"sourcesContent\": ["); + output = try JSPrinter.quoteForJSON(source.contents, output, ascii_only); + try output.append("],\n \"mappings\": "); + output = try JSPrinter.quoteForJSON(chunk.buffer.list.items, output, ascii_only); + try output.append(", \"names\": []\n}"); + + return output; + } + pub const Builder = struct { input_source_map: ?*SourceMap = null, source_map: MutableString, @@ -433,7 +589,7 @@ pub const Chunk = struct { .buffer = b.source_map, .end_state = b.prev_state, .final_generated_column = b.generated_column, - .should_ignore = strings.containsAnyBesidesChar(b.source_map.list.items, ';'), + .should_ignore = !strings.containsAnyBesidesChar(b.source_map.list.items, ';'), }; } @@ -441,22 +597,40 @@ pub const Chunk = struct { // generated line and column numbers pub fn updateGeneratedLineAndColumn(b: *Builder, output: []const u8) void { const slice = output[b.last_generated_update..]; - - var iter = strings.CodepointIterator.init(slice); - var cursor = strings.CodepointIterator.Cursor{}; - while (iter.next(&cursor)) { - switch (cursor.c) { + var needs_mapping = b.cover_lines_without_mappings and !b.line_starts_with_mapping and b.has_prev_state; + + var i: usize = 0; + const n = @intCast(usize, slice.len); + var c: i32 = 0; + while (i < n) { + const len = strings.wtf8ByteSequenceLength(slice[i]); + c = strings.decodeWTF8RuneT(slice[i..].ptr[0..4], len, i32, strings.unicode_replacement); + i += @as(usize, len); + + switch (c) { + 14...127 => { + if (strings.indexOfNewlineOrNonASCII(slice, @intCast(u32, i))) |j| { + b.generated_column += @intCast(i32, (@as(usize, j) - i) + 1); + i = j; + continue; + } else { + b.generated_column += @intCast(i32, slice[i..].len); + i = n; + break; + } + }, '\r', '\n', 0x2028, 0x2029 => { // windows newline - if (cursor.c == '\r') { - const newline_check = b.last_generated_update + cursor.i + 1; - if (newline_check < output.len and output[newline_check] == '\n') + if (c == '\r') { + const newline_check = b.last_generated_update + i; + if (newline_check < output.len and output[newline_check] == '\n') { continue; + } } // If we're about to move to the next line and the previous line didn't have // any mappings, add a mapping at the start of the previous line. - if (b.cover_lines_without_mappings and !b.line_starts_with_mapping and b.has_prev_state) { + if (needs_mapping) { b.appendMappingWithoutRemapping(.{ .generated_line = b.prev_state.generated_line, .generated_column = 0, @@ -473,10 +647,13 @@ pub const Chunk = struct { // This new line doesn't have a mapping yet b.line_starts_with_mapping = false; + + needs_mapping = b.cover_lines_without_mappings and !b.line_starts_with_mapping and b.has_prev_state; }, + else => { // Mozilla's "source-map" library counts columns using UTF-16 code units - b.generated_column += @as(i32, @boolToInt(cursor.c > 0xFFFF)) + 1; + b.generated_column += @as(i32, @boolToInt(c > 0xFFFF)) + 1; }, } } diff --git a/src/string_immutable.zig b/src/string_immutable.zig index 28a4ba11d..e551333ae 100644 --- a/src/string_immutable.zig +++ b/src/string_immutable.zig @@ -671,23 +671,24 @@ pub fn toUTF8Alloc(allocator: std.mem.Allocator, js: []const u16) !string { return try toUTF8AllocWithType(allocator, []const u16, js); } -pub inline fn appendUTF8MachineWordToUTF16MachineWord(output: *[4]u16, input: *const [4]u8) void { - output[0] = input[0]; - output[1] = input[1]; - output[2] = input[2]; - output[3] = input[3]; +pub inline fn appendUTF8MachineWordToUTF16MachineWord(output: *[@sizeOf(usize) / 2]u16, input: *const [@sizeOf(usize) / 2]u8) void { + comptime var i: usize = 0; + inline while (i < @sizeOf(usize) / 2) : (i += 1) { + output[i] = input[i]; + } } pub inline fn copyU8IntoU16(output_: []u16, input_: []const u8) void { var output = output_; var input = input_; + const word = @sizeOf(usize) / 2; if (comptime Environment.allow_assert) { std.debug.assert(input.len <= output.len); } - while (input.len >= 4) { - appendUTF8MachineWordToUTF16MachineWord(output[0..4], input[0..4]); - output = output[4..]; - input = input[4..]; + while (input.len >= word) { + appendUTF8MachineWordToUTF16MachineWord(output[0..word], input[0..word]); + output = output[word..]; + input = input[word..]; } for (input) |c, i| { @@ -695,6 +696,33 @@ pub inline fn copyU8IntoU16(output_: []u16, input_: []const u8) void { } } +// pub inline fn copy(output_: []u8, input_: []const u8) void { +// var output = output_; +// var input = input_; +// if (comptime Environment.allow_assert) { +// std.debug.assert(input.len <= output.len); +// } + +// if (input.len > @sizeOf(usize) * 4) { +// comptime var i: usize = 0; +// inline while (i < 4) : (i += 1) { +// appendUTF8MachineWord(output[i * @sizeOf(usize) ..][0..@sizeOf(usize)], input[i * @sizeOf(usize) ..][0..@sizeOf(usize)]); +// } +// output = output[4 * @sizeOf(usize) ..]; +// input = input[4 * @sizeOf(usize) ..]; +// } + +// while (input.len >= @sizeOf(usize)) { +// appendUTF8MachineWord(output[0..@sizeOf(usize)], input[0..@sizeOf(usize)]); +// output = output[@sizeOf(usize)..]; +// input = input[@sizeOf(usize)..]; +// } + +// for (input) |c, i| { +// output[i] = c; +// } +// } + pub inline fn copyU16IntoU8(output_: []u8, comptime InputType: type, input_: InputType) void { var output = output_; var input = input_; @@ -1335,17 +1363,17 @@ pub inline fn decodeWTF8RuneTMultibyte(p: *const [4]u8, len: u3, comptime T: typ unreachable; } -const ascii_vector_size = if (Environment.isWasm) 8 else 16; -const ascii_u16_vector_size = if (Environment.isWasm) 4 else 8; -const AsciiVectorInt = std.meta.Int(.unsigned, ascii_vector_size); -const AsciiVectorIntU16 = std.meta.Int(.unsigned, ascii_u16_vector_size); -const max_16_ascii = @splat(ascii_vector_size, @as(u8, 127)); -const min_16_ascii = @splat(ascii_vector_size, @as(u8, 0x20)); -const max_u16_ascii = @splat(ascii_u16_vector_size, @as(u16, 127)); -const AsciiVector = std.meta.Vector(ascii_vector_size, u8); -const AsciiVectorU1 = std.meta.Vector(ascii_vector_size, u1); -const AsciiU16Vector = std.meta.Vector(ascii_u16_vector_size, u16); -const max_4_ascii = @splat(4, @as(u8, 127)); +pub const ascii_vector_size = if (Environment.isWasm) 8 else 16; +pub const ascii_u16_vector_size = if (Environment.isWasm) 4 else 8; +pub const AsciiVectorInt = std.meta.Int(.unsigned, ascii_vector_size); +pub const AsciiVectorIntU16 = std.meta.Int(.unsigned, ascii_u16_vector_size); +pub const max_16_ascii = @splat(ascii_vector_size, @as(u8, 127)); +pub const min_16_ascii = @splat(ascii_vector_size, @as(u8, 0x20)); +pub const max_u16_ascii = @splat(ascii_u16_vector_size, @as(u16, 127)); +pub const AsciiVector = std.meta.Vector(ascii_vector_size, u8); +pub const AsciiVectorU1 = std.meta.Vector(ascii_vector_size, u1); +pub const AsciiU16Vector = std.meta.Vector(ascii_u16_vector_size, u16); +pub const max_4_ascii = @splat(4, @as(u8, 127)); pub fn isAllASCII(slice: []const u8) bool { var remaining = slice; @@ -1436,6 +1464,49 @@ pub fn firstNonASCII(slice: []const u8) ?u32 { return null; } +pub fn indexOfNewlineOrNonASCII(slice_: []const u8, offset: u32) ?u32 { + return indexOfNewlineOrNonASCIICheckStart(slice_, offset, true); +} + +pub fn indexOfNewlineOrNonASCIICheckStart(slice_: []const u8, offset: u32, comptime check_start: bool) ?u32 { + const slice = slice_[offset..]; + var remaining = slice; + + if (remaining.len == 0) + return null; + + if (comptime check_start) { + // this shows up in profiling + if (remaining[0] > 127 or remaining[0] < 0x20 or remaining[0] == '\r' or remaining[0] == '\n') { + return offset; + } + } + + if (comptime Environment.isAarch64 or Environment.isX64) { + while (remaining.len >= ascii_vector_size) { + const vec: AsciiVector = remaining[0..ascii_vector_size].*; + const cmp = @bitCast(AsciiVectorU1, (vec > max_16_ascii)) | @bitCast(AsciiVectorU1, (vec < min_16_ascii)) | + @bitCast(AsciiVectorU1, vec == @splat(ascii_vector_size, @as(u8, '\r'))) | + @bitCast(AsciiVectorU1, vec == @splat(ascii_vector_size, @as(u8, '\n'))); + const bitmask = @ptrCast(*const AsciiVectorInt, &cmp).*; + const first = @ctz(AsciiVectorInt, bitmask); + if (first < ascii_vector_size) { + return @as(u32, first) + @intCast(u32, slice.len - remaining.len) + offset; + } + + remaining = remaining[ascii_vector_size..]; + } + } + + for (remaining) |char, i| { + if (char > 127 or char < 0x20 or char == '\n' or char == '\r') { + return @truncate(u32, i + (slice.len - remaining.len)) + offset; + } + } + + return null; +} + pub fn indexOfNeedsEscape(slice: []const u8) ?u32 { var remaining = slice; if (remaining.len == 0) |