From 1d9c98248893d9ad002ccdc6a83c1347adb68c22 Mon Sep 17 00:00:00 2001 From: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> Date: Sun, 27 Nov 2022 07:28:30 -0800 Subject: [console.log] Fix printing latin1 supplement characters at runtime Fixes https://github.com/oven-sh/bun/issues/1031#issuecomment-1236092120 --- src/bun.js/bindings/exports.zig | 31 ++++++++++-------- src/bun.js/bindings/headers-cpp.h | 2 +- src/bun.js/bindings/headers.h | 3 +- src/bun.js/bindings/headers.zig | 1 + src/js_printer.zig | 14 ++++++-- src/string_immutable.zig | 69 ++++++++++++++++++++++++++++----------- 6 files changed, 83 insertions(+), 37 deletions(-) diff --git a/src/bun.js/bindings/exports.zig b/src/bun.js/bindings/exports.zig index 917a62ee0..043fbdbc7 100644 --- a/src/bun.js/bindings/exports.zig +++ b/src/bun.js/bindings/exports.zig @@ -1869,19 +1869,11 @@ pub const ZigConsoleClient = struct { switch (comptime Format) { .StringPossiblyFormatted => { - var str = ZigString.init(""); - value.toZigString(&str, this.globalThis); + var str = value.toSlice(this.globalThis, bun.default_allocator); + defer str.deinit(); this.addForNewLine(str.len); - - if (!str.is16Bit()) { - const sliced = str.toSlice(bun.default_allocator); - defer sliced.deinit(); - const slice = sliced.slice(); - this.writeWithFormatting(Writer, writer_, @TypeOf(slice), slice, this.globalThis, enable_ansi_colors); - } else { - // TODO: UTF16 - writer.print("{}", .{str}); - } + const slice = str.slice(); + this.writeWithFormatting(Writer, writer_, @TypeOf(slice), slice, this.globalThis, enable_ansi_colors); }, .String => { var str = ZigString.init(""); @@ -1915,7 +1907,20 @@ pub const ZigConsoleClient = struct { writer.print(comptime Output.prettyFmt("", enable_ansi_colors), .{}); } - writer.print("{}", .{str}); + if (str.is16Bit()) { + // streaming print + writer.print("{s}", .{str}); + } else if (strings.isAllASCII(str.slice())) { + // fast path + writer.writeAll(str.slice()); + } else if (str.len > 0) { + // slow path + var buf = strings.allocateLatin1IntoUTF8(bun.default_allocator, []const u8, str.slice()) catch &[_]u8{}; + if (buf.len > 0) { + defer bun.default_allocator.free(buf); + writer.writeAll(buf); + } + } if (jsType == .RegExpObject and enable_ansi_colors) { writer.print(comptime Output.prettyFmt("", enable_ansi_colors), .{}); diff --git a/src/bun.js/bindings/headers-cpp.h b/src/bun.js/bindings/headers-cpp.h index 684656083..216a45292 100644 --- a/src/bun.js/bindings/headers-cpp.h +++ b/src/bun.js/bindings/headers-cpp.h @@ -1,4 +1,4 @@ -//-- AUTOGENERATED FILE -- 1669352137 +//-- AUTOGENERATED FILE -- 1669539205 // clang-format off #pragma once diff --git a/src/bun.js/bindings/headers.h b/src/bun.js/bindings/headers.h index 419a8965a..9d335ee90 100644 --- a/src/bun.js/bindings/headers.h +++ b/src/bun.js/bindings/headers.h @@ -1,5 +1,5 @@ // clang-format off -//-- AUTOGENERATED FILE -- 1669352137 +//-- AUTOGENERATED FILE -- 1669539205 #pragma once #include @@ -491,6 +491,7 @@ CPP_DECL bool JSC__JSValue__eqlCell(JSC__JSValue JSValue0, JSC__JSCell* arg1); CPP_DECL bool JSC__JSValue__eqlValue(JSC__JSValue JSValue0, JSC__JSValue JSValue1); CPP_DECL JSC__JSValue JSC__JSValue__fastGet_(JSC__JSValue JSValue0, JSC__JSGlobalObject* arg1, unsigned char arg2); CPP_DECL void JSC__JSValue__forEach(JSC__JSValue JSValue0, JSC__JSGlobalObject* arg1, void* arg2, void (* ArgFn3)(JSC__VM* arg0, JSC__JSGlobalObject* arg1, void* arg2, JSC__JSValue JSValue3)); +CPP_DECL void JSC__JSValue__forEachProperty(JSC__JSValue JSValue0, JSC__JSGlobalObject* arg1, void* arg2, void (* ArgFn3)(JSC__JSGlobalObject* arg0, void* arg1, ZigString* arg2, JSC__JSValue JSValue3, bool arg4)); CPP_DECL JSC__JSValue JSC__JSValue__fromEntries(JSC__JSGlobalObject* arg0, ZigString* arg1, ZigString* arg2, size_t arg3, bool arg4); CPP_DECL JSC__JSValue JSC__JSValue__fromInt64NoTruncate(JSC__JSGlobalObject* arg0, int64_t arg1); CPP_DECL JSC__JSValue JSC__JSValue__fromUInt64NoTruncate(JSC__JSGlobalObject* arg0, uint64_t arg1); diff --git a/src/bun.js/bindings/headers.zig b/src/bun.js/bindings/headers.zig index 7b649f26f..20f767d73 100644 --- a/src/bun.js/bindings/headers.zig +++ b/src/bun.js/bindings/headers.zig @@ -284,6 +284,7 @@ pub extern fn JSC__JSValue__eqlCell(JSValue0: JSC__JSValue, arg1: [*c]JSC__JSCel pub extern fn JSC__JSValue__eqlValue(JSValue0: JSC__JSValue, JSValue1: JSC__JSValue) bool; pub extern fn JSC__JSValue__fastGet_(JSValue0: JSC__JSValue, arg1: ?*JSC__JSGlobalObject, arg2: u8) JSC__JSValue; pub extern fn JSC__JSValue__forEach(JSValue0: JSC__JSValue, arg1: ?*JSC__JSGlobalObject, arg2: ?*anyopaque, ArgFn3: ?fn ([*c]JSC__VM, ?*JSC__JSGlobalObject, ?*anyopaque, JSC__JSValue) callconv(.C) void) void; +pub extern fn JSC__JSValue__forEachProperty(JSValue0: JSC__JSValue, arg1: ?*JSC__JSGlobalObject, arg2: ?*anyopaque, ArgFn3: ?fn (?*JSC__JSGlobalObject, ?*anyopaque, [*c]ZigString, JSC__JSValue, bool) callconv(.C) void) void; pub extern fn JSC__JSValue__fromEntries(arg0: ?*JSC__JSGlobalObject, arg1: [*c]ZigString, arg2: [*c]ZigString, arg3: usize, arg4: bool) JSC__JSValue; pub extern fn JSC__JSValue__fromInt64NoTruncate(arg0: ?*JSC__JSGlobalObject, arg1: i64) JSC__JSValue; pub extern fn JSC__JSValue__fromUInt64NoTruncate(arg0: ?*JSC__JSGlobalObject, arg1: u64) JSC__JSValue; diff --git a/src/js_printer.zig b/src/js_printer.zig index 2e6bee7f3..33f20b557 100644 --- a/src/js_printer.zig +++ b/src/js_printer.zig @@ -259,12 +259,20 @@ pub fn writeJSONString(input: []const u8, comptime Writer: type, writer: Writer, }; if (canPrintWithoutEscape(i32, c, false)) { const remain = text[@as(usize, width)..]; + if (encoding != .utf8 and width > 0) { + var codepoint_bytes: [4]u8 = undefined; + std.mem.writeIntNative(i32, &codepoint_bytes, c); + try writer.writeAll( + codepoint_bytes[0..strings.encodeWTF8Rune(codepoint_bytes[0..4], c)], + ); + } + if (strings.indexOfNeedsEscape(remain)) |j| { - try writer.writeAll(text[0 .. j + @as(usize, width)]); - text = text[j + @as(usize, width) ..]; + try writer.writeAll(remain[0..j]); + text = remain[j..]; continue; } else { - try writer.writeAll(text); + try writer.writeAll(remain); break; } } diff --git a/src/string_immutable.zig b/src/string_immutable.zig index c246e77cd..1eb46ee96 100644 --- a/src/string_immutable.zig +++ b/src/string_immutable.zig @@ -1600,10 +1600,10 @@ pub fn copyLatin1IntoUTF16(comptime Buffer: type, buf_: Buffer, comptime Type: t latin1 = latin1[to_write..]; buf = buf[to_write..]; - if (latin1.len > 0 and buf.len >= 2) { - buf[0..2].* = latin1ToCodepointBytesAssumeNotASCII16(latin1[0]); + if (latin1.len > 0 and buf.len >= 1) { + buf[0] = latin1ToCodepointBytesAssumeNotASCII16(latin1[0]); latin1 = latin1[1..]; - buf = buf[2..]; + buf = buf[1..]; } } @@ -2308,27 +2308,54 @@ test "copyLatin1IntoUTF8 - latin1" { pub fn latin1ToCodepointAssumeNotASCII(char: u8, comptime CodePointType: type) CodePointType { return @intCast( CodePointType, - @bitCast( - u16, - latin1ToCodepointBytesAssumeNotASCII(char), - ), + latin1ToCodepointBytesAssumeNotASCII16(char), ); } -pub fn latin1ToCodepointBytesAssumeNotASCIIWIthCharType(comptime Char: type, char: u32) [2]Char { - assert(char > 127); - return [2]Char{ - @as(Char, @truncate(u8, 0xc0 | char >> 6)), - @as(Char, @truncate(u8, 0x80 | (char & 0x3f))), - }; -} +const latin1_to_utf16_conversion_table = [256]u16{ + 0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, // 00-07 + 0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, // 08-0F + 0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, // 10-17 + 0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, // 18-1F + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, // 20-27 + 0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F, // 28-2F + 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, // 30-37 + 0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, // 38-3F + 0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, // 40-47 + 0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, // 48-4F + 0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, // 50-57 + 0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, // 58-5F + 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, // 60-67 + 0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, // 68-6F + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, // 70-77 + 0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F, // 78-7F + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, // 80-87 + 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, // 88-8F + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, // 90-97 + 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178, // 98-9F + 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, // A0-A7 + 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, // A8-AF + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, // B0-B7 + 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, // B8-BF + 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, // C0-C7 + 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, // C8-CF + 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, // D0-D7 + 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF, // D8-DF + 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, // E0-E7 + 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, // E8-EF + 0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, // F0-F7 + 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF, // F8-FF +}; pub fn latin1ToCodepointBytesAssumeNotASCII(char: u32) [2]u8 { - return latin1ToCodepointBytesAssumeNotASCIIWIthCharType(u8, char); + const as_utf16 = latin1ToCodepointBytesAssumeNotASCII16(char); + var bytes = [4]u8{ 0, 0, 0, 0 }; + _ = encodeWTF8Rune(&bytes, @intCast(i32, as_utf16)); + return bytes[0..2].*; } -pub fn latin1ToCodepointBytesAssumeNotASCII16(char: u32) [2]u16 { - return latin1ToCodepointBytesAssumeNotASCIIWIthCharType(u16, char); +pub fn latin1ToCodepointBytesAssumeNotASCII16(char: u32) u16 { + return latin1_to_utf16_conversion_table[@truncate(u8, char)]; } pub fn copyUTF16IntoUTF8(buf: []u8, comptime Type: type, utf16: Type) EncodeIntoResult { @@ -2339,7 +2366,11 @@ pub fn copyUTF16IntoUTF8(buf: []u8, comptime Type: type, utf16: Type) EncodeInto if (comptime Type == []const u16) { if (bun.FeatureFlags.use_simdutf) { const trimmed = bun.simdutf.trim.utf16(utf16_remaining); - const out_len = bun.simdutf.length.utf8.from.utf16.le(trimmed); + const out_len = if (buf.len <= (trimmed.len * 3 + 2)) + bun.simdutf.length.utf8.from.utf16.le(trimmed) + else + buf.len; + if (remaining.len >= out_len) { const result = bun.simdutf.convert.utf16.to.utf8.with_errors.le(trimmed, remaining[0..out_len]); return EncodeIntoResult{ @@ -2831,7 +2862,7 @@ pub fn indexOfNeedsEscape(slice: []const u8) ?u32 { if (remaining.len == 0) return null; - if (remaining[0] > 127 or remaining[0] < 0x20 or remaining[0] == '\\' or remaining[0] == '"') { + if (remaining[0] >= 127 or remaining[0] < 0x20 or remaining[0] == '\\' or remaining[0] == '"') { return 0; } -- cgit v1.2.3