diff options
| -rw-r--r-- | src/bun.js/bindings/bindings.cpp | 69 | ||||
| -rw-r--r-- | src/bun.js/bindings/bindings.zig | 15 | ||||
| -rw-r--r-- | src/bun.js/bindings/exports.zig | 6 | ||||
| -rw-r--r-- | src/bun.js/bindings/headers-cpp.h | 2 | ||||
| -rw-r--r-- | src/bun.js/bindings/headers.h | 4 | ||||
| -rw-r--r-- | src/bun.js/bindings/headers.zig | 2 | ||||
| -rw-r--r-- | src/bun.js/javascript.zig | 6 | ||||
| -rw-r--r-- | src/bun.js/webcore/response.zig | 12 | ||||
| -rw-r--r-- | test/bun.js/body-stream.test.ts | 268 |
9 files changed, 251 insertions, 133 deletions
diff --git a/src/bun.js/bindings/bindings.cpp b/src/bun.js/bindings/bindings.cpp index 215519302..dd3cfa3ca 100644 --- a/src/bun.js/bindings/bindings.cpp +++ b/src/bun.js/bindings/bindings.cpp @@ -225,21 +225,35 @@ typedef struct PicoHTTPHeaders { const PicoHTTPHeader* ptr; size_t len; } PicoHTTPHeaders; -WebCore::FetchHeaders* WebCore__FetchHeaders__createFromPicoHeaders_(JSC__JSGlobalObject* arg0, const void* arg1) +WebCore::FetchHeaders* WebCore__FetchHeaders__createFromPicoHeaders_(const void* arg1) { PicoHTTPHeaders pico_headers = *reinterpret_cast<const PicoHTTPHeaders*>(arg1); auto* headers = new WebCore::FetchHeaders({ WebCore::FetchHeaders::Guard::None, {} }); if (pico_headers.len > 0) { - Vector<KeyValuePair<String, String>> pairs; - pairs.reserveCapacity(pico_headers.len); - for (size_t i = 0; i < pico_headers.len; i++) { - WTF::String name = WTF::String(pico_headers.ptr[i].name, pico_headers.ptr[i].name_len); - WTF::String value = WTF::String(pico_headers.ptr[i].value, pico_headers.ptr[i].value_len); - pairs.uncheckedAppend(KeyValuePair<String, String>(name, value)); + HTTPHeaderMap map = HTTPHeaderMap(); + + for (size_t j = 0; j < pico_headers.len; j++) { + PicoHTTPHeader header = pico_headers.ptr[j]; + if (header.value_len == 0) + continue; + + StringView nameView = StringView(reinterpret_cast<const char*>(header.name), header.name_len); + + LChar* data = nullptr; + auto value = String::createUninitialized(header.value_len, data); + memcpy(data, header.value, header.value_len); + + HTTPHeaderName name; + + if (WebCore::findHTTPHeaderName(nameView, name)) { + map.add(name, WTFMove(value)); + } else { + map.setUncommonHeader(nameView.toString().isolatedCopy(), WTFMove(value)); + } } - headers->fill(WebCore::FetchHeaders::Init(WTFMove(pairs))); - pairs.releaseBuffer(); + + headers->setInternalHeaders(WTFMove(map)); } return headers; @@ -247,53 +261,28 @@ WebCore::FetchHeaders* WebCore__FetchHeaders__createFromPicoHeaders_(JSC__JSGlob WebCore::FetchHeaders* WebCore__FetchHeaders__createFromUWS(JSC__JSGlobalObject* arg0, void* arg1) { uWS::HttpRequest req = *reinterpret_cast<uWS::HttpRequest*>(arg1); - std::bitset<255> seenHeaderSizes; - // uWebSockets limits to 50 headers - uint32_t nameHashes[55]; size_t i = 0; auto* headers = new WebCore::FetchHeaders({ WebCore::FetchHeaders::Guard::None, {} }); HTTPHeaderMap map = HTTPHeaderMap(); -outer: for (const auto& header : req) { StringView nameView = StringView(reinterpret_cast<const LChar*>(header.first.data()), header.first.length()); - - uint32_t hash = nameView.hash(); - nameHashes[i++] = hash; size_t name_len = nameView.length(); - if (UNLIKELY(name_len >= 255)) { - auto value = WTF::StringView(reinterpret_cast<const LChar*>(header.second.data()), header.second.length()).toString(); - map.add(nameView.toString(), value); - continue; - } - - if (seenHeaderSizes[name_len]) { - if (i > 56) - __builtin_unreachable(); - - for (size_t j = 0; j < i - 1; j++) { - if (nameHashes[j] == hash) { - // When the same header is seen twice, we need to merge them - // Merging already allocates - // so we can skip that step here - map.add(nameView.toString(), WTF::String(WTF::StringImpl::createWithoutCopying(header.second.data(), header.second.length()))); - goto outer; - } - } - } + LChar* data = nullptr; + auto value = String::createUninitialized(header.second.length(), data); + memcpy(data, header.second.data(), header.second.length()); HTTPHeaderName name; - auto value = WTF::StringView(reinterpret_cast<const LChar*>(header.second.data()), header.second.length()).toString(); if (WebCore::findHTTPHeaderName(nameView, name)) { - map.add(name, value); + map.add(name, WTFMove(value)); } else { - map.setUncommonHeader(nameView.toString().isolatedCopy(), value); + map.setUncommonHeader(nameView.toString().isolatedCopy(), WTFMove(value)); } - seenHeaderSizes[name_len] = true; + // seenHeaderSizes[name_len] = true; if (i > 56) __builtin_unreachable(); diff --git a/src/bun.js/bindings/bindings.zig b/src/bun.js/bindings/bindings.zig index 854b51a77..b221f7e87 100644 --- a/src/bun.js/bindings/bindings.zig +++ b/src/bun.js/bindings/bindings.zig @@ -130,6 +130,8 @@ pub const ZigString = extern struct { return this.len * 2; } + /// Count the number of code points in the string. + /// This function is slow. Use maxUITF8ByteLength() to get a quick estimate pub fn utf8ByteLength(this: ZigString) usize { if (this.isUTF8()) { return this.len; @@ -712,23 +714,19 @@ pub const FetchHeaders = opaque { } pub fn createFromPicoHeaders( - global: *JSGlobalObject, pico_headers: anytype, ) *FetchHeaders { const out = PicoHeaders{ .ptr = pico_headers.ptr, .len = pico_headers.len }; const result = shim.cppFn("createFromPicoHeaders_", .{ - global, &out, }); return result; } pub fn createFromPicoHeaders_( - global: *JSGlobalObject, pico_headers: *const anyopaque, ) *FetchHeaders { return shim.cppFn("createFromPicoHeaders_", .{ - global, pico_headers, }); } @@ -2772,6 +2770,15 @@ pub const JSValue = enum(JSValueReprInt) { JSC.C.JSValueUnprotect(JSC.VirtualMachine.vm.global, this.asObjectRef()); } + pub fn JSONValueFromString( + global: *JSGlobalObject, + str: [*]const u8, + len: usize, + ascii: bool, + ) JSValue { + return cppFn("JSONValueFromString", .{ global, str, len, ascii }); + } + /// Create an object with exactly two properties pub fn createObject2(global: *JSGlobalObject, key1: *const ZigString, key2: *const ZigString, value1: JSValue, value2: JSValue) JSValue { return cppFn("createObject2", .{ global, key1, key2, value1, value2 }); diff --git a/src/bun.js/bindings/exports.zig b/src/bun.js/bindings/exports.zig index d490cd5a9..c6d6ef867 100644 --- a/src/bun.js/bindings/exports.zig +++ b/src/bun.js/bindings/exports.zig @@ -271,9 +271,9 @@ export fn ZigString__free(raw: [*]const u8, len: usize, allocator_: ?*anyopaque) } export fn ZigString__free_global(ptr: [*]const u8, len: usize) void { - if (comptime Environment.allow_assert) { - std.debug.assert(Mimalloc.mi_check_owned(ZigString.init(ptr[0..len]).slice().ptr)); - } + // if (comptime Environment.allow_assert) { + // std.debug.assert(Mimalloc.mi_check_owned(ptr)); + // } // we must untag the string pointer Mimalloc.mi_free(@intToPtr(*anyopaque, @ptrToInt(ZigString.init(ptr[0..len]).slice().ptr))); } diff --git a/src/bun.js/bindings/headers-cpp.h b/src/bun.js/bindings/headers-cpp.h index 976d4e940..617b1b63b 100644 --- a/src/bun.js/bindings/headers-cpp.h +++ b/src/bun.js/bindings/headers-cpp.h @@ -1,4 +1,4 @@ -//-- AUTOGENERATED FILE -- 1664421569 +//-- AUTOGENERATED FILE -- 1664608671 // clang-format off #pragma once diff --git a/src/bun.js/bindings/headers.h b/src/bun.js/bindings/headers.h index 365ed5c8e..b42daca48 100644 --- a/src/bun.js/bindings/headers.h +++ b/src/bun.js/bindings/headers.h @@ -1,5 +1,5 @@ // clang-format off -//-- AUTOGENERATED FILE -- 1664421569 +//-- AUTOGENERATED FILE -- 1664608671 #pragma once #include <stddef.h> @@ -281,7 +281,7 @@ CPP_DECL void WebCore__FetchHeaders__copyTo(WebCore__FetchHeaders* arg0, StringP CPP_DECL void WebCore__FetchHeaders__count(WebCore__FetchHeaders* arg0, uint32_t* arg1, uint32_t* arg2); CPP_DECL WebCore__FetchHeaders* WebCore__FetchHeaders__createEmpty(); CPP_DECL WebCore__FetchHeaders* WebCore__FetchHeaders__createFromJS(JSC__JSGlobalObject* arg0, JSC__JSValue JSValue1); -CPP_DECL WebCore__FetchHeaders* WebCore__FetchHeaders__createFromPicoHeaders_(JSC__JSGlobalObject* arg0, const void* arg1); +CPP_DECL WebCore__FetchHeaders* WebCore__FetchHeaders__createFromPicoHeaders_(const void* arg0); CPP_DECL WebCore__FetchHeaders* WebCore__FetchHeaders__createFromUWS(JSC__JSGlobalObject* arg0, void* arg1); CPP_DECL JSC__JSValue WebCore__FetchHeaders__createValue(JSC__JSGlobalObject* arg0, StringPointer* arg1, StringPointer* arg2, const ZigString* arg3, uint32_t arg4); CPP_DECL void WebCore__FetchHeaders__deref(WebCore__FetchHeaders* arg0); diff --git a/src/bun.js/bindings/headers.zig b/src/bun.js/bindings/headers.zig index ef6257499..24d188720 100644 --- a/src/bun.js/bindings/headers.zig +++ b/src/bun.js/bindings/headers.zig @@ -117,7 +117,7 @@ pub extern fn WebCore__FetchHeaders__copyTo(arg0: ?*bindings.FetchHeaders, arg1: pub extern fn WebCore__FetchHeaders__count(arg0: ?*bindings.FetchHeaders, arg1: [*c]u32, arg2: [*c]u32) void; pub extern fn WebCore__FetchHeaders__createEmpty(...) ?*bindings.FetchHeaders; pub extern fn WebCore__FetchHeaders__createFromJS(arg0: ?*JSC__JSGlobalObject, JSValue1: JSC__JSValue) ?*bindings.FetchHeaders; -pub extern fn WebCore__FetchHeaders__createFromPicoHeaders_(arg0: ?*JSC__JSGlobalObject, arg1: ?*const anyopaque) ?*bindings.FetchHeaders; +pub extern fn WebCore__FetchHeaders__createFromPicoHeaders_(arg0: ?*const anyopaque) ?*bindings.FetchHeaders; pub extern fn WebCore__FetchHeaders__createFromUWS(arg0: ?*JSC__JSGlobalObject, arg1: ?*anyopaque) ?*bindings.FetchHeaders; pub extern fn WebCore__FetchHeaders__createValue(arg0: ?*JSC__JSGlobalObject, arg1: [*c]StringPointer, arg2: [*c]StringPointer, arg3: [*c]const ZigString, arg4: u32) JSC__JSValue; pub extern fn WebCore__FetchHeaders__deref(arg0: ?*bindings.FetchHeaders) void; diff --git a/src/bun.js/javascript.zig b/src/bun.js/javascript.zig index 57e3cebd1..8eeca5555 100644 --- a/src/bun.js/javascript.zig +++ b/src/bun.js/javascript.zig @@ -368,8 +368,6 @@ pub const VirtualMachine = struct { global_api_constructors: [GlobalConstructors.len]JSC.JSValue = undefined, origin_timer: std.time.Timer = undefined, - active_tasks: usize = 0, - macro_event_loop: EventLoop = EventLoop{}, regular_event_loop: EventLoop = EventLoop{}, event_loop: *EventLoop = undefined, @@ -379,6 +377,8 @@ pub const VirtualMachine = struct { source_mappings: SavedSourceMap = undefined, + active_tasks: usize = 0, + rare_data: ?*JSC.RareData = null, poller: JSC.Poller = JSC.Poller{}, us_loop_reference_count: usize = 0, @@ -2044,7 +2044,7 @@ pub const EventListenerMixin = struct { fetch_event.* = FetchEvent{ .request_context = request_context, - .request = try Request.fromRequestContext(request_context, vm.global), + .request = try Request.fromRequestContext(request_context), .onPromiseRejectionCtx = @as(*anyopaque, ctx), .onPromiseRejectionHandler = FetchEventRejectionHandler.onRejection, }; diff --git a/src/bun.js/webcore/response.zig b/src/bun.js/webcore/response.zig index 361d58315..ff5ad7fcd 100644 --- a/src/bun.js/webcore/response.zig +++ b/src/bun.js/webcore/response.zig @@ -639,7 +639,7 @@ pub const Fetch = struct { .redirected = this.result.redirected, .body = .{ .init = .{ - .headers = FetchHeaders.createFromPicoHeaders(this.global_this, http_response.headers), + .headers = FetchHeaders.createFromPicoHeaders(http_response.headers), .status_code = @truncate(u16, http_response.status_code), }, .value = body_value, @@ -3909,6 +3909,12 @@ pub const AnyBlob = union(enum) { } var str = this.InternalBlob.toStringOwned(global); + + // the GC will collect the string + this.* = .{ + .InlineBlob = .{}, + }; + return str.parseJSON(global); }, } @@ -5057,12 +5063,12 @@ pub const Request = struct { try writer.writeAll("}"); } - pub fn fromRequestContext(ctx: *RequestContext, global: *JSGlobalObject) !Request { + pub fn fromRequestContext(ctx: *RequestContext) !Request { var req = Request{ .url = std.mem.span(ctx.getFullURL()), .body = Body.Value.empty, .method = ctx.method, - .headers = FetchHeaders.createFromPicoHeaders(global, ctx.request.headers), + .headers = FetchHeaders.createFromPicoHeaders(ctx.request.headers), .url_was_allocated = true, }; return req; diff --git a/test/bun.js/body-stream.test.ts b/test/bun.js/body-stream.test.ts index 33d242630..68190e8f3 100644 --- a/test/bun.js/body-stream.test.ts +++ b/test/bun.js/body-stream.test.ts @@ -6,6 +6,131 @@ import { readFileSync } from "fs"; var port = 40001; +{ + const BodyMixin = [ + Request.prototype.arrayBuffer, + Request.prototype.blob, + Request.prototype.text, + Request.prototype.json, + ]; + const useRequestObjectValues = [true, false]; + + for (let RequestPrototyeMixin of BodyMixin) { + for (let useRequestObject of useRequestObjectValues) { + describe(`Request.prototoype.${RequestPrototyeMixin.name}() ${ + useRequestObject ? "fetch(req)" : "fetch(url)" + }`, () => { + const inputFixture = [ + [JSON.stringify("Hello World"), JSON.stringify("Hello World")], + [ + JSON.stringify("Hello World 123"), + Buffer.from(JSON.stringify("Hello World 123")).buffer, + ], + [ + JSON.stringify("Hello World 456"), + Buffer.from(JSON.stringify("Hello World 456")), + ], + [ + JSON.stringify( + "EXTREMELY LONG VERY LONG STRING WOW SO LONG YOU WONT BELIEVE IT! ".repeat( + 100 + ) + ), + Buffer.from( + JSON.stringify( + "EXTREMELY LONG VERY LONG STRING WOW SO LONG YOU WONT BELIEVE IT! ".repeat( + 100 + ) + ) + ), + ], + [ + JSON.stringify( + "EXTREMELY LONG 🔥 UTF16 🔥 VERY LONG STRING WOW SO LONG YOU WONT BELIEVE IT! ".repeat( + 100 + ) + ), + Buffer.from( + JSON.stringify( + "EXTREMELY LONG 🔥 UTF16 🔥 VERY LONG STRING WOW SO LONG YOU WONT BELIEVE IT! ".repeat( + 100 + ) + ) + ), + ], + ]; + for (const [name, input] of inputFixture) { + test(`${name.slice( + 0, + Math.min(name.length ?? name.byteLength, 64) + )}`, async () => { + await runInServer( + { + async fetch(req) { + var result = await RequestPrototyeMixin.call(req); + if (RequestPrototyeMixin === Request.prototype.json) { + result = JSON.stringify(result); + } + if (typeof result === "string") { + expect(result.length).toBe(name.length); + expect(result).toBe(name); + } else if (result && result instanceof Blob) { + expect(result.size).toBe( + new TextEncoder().encode(name).byteLength + ); + expect(await result.text()).toBe(name); + } else { + expect(result.byteLength).toBe( + Buffer.from(input).byteLength + ); + expect(Bun.SHA1.hash(result, "base64")).toBe( + Bun.SHA1.hash(input, "base64") + ); + } + return new Response(result, { + headers: req.headers, + }); + }, + }, + async (url) => { + var response; + + if (useRequestObject) { + response = await fetch( + new Request({ + body: input, + method: "POST", + url: url, + headers: { + "content-type": "text/plain", + }, + }) + ); + } else { + response = await fetch(url, { + body: input, + method: "POST", + headers: { + "content-type": "text/plain", + }, + }); + } + + expect(response.status).toBe(200); + expect(response.headers.get("content-length")).toBe( + String(Buffer.from(input).byteLength) + ); + expect(response.headers.get("content-type")).toBe("text/plain"); + expect(await response.text()).toBe(name); + } + ); + }); + } + }); + } + } +} + async function runInServer( opts: ServeOptions, cb: (url: string) => void | Promise<void> @@ -63,27 +188,28 @@ describe("reader", function () { // - 1 byte // - less than the InlineBlob limit // - multiple chunks + // - backpressure for (let inputLength of [ 0, 1, 2, 12, - 63, - 128, + 95, + 1024, + 1024 * 1024, 1024 * 1024 * 2, - 1024 * 1024 * 4, ]) { var bytes = new Uint8Array(inputLength); { const chunk = Math.min(bytes.length, 256); for (var i = 0; i < chunk; i++) { - bytes[i] = i % 256; + bytes[i] = 255 - i; } } if (bytes.length > 255) fillRepeating(bytes, 0, bytes.length); - for (var huge_ of [ + for (const huge_ of [ bytes, bytes.buffer, new DataView(bytes.buffer), @@ -114,11 +240,11 @@ describe("reader", function () { new Float32Array(bytes).subarray(0, 1), ]) { gc(); - - it(`works with ${huge_.constructor.name}(${ - huge_.byteLength ?? huge_.size - }:${inputLength})`, async () => { - var huge = huge_; + const thisArray = huge_; + it(`works with ${thisArray.constructor.name}(${ + thisArray.byteLength ?? thisArray.size + }:${inputLength}) via req.body.getReader() in chunks`, async () => { + var huge = thisArray; var called = false; gc(); @@ -148,6 +274,7 @@ describe("reader", function () { expect(req.headers.get("user-agent")).toBe( navigator.userAgent ); + var reader = req.body.getReader(); called = true; var buffers = []; @@ -185,6 +312,7 @@ describe("reader", function () { headers: { "content-type": "text/plain", "x-custom": "hello", + "x-typed-array": thisArray.constructor.name, }, }); huge = undefined; @@ -207,12 +335,13 @@ describe("reader", function () { }); for (let isDirectStream of [true, false]) { - const inner = () => { - for (let position of ["begin" /*"end"*/]) { - it(`streaming back ${huge_.constructor.name}(${ - huge_.byteLength ?? huge_.size + const positions = ["begin", "end"]; + const inner = (thisArray) => { + for (let position of positions) { + it(`streaming back ${thisArray.constructor.name}(${ + thisArray.byteLength ?? thisArray.size }:${inputLength}) starting request.body.getReader() at ${position}`, async () => { - var huge = huge_; + var huge = thisArray; var called = false; gc(); @@ -232,7 +361,15 @@ describe("reader", function () { try { var reader; - if (position === "begin") reader = req.body.getReader(); + if (position === "begin") { + reader = req.body.getReader(); + } + + if (position === "end") { + await 1; + reader = req.body.getReader(); + } + expect(req.headers.get("x-custom")).toBe("hello"); expect(req.headers.get("content-type")).toBe( "text/plain" @@ -249,32 +386,39 @@ describe("reader", function () { expect(req.headers.get("user-agent")).toBe( navigator.userAgent ); - if (position === "end") { - await 1; - await 123; - await new Promise((resolve, reject) => { - setTimeout(resolve, 1); - }); - reader = req.body.getReader(); - } + const direct = { + type: "direct", + async pull(controller) { + while (true) { + const { done, value } = await reader.read(); + if (done) { + called = true; + controller.end(); - return new Response( - new ReadableStream({ - type: "direct", - async pull(controller) { - while (true) { - const { done, value } = await reader.read(); - if (done) { - called = true; - controller.end(); - - return; - } - controller.write(value); + return; } - }, - }), + controller.write(value); + } + }, + }; + + const web = { + async pull(controller) { + while (true) { + const { done, value } = await reader.read(); + if (done) { + called = true; + controller.close(); + return; + } + controller.enqueue(value); + } + }, + }; + + return new Response( + new ReadableStream(isDirectStream ? direct : web), { headers: req.headers, } @@ -293,6 +437,7 @@ describe("reader", function () { headers: { "content-type": "text/plain", "x-custom": "hello", + "x-typed-array": thisArray.constructor.name, }, }); huge = undefined; @@ -307,6 +452,12 @@ describe("reader", function () { ); gc(); + if (!response.headers.has("content-type")) { + console.error( + Object.fromEntries(response.headers.entries()) + ); + } + expect(response.headers.get("content-type")).toBe( "text/plain" ); @@ -321,9 +472,9 @@ describe("reader", function () { }; if (isDirectStream) { - describe("direct stream", () => inner()); + describe(" direct stream", () => inner(thisArray)); } else { - describe("default stream", () => inner()); + describe("default stream", () => inner(thisArray)); } } } @@ -333,38 +484,3 @@ describe("reader", function () { throw e; } }); - -{ - const inputFixture = [ - ["Hello World", "Hello World"], - ["Hello World 123", Buffer.from("Hello World 123").buffer], - ["Hello World 456", Buffer.from("Hello World 456")], - ]; - describe("echo", () => { - for (const [name, input] of inputFixture) { - test(`${name}`, async () => { - return await runInServer( - { - fetch(req) { - return new Response(req.body, { headers: req.headers }); - }, - }, - async (url) => { - var request = new Request({ - body: input, - method: "POST", - url: url, - headers: { - "content-type": "text/plain", - }, - }); - var response = await fetch(request); - expect(response.status).toBe(200); - expect(response.headers.get("content-type")).toBe("text/plain"); - expect(await response.text()).toBe(name); - } - ); - }); - } - }); -} |