enhancement(fetch): Merge parameters from request parameter with the second parameter for fetch, move verbose and proxy options to second parameter, add non-TLS tests for fetch (#1862)

* initial steps for proxy-server

* added http_proxy in fetch, move 3rd argument to 3nd argument options, add some non-TLS proxy tests

* some changes

* use only 1 buffer for url+proxy, merge headers on fetch

* initial steps

* change back to override headers instead of merging in fetch

* fix build response.zig

* fix conditional in merged headers on fetch

* updated with main and make proxy disabled if null is passed

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

2 files changed, 277 insertions, 98 deletions

diff --git a/src/bun.js/webcore/request.zig b/src/bun.js/webcore/request.zig
index 7457b941f..ff6816351 100644
--- a/src/bun.js/webcore/request.zig
+++ b/src/bun.js/webcore/request.zig
@@ -268,12 +268,17 @@ pub const Request = struct {
             return this.url.len;
 
         if (this.uws_request) |req| {
-            const fmt = ZigURL.HostFormatter{
-                .is_https = this.https,
-                .host = req.header("host") orelse "",
-            };
-
-            return this.getProtocol().len + req.url().len + std.fmt.count("{any}", .{fmt});
+            const req_url = req.url();
+            if (req_url.len > 0 and req_url[0] == '/') {
+                if (req.header("host")) |host| {
+                    const fmt = ZigURL.HostFormatter{
+                        .is_https = this.https,
+                        .host = host,
+                    };
+                    return this.getProtocol().len + req_url.len + std.fmt.count("{any}", .{fmt});
+                }
+            }
+            return req_url.len;
         }
 
         return 0;
@@ -291,28 +296,31 @@ pub const Request = struct {
 
         if (this.uws_request) |req| {
             const req_url = req.url();
-            if (req.header("host")) |host| {
-                const fmt = ZigURL.HostFormatter{
-                    .is_https = this.https,
-                    .host = host,
-                };
-                const url = try std.fmt.allocPrint(bun.default_allocator, "{s}{any}{s}", .{
-                    this.getProtocol(),
-                    fmt,
-                    req_url,
-                });
-                if (comptime Environment.allow_assert) {
-                    std.debug.assert(this.sizeOfURL() == url.len);
-                }
-                this.url = url;
-                this.url_was_allocated = true;
-            } else {
-                if (comptime Environment.allow_assert) {
-                    std.debug.assert(this.sizeOfURL() == req_url.len);
+            if (req_url.len > 0 and req_url[0] == '/') {
+                if (req.header("host")) |host| {
+                    const fmt = ZigURL.HostFormatter{
+                        .is_https = this.https,
+                        .host = host,
+                    };
+                    const url = try std.fmt.allocPrint(bun.default_allocator, "{s}{any}{s}", .{
+                        this.getProtocol(),
+                        fmt,
+                        req_url,
+                    });
+                    if (comptime Environment.allow_assert) {
+                        std.debug.assert(this.sizeOfURL() == url.len);
+                    }
+                    this.url = url;
+                    this.url_was_allocated = true;
+                    return;
                 }
-                this.url = try bun.default_allocator.dupe(u8, req_url);
-                this.url_was_allocated = true;
             }
+
+            if (comptime Environment.allow_assert) {
+                std.debug.assert(this.sizeOfURL() == req_url.len);
+            }
+            this.url = try bun.default_allocator.dupe(u8, req_url);
+            this.url_was_allocated = true;
         }
     }
 
diff --git a/src/bun.js/webcore/response.zig b/src/bun.js/webcore/response.zig
index 7005888bc..22f9a4d15 100644
--- a/src/bun.js/webcore/response.zig
+++ b/src/bun.js/webcore/response.zig
@@ -575,18 +575,18 @@ pub const Fetch = struct {
         concurrent_task: JSC.ConcurrentTask = .{},
         poll_ref: JSC.PollRef = .{},
 
-        /// Memory is owned by FetchTasklet
-        /// We always clone this one
-        url_str: []const u8 = "",
+        /// This is url + proxy memory buffer and is owned by FetchTasklet
+        /// We always clone url and proxy (if informed)
+        url_proxy_buffer: []const u8 = "",
 
         pub fn init(_: std.mem.Allocator) anyerror!FetchTasklet {
             return FetchTasklet{};
         }
 
         fn clearData(this: *FetchTasklet) void {
-            if (this.url_str.len > 0) {
-                bun.default_allocator.free(bun.constStrToU8(this.url_str));
-                this.url_str.len = 0;
+            if (this.url_proxy_buffer.len > 0) {
+                bun.default_allocator.free(this.url_proxy_buffer);
+                this.url_proxy_buffer.len = 0;
             }
 
             this.request_headers.entries.deinit(bun.default_allocator);
@@ -723,30 +723,28 @@ pub const Fetch = struct {
                 .global_this = globalThis,
                 .request_headers = fetch_options.headers,
                 .ref = JSC.napi.Ref.create(globalThis, promise),
-                .url_str = fetch_options.url.href,
+                .url_proxy_buffer = fetch_options.url_proxy_buffer,
             };
 
             if (fetch_tasklet.request_body.store()) |store| {
                 store.ref();
             }
-            
-            fetch_tasklet.http.?.* = HTTPClient.AsyncHTTP.init(
-                allocator,
-                fetch_options.method,
-                fetch_options.url,
-                fetch_options.headers.entries,
-                fetch_options.headers.buf.items,
-                &fetch_tasklet.response_buffer,
-                fetch_tasklet.request_body.slice(),
-                fetch_options.timeout,
-                HTTPClient.HTTPClientResult.Callback.New(
-                    *FetchTasklet,
-                    FetchTasklet.callback,
-                ).init(
-                    fetch_tasklet,
-                ),
-                jsc_vm.bundler.env.getHttpProxy(fetch_options.url)
-            );
+
+            var proxy: ?ZigURL = null;
+            if (fetch_options.proxy) |proxy_opt| {
+                if (!proxy_opt.isEmpty()) { //if is empty just ignore proxy
+                    proxy = fetch_options.proxy orelse jsc_vm.bundler.env.getHttpProxy(fetch_options.url);
+                }
+            } else {
+                proxy = jsc_vm.bundler.env.getHttpProxy(fetch_options.url);
+            }
+
+            fetch_tasklet.http.?.* = HTTPClient.AsyncHTTP.init(allocator, fetch_options.method, fetch_options.url, fetch_options.headers.entries, fetch_options.headers.buf.items, &fetch_tasklet.response_buffer, fetch_tasklet.request_body.slice(), fetch_options.timeout, HTTPClient.HTTPClientResult.Callback.New(
+                *FetchTasklet,
+                FetchTasklet.callback,
+            ).init(
+                fetch_tasklet,
+            ), proxy);
 
             if (!fetch_options.follow_redirects) {
                 fetch_tasklet.http.?.client.remaining_redirect_count = 0;
@@ -758,17 +756,7 @@ pub const Fetch = struct {
             return fetch_tasklet;
         }
 
-        const FetchOptions = struct {
-            method: Method,
-            headers: Headers,
-            body: AnyBlob,
-            timeout: usize,
-            disable_timeout: bool,
-            disable_keepalive: bool,
-            url: ZigURL,
-            verbose: bool = false,
-            follow_redirects: bool = true,
-        };
+        const FetchOptions = struct { method: Method, headers: Headers, body: AnyBlob, timeout: usize, disable_timeout: bool, disable_keepalive: bool, url: ZigURL, verbose: bool = false, follow_redirects: bool = true, proxy: ?ZigURL = null, url_proxy_buffer: []const u8 = "" };
 
         pub fn queue(
             allocator: std.mem.Allocator,
@@ -828,27 +816,129 @@ pub const Fetch = struct {
         var disable_timeout = false;
         var disable_keepalive = false;
         var verbose = false;
+        var proxy: ?ZigURL = null;
         var follow_redirects = true;
+        var url_proxy_buffer: []const u8 = undefined;
+
         if (first_arg.as(Request)) |request| {
-            url = ZigURL.parse(getAllocator(ctx).dupe(u8, request.url) catch unreachable);
-            method = request.method;
-            if (request.headers) |head| {
-                headers = Headers.from(head, bun.default_allocator) catch unreachable;
-            }
-            body = request.body.useAsAnyBlob();
-        } else if (first_arg.toStringOrNull(globalThis)) |jsstring| {
-            var url_slice = jsstring.toSlice(globalThis, bun.default_allocator).cloneIfNeeded(bun.default_allocator) catch {
-                JSC.JSError(bun.default_allocator, "Out of memory", .{}, ctx, exception);
-                return null;
-            };
+            if (arguments.len >= 2) {
+                const options = arguments[1].?.value();
+                if (options.isObject() or options.jsType() == .DOMWrapper) {
+                    if (options.fastGet(ctx.ptr(), .method)) |method_| {
+                        var slice_ = method_.toSlice(ctx.ptr(), getAllocator(ctx));
+                        defer slice_.deinit();
+                        method = Method.which(slice_.slice()) orelse .GET;
+                    } else {
+                        method = request.method;
+                    }
 
-            if (url_slice.len == 0) {
-                const fetch_error = fetch_error_blank_url;
-                return JSPromise.rejectedPromiseValue(globalThis, ZigString.init(fetch_error).toErrorInstance(globalThis)).asRef();
-            }
+                    if (options.fastGet(ctx.ptr(), .headers)) |headers_| {
+                        if (headers_.as(FetchHeaders)) |headers__| {
+                            headers = Headers.from(headers__, bun.default_allocator) catch unreachable;
+                            // TODO: make this one pass
+                        } else if (FetchHeaders.createFromJS(ctx.ptr(), headers_)) |headers__| {
+                            headers = Headers.from(headers__, bun.default_allocator) catch unreachable;
+                            headers__.deref();
+                        } else if (request.headers) |head| {
+                            headers = Headers.from(head, bun.default_allocator) catch unreachable;
+                        }
+                    } else if (request.headers) |head| {
+                        headers = Headers.from(head, bun.default_allocator) catch unreachable;
+                    }
 
-            url = ZigURL.parse(url_slice.slice());
+                    if (options.fastGet(ctx.ptr(), .body)) |body__| {
+                        if (Body.Value.fromJS(ctx.ptr(), body__)) |body_const| {
+                            var body_value = body_const;
+                            // TODO: buffer ReadableStream?
+                            // we have to explicitly check for InternalBlob
+                            body = body_value.useAsAnyBlob();
+                        } else {
+                            // an error was thrown
+                            return JSC.JSValue.jsUndefined().asObjectRef();
+                        }
+                    } else {
+                        body = request.body.useAsAnyBlob();
+                    }
 
+                    if (options.get(ctx, "timeout")) |timeout_value| {
+                        if (timeout_value.isBoolean()) {
+                            disable_timeout = !timeout_value.asBoolean();
+                        } else if (timeout_value.isNumber()) {
+                            disable_timeout = timeout_value.to(i32) == 0;
+                        }
+                    }
+
+                    if (options.get(ctx, "redirect")) |redirect_value| {
+                        if (redirect_value.getZigString(globalThis).eqlComptime("manual")) {
+                            follow_redirects = false;
+                        }
+                    }
+
+                    if (options.get(ctx, "keepalive")) |keepalive_value| {
+                        if (keepalive_value.isBoolean()) {
+                            disable_keepalive = !keepalive_value.asBoolean();
+                        } else if (keepalive_value.isNumber()) {
+                            disable_keepalive = keepalive_value.to(i32) == 0;
+                        }
+                    }
+                    if (options.get(globalThis, "verbose")) |verb| {
+                        verbose = verb.toBoolean();
+                    }
+                    if (options.get(globalThis, "proxy")) |proxy_arg| {
+                        if (!proxy_arg.isUndefined()) {
+                            if (proxy_arg.isNull()) {
+                                //if null we add an empty proxy to be ignore all proxy
+                                //only allocate url
+                                url = ZigURL.parse(getAllocator(ctx).dupe(u8, request.url) catch unreachable);
+                                url_proxy_buffer = url.href;
+                                proxy = ZigURL{}; //empty proxy
+                            } else {
+                                var proxy_str = proxy_arg.toStringOrNull(globalThis) orelse return null;
+                                // proxy + url 1 allocation
+                                var proxy_url_zig = proxy_str.getZigString(globalThis);
+
+                                // ignore proxy if it is len = 0
+                                if (proxy_url_zig.len == 0) {
+                                    url = ZigURL.parse(getAllocator(ctx).dupe(u8, request.url) catch unreachable);
+                                    url_proxy_buffer = url.href;
+                                } else {
+                                    var total_len = request.url.len + proxy_url_zig.len;
+
+                                    const allocator = getAllocator(ctx);
+
+                                    var buffer = allocator.alloc(u8, total_len) catch {
+                                        JSC.JSError(bun.default_allocator, "Out of memory", .{}, ctx, exception);
+                                        return null;
+                                    };
+
+                                    var url_slice = buffer[0..request.url.len];
+                                    std.mem.copy(u8, url_slice, request.url);
+                                    var proxy_url_slice = buffer[request.url.len..buffer.len];
+                                    std.mem.copy(u8, proxy_url_slice, proxy_url_zig.ptr[0..proxy_url_zig.len]);
+
+                                    url = ZigURL.parse(url_slice);
+                                    proxy = ZigURL.parse(proxy_url_slice);
+                                    url_proxy_buffer = buffer;
+                                }
+                            }
+                        }
+                    } else {
+                        // no proxy only url
+                        url = ZigURL.parse(getAllocator(ctx).dupe(u8, request.url) catch unreachable);
+                        url_proxy_buffer = url.href;
+                    }
+                }
+            } else {
+                method = request.method;
+                if (request.headers) |head| {
+                    headers = Headers.from(head, bun.default_allocator) catch unreachable;
+                }
+                body = request.body.useAsAnyBlob();
+                // no proxy only url
+                url = ZigURL.parse(getAllocator(ctx).dupe(u8, request.url) catch unreachable);
+                url_proxy_buffer = url.href;
+            }
+        } else if (first_arg.toStringOrNull(globalThis)) |jsstring| {
             if (arguments.len >= 2) {
                 const options = arguments[1].?.value();
                 if (options.isObject() or options.jsType() == .DOMWrapper) {
@@ -901,17 +991,108 @@ pub const Fetch = struct {
                             disable_keepalive = keepalive_value.to(i32) == 0;
                         }
                     }
-                }
 
-                // non-standard debug things
-                if (arguments.len == 3) {
-                    const special = arguments[2].?.value();
-                    if (!special.isEmptyOrUndefinedOrNull() and special.isObject()) {
-                        if (special.get(globalThis, "verbose")) |verb| {
-                            verbose = verb.toBoolean();
+                    if (options.get(globalThis, "verbose")) |verb| {
+                        verbose = verb.toBoolean();
+                    }
+                    if (options.get(globalThis, "proxy")) |proxy_arg| {
+                        if (!proxy_arg.isUndefined()) {
+                            var proxy_str = proxy_arg.toStringOrNull(globalThis) orelse return null;
+                            // proxy + url 1 allocation
+                            var url_zig = proxy_str.getZigString(globalThis);
+
+                            if (url_zig.len == 0) {
+                                const fetch_error = fetch_error_blank_url;
+                                return JSPromise.rejectedPromiseValue(globalThis, ZigString.init(fetch_error).toErrorInstance(globalThis)).asRef();
+                            }
+
+                            if (proxy_arg.isNull()) {
+                                //if null we add an empty proxy to be ignore all proxy
+                                //only allocate url
+                                const url_slice = url_zig.toSlice(bun.default_allocator).cloneIfNeeded(bun.default_allocator) catch {
+                                    JSC.JSError(bun.default_allocator, "Out of memory", .{}, ctx, exception);
+                                    return null;
+                                };
+                                url = ZigURL.parse(url_slice.slice());
+                                url_proxy_buffer = url.href;
+                                proxy = ZigURL{}; //empty proxy
+
+                            } else {
+                                var proxy_url_zig = proxy_str.getZigString(globalThis);
+
+                                // proxy is actual 0 len so ignores it
+                                if (proxy_url_zig.len == 0) {
+                                    const url_slice = url_zig.toSlice(bun.default_allocator).cloneIfNeeded(bun.default_allocator) catch {
+                                        JSC.JSError(bun.default_allocator, "Out of memory", .{}, ctx, exception);
+                                        return null;
+                                    };
+                                    url = ZigURL.parse(url_slice.slice());
+                                    url_proxy_buffer = url.href;
+                                } else {
+                                    const allocator = getAllocator(ctx);
+
+                                    var total_len = url_zig.len + proxy_url_zig.len;
+                                    var buffer = allocator.alloc(u8, total_len) catch {
+                                        JSC.JSError(bun.default_allocator, "Out of memory", .{}, ctx, exception);
+                                        return null;
+                                    };
+
+                                    var url_slice = buffer[0..url_zig.len];
+
+                                    std.mem.copy(u8, url_slice, url_zig.ptr[0..url_zig.len]);
+                                    var proxy_url_slice = buffer[url_zig.len..buffer.len];
+                                    std.mem.copy(u8, proxy_url_slice, proxy_url_zig.ptr[0..proxy_url_zig.len]);
+
+                                    url = ZigURL.parse(url_slice);
+                                    proxy = ZigURL.parse(proxy_url_slice);
+                                    url_proxy_buffer = buffer;
+                                }
+                            }
+                        } else {
+                            //no proxy only url
+                            var url_slice = jsstring.toSlice(globalThis, bun.default_allocator).cloneIfNeeded(bun.default_allocator) catch {
+                                JSC.JSError(bun.default_allocator, "Out of memory", .{}, ctx, exception);
+                                return null;
+                            };
+
+                            if (url_slice.len == 0) {
+                                const fetch_error = fetch_error_blank_url;
+                                return JSPromise.rejectedPromiseValue(globalThis, ZigString.init(fetch_error).toErrorInstance(globalThis)).asRef();
+                            }
+
+                            url = ZigURL.parse(url_slice.slice());
+                            url_proxy_buffer = url.href;
+                        }
+                    } else {
+                        //no proxy only url
+                        var url_slice = jsstring.toSlice(globalThis, bun.default_allocator).cloneIfNeeded(bun.default_allocator) catch {
+                            JSC.JSError(bun.default_allocator, "Out of memory", .{}, ctx, exception);
+                            return null;
+                        };
+
+                        if (url_slice.len == 0) {
+                            const fetch_error = fetch_error_blank_url;
+                            return JSPromise.rejectedPromiseValue(globalThis, ZigString.init(fetch_error).toErrorInstance(globalThis)).asRef();
                         }
+
+                        url = ZigURL.parse(url_slice.slice());
+                        url_proxy_buffer = url.href;
                     }
                 }
+            } else {
+                //no proxy only url
+                var url_slice = jsstring.toSlice(globalThis, bun.default_allocator).cloneIfNeeded(bun.default_allocator) catch {
+                    JSC.JSError(bun.default_allocator, "Out of memory", .{}, ctx, exception);
+                    return null;
+                };
+
+                if (url_slice.len == 0) {
+                    const fetch_error = fetch_error_blank_url;
+                    return JSPromise.rejectedPromiseValue(globalThis, ZigString.init(fetch_error).toErrorInstance(globalThis)).asRef();
+                }
+
+                url = ZigURL.parse(url_slice.slice());
+                url_proxy_buffer = url.href;
             }
         } else {
             const fetch_error = fetch_type_error_strings.get(js.JSValueGetType(ctx, arguments[0]));
@@ -925,19 +1106,9 @@ pub const Fetch = struct {
         _ = FetchTasklet.queue(
             default_allocator,
             globalThis,
-            .{
-                .method = method,
-                .url = url,
-                .headers = headers orelse Headers{
-                    .allocator = bun.default_allocator,
-                },
-                .body = body,
-                .timeout = std.time.ns_per_hour,
-                .disable_keepalive = disable_keepalive,
-                .disable_timeout = disable_timeout,
-                .follow_redirects = follow_redirects,
-                .verbose = verbose,
-            },
+            .{ .method = method, .url = url, .headers = headers orelse Headers{
+                .allocator = bun.default_allocator,
+            }, .body = body, .timeout = std.time.ns_per_hour, .disable_keepalive = disable_keepalive, .disable_timeout = disable_timeout, .follow_redirects = follow_redirects, .verbose = verbose, .proxy = proxy, .url_proxy_buffer = url_proxy_buffer },
             JSC.JSValue.fromRef(deferred_promise),
         ) catch unreachable;
         return deferred_promise;