diff options
author | 2021-10-13 00:27:35 -0700 | |
---|---|---|
committer | 2021-10-13 00:27:35 -0700 | |
commit | 88a5e2d34d25c3ac3d13a432bbd85daa075c4bcb (patch) | |
tree | 53e40562b7bbe006304daa341a5ce3dfbdeca81e | |
parent | afc346d6f125a41fb6ff823d04d3ffd85ab36dcd (diff) | |
download | bun-88a5e2d34d25c3ac3d13a432bbd85daa075c4bcb.tar.gz bun-88a5e2d34d25c3ac3d13a432bbd85daa075c4bcb.tar.zst bun-88a5e2d34d25c3ac3d13a432bbd85daa075c4bcb.zip |
Add TLS 1.3 support, improve fetch() HTTPS performance
-rw-r--r-- | Makefile | 298 | ||||
-rw-r--r-- | misctools/fetch.zig | 10 | ||||
-rw-r--r-- | src/cli.zig | 58 | ||||
-rw-r--r-- | src/cli/colon_list_type.zig | 36 | ||||
-rw-r--r-- | src/cli/create_command.zig | 31 | ||||
-rw-r--r-- | src/deps/picohttp.zig | 17 | ||||
-rw-r--r-- | src/deps/picohttpparser.zig | 21 | ||||
m--------- | src/deps/s2n-tls | 0 | ||||
-rw-r--r-- | src/deps/zig-clap/clap.zig | 2 | ||||
-rw-r--r-- | src/feature_flags.zig | 2 | ||||
-rw-r--r-- | src/http.zig | 49 | ||||
-rw-r--r-- | src/http/method.zig | 49 | ||||
-rw-r--r-- | src/http_client.zig | 207 | ||||
-rw-r--r-- | src/javascript/jsc/webcore/response.zig | 4 | ||||
-rw-r--r-- | src/libarchive/libarchive.zig | 8 | ||||
-rw-r--r-- | src/s2n.zig | 246 |
16 files changed, 751 insertions, 287 deletions
@@ -1,6 +1,8 @@ OS_NAME := $(shell uname -s | tr '[:upper:]' '[:lower:]') ARCH_NAME_RAW := $(shell uname -m) +make-lazy = $(eval $1 = $$(eval $1 := $(value $(1)))$$($1)) + ARCH_NAME := ifeq ($(ARCH_NAME_RAW),arm64) ARCH_NAME = aarch64 @@ -8,21 +10,28 @@ else ARCH_NAME = x64 endif -TRIPLET := $(OS_NAME)-$(ARCH_NAME) -PACKAGE_NAME := bun-cli-$(TRIPLET) -PACKAGES_REALPATH := $(shell realpath packages) -PACKAGE_DIR := $(PACKAGES_REALPATH)/$(PACKAGE_NAME) -DEBUG_PACKAGE_DIR := $(PACKAGES_REALPATH)/debug-$(PACKAGE_NAME) -BIN_DIR := $(PACKAGE_DIR)/bin -RELEASE_BUN := $(PACKAGE_DIR)/bin/bun -DEBUG_BIN := $(DEBUG_PACKAGE_DIR)/bin -DEBUG_BUN := $(DEBUG_BIN)/bun-debug -BUILD_ID := $(shell cat ./build-id) -PACKAGE_JSON_VERSION := 0.0.$(BUILD_ID) -BUN_BUILD_TAG := bun-v$(PACKAGE_JSON_VERSION) -CC := clang -CXX := clang++ -DEPS_DIR := $(shell pwd)/src/deps +TRIPLET = $(OS_NAME)-$(ARCH_NAME) +PACKAGE_NAME = bun-cli-$(TRIPLET) +PACKAGES_REALPATH = $(shell realpath packages) +PACKAGE_DIR = $(PACKAGES_REALPATH)/$(PACKAGE_NAME) +DEBUG_PACKAGE_DIR = $(PACKAGES_REALPATH)/debug-$(PACKAGE_NAME) +BIN_DIR = $(PACKAGE_DIR)/bin +RELEASE_BUN = $(PACKAGE_DIR)/bin/bun +DEBUG_BIN = $(DEBUG_PACKAGE_DIR)/bin +DEBUG_BUN = $(DEBUG_BIN)/bun-debug +BUILD_ID = $(shell cat ./build-id) +PACKAGE_JSON_VERSION = 0.0.$(BUILD_ID) +BUN_BUILD_TAG = bun-v$(PACKAGE_JSON_VERSION) +CC ?= $(shell realpath clang) +CXX ?= $(shell realpath clang++) +DEPS_DIR = $(shell pwd)/src/deps +CPUS ?= $(shell nproc) +USER ?= $(echo $USER) + +LIBCRYPTO_STATIC_LIB = $(shell brew list openssl@1.1|grep libcrypto.a) +LIBCRYPTO_PREFIX_LIB_DIR = $(shell dirname $(LIBCRYPTO_STATIC_LIB)) +LIBCRYPTO_PREFIX_DIR = $(shell dirname $(LIBCRYPTO_PREFIX_LIB_DIR)) +LIBCRYPTO_INCLUDE_DIR = $(LIBCRYPTO_PREFIX_DIR)/include BUN_TMP_DIR := /tmp/make-bun @@ -65,7 +74,105 @@ endif STRIP ?= $(shell which llvm-strip || which llvm-strip-12 || echo "Missing llvm-strip. Please pass it in the STRIP environment var"; exit 1;) ifeq ($(OS_NAME),darwin) - HOMEBREW_PREFIX := $(shell brew --prefix)/ + HOMEBREW_PREFIX = $(shell brew --prefix)/ +endif + + +SRC_DIR := src/javascript/jsc/bindings +OBJ_DIR := src/javascript/jsc/bindings-obj +SRC_FILES := $(wildcard $(SRC_DIR)/*.cpp) +OBJ_FILES := $(patsubst $(SRC_DIR)/%.cpp,$(OBJ_DIR)/%.o,$(SRC_FILES)) +MAC_INCLUDE_DIRS := -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/JavaScriptCore/PrivateHeaders \ + -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/WTF/Headers \ + -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/ICU/Headers \ + -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/ \ + -Isrc/javascript/jsc/bindings/ \ + -Isrc/javascript/jsc/WebKit/Source/bmalloc + +LINUX_INCLUDE_DIRS := -I$(JSC_INCLUDE_DIR) \ + -Isrc/javascript/jsc/bindings/ + +INCLUDE_DIRS := + +ifeq ($(OS_NAME),linux) + INCLUDE_DIRS += $(LINUX_INCLUDE_DIRS) +endif + +ifeq ($(OS_NAME),darwin) + INCLUDE_DIRS += $(MAC_INCLUDE_DIRS) +endif + + + +MACOS_ICU_FILES := $(HOMEBREW_PREFIX)opt/icu4c/lib/libicudata.a \ + $(HOMEBREW_PREFIX)opt/icu4c/lib/libicui18n.a \ + $(HOMEBREW_PREFIX)opt/icu4c/lib/libicuuc.a + +MACOS_ICU_INCLUDE := $(HOMEBREW_PREFIX)opt/icu4c/include + +ICU_FLAGS := + +# TODO: find a way to make this more resilient +# Ideally, we could just look up the linker search paths +LIB_ICU_PATH ?= /usr/lib/x86_64-linux-gnu + +ifeq ($(OS_NAME),linux) + ICU_FLAGS += $(LIB_ICU_PATH)/libicuuc.a $(LIB_ICU_PATH)/libicudata.a $(LIB_ICU_PATH)/libicui18n.a +endif + +ifeq ($(OS_NAME),darwin) +ICU_FLAGS += -l icucore \ + $(MACOS_ICU_FILES) \ + -I$(MACOS_ICU_INCLUDE) +endif + + + +CLANG_FLAGS = $(INCLUDE_DIRS) \ + -std=gnu++17 \ + -DSTATICALLY_LINKED_WITH_JavaScriptCore=1 \ + -DSTATICALLY_LINKED_WITH_WTF=1 \ + -DSTATICALLY_LINKED_WITH_BMALLOC=1 \ + -DBUILDING_WITH_CMAKE=1 \ + -DNDEBUG=1 \ + -DNOMINMAX \ + -DIS_BUILD \ + -g \ + -DENABLE_INSPECTOR_ALTERNATE_DISPATCHERS=0 \ + -DBUILDING_JSCONLY__ \ + -DASSERT_ENABLED=0 \ + -fPIE + +# This flag is only added to webkit builds on Apple platforms +# It has something to do with ICU +ifeq ($(OS_NAME), darwin) +CLANG_FLAGS += -DDU_DISABLE_RENAMING=1 +endif + +BUN_LLD_FLAGS = $(OBJ_FILES) \ + ${ICU_FLAGS} \ + ${JSC_FILES} \ + src/deps/mimalloc/libmimalloc.a \ + src/deps/zlib/libz.a \ + src/deps/libarchive.a \ + src/deps/libs2n.a \ + src/deps/libcrypto.a \ + src/deps/picohttpparser.o \ + $(CLANG_FLAGS) \ + +ifeq ($(OS_NAME), linux) +BUN_LLD_FLAGS += -lstdc++fs \ + -pthread \ + -ldl \ + -lc \ + -Wl,-z,now \ + -Wl,--as-needed \ + -Wl,-z,stack-size=12800000 \ + -Wl,-z,notext \ + -ffunction-sections \ + -fdata-sections \ + -Wl,--gc-sections \ + -fuse-ld=lld endif bun: vendor build-obj bun-link-lld-release @@ -77,7 +184,7 @@ libarchive: cd src/deps/libarchive; \ make clean; \ cmake . -DENABLE_ZLIB=OFF -DENABLE_OPENSSL=OFF; \ - make -j${shell nproc}; \ + make -j${CPUS}; \ cp libarchive/libarchive.a $(DEPS_DIR)/libarchive.a; vendor: require init-submodules vendor-without-check @@ -85,11 +192,6 @@ vendor: require init-submodules vendor-without-check zlib: cd src/deps/zlib; cmake .; make; -openssl: - cd src/deps/zlib; \ - ./configure --with-zlib-lib=$(ZLIB_LIB_DIR) --with-zlib-include=$(ZLIB_INCLUDE_DIR); \ - make -j${shell nproc}; - require: @echo "Checking if the required utilities are available..." @realpath --version >/dev/null 2>&1 || (echo "ERROR: realpath is required."; exit 1) @@ -136,8 +238,64 @@ runtime_js: bun_error: @cd packages/bun-error; npm install; npm run --silent build +fetch: + cd misctools; zig build-obj -Drelease-fast ./fetch.zig -fcompiler-rt --main-pkg-path ../ + $(CXX) ./misctools/fetch.o -g -O3 -o ./misctools/fetch \ + src/deps/mimalloc/libmimalloc.a \ + src/deps/zlib/libz.a \ + src/deps/libarchive.a \ + src/deps/libs2n.a \ + src/deps/picohttpparser.o \ + src/deps/libcrypto.a +fetch-debug: + cd misctools; zig build-obj ./fetch.zig -fcompiler-rt --main-pkg-path ../ + $(CXX) ./misctools/fetch.o -g -o ./misctools/fetch \ + src/deps/mimalloc/libmimalloc.a \ + src/deps/zlib/libz.a \ + src/deps/libarchive.a \ + src/deps/libs2n.a \ + src/deps/picohttpparser.o \ + src/deps/libcrypto.a +s2n-mac: + cd $(DEPS_DIR)/s2n-tls; \ + make clean; \ + CC=$(CC) CXX=$(CXX) cmake . -Bbuild -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DLibCrypto_INCLUDE_DIR=$(LIBCRYPTO_INCLUDE_DIR) \ + -DLibCrypto_STATIC_LIBRARY=$(LIBCRYPTO_STATIC_LIB) \ + -DLibCrypto_LIBRARY=$(LIBCRYPTO_STATIC_LIB) \ + -DCMAKE_PREFIX_PATH=$(shell brew --prefix openssl@1.1); \ + CC=$(CC) CXX=$(CXX) cmake --build ./build -j$(CPUS); \ + CC=$(CC) CXX=$(CXX) CTEST_PARALLEL_LEVEL=$(CPUS) ninja -C build + cp $(DEPS_DIR)/s2n-tls/build/lib/libs2n.a $(DEPS_DIR)/libs2n.a + unlink $(DEPS_DIR)/libcrypto.a || echo ""; + ln $(LIBCRYPTO_STATIC_LIB) $(DEPS_DIR)/libcrypto.a || echo ""; + +s2n-mac-debug: + cd $(DEPS_DIR)/s2n-tls; \ + make clean; \ + CC=$(CC) CXX=$(CXX) cmake . -Bbuild -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_SHARED_LIBS=OFF \ + -DLibCrypto_INCLUDE_DIR=$(LIBCRYPTO_INCLUDE_DIR) \ + -DLibCrypto_STATIC_LIBRARY=$(LIBCRYPTO_STATIC_LIB) \ + -DLibCrypto_LIBRARY=$(LIBCRYPTO_STATIC_LIB) \ + -DCMAKE_PREFIX_PATH=$(shell brew --prefix openssl@1.1); \ + CC=$(CC) CXX=$(CXX) cmake --build ./build -j$(CPUS); \ + CC=$(CC) CXX=$(CXX) CTEST_PARALLEL_LEVEL=$(CPUS) ninja -C build test + cp $(DEPS_DIR)/s2n-tls/build/lib/libs2n.a $(DEPS_DIR)/libs2n.a + unlink $(DEPS_DIR)/libcrypto.a || echo ""; + ln $(LIBCRYPTO_STATIC_LIB) $(DEPS_DIR)/libcrypto.a || echo ""; + +libcrypto_path: + @echo ${LIBCRYPTO_STATIC_LIB} + +ifeq ($(OS_NAME),darwin) +s2n: s2n-mac +endif jsc: jsc-build jsc-bindings jsc-build: $(JSC_BUILD_STEPS) @@ -259,105 +417,11 @@ clean: clean-bindings -SRC_DIR := src/javascript/jsc/bindings -OBJ_DIR := src/javascript/jsc/bindings-obj -SRC_FILES := $(wildcard $(SRC_DIR)/*.cpp) -OBJ_FILES := $(patsubst $(SRC_DIR)/%.cpp,$(OBJ_DIR)/%.o,$(SRC_FILES)) -MAC_INCLUDE_DIRS := -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/JavaScriptCore/PrivateHeaders \ - -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/WTF/Headers \ - -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/ICU/Headers \ - -Isrc/javascript/jsc/WebKit/WebKitBuild/Release/ \ - -Isrc/javascript/jsc/bindings/ \ - -Isrc/javascript/jsc/WebKit/Source/bmalloc - -LINUX_INCLUDE_DIRS := -I$(JSC_INCLUDE_DIR) \ - -Isrc/javascript/jsc/bindings/ - -INCLUDE_DIRS := - -ifeq ($(OS_NAME),linux) - INCLUDE_DIRS += $(LINUX_INCLUDE_DIRS) -endif - -ifeq ($(OS_NAME),darwin) - INCLUDE_DIRS += $(MAC_INCLUDE_DIRS) -endif - -CLANG_FLAGS := $(INCLUDE_DIRS) \ - -std=gnu++17 \ - -DSTATICALLY_LINKED_WITH_JavaScriptCore=1 \ - -DSTATICALLY_LINKED_WITH_WTF=1 \ - -DSTATICALLY_LINKED_WITH_BMALLOC=1 \ - -DBUILDING_WITH_CMAKE=1 \ - -DNDEBUG=1 \ - -DNOMINMAX \ - -DIS_BUILD \ - -g \ - -DENABLE_INSPECTOR_ALTERNATE_DISPATCHERS=0 \ - -DBUILDING_JSCONLY__ \ - -DASSERT_ENABLED=0 \ - -fPIE - -# This flag is only added to webkit builds on Apple platforms -# It has something to do with ICU -ifeq ($(OS_NAME), darwin) -CLANG_FLAGS += -DDU_DISABLE_RENAMING=1 -endif jsc-bindings-mac: $(OBJ_FILES) -MACOS_ICU_FILES := $(HOMEBREW_PREFIX)opt/icu4c/lib/libicudata.a \ - $(HOMEBREW_PREFIX)opt/icu4c/lib/libicui18n.a \ - $(HOMEBREW_PREFIX)opt/icu4c/lib/libicuuc.a - -MACOS_ICU_INCLUDE := $(HOMEBREW_PREFIX)opt/icu4c/include - -ICU_FLAGS := - -# TODO: find a way to make this more resilient -# Ideally, we could just look up the linker search paths -LIB_ICU_PATH ?= /usr/lib/x86_64-linux-gnu - -ifeq ($(OS_NAME),linux) - ICU_FLAGS += $(LIB_ICU_PATH)/libicuuc.a $(LIB_ICU_PATH)/libicudata.a $(LIB_ICU_PATH)/libicui18n.a -endif - -ifeq ($(OS_NAME),darwin) -ICU_FLAGS += -l icucore \ - $(MACOS_ICU_FILES) \ - -I$(MACOS_ICU_INCLUDE) -endif - -BUN_LLD_FLAGS := $(OBJ_FILES) \ - ${ICU_FLAGS} \ - ${JSC_FILES} \ - src/deps/picohttpparser.o \ - src/deps/mimalloc/libmimalloc.a \ - src/deps/zlib/libz.a \ - src/deps/libarchive.a \ - src/deps/openssl/libssl.a \ - src/deps/openssl/libcrypto.a \ - $(CLANG_FLAGS) \ - - -ifeq ($(OS_NAME), linux) -BUN_LLD_FLAGS += -lstdc++fs \ - -pthread \ - -ldl \ - -lc \ - -Wl,-z,now \ - -Wl,--as-needed \ - -Wl,-z,stack-size=12800000 \ - -Wl,-z,notext \ - -ffunction-sections \ - -fdata-sections \ - -Wl,--gc-sections \ - -fuse-ld=lld -endif - - mimalloc: cd src/deps/mimalloc; cmake .; make; @@ -401,7 +465,7 @@ sizegen: $(BUN_TMP_DIR)/sizegen > src/javascript/jsc/bindings/sizes.zig picohttp: - $(CC) -O3 -g -fPIE -c src/deps/picohttpparser.c -Isrc/deps -o src/deps/picohttpparser.o; cd ../../ + $(CC) -march=native -O3 -g -fPIE -c src/deps/picohttpparser/picohttpparser.c -Isrc/deps -o src/deps/picohttpparser.o; cd ../../ analytics: ./node_modules/.bin/peechy --schema src/analytics/schema.peechy --zig src/analytics/analytics_schema.zig diff --git a/misctools/fetch.zig b/misctools/fetch.zig index ad704151a..9780af398 100644 --- a/misctools/fetch.zig +++ b/misctools/fetch.zig @@ -161,9 +161,19 @@ pub fn main() anyerror!void { var args = try Arguments.parse(default_allocator); var client = HTTPClient.init(default_allocator, args.method, args.url, args.headers, args.headers_buf); client.verbose = args.verbose; + client.disable_shutdown = true; var body_out_str = try MutableString.init(default_allocator, 1024); var response = try client.send(args.body, &body_out_str); Output.disableBuffering(); try Output.writer().writeAll(body_out_str.list.items); + + switch (response.status_code) { + 200, 302 => {}, + else => { + if (!client.verbose) { + Output.prettyErrorln("Response: {}", .{response}); + } + }, + } } diff --git a/src/cli.zig b/src/cli.zig index e26d94f6d..13d9ef0aa 100644 --- a/src/cli.zig +++ b/src/cli.zig @@ -32,6 +32,7 @@ const BunCommand = @import("./cli/bun_command.zig").BunCommand; const DevCommand = @import("./cli/dev_command.zig").DevCommand; const DiscordCommand = @import("./cli/discord_command.zig").DiscordCommand; const BuildCommand = @import("./cli/build_command.zig").BuildCommand; +const CreateCommand = @import("./cli/create_command.zig").CreateCommand; const RunCommand = @import("./cli/run_command.zig").RunCommand; var start_time: i128 = undefined; @@ -62,39 +63,7 @@ pub const Cli = struct { }; const LoaderMatcher = strings.ExactSizeMatcher(4); -pub fn ColonListType(comptime t: type, value_resolver: anytype) type { - return struct { - pub fn init(allocator: *std.mem.Allocator, count: usize) !@This() { - var keys = try allocator.alloc(string, count); - var values = try allocator.alloc(t, count); - - return @This(){ .keys = keys, .values = values }; - } - keys: []string, - values: []t, - - pub fn load(self: *@This(), input: []const string) !void { - for (input) |str, i| { - // Support either ":" or "=" as the separator, preferring whichever is first. - // ":" is less confusing IMO because that syntax is used with flags - // but "=" is what esbuild uses and I want this to be somewhat familiar for people using esbuild - const midpoint = std.math.min(strings.indexOfChar(str, ':') orelse std.math.maxInt(usize), strings.indexOfChar(str, '=') orelse std.math.maxInt(usize)); - if (midpoint == std.math.maxInt(usize)) { - return error.InvalidSeparator; - } - - self.keys[i] = str[0..midpoint]; - self.values[i] = try value_resolver(str[midpoint + 1 .. str.len]); - } - } - - pub fn resolve(allocator: *std.mem.Allocator, input: []const string) !@This() { - var list = try init(allocator, input.len); - try list.load(input); - return list; - } - }; -} +const ColonListType = @import("./cli/colon_list_type.zig").ColonListType; pub const LoaderColonList = ColonListType(Api.Loader, Arguments.loader_resolver); pub const DefineColonList = ColonListType(string, Arguments.noop_resolver); @@ -197,7 +166,10 @@ pub const Arguments = struct { pub fn parse(allocator: *std.mem.Allocator, comptime cmd: Command.Tag) !Api.TransformOptions { var diag = clap.Diagnostic{}; - var args = clap.parse(clap.Help, ¶ms, .{ .diagnostic = &diag }) catch |err| { + var args = clap.parse(clap.Help, ¶ms, .{ + .diagnostic = &diag, + .allocator = allocator, + }) catch |err| { // Report useful error and exit diag.report(Output.errorWriter(), err) catch {}; return err; @@ -469,10 +441,11 @@ const HelpCommand = struct { const dirname = std.fs.path.basename(cwd); if (FeatureFlags.dev_only) { const fmt = - \\> <r> <b><green>dev <r><d> ./a.ts ./b.jsx<r> Start a Bun Dev Server - \\> <r> <b><magenta>bun <r><d> ./a.ts ./b.jsx<r> Bundle dependencies of input files into a <r><magenta>.bun<r> - \\> <r> <b><blue>discord<r> Open Bun's Discord server - \\> <r> <b><d>help <r> Print this help menu + \\> <r> <b><green>dev <r><d> ./a.ts ./b.jsx<r> Start a Bun Dev Server + \\> <r> <b><magenta>bun <r><d> ./a.ts ./b.jsx<r> Bundle dependencies of input files into a <r><magenta>.bun<r> + \\> <r> <b><cyan>create <r><d> next<r> <r> Start a new project from a template <d>(shorthand: c)<r> + \\> <r> <b><blue>discord <r> Open Bun's Discord server + \\> <r> <b><d>help <r> Print this help menu \\ ; @@ -485,6 +458,7 @@ const HelpCommand = struct { \\> <r> <b><white>init<r> Setup Bun in \"{s}\" \\> <r> <b><green>dev <r><d> ./a.ts ./b.jsx<r> Start a Bun Dev Server \\<d>*<r> <b><cyan>build <r><d> ./a.ts ./b.jsx<r> Make JavaScript-like code runnable & bundle CSS + \\> <r> <b><cyan>create<r><d> next<r> Use a template from https://github.com/jarred-sumner/bun/tree/main/examples<r> \\> <r> <b><magenta>bun <r><d> ./a.ts ./b.jsx<r> Bundle dependencies of input files into a <r><magenta>.bun<r> \\> <r> <green>run <r><d> ./a.ts <r> Run a JavaScript-like file with Bun.js \\> <r> <b><blue>discord<r> Open Bun's Discord server @@ -571,6 +545,7 @@ pub const Command = struct { RootCommandMatcher.case("init") => .InitCommand, RootCommandMatcher.case("bun") => .BunCommand, RootCommandMatcher.case("discord") => .DiscordCommand, + RootCommandMatcher.case("c"), RootCommandMatcher.case("create") => .CreateCommand, RootCommandMatcher.case("b"), RootCommandMatcher.case("build") => .BuildCommand, RootCommandMatcher.case("r"), RootCommandMatcher.case("run") => .RunCommand, @@ -585,6 +560,7 @@ pub const Command = struct { RootCommandMatcher.case("bun") => .BunCommand, RootCommandMatcher.case("discord") => .DiscordCommand, RootCommandMatcher.case("d"), RootCommandMatcher.case("dev") => .DevCommand, + RootCommandMatcher.case("c"), RootCommandMatcher.case("create") => .CreateCommand, RootCommandMatcher.case("help") => .HelpCommand, else => .AutoCommand, @@ -617,6 +593,11 @@ pub const Command = struct { try BuildCommand.exec(ctx); }, + .CreateCommand => { + const ctx = try Command.Context.create(allocator, log, .CreateCommand); + + try CreateCommand.exec(ctx); + }, .RunCommand => { const ctx = try Command.Context.create(allocator, log, .RunCommand); @@ -661,5 +642,6 @@ pub const Command = struct { RunCommand, AutoCommand, HelpCommand, + CreateCommand, }; }; diff --git a/src/cli/colon_list_type.zig b/src/cli/colon_list_type.zig new file mode 100644 index 000000000..bb243feff --- /dev/null +++ b/src/cli/colon_list_type.zig @@ -0,0 +1,36 @@ +usingnamespace @import("../global.zig"); +const std = @import("std"); + +pub fn ColonListType(comptime t: type, value_resolver: anytype) type { + return struct { + pub fn init(allocator: *std.mem.Allocator, count: usize) !@This() { + var keys = try allocator.alloc(string, count); + var values = try allocator.alloc(t, count); + + return @This(){ .keys = keys, .values = values }; + } + keys: []string, + values: []t, + + pub fn load(self: *@This(), input: []const string) !void { + for (input) |str, i| { + // Support either ":" or "=" as the separator, preferring whichever is first. + // ":" is less confusing IMO because that syntax is used with flags + // but "=" is what esbuild uses and I want this to be somewhat familiar for people using esbuild + const midpoint = std.math.min(strings.indexOfChar(str, ':') orelse std.math.maxInt(usize), strings.indexOfChar(str, '=') orelse std.math.maxInt(usize)); + if (midpoint == std.math.maxInt(usize)) { + return error.InvalidSeparator; + } + + self.keys[i] = str[0..midpoint]; + self.values[i] = try value_resolver(str[midpoint + 1 .. str.len]); + } + } + + pub fn resolve(allocator: *std.mem.Allocator, input: []const string) !@This() { + var list = try init(allocator, input.len); + try list.load(input); + return list; + } + }; +} diff --git a/src/cli/create_command.zig b/src/cli/create_command.zig new file mode 100644 index 000000000..14e6ac428 --- /dev/null +++ b/src/cli/create_command.zig @@ -0,0 +1,31 @@ +usingnamespace @import("../global.zig"); +const std = @import("std"); + +const lex = @import("../js_lexer.zig"); +const logger = @import("../logger.zig"); +const alloc = @import("../alloc.zig"); +const options = @import("../options.zig"); +const js_parser = @import("../js_parser.zig"); +const js_ast = @import("../js_ast.zig"); +const linker = @import("../linker.zig"); +usingnamespace @import("../ast/base.zig"); +usingnamespace @import("../defines.zig"); +const panicky = @import("../panic_handler.zig"); +const allocators = @import("../allocators.zig"); +const sync = @import(".././sync.zig"); +const Api = @import("../api/schema.zig").Api; +const resolve_path = @import("../resolver/resolve_path.zig"); +const configureTransformOptionsForBun = @import("../javascript/jsc/config.zig").configureTransformOptionsForBun; +const Command = @import("../cli.zig").Command; +const bundler = @import("../bundler.zig"); +const NodeModuleBundle = @import("../node_module_bundle.zig").NodeModuleBundle; +const fs = @import("../fs.zig"); + +pub const CreateCommand = struct { + pub const Args = struct { + template_name: string, + directory_name: string, + }; + + pub fn exec(ctx: Command.Context) !void {} +}; diff --git a/src/deps/picohttp.zig b/src/deps/picohttp.zig index cdd26b710..374793f38 100644 --- a/src/deps/picohttp.zig +++ b/src/deps/picohttp.zig @@ -1,7 +1,8 @@ const std = @import("std"); -const c = @cImport(@cInclude("picohttpparser.h")); +const c = @import("picohttpparser.zig"); const ExactSizeMatcher = @import("../exact_size_matcher.zig").ExactSizeMatcher; const Match = ExactSizeMatcher(2); +const Output = @import("../global.zig").Output; const fmt = std.fmt; @@ -16,10 +17,18 @@ pub const Header = struct { } pub fn format(self: Header, comptime layout: []const u8, opts: fmt.FormatOptions, writer: anytype) !void { - if (self.isMultiline()) { - try fmt.format(writer, "{s}", .{self.value}); + if (Output.enable_ansi_colors) { + if (self.isMultiline()) { + try fmt.format(writer, comptime Output.prettyFmt("<r><cyan>{s}", true), .{self.value}); + } else { + try fmt.format(writer, comptime Output.prettyFmt("<r><cyan>{s}<r><d>: <r>{s}", true), .{ self.name, self.value }); + } } else { - try fmt.format(writer, "{s}: {s}", .{ self.name, self.value }); + if (self.isMultiline()) { + try fmt.format(writer, comptime Output.prettyFmt("<r><cyan>{s}", false), .{self.value}); + } else { + try fmt.format(writer, comptime Output.prettyFmt("<r><cyan>{s}<r><d>: <r>{s}", false), .{ self.name, self.value }); + } } } diff --git a/src/deps/picohttpparser.zig b/src/deps/picohttpparser.zig new file mode 100644 index 000000000..d83d7cd31 --- /dev/null +++ b/src/deps/picohttpparser.zig @@ -0,0 +1,21 @@ +pub usingnamespace @import("std").zig.c_builtins; + +pub const struct_phr_header = extern struct { + name: [*c]const u8, + name_len: usize, + value: [*c]const u8, + value_len: usize, +}; +pub extern fn phr_parse_request(buf: [*c]const u8, len: usize, method: [*c][*c]const u8, method_len: [*c]usize, path: [*c][*c]const u8, path_len: [*c]usize, minor_version: [*c]c_int, headers: [*c]struct_phr_header, num_headers: [*c]usize, last_len: usize) c_int; +pub extern fn phr_parse_response(_buf: [*c]const u8, len: usize, minor_version: [*c]c_int, status: [*c]c_int, msg: [*c][*c]const u8, msg_len: [*c]usize, headers: [*c]struct_phr_header, num_headers: [*c]usize, last_len: usize) c_int; +pub extern fn phr_parse_headers(buf: [*c]const u8, len: usize, headers: [*c]struct_phr_header, num_headers: [*c]usize, last_len: usize) c_int; +pub const struct_phr_chunked_decoder = extern struct { + bytes_left_in_chunk: usize, + consume_trailer: u8, + _hex_count: u8, + _state: u8, +}; +pub extern fn phr_decode_chunked(decoder: [*c]struct_phr_chunked_decoder, buf: [*c]u8, bufsz: [*c]usize) isize; +pub extern fn phr_decode_chunked_is_in_data(decoder: [*c]struct_phr_chunked_decoder) c_int; +pub const phr_header = struct_phr_header; +pub const phr_chunked_decoder = struct_phr_chunked_decoder; diff --git a/src/deps/s2n-tls b/src/deps/s2n-tls -Subproject 0dd0b56bbb8daac39b8bf1b0884dcc7d5951c95 +Subproject 4c31c6e790f14b299bb16de550a2132666e69a6 diff --git a/src/deps/zig-clap/clap.zig b/src/deps/zig-clap/clap.zig index e6bf56f08..907ece37f 100644 --- a/src/deps/zig-clap/clap.zig +++ b/src/deps/zig-clap/clap.zig @@ -63,6 +63,8 @@ pub fn Param(comptime Id: type) type { /// Takes a string and parses it to a Param(Help). /// This is the reverse of 'help' but for at single parameter only. pub fn parseParam(line: []const u8) !Param(Help) { + @setEvalBranchQuota(9999); + var found_comma = false; var it = mem.tokenize(u8, line, " \t"); var param_str = it.next() orelse return error.NoParamFound; diff --git a/src/feature_flags.zig b/src/feature_flags.zig index 777753561..ebaf8e0f8 100644 --- a/src/feature_flags.zig +++ b/src/feature_flags.zig @@ -77,3 +77,5 @@ pub const force_macro = false; pub const include_filename_in_jsx = false; pub const verbose_analytics = false; + +pub const disable_compression_in_http_client = false; diff --git a/src/http.zig b/src/http.zig index 5cc7376cb..4ceccfaa4 100644 --- a/src/http.zig +++ b/src/http.zig @@ -60,54 +60,7 @@ const ZigURL = @import("./query_string_map.zig").URL; const HTTPStatusCode = u10; const URLPath = @import("./http/url_path.zig"); - -pub const Method = enum { - GET, - HEAD, - PATCH, - PUT, - POST, - OPTIONS, - CONNECT, - TRACE, - - pub fn which(str: []const u8) ?Method { - if (str.len < 3) { - return null; - } - const Match = strings.ExactSizeMatcher(2); - // we already did the length check - switch (Match.match(str[0..2])) { - Match.case("GE"), Match.case("ge") => { - return .GET; - }, - Match.case("HE"), Match.case("he") => { - return .HEAD; - }, - Match.case("PA"), Match.case("pa") => { - return .PATCH; - }, - Match.case("PO"), Match.case("po") => { - return .POST; - }, - Match.case("PU"), Match.case("pu") => { - return .PUT; - }, - Match.case("OP"), Match.case("op") => { - return .OPTIONS; - }, - Match.case("CO"), Match.case("co") => { - return .CONNECT; - }, - Match.case("TR"), Match.case("tr") => { - return .TRACE; - }, - else => { - return null; - }, - } - } -}; +const Method = @import("./http/method.zig").Method; pub const RequestContext = struct { request: Request, diff --git a/src/http/method.zig b/src/http/method.zig new file mode 100644 index 000000000..f306e522e --- /dev/null +++ b/src/http/method.zig @@ -0,0 +1,49 @@ +usingnamespace @import("../global.zig"); + +pub const Method = enum { + GET, + HEAD, + PATCH, + PUT, + POST, + OPTIONS, + CONNECT, + TRACE, + + pub fn which(str: []const u8) ?Method { + if (str.len < 3) { + return null; + } + const Match = strings.ExactSizeMatcher(2); + // we already did the length check + switch (Match.match(str[0..2])) { + Match.case("GE"), Match.case("ge") => { + return .GET; + }, + Match.case("HE"), Match.case("he") => { + return .HEAD; + }, + Match.case("PA"), Match.case("pa") => { + return .PATCH; + }, + Match.case("PO"), Match.case("po") => { + return .POST; + }, + Match.case("PU"), Match.case("pu") => { + return .PUT; + }, + Match.case("OP"), Match.case("op") => { + return .OPTIONS; + }, + Match.case("CO"), Match.case("co") => { + return .CONNECT; + }, + Match.case("TR"), Match.case("tr") => { + return .TRACE; + }, + else => { + return null; + }, + } + } +}; diff --git a/src/http_client.zig b/src/http_client.zig index c14e5d1c2..1a67aa674 100644 --- a/src/http_client.zig +++ b/src/http_client.zig @@ -1,17 +1,17 @@ // @link "/Users/jarred/Code/bun/src/deps/zlib/libz.a" -// @link "/Users/jarred/Code/bun/src/deps/picohttpparser.o" -const picohttp = @import("picohttp"); +const picohttp = @import("./deps/picohttp.zig"); usingnamespace @import("./global.zig"); const std = @import("std"); const Headers = @import("./javascript/jsc/webcore/response.zig").Headers; const URL = @import("./query_string_map.zig").URL; -const Method = @import("./http.zig").Method; -const iguanaTLS = @import("iguanaTLS"); +const Method = @import("./http/method.zig").Method; +const iguanaTLS = @import("./deps/iguanaTLS/src/main.zig"); const Api = @import("./api/schema.zig").Api; const Lock = @import("./lock.zig").Lock; const HTTPClient = @This(); const SOCKET_FLAGS = os.SOCK_CLOEXEC; +const S2n = @import("./s2n.zig"); fn writeRequest( comptime Writer: type, @@ -40,6 +40,11 @@ url: URL, allocator: *std.mem.Allocator, verbose: bool = isTest, tcp_client: tcp.Client = undefined, +body_size: u32 = 0, +read_count: u32 = 0, +remaining_redirect_count: i8 = 127, +redirect_buf: [2048]u8 = undefined, +disable_shutdown: bool = false, pub fn init(allocator: *std.mem.Allocator, method: Method, url: URL, header_entries: Headers.Entries, header_buf: string) HTTPClient { return HTTPClient{ @@ -98,15 +103,29 @@ const content_length_header_hash = hashHeaderName("Content-Length"); const connection_header = picohttp.Header{ .name = "Connection", .value = "close" }; const accept_header = picohttp.Header{ .name = "Accept", .value = "*/*" }; const accept_header_hash = hashHeaderName("Accept"); -const accept_encoding_header = picohttp.Header{ .name = "Accept-Encoding", .value = "deflate, gzip" }; + +const accept_encoding_no_compression = "identity"; +const accept_encoding_compression = "deflate, gzip"; +const accept_encoding_header_compression = picohttp.Header{ .name = "Accept-Encoding", .value = accept_encoding_compression }; +const accept_encoding_header_no_compression = picohttp.Header{ .name = "Accept-Encoding", .value = accept_encoding_no_compression }; + +const accept_encoding_header = if (FeatureFlags.disable_compression_in_http_client) + accept_encoding_header_no_compression +else + accept_encoding_header_compression; + const accept_encoding_header_hash = hashHeaderName("Accept-Encoding"); + const user_agent_header = picohttp.Header{ .name = "User-Agent", .value = "Bun.js " ++ Global.package_json_version }; const user_agent_header_hash = hashHeaderName("User-Agent"); +const location_header_hash = hashHeaderName("Location"); pub fn headerStr(this: *const HTTPClient, ptr: Api.StringPointer) string { return this.header_buf[ptr.offset..][0..ptr.length]; } +threadlocal var server_name_buf: [1024]u8 = undefined; + pub fn buildRequest(this: *const HTTPClient, body_len: usize) picohttp.Request { var header_count: usize = 0; var header_entries = this.header_entries.slice(); @@ -222,19 +241,38 @@ pub fn connect( threadlocal var http_req_buf: [65436]u8 = undefined; -pub inline fn send(this: *HTTPClient, body: []const u8, body_out_str: *MutableString) !picohttp.Response { - if (this.url.isHTTPS()) { - return this.sendHTTPS(body, body_out_str); - } else { - return this.sendHTTP(body, body_out_str); +pub fn send(this: *HTTPClient, body: []const u8, body_out_str: *MutableString) !picohttp.Response { + // this prevents stack overflow + redirect: while (this.remaining_redirect_count >= -1) { + if (this.url.isHTTPS()) { + return this.sendHTTPS(body, body_out_str) catch |err| { + switch (err) { + error.Redirect => { + this.remaining_redirect_count -= 1; + continue :redirect; + }, + else => return err, + } + }; + } else { + return this.sendHTTP(body, body_out_str) catch |err| { + switch (err) { + error.Redirect => { + this.remaining_redirect_count -= 1; + continue :redirect; + }, + else => return err, + } + }; + } } + + return error.TooManyRedirects; } pub fn sendHTTP(this: *HTTPClient, body: []const u8, body_out_str: *MutableString) !picohttp.Response { this.tcp_client = try this.connect(); - defer { - std.os.closeSocket(this.tcp_client.socket.fd); - } + defer std.os.closeSocket(this.tcp_client.socket.fd); var request = buildRequest(this, body.len); if (this.verbose) { Output.prettyErrorln("{s}", .{request}); @@ -308,6 +346,7 @@ pub fn processResponse(this: *HTTPClient, comptime is_https: bool, comptime Clie { var req_buf_len: usize = 0; var req_buf_read: usize = std.math.maxInt(usize); + defer this.read_count += @intCast(u32, req_buf_len); var response_length: usize = 0; restart: while (req_buf_read != 0) { @@ -337,11 +376,13 @@ pub fn processResponse(this: *HTTPClient, comptime is_https: bool, comptime Clie var content_length: u32 = 0; var encoding = Encoding.identity; - for (response.headers) |header| { - if (this.verbose) { - Output.prettyErrorln("Response: {s}", .{response}); - } + var location: string = ""; + if (this.verbose) { + Output.prettyErrorln("Response: {s}", .{response}); + } + + for (response.headers) |header| { switch (hashHeaderName(header.name)) { content_length_header_hash => { content_length = std.fmt.parseInt(u32, header.value, 10) catch 0; @@ -357,13 +398,48 @@ pub fn processResponse(this: *HTTPClient, comptime is_https: bool, comptime Clie return error.UnsupportedEncoding; } }, + location_header_hash => { + location = header.value; + }, + + else => {}, + } + } + + if (location.len > 0 and this.remaining_redirect_count > 0) { + switch (response.status_code) { + 302, 301, 307, 308, 303 => { + if (strings.indexOf(location, "://")) |i| { + const protocol_name = location[0..i]; + if (strings.eqlComptime(protocol_name, "http") or strings.eqlComptime(protocol_name, "https")) {} else { + return error.UnsupportedRedirectProtocol; + } + + std.mem.copy(u8, &this.redirect_buf, location); + this.url = URL.parse(location); + } else { + const original_url = this.url; + this.url = URL.parse(std.fmt.bufPrint( + &this.redirect_buf, + "{s}://{s}{s}", + .{ original_url.displayProtocol(), original_url.displayHostname(), location }, + ) catch return error.RedirectURLTooLong); + } + + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/303 + if (response.status_code == 303) { + this.method = .GET; + } + + return error.Redirect; + }, else => {}, } } if (content_length > 0) { var remaining_content_length = content_length; - var remainder = http_req_buf[@intCast(u32, response.bytes_read)..]; + var remainder = http_req_buf[@intCast(usize, response.bytes_read)..]; remainder = remainder[0..std.math.min(remainder.len, content_length)]; const Zlib = @import("./zlib.zig"); @@ -390,7 +466,8 @@ pub fn processResponse(this: *HTTPClient, comptime is_https: bool, comptime Clie if (comptime !is_https) { if (remainder.len > 0) { std.mem.copy(u8, buffer.list.items, remainder); - body_size = @intCast(u32, remainder.len); + body_size = remainder.len; + this.read_count += @intCast(u32, body_size); remaining_content_length -= @intCast(u32, remainder.len); } } @@ -399,6 +476,7 @@ pub fn processResponse(this: *HTTPClient, comptime is_https: bool, comptime Clie const size = @intCast(u32, try client.read( buffer.list.items[body_size..], )); + this.read_count += size; if (size == 0) break; body_size += size; @@ -429,31 +507,16 @@ pub fn processResponse(this: *HTTPClient, comptime is_https: bool, comptime Clie pub fn sendHTTPS(this: *HTTPClient, body_str: []const u8, body_out_str: *MutableString) !picohttp.Response { var connection = try this.connect(); + S2n.boot(default_allocator); + const hostname = this.url.displayHostname(); + std.mem.copy(u8, &server_name_buf, hostname); + server_name_buf[hostname.len] = 0; + var server_name = server_name_buf[0..hostname.len :0]; - var arena = std.heap.ArenaAllocator.init(this.allocator); - defer arena.deinit(); - - var rand = blk: { - var seed: [std.rand.DefaultCsprng.secret_seed_length]u8 = undefined; - try std.os.getrandom(&seed); - break :blk &std.rand.DefaultCsprng.init(seed).random; - }; - - var client = try iguanaTLS.client_connect( - .{ - .rand = rand, - .temp_allocator = &arena.allocator, - .reader = connection.reader(SOCKET_FLAGS), - .writer = connection.writer(SOCKET_FLAGS), - .cert_verifier = .none, - .protocols = &[_][]const u8{"http/1.1"}, - }, - this.url.hostname, - ); - - defer { - client.close_notify() catch {}; - } + var client = S2n.Connection.init(connection.socket.fd); + try client.start(server_name); + client.disable_shutdown = this.disable_shutdown; + defer client.close() catch {}; var request = buildRequest(this, body_str.len); if (this.verbose) { @@ -568,7 +631,6 @@ test "sendHTTPS - identity" { try std.testing.expectEqualStrings(body_out_str.list.items, @embedFile("fixtures_example.com.html")); } -// zig test src/http_client.zig --test-filter "sendHTTPS - gzip" -lc -lc++ /Users/jarred/Code/bun/src/deps/zlib/libz.a /Users/jarred/Code/bun/src/deps/picohttpparser.o --cache-dir /Users/jarred/Code/bun/zig-cache --global-cache-dir /Users/jarred/.cache/zig --name bun --pkg-begin clap /Users/jarred/Code/bun/src/deps/zig-clap/clap.zig --pkg-end --pkg-begin picohttp /Users/jarred/Code/bun/src/deps/picohttp.zig --pkg-end --pkg-begin iguanaTLS /Users/jarred/Code/bun/src/deps/iguanaTLS/src/main.zig --pkg-end -I /Users/jarred/Code/bun/src/deps -I /Users/jarred/Code/bun/src/deps/mimalloc -I /usr/local/opt/icu4c/include -L src/deps/mimalloc -L /usr/local/opt/icu4c/lib --main-pkg-path /Users/jarred/Code/bun --enable-cache -femit-bin=zig-out/bin/test --test-no-exec test "sendHTTPS - gzip" { Output.initTest(); defer Output.flush(); @@ -596,4 +658,61 @@ test "sendHTTPS - gzip" { try std.testing.expectEqualStrings(body_out_str.list.items, @embedFile("fixtures_example.com.html")); } +// zig test src/http_client.zig --test-filter "sendHTTPS - deflate" -lc -lc++ /Users/jarred/Code/bun/src/deps/zlib/libz.a /Users/jarred/Code/bun/src/deps/picohttpparser.o --cache-dir /Users/jarred/Code/bun/zig-cache --global-cache-dir /Users/jarred/.cache/zig --name bun --pkg-begin clap /Users/jarred/Code/bun/src/deps/zig-clap/clap.zig --pkg-end --pkg-begin picohttp /Users/jarred/Code/bun/src/deps/picohttp.zig --pkg-end --pkg-begin iguanaTLS /Users/jarred/Code/bun/src/deps/iguanaTLS/src/main.zig --pkg-end -I /Users/jarred/Code/bun/src/deps -I /Users/jarred/Code/bun/src/deps/mimalloc -I /usr/local/opt/icu4c/include -L src/deps/mimalloc -L /usr/local/opt/icu4c/lib --main-pkg-path /Users/jarred/Code/bun --enable-cache -femit-bin=zig-out/bin/test +test "sendHTTPS - deflate" { + Output.initTest(); + defer Output.flush(); + + var headers = try std.heap.c_allocator.create(Headers); + headers.* = Headers{ + .entries = @TypeOf(headers.entries){}, + .buf = @TypeOf(headers.buf){}, + .used = 0, + .allocator = std.heap.c_allocator, + }; + + headers.appendHeader("Accept-Encoding", "deflate", false, false, false); + + var client = HTTPClient.init( + std.heap.c_allocator, + .GET, + URL.parse("https://example.com/"), + headers.entries, + headers.buf.items, + ); + var body_out_str = try MutableString.init(std.heap.c_allocator, 0); + var response = try client.sendHTTPS("", &body_out_str); + try std.testing.expectEqual(response.status_code, 200); + try std.testing.expectEqualStrings(body_out_str.list.items, @embedFile("fixtures_example.com.html")); +} + // zig test src/http_client.zig --test-filter "sendHTTP" -lc -lc++ /Users/jarred/Code/bun/src/deps/zlib/libz.a /Users/jarred/Code/bun/src/deps/picohttpparser.o --cache-dir /Users/jarred/Code/bun/zig-cache --global-cache-dir /Users/jarred/.cache/zig --name bun --pkg-begin clap /Users/jarred/Code/bun/src/deps/zig-clap/clap.zig --pkg-end --pkg-begin picohttp /Users/jarred/Code/bun/src/deps/picohttp.zig --pkg-end --pkg-begin iguanaTLS /Users/jarred/Code/bun/src/deps/iguanaTLS/src/main.zig --pkg-end -I /Users/jarred/Code/bun/src/deps -I /Users/jarred/Code/bun/src/deps/mimalloc -I /usr/local/opt/icu4c/include -L src/deps/mimalloc -L /usr/local/opt/icu4c/lib --main-pkg-path /Users/jarred/Code/bun --enable-cache -femit-bin=zig-out/bin/test + +test "send - redirect" { + Output.initTest(); + defer Output.flush(); + + var headers = try std.heap.c_allocator.create(Headers); + headers.* = Headers{ + .entries = @TypeOf(headers.entries){}, + .buf = @TypeOf(headers.buf){}, + .used = 0, + .allocator = std.heap.c_allocator, + }; + + headers.appendHeader("Accept-Encoding", "gzip", false, false, false); + + var client = HTTPClient.init( + std.heap.c_allocator, + .GET, + URL.parse("https://www.bun.sh/"), + headers.entries, + headers.buf.items, + ); + try std.testing.expectEqualStrings(client.url.hostname, "www.bun.sh"); + var body_out_str = try MutableString.init(std.heap.c_allocator, 0); + var response = try client.send("", &body_out_str); + try std.testing.expectEqual(response.status_code, 200); + try std.testing.expectEqual(client.url.hostname, "bun.sh"); + try std.testing.expectEqualStrings(body_out_str.list.items, @embedFile("fixtures_example.com.html")); +} diff --git a/src/javascript/jsc/webcore/response.zig b/src/javascript/jsc/webcore/response.zig index ff399990e..76be74d84 100644 --- a/src/javascript/jsc/webcore/response.zig +++ b/src/javascript/jsc/webcore/response.zig @@ -6,6 +6,8 @@ usingnamespace @import("../javascript.zig"); usingnamespace @import("../bindings/bindings.zig"); const ZigURL = @import("../../../query_string_map.zig").URL; const HTTPClient = @import("../../../http_client.zig"); +const Method = @import("../../../http/method.zig").Method; + const picohttp = @import("picohttp"); pub const Response = struct { pub const Class = NewClass( @@ -500,7 +502,7 @@ pub const Fetch = struct { defer js.JSStringRelease(string_ref); var method_name_buf: [16]u8 = undefined; var method_name = method_name_buf[0..js.JSStringGetUTF8CString(string_ref, &method_name_buf, method_name_buf.len)]; - http_client.method = http.Method.which(method_name) orelse http_client.method; + http_client.method = Method.which(method_name) orelse http_client.method; } } }, diff --git a/src/libarchive/libarchive.zig b/src/libarchive/libarchive.zig index c6f01402b..3d1a00be8 100644 --- a/src/libarchive/libarchive.zig +++ b/src/libarchive/libarchive.zig @@ -448,7 +448,7 @@ pub const Archive = struct { error.FileNotFound => { const subdir = try dir.makeOpenPath(dirname, .{ .iterate = true }); defer if (comptime close_handles) subdir.close(); - break :brk try subdir.createFile(std.fs.path.basename(pathname), .{ .truncate = true }); + break :brk try dir.createFileZ(pathname, .{ .truncate = true }); }, else => { return err; @@ -458,12 +458,6 @@ pub const Archive = struct { defer if (comptime close_handles) file.close(); _ = lib.archive_read_data_into_fd(archive, file.handle); _ = C.fchmod(file.handle, lib.archive_entry_perm(entry)); - - // switch (status) { - // Status.eof => break :copy_data, - // Status.ok => { - // else => return error.Fail, - // } } }, } diff --git a/src/s2n.zig b/src/s2n.zig index fd8d23cd0..3c8d13014 100644 --- a/src/s2n.zig +++ b/src/s2n.zig @@ -1,19 +1,193 @@ pub usingnamespace @import("std").zig.c_builtins; const std = @import("std"); +const Output = @import("./global.zig").Output; + +const alpn_protocols = "http/1.1"; + +pub inline fn s2nassert(value: c_int) void { + std.debug.assert(value == 0); +} + pub fn boot(allcoator: *std.mem.Allocator) void { if (booted) return; booted = true; Allocator.allocator = allcoator; + CacheStore.instance = CacheStore{ .allocator = allcoator, .map = @TypeOf(CacheStore.instance.map).init(allcoator) }; + + // Important for any website using Cloudflare. + // Do to our modifications in the library, this must be called _first_ + // Before we initialize s2n. + // It can never be changed after initialization or we risk undefined memory bugs. + if (s2n_get_highest_fully_supported_tls_version() == S2N_TLS13) { + // This conditional should always return true since we statically compile libCrypto. + _ = s2n_enable_tls13(); + + // Sadly, this TLS 1.3 implementation is slower than TLS 1.2. + // ❯ hyperfine "./fetch https://example.com" "./fetchtls13 https://example.com" + // Benchmark #1: ./fetch https://example.com + // Time (mean ± σ): 83.6 ms ± 5.4 ms [User: 15.1 ms, System: 4.7 ms] + // Range (min … max): 73.5 ms … 97.5 ms 35 runs + + // Benchmark #2: ./fetchtls13 https://example.com + // Time (mean ± σ): 94.9 ms ± 3.2 ms [User: 15.8 ms, System: 4.8 ms] + // Range (min … max): 90.7 ms … 104.6 ms 29 runs - _ = s2n_disable_atexit(); - _ = s2n_mem_set_callbacks(Allocator.initCallback, Allocator.deinitCallback, Allocator.mallocCallback, Allocator.freeCallback); + // Summary + // './fetch https://example.com' ran + // 1.14 ± 0.08 times faster than './fetchtls13 https://example.com' - _ = s2n_init(); - global_s2n_config = s2n_config_new(); - _ = s2n_config_disable_x509_verification(global_s2n_config); + } + + // We don't actually need the memory allocator, it will automatically use mimalloc...I don't know how! + // Also, the implementation + // s2nassert(s2n_mem_set_callbacks(Allocator.initCallback, Allocator.deinitCallback, Allocator.mallocCallback, Allocator.freeCallback)); + + s2nassert(s2n_disable_atexit()); + s2nassert(s2n_init()); + global_s2n_config = s2n_fetch_default_config(); + // s2nassert(s2n_config_set_verify_host_callback(global_s2n_config, verify_host_callback, null)); + // s2nassert(s2n_config_set_check_stapled_ocsp_response(global_s2n_config, 0)); + // s2nassert(s2n_config_set_cipher_preferences(global_s2n_config, "default")); + // s2nassert(s2n_config_disable_x509_verification(global_s2n_config)); + var protocol: [*c]const u8 = "http/1.1"; + var protocols = &protocol; + s2nassert(s2n_config_set_protocol_preferences(global_s2n_config, protocols, 1)); + s2nassert(s2n_config_send_max_fragment_length(global_s2n_config, S2N_TLS_MAX_FRAG_LEN_4096)); + + // s2n_config_set_ticket_decrypt_key_lifetime(global_s2n_config, 9999999); + + s2nassert( + s2n_config_set_cache_store_callback(global_s2n_config, CacheStore.store, &CacheStore.instance), + ); + s2nassert( + s2n_config_set_cache_retrieve_callback(global_s2n_config, CacheStore.retrieve, &CacheStore.instance), + ); + s2nassert( + s2n_config_set_cache_delete_callback(global_s2n_config, CacheStore.delete, &CacheStore.instance), + ); + s2nassert( + s2n_config_set_session_cache_onoff(global_s2n_config, 1), + ); + + // s2nassert(s2n_config_init_session_ticket_keys()); + // s2nassert(s2n_config_set_client_auth_type(global_s2n_config, S2N_STATUS_REQUEST_NONE)); } +pub const CacheStore = struct { + const CacheEntry = struct { + key: []u8, + value: []u8, + seconds: u64, + + pub fn init( + allocator: *std.mem.Allocator, + key: *const c_void, + size: u64, + value: *const c_void, + value_size: u64, + seconds: u64, + ) CacheEntry { + const key_bytes = keyBytes(key, size); + const value_bytes = keyBytes(key, value_size); + + var total_bytes = allocator.alloc(u8, key_bytes.len + value_bytes.len) catch unreachable; + @memcpy(total_bytes.ptr, key_bytes.ptr, key_bytes.len); + @memcpy(total_bytes[key_bytes.len..].ptr, value_bytes.ptr, value_bytes.len); + + return CacheEntry{ .key = total_bytes[0..key_bytes.len], .value = total_bytes[key_bytes.len..], .seconds = seconds }; + } + }; + + const Context = struct { + pub fn hash(this: @This(), key: u64) u64 { + return key; + } + + pub fn eql(this: @This(), a: u64, b: u64) bool { + return a == b; + } + }; + + allocator: *std.mem.Allocator, + map: std.HashMap(u64, CacheEntry, Context, 80), + + pub inline fn keyBytes(key: *const c_void, size: u64) []u8 { + const ptr = @intToPtr([*]u8, @ptrToInt(key)); + + return ptr[0..size]; + } + + inline fn hashKey(key: *const c_void, size: u64) u64 { + const bytes = keyBytes(key, size); + return std.hash.Wyhash.hash(0, bytes); + } + + pub fn retrieve( + conn: *s2n_connection, + ctx: ?*c_void, + key: *const c_void, + key_size: u64, + value: *c_void, + value_size: *u64, + ) callconv(.C) c_int { + const hash = hashKey(key, key_size); + + if (instance.map.getAdapted(hash, Context{})) |entry| { + const now = @intCast(usize, std.time.timestamp()); + if (now > entry.seconds) { + _ = instance.map.removeAdapted(hash, Context{}); + return 0; + } + + var value_bytes = keyBytes(value, value_size.*); + if (value_bytes.len < entry.value.len) return -1; + std.mem.copy(u8, value_bytes, entry.value); + value_size.* = entry.value.len; + return 0; + } + + return 0; + } + + pub fn store( + conn: *s2n_connection, + ctx: ?*c_void, + seconds: u64, + key: *const c_void, + key_size: u64, + value: *const c_void, + value_size: u64, + ) callconv(.C) c_int { + var map_entry = instance.map.getOrPutAdapted(hashKey(key, key_size), Context{}) catch unreachable; + + if (!map_entry.found_existing) { + map_entry.value_ptr.* = CacheEntry.init(instance.allocator, key, key_size, value, value_size, @intCast(usize, std.time.timestamp()) + seconds); + } + + return S2N_SUCCESS; + } + + pub fn delete( + conn: *s2n_connection, + ctx: ?*c_void, + key: *const c_void, + key_size: u64, + ) callconv(.C) c_int { + _ = instance.map.remove(hashKey(key, key_size)); + return 0; + } + + pub var instance: CacheStore = undefined; +}; + +pub fn verify_host_callback(ptr: [*c]const u8, len: usize, ctx: ?*c_void) callconv(.C) u8 { + return 1; +} + +pub extern fn s2n_enable_tls13() c_int; +pub extern fn s2n_fetch_default_config() *s2n_config; +pub extern fn s2n_get_highest_fully_supported_tls_version() c_int; pub extern fn s2n_errno_location() [*c]c_int; pub const S2N_ERR_T_OK: c_int = 0; pub const S2N_ERR_T_IO: c_int = 1; @@ -37,9 +211,9 @@ pub extern fn s2n_config_free(config: *struct_s2n_config) c_int; pub extern fn s2n_config_free_dhparams(config: *struct_s2n_config) c_int; pub extern fn s2n_config_free_cert_chain_and_key(config: *struct_s2n_config) c_int; pub const s2n_clock_time_nanoseconds = ?fn (?*c_void, [*c]u64) callconv(.C) c_int; -pub const s2n_cache_retrieve_callback = ?fn (*struct_s2n_connection, ?*c_void, ?*const c_void, u64, ?*c_void, [*c]u64) callconv(.C) c_int; -pub const s2n_cache_store_callback = ?fn (*struct_s2n_connection, ?*c_void, u64, ?*const c_void, u64, ?*const c_void, u64) callconv(.C) c_int; -pub const s2n_cache_delete_callback = ?fn (*struct_s2n_connection, ?*c_void, ?*const c_void, u64) callconv(.C) c_int; +pub const s2n_cache_retrieve_callback = ?fn (*struct_s2n_connection, ?*c_void, *const c_void, u64, *c_void, *u64) callconv(.C) c_int; +pub const s2n_cache_store_callback = ?fn (*struct_s2n_connection, ?*c_void, u64, *const c_void, u64, *const c_void, u64) callconv(.C) c_int; +pub const s2n_cache_delete_callback = ?fn (*struct_s2n_connection, ?*c_void, *const c_void, u64) callconv(.C) c_int; pub extern fn s2n_config_set_wall_clock(config: *struct_s2n_config, clock_fn: s2n_clock_time_nanoseconds, ctx: ?*c_void) c_int; pub extern fn s2n_config_set_monotonic_clock(config: *struct_s2n_config, clock_fn: s2n_clock_time_nanoseconds, ctx: ?*c_void) c_int; pub extern fn s2n_strerror(@"error": c_int, lang: [*c]const u8) [*c]const u8; @@ -164,8 +338,8 @@ pub extern fn s2n_connection_set_write_fd(conn: *struct_s2n_connection, writefd: pub extern fn s2n_connection_get_read_fd(conn: *struct_s2n_connection, readfd: [*c]c_int) c_int; pub extern fn s2n_connection_get_write_fd(conn: *struct_s2n_connection, writefd: [*c]c_int) c_int; pub extern fn s2n_connection_use_corked_io(conn: *struct_s2n_connection) c_int; -pub const s2n_recv_fn = fn (?*c_void, [*c]u8, u32) callconv(.C) c_int; -pub const s2n_send_fn = fn (?*c_void, [*c]const u8, u32) callconv(.C) c_int; +pub const s2n_recv_fn = fn (*s2n_connection, [*c]u8, u32) callconv(.C) c_int; +pub const s2n_send_fn = fn (*s2n_connection, [*c]const u8, u32) callconv(.C) c_int; pub extern fn s2n_connection_set_recv_ctx(conn: *struct_s2n_connection, ctx: ?*c_void) c_int; pub extern fn s2n_connection_set_send_ctx(conn: *struct_s2n_connection, ctx: ?*c_void) c_int; pub extern fn s2n_connection_set_recv_cb(conn: *struct_s2n_connection, recv: ?s2n_recv_fn) c_int; @@ -395,6 +569,7 @@ pub const Connection = struct { conn: *s2n_connection = undefined, fd: std.os.socket_t, node: *Pool.List.Node, + disable_shutdown: bool = false, pub const Pool = struct { pub const List = std.SinglyLinkedList(*s2n_connection); @@ -429,27 +604,40 @@ pub const Connection = struct { const errno = s2nErrorNo; - pub fn s2n_recv_function(conn: *s2n_connection, buf: *c, len: u32) c_int { - return std.os.recv(fd, buf, len, 0); - } - pub fn s2n_send_function(conn: *s2n_connection, buf: *c, len: u32) c_int { - return std.os.send(fd, buf, SOCKET_FLAGS); - } + // pub fn s2n_recv_function(conn: *s2n_connection, buf: [*c]u8, len: u32) callconv(.C) c_int { + // if (buf == null) return 0; + // var fd: c_int = 0; + // _ = s2n_connection_get_read_fd(conn, &fd); + // return @intCast(c_int, std.os.system.recvfrom(fd, buf, len, std.os.SOCK_CLOEXEC, null, null)); + // } + // pub fn s2n_send_function(conn: *s2n_connection, buf: [*c]const u8, len: u32) callconv(.C) c_int { + // if (buf == null) return 0; + // var fd: c_int = 0; + // _ = s2n_connection_get_write_fd(conn, &fd); + + // return @intCast(c_int, std.os.system.sendto(fd, buf.?, len, std.os.SOCK_CLOEXEC, null, 0)); + // } - pub fn start(this: *Connection) !void { + pub fn start(this: *Connection, server_name: [:0]const u8) !void { this.node = Pool.get(); this.conn = this.node.data; - _ = s2n_connection_set_config(this.conn, global_s2n_config); - _ = s2n_connection_set_fd(this.conn, @intCast(c_int, this.fd)); - _ = s2n_connection_set_blinding(this.conn, S2N_SELF_SERVICE_BLINDING); - _ = s2n_connection_prefer_low_latency(this.conn); - _ = s2n_connection_set_ctx(this.conn, this); - - s2n_connection_set_recv_cb(this.conn, s2n_recv_function); - s2n_connection_set_send_cb(this.conn, s2n_send_function); + s2nassert(s2n_connection_set_ctx(this.conn, this)); + s2nassert(s2n_connection_set_config(this.conn, global_s2n_config)); + s2nassert(s2n_connection_set_read_fd(this.conn, @intCast(c_int, this.fd))); + s2nassert(s2n_connection_set_write_fd(this.conn, @intCast(c_int, this.fd))); + s2nassert(s2n_connection_set_blinding(this.conn, S2N_SELF_SERVICE_BLINDING)); + // s2nassert(s2n_connection_set_dynamic_record(this.conn)); + s2nassert(s2n_set_server_name(this.conn, server_name.ptr)); + + // _ = s2n_connection_set_recv_cb(this.conn, s2n_recv_function); + // _ = s2n_connection_set_send_cb(this.conn, s2n_send_function); const rc = s2n_negotiate(this.conn, &blocked_status); + if (rc < 0) { + Output.printErrorln("Alert: {d}", .{s2n_connection_get_alert(this.conn)}); + Output.prettyErrorln("ERROR: {s}", .{s2n_strerror_debug(rc, "EN")}); + } - defer s2n_connection_free_handshake(this.conn); + defer s2nassert(s2n_connection_free_handshake(this.conn)); switch (try s2nErrorNo(rc)) { .SUCCESS => return, @@ -468,8 +656,10 @@ pub const Connection = struct { } pub fn close(this: *Connection) !void { - _ = s2n_shutdown(this.conn, &blocked_status); - Pool.put(this.node); + if (!this.disable_shutdown) { + _ = s2n_shutdown(this.conn, &blocked_status); + Pool.put(this.node); + } std.os.closeSocket(this.fd); } |