diff options
author | 2023-09-11 20:05:48 -0700 | |
---|---|---|
committer | 2023-09-12 16:23:14 -0700 | |
commit | 3a58ef4a0ba23f1d245b81296017dc1c7aa8cfdd (patch) | |
tree | 9167aef82f7ff5771d1cd115b12887f5cdcc2122 | |
parent | 9c2e8f1d1b5828090bd6c82fcd4a72c6df67f8c6 (diff) | |
download | bun-3a58ef4a0ba23f1d245b81296017dc1c7aa8cfdd.tar.gz bun-3a58ef4a0ba23f1d245b81296017dc1c7aa8cfdd.tar.zst bun-3a58ef4a0ba23f1d245b81296017dc1c7aa8cfdd.zip |
stuff
-rw-r--r-- | .vscode/launch.json | 14 | ||||
-rw-r--r-- | src/bun.js/api/bun/socket.zig | 4 | ||||
-rw-r--r-- | src/bun.js/base.zig | 42 | ||||
-rw-r--r-- | src/bun.js/event_loop.zig | 22 | ||||
-rw-r--r-- | src/bun.js/test/jest.zig | 2 | ||||
-rw-r--r-- | src/bundler/bundle_v2.zig | 2 | ||||
-rw-r--r-- | src/cli/run_command.zig | 461 | ||||
-rw-r--r-- | src/deps/uws.zig | 19 | ||||
-rw-r--r-- | src/install/install.zig | 87 | ||||
-rw-r--r-- | src/install/lockfile.zig | 53 | ||||
-rw-r--r-- | src/io/io_darwin.zig | 2 | ||||
-rw-r--r-- | src/io/io_linux.zig | 2 | ||||
-rw-r--r-- | src/network_thread.zig | 2 |
13 files changed, 545 insertions, 167 deletions
diff --git a/.vscode/launch.json b/.vscode/launch.json index 862c583a5..403ca8d7b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,6 +9,20 @@ { "type": "lldb", "request": "launch", + "name": "sharp", + "program": "bun-debug", + "args": ["install", "sharp"], + // The cwd here must be the same as in CI. Or you will cause test failures that only happen in CI. + "cwd": "/tmp/scratchpad_20230911T213851", + "env": { + "FORCE_COLOR": "1", + "BUN_GARBAGE_COLLECTOR_LEVEL": "2" + }, + "console": "internalConsole" + }, + { + "type": "lldb", + "request": "launch", "name": "bun test [file]", "program": "bun-debug", "args": ["test", "${file}"], diff --git a/src/bun.js/api/bun/socket.zig b/src/bun.js/api/bun/socket.zig index 4cdcbdac8..2ad44ffb0 100644 --- a/src/bun.js/api/bun/socket.zig +++ b/src/bun.js/api/bun/socket.zig @@ -582,7 +582,7 @@ pub const Listener = struct { var socket_context = uws.us_create_bun_socket_context( @intFromBool(ssl_enabled), - uws.Loop.get().?, + uws.Loop.get(), @sizeOf(usize), ctx_opts, ) orelse { @@ -919,7 +919,7 @@ pub const Listener = struct { globalObject.bunVM().eventLoop().ensureWaker(); - var socket_context = uws.us_create_bun_socket_context(@intFromBool(ssl_enabled), uws.Loop.get().?, @sizeOf(usize), ctx_opts).?; + var socket_context = uws.us_create_bun_socket_context(@intFromBool(ssl_enabled), uws.Loop.get(), @sizeOf(usize), ctx_opts).?; var connection: Listener.UnixOrHost = if (port) |port_| .{ .host = .{ .host = (hostname_or_unix.cloneIfNeeded(bun.default_allocator) catch unreachable).slice(), .port = port_ }, } else .{ diff --git a/src/bun.js/base.zig b/src/bun.js/base.zig index a6df36c4f..d050804e3 100644 --- a/src/bun.js/base.zig +++ b/src/bun.js/base.zig @@ -23,6 +23,7 @@ const uws = @import("root").bun.uws; const Body = WebCore.Body; const TaggedPointerTypes = @import("../tagged_pointer.zig"); const TaggedPointerUnion = TaggedPointerTypes.TaggedPointerUnion; +const PackageManager = @import("../install/install.zig").PackageManager; pub const ExceptionValueRef = [*c]js.JSValueRef; pub const JSValueRef = js.JSValueRef; @@ -1720,6 +1721,10 @@ pub const FilePoll = struct { pub var owner: Owner = Owner.init(@as(*Deactivated, @ptrFromInt(@as(usize, 0xDEADBEEF)))); }; + const RunCommand = @import("../../src/cli/run_command.zig").RunCommand; + const PostinstallSubprocess = RunCommand.PostinstallSubprocess; + const PostinstallSubprocessPid = RunCommand.PostinstallSubprocess.PidPollData; + pub const Owner = bun.TaggedPointerUnion(.{ FileReader, FileSink, @@ -1729,6 +1734,8 @@ pub const FilePoll = struct { Deactivated, DNSResolver, GetAddrInfoRequest, + PostinstallSubprocess, + PostinstallSubprocessPid, }); fn updateFlags(poll: *FilePoll, updated: Flags.Set) void { @@ -1827,7 +1834,6 @@ pub const FilePoll = struct { @field(Owner.Tag, "Subprocess") => { log("onUpdate " ++ kqueue_or_epoll ++ " (fd: {d}) Subprocess", .{poll.fd}); var loader = ptr.as(JSC.Subprocess); - loader.onExitNotification(); }, @field(Owner.Tag, "FileSink") => { @@ -1835,18 +1841,26 @@ pub const FilePoll = struct { var loader = ptr.as(JSC.WebCore.FileSink); loader.onPoll(size_or_offset, 0); }, - @field(Owner.Tag, "DNSResolver") => { log("onUpdate " ++ kqueue_or_epoll ++ " (fd: {d}) DNSResolver", .{poll.fd}); var loader: *DNSResolver = ptr.as(DNSResolver); loader.onDNSPoll(poll); }, - @field(Owner.Tag, "GetAddrInfoRequest") => { log("onUpdate " ++ kqueue_or_epoll ++ " (fd: {d}) GetAddrInfoRequest", .{poll.fd}); var loader: *GetAddrInfoRequest = ptr.as(GetAddrInfoRequest); loader.onMachportChange(); }, + @field(Owner.Tag, "PostinstallSubprocess") => { + log("onUpdate " ++ kqueue_or_epoll ++ " (fd: {d}) PostinstallSubprocess Output", .{poll.fd}); + var loader: *PostinstallSubprocess = ptr.as(PostinstallSubprocess); + loader.onOutputUpdate(size_or_offset, poll.fileDescriptor()); + }, + @field(Owner.Tag, "PidPollData") => { + log("onUpdate " ++ kqueue_or_epoll ++ " (fd: {d}) PostinstallSubprocess Pid", .{poll.fd}); + var loader: *PostinstallSubprocess = ptr.as(PostinstallSubprocess); + loader.onProcessUpdate(size_or_offset); + }, else => { log("onUpdate " ++ kqueue_or_epoll ++ " (fd: {d}) disconnected?", .{poll.fd}); @@ -1854,6 +1868,10 @@ pub const FilePoll = struct { } } + pub inline fn fileDescriptor(this: *FilePoll) bun.FileDescriptor { + return @intCast(this.fd); + } + pub const Flags = enum { // What are we asking the event loop about? @@ -2082,6 +2100,24 @@ pub const FilePoll = struct { return poll; } + pub fn initWithPackageManager(m: *PackageManager, fd: bun.FileDescriptor, flags: Flags.Struct, owner: anytype) *FilePoll { + return initWithPackageManagerWithOwner(m, fd, flags, Owner.init(owner)); + } + + pub fn initWithPackageManagerWithOwner(manager: *PackageManager, fd: bun.FileDescriptor, flags: Flags.Struct, owner: Owner) *FilePoll { + var poll = manager.file_poll_store.get(); + poll.fd = @intCast(fd); + poll.flags = Flags.Set.init(flags); + poll.owner = owner; + poll.next_to_free = null; + + if (KQueueGenerationNumber != u0) { + max_generation_number +%= 1; + poll.generation_number = max_generation_number; + } + return poll; + } + pub inline fn canRef(this: *const FilePoll) bool { if (this.flags.contains(.disable)) return false; diff --git a/src/bun.js/event_loop.zig b/src/bun.js/event_loop.zig index c7de557f4..81496ab0a 100644 --- a/src/bun.js/event_loop.zig +++ b/src/bun.js/event_loop.zig @@ -913,8 +913,7 @@ pub const EventLoop = struct { pub fn ensureWaker(this: *EventLoop) void { JSC.markBinding(@src()); if (this.virtual_machine.event_loop_handle == null) { - var actual = uws.Loop.get().?; - this.virtual_machine.event_loop_handle = actual; + this.virtual_machine.event_loop_handle = uws.Loop.get(); this.virtual_machine.gc_controller.init(this.virtual_machine); // _ = actual.addPostHandler(*JSC.EventLoop, this, JSC.EventLoop.afterUSocketsTick); // _ = actual.addPreHandler(*JSC.VM, this.virtual_machine.global.vm(), JSC.VM.drainMicrotasks); @@ -955,7 +954,7 @@ pub const MiniEventLoop = struct { return .{ .tasks = Queue.init(allocator), .allocator = allocator, - .loop = uws.Loop.get().?, + .loop = uws.Loop.get(), }; } @@ -1006,6 +1005,12 @@ pub const MiniEventLoop = struct { } } + pub fn drainTasks(this: *MiniEventLoop, context: *anyopaque) void { + while (this.tasks.readItem()) |task| { + task.run(context); + } + } + pub fn enqueueTask( this: *MiniEventLoop, comptime Context: type, @@ -1094,4 +1099,15 @@ pub const AnyEventLoop = union(enum) { }, } } + + pub fn drainTasks(this: *AnyEventLoop, context: *anyopaque) void { + switch (this.*) { + .jsc => { + unreachable; + }, + .mini => { + this.mini.drainTasks(context); + }, + } + } }; diff --git a/src/bun.js/test/jest.zig b/src/bun.js/test/jest.zig index 3617d5961..f3c9ffa26 100644 --- a/src/bun.js/test/jest.zig +++ b/src/bun.js/test/jest.zig @@ -127,7 +127,7 @@ pub const TestRunner = struct { if (milliseconds > 0) { if (this.test_timeout_timer == null) { - this.test_timeout_timer = bun.uws.Timer.createFallthrough(bun.uws.Loop.get().?, this); + this.test_timeout_timer = bun.uws.Timer.createFallthrough(bun.uws.Loop.get(), this); } if (this.last_test_timeout_timer_duration != milliseconds) { diff --git a/src/bundler/bundle_v2.zig b/src/bundler/bundle_v2.zig index 67cdee041..d75c1923c 100644 --- a/src/bundler/bundle_v2.zig +++ b/src/bundler/bundle_v2.zig @@ -1163,7 +1163,7 @@ pub const BundleV2 = struct { thread.detach(); } else { BundleThread.instance.queue.push(completion); - BundleThread.instance.waker.wake() catch {}; + BundleThread.instance.waker.wake(); } completion.poll_ref.ref(globalThis.bunVM()); diff --git a/src/cli/run_command.zig b/src/cli/run_command.zig index 494913cd5..909b12d09 100644 --- a/src/cli/run_command.zig +++ b/src/cli/run_command.zig @@ -10,6 +10,7 @@ const default_allocator = bun.default_allocator; const C = bun.C; const std = @import("std"); const uws = @import("../deps/uws.zig"); +const JSC = bun.JSC; const lex = bun.js_lexer; const logger = @import("root").bun.logger; @@ -44,6 +45,8 @@ const yarn_commands: []u64 = @import("./list-of-yarn-commands.zig").all_yarn_com const ShellCompletions = @import("./shell_completions.zig"); const PosixSpawn = @import("../bun.js/api/bun/spawn.zig").PosixSpawn; +const PackageManager = @import("../install/install.zig").PackageManager; + pub const RunCommand = struct { const shells_to_search = &[_]string{ "bash", @@ -227,38 +230,168 @@ pub const RunCommand = struct { const log = Output.scoped(.RUN, false); - pub const SpawnedScript = struct { - pid: std.os.pid_t, + pub const PostinstallSubprocess = struct { + script_name: []const u8, + package_name: []const u8, + output_buffer: bun.ByteList, - output_fd: std.os.fd_t, + pid_poll: *JSC.FilePoll, + stdout_poll: *JSC.FilePoll, + stderr_poll: *JSC.FilePoll, + package_manager: *PackageManager, + + /// A "nothing" struct that lets us reuse the same pointer + /// but with a different tag for the file poll + pub const PidPollData = struct { process: PostinstallSubprocess }; + + pub fn init( + manager: *PackageManager, + script_name: []const u8, + package_name: []const u8, + stdout_fd: bun.FileDescriptor, + stderr_fd: bun.FileDescriptor, + pid_fd: bun.FileDescriptor, + ) !?*PostinstallSubprocess { + var this = try manager.allocator.create(PostinstallSubprocess); + errdefer this.deinit(manager.allocator); + + this.* = .{ + .package_name = package_name, + .script_name = script_name, + .package_manager = manager, + .pid_poll = undefined, + .stdout_poll = undefined, + .stderr_poll = undefined, + .output_buffer = .{}, + }; + + this.pid_poll = JSC.FilePoll.initWithPackageManager( + manager, + pid_fd, + .{}, + @as(*PidPollData, @ptrCast(this)), + ); + this.stdout_poll = JSC.FilePoll.initWithPackageManager(manager, stdout_fd, .{}, this); + this.stderr_poll = JSC.FilePoll.initWithPackageManager(manager, stderr_fd, .{}, this); + + try this.stdout_poll.register(manager.uws_event_loop, .readable, false).throw(); + try this.stderr_poll.register(manager.uws_event_loop, .readable, false).throw(); + + switch (this.pid_poll.register( + manager.uws_event_loop, + .process, + true, + )) { + .result => {}, + .err => |err| { + if (err.getErrno() != .SRCH) { + @panic("This shouldn't happen"); + } + + this.package_manager.pending_tasks -= 1; + this.onProcessUpdate(0); + return null; + }, + } + + return this; + } + + pub fn onOutputUpdate(this: *PostinstallSubprocess, size: i64, fd: bun.FileDescriptor) void { + this.output_buffer.ensureUnusedCapacity(this.package_manager.allocator, @intCast(size)) catch @panic("Failed to allocate memory for output buffer"); + + var remaining = size; + while (remaining > 0) { + const n: u32 = @truncate(std.os.read( + fd, + this.output_buffer.ptr[this.output_buffer.len..this.output_buffer.cap], + ) catch return); + this.output_buffer.len += n; + remaining -|= n; + } + } + + pub fn readAllThenDump(this: *PostinstallSubprocess) void { + // wait for both file polls to be done first! + + Output.errorWriter().writeAll(this.output_buffer.slice()) catch {}; + } + + pub fn onProcessUpdate(this: *PostinstallSubprocess, _: i64) void { + Output.debug("onProcessUpdate", .{}); + switch (PosixSpawn.waitpid(this.pid_poll.fileDescriptor(), std.os.W.NOHANG)) { + .err => |err| { + Output.prettyErrorln("<r><red>error<r>: Failed to run <b>{s}<r> script from \"<b>{s}<r>\" due to error <b>{d} {s}<r>", .{ this.script_name, this.package_name, err.errno, @tagName(err.getErrno()) }); + Output.flush(); + this.package_manager.pending_tasks -= 1; + }, + .result => |result| { + if (result.pid == 0) { + Output.prettyErrorln("<r><red>error<r>: Failed to run <b>{s}<r> script from \"<b>{s}<r>\" due to error <b>{d} {s}<r>", .{ this.script_name, this.package_name, 0, "Unknown" }); + Output.flush(); + + this.package_manager.pending_tasks -= 1; + return; + } + if (std.os.W.IFEXITED(result.status)) { + defer this.deinit(this.package_manager.allocator); + + const code = std.os.W.EXITSTATUS(result.status); + if (code > 0) { + this.readAllThenDump(); + Output.prettyErrorln("<r><red>error<r><d>:<r> <b>{s}<r> script from \"<b>{s}<r>\" exited with {any}<r>", .{ this.script_name, this.package_name, bun.SignalCode.from(code) }); + Output.flush(); + Global.exit(code); + } + Output.debug("EXIT WITH CODE {d}?", .{code}); + + this.package_manager.pending_tasks -= 1; + return; + } + if (std.os.W.IFSIGNALED(result.status)) { + const signal = std.os.W.TERMSIG(result.status); + + this.readAllThenDump(); + + Output.prettyErrorln("<r><red>error<r><d>:<r> <b>{s}<r> script from \"<b>{s}<r>\" exited with {any}<r>", .{ this.script_name, this.package_name, bun.SignalCode.from(signal) }); + Output.flush(); + Global.exit(1); + } + if (std.os.W.IFSTOPPED(result.status)) { + const signal = std.os.W.STOPSIG(result.status); + + this.readAllThenDump(); - pub fn deinit(this: *SpawnedScript, alloc: std.mem.Allocator) void { - this.output_buffer.deinitWithAllocator(alloc); + Output.prettyErrorln("<r><red>error<r><d>:<r> <b>{s}<r> script from \"<b>{s}<r>\" was stopped by signal {any}<r>", .{ this.script_name, this.package_name, bun.SignalCode.from(signal) }); + Output.flush(); + Global.exit(1); + } + }, + } + } + + pub fn deinit(this: *PostinstallSubprocess, alloc: std.mem.Allocator) void { + std.os.close(this.stdout_poll.fileDescriptor()); + std.os.close(this.stderr_poll.fileDescriptor()); + // ?! close pid poll? + // this.output_buffer.deinitWithAllocator(alloc); alloc.destroy(this); } }; inline fn spawnScript( - allocator: std.mem.Allocator, + ctx: *PackageManager, name: string, + package_name: string, cwd: string, env: *DotEnv.Loader, argv: [*:null]?[*:0]const u8, - ) !?*SpawnedScript { + ) !?*PostinstallSubprocess { var flags: i32 = bun.C.POSIX_SPAWN_SETSIGDEF | bun.C.POSIX_SPAWN_SETSIGMASK; if (comptime Environment.isMac) { flags |= bun.C.POSIX_SPAWN_CLOEXEC_DEFAULT; } - var spawned = try allocator.create(SpawnedScript); - errdefer spawned.deinit(allocator); - - spawned.* = .{ - .pid = undefined, - .output_buffer = try bun.ByteList.initCapacity(allocator, 1024), - .output_fd = -1, - }; - var attr = try PosixSpawn.Attr.init(); defer attr.deinit(); try attr.set(@intCast(flags)); @@ -269,51 +402,89 @@ pub const RunCommand = struct { try actions.openZ(bun.STDIN_FD, "/dev/null", std.os.O.RDONLY, 0o664); // Have both stdout and stderr write to the same buffer - const fds = try std.os.pipe2(0); - try actions.dup2(fds[1], bun.STDOUT_FD); - try actions.dup2(fds[1], bun.STDERR_FD); - spawned.output_fd = fds[0]; + const fdsOut = try std.os.pipe2(0); + try actions.dup2(fdsOut[1], bun.STDOUT_FD); - // ?! do i call close() on one of these - // std.os.close(fds[1]); + const fdsErr = try std.os.pipe2(0); + try actions.dup2(fdsErr[1], bun.STDERR_FD); try actions.chdir(cwd); - var arena = bun.ArenaAllocator.init(allocator); + var arena = bun.ArenaAllocator.init(ctx.allocator); defer arena.deinit(); - switch (PosixSpawn.spawnZ( - argv[0].?, - actions, - attr, - argv, - try env.map.createNullDelimitedEnvMap(arena.allocator()), - )) { - .err => |err| { - Output.prettyErrorln("<r><red>error<r>: Failed to spawn script <b>{s}<r> due to error <b>{d} {s}<r>", .{ name, err.errno, @tagName(err.getErrno()) }); - Output.flush(); - return null; - }, - .result => |pid| { - spawned.pid = pid; - return spawned; - }, - } + const pid = brk: { + defer { + _ = bun.sys.close(fdsOut[1]); + _ = bun.sys.close(fdsErr[1]); + } + switch (PosixSpawn.spawnZ( + argv[0].?, + actions, + attr, + argv, + try env.map.createNullDelimitedEnvMap(arena.allocator()), + )) { + .err => |err| { + Output.prettyErrorln("<r><red>error<r>: Failed to spawn script <b>{s}<r> due to error <b>{d} {s}<r>", .{ name, err.errno, @tagName(err.getErrno()) }); + Output.flush(); + return null; + }, + .result => |pid| break :brk pid, + } + }; + + const pidfd: std.os.fd_t = brk: { + if (!Environment.isLinux) { + break :brk pid; + } + + const kernel = @import("../analytics.zig").GenerateHeader.GeneratePlatform.kernelVersion(); + + // pidfd_nonblock only supported in 5.10+ + const pidfd_flags: u32 = if (kernel.orderWithoutTag(.{ .major = 5, .minor = 10, .patch = 0 }).compare(.gte)) + std.os.O.NONBLOCK + else + 0; + + const fd = std.os.linux.pidfd_open( + pid, + pidfd_flags, + ); + + switch (std.os.linux.getErrno(fd)) { + .SUCCESS => break :brk @as(std.os.fd_t, @intCast(fd)), + else => |err| { + var status: u32 = 0; + // ensure we don't leak the child process on error + _ = std.os.linux.waitpid(pid, &status, 0); + + Output.prettyErrorln("<r><red>error<r>: Failed to spawn script <b>{s}<r> due to error <b>{d} {s}<r>", .{ name, err, @tagName(err) }); + Output.flush(); + + return null; + }, + } + }; + + return try PostinstallSubprocess.init(ctx, name, package_name, fdsOut[0], fdsErr[0], pidfd); } + /// Used to execute postinstall scripts pub fn spawnPackageScript( - allocator: std.mem.Allocator, + ctx: *PackageManager, original_script: string, name: string, + package_name: string, cwd: string, - env: *DotEnv.Loader, passthrough: []const string, silent: bool, - ) !?*SpawnedScript { + ) !void { + const env = ctx.env; const shell_bin = findShell(env.map.get("PATH") orelse "", cwd) orelse return error.MissingShell; var script = original_script; - var copy_script = try std.ArrayList(u8).initCapacity(allocator, script.len + 1); + var copy_script = try std.ArrayList(u8).initCapacity(ctx.allocator, script.len + 1); // We're going to do this slowly. // Find exact matches of yarn, pnpm, npm @@ -323,14 +494,14 @@ pub const RunCommand = struct { var combined_script: [:0]u8 = copy_script.items[0 .. copy_script.items.len - 1 :0]; - log("Script: \"{s}\"", .{combined_script}); + log("Script from pkg \"{s}\" : \"{s}\"", .{ package_name, combined_script }); if (passthrough.len > 0) { var combined_script_len = script.len; for (passthrough) |p| { combined_script_len += p.len + 1; } - var combined_script_buf = try allocator.allocSentinel(u8, combined_script_len, 0); + var combined_script_buf = try ctx.allocator.allocSentinel(u8, combined_script_len, 0); bun.copy(u8, combined_script_buf, script); var remaining_script_buf = combined_script_buf[script.len..]; for (passthrough) |part| { @@ -347,61 +518,78 @@ pub const RunCommand = struct { Output.flush(); } - var argv = try allocator.allocSentinel(?[*:0]const u8, 3, null); - defer allocator.free(argv); + var argv = try ctx.allocator.allocSentinel(?[*:0]const u8, 3, null); + defer ctx.allocator.free(argv); argv[0] = shell_bin; argv[1] = "-c"; argv[2] = combined_script; - return spawnScript(allocator, name, cwd, env, argv) catch |err| { + _ = spawnScript(ctx, name, package_name, cwd, env, argv) catch |err| { Output.prettyErrorln("<r><red>error<r>: Failed to run script <b>{s}<r> due to error <b>{s}<r>", .{ name, @errorName(err) }); Output.flush(); - return null; + return; }; } - pub fn waitForPackageScript(script: *SpawnedScript, name: string, is_sync: bool, alloc: std.mem.Allocator) bool { - log("Waiting for script {s}, {d}", .{ name, script.pid }); - while (true) { - switch (PosixSpawn.waitpid(script.pid, if (is_sync) 0 else std.os.W.NOHANG)) { - .err => |err| { - Output.prettyErrorln("<r><red>error<r>: Failed to run script <b>{s}<r> due to error <b>{d} {s}<r>", .{ name, err.errno, @tagName(err.getErrno()) }); - Output.flush(); - return true; - }, - .result => |result| { - defer script.deinit(alloc); - - if (result.pid == 0) return false; - if (std.os.W.IFEXITED(result.status)) { - const code = std.os.W.EXITSTATUS(result.status); - if (code > 0) { - if (code != 2) { - Output.prettyErrorln("<r><red>error<r><d>:<r> script <b>\"{s}\"<r> exited with {any}<r>", .{ name, bun.SignalCode.from(code) }); - Output.flush(); - } - Global.exit(code); - } - return true; - } - if (std.os.W.IFSIGNALED(result.status)) { - const signal = std.os.W.TERMSIG(result.status); - Output.prettyErrorln("<r><red>error<r><d>:<r> script <b>\"{s}\"<r> exited with {any}<r>", .{ name, bun.SignalCode.from(signal) }); - Output.flush(); - Global.exit(1); - } - if (std.os.W.IFSTOPPED(result.status)) { - const signal = std.os.W.STOPSIG(result.status); - Output.prettyErrorln("<r><red>error<r><d>:<r> script <b>\"{s}\"<r> was stopped by signal {any}<r>", .{ name, bun.SignalCode.from(signal) }); - Output.flush(); - Global.exit(1); - } - }, - } - } - } - - pub fn runPackageScript( + // pub fn waitForPackageScript(script: *SpawnedScript, name: string, is_sync: bool, alloc: std.mem.Allocator) bool { + // log("Waiting for script {s}, {d}", .{ name, script.pid }); + // // script.readNonBlocking(alloc) catch |err| { + // // Output.prettyErrorln("<r><red>error<r>: Failed to read output of script <b>{s}<r> due to error <b>{s}<r>", .{ name, @errorName(err) }); + // // Output.flush(); + // // // ?! kill process? + // // return true; + // // }; + // while (true) { + // switch (PosixSpawn.waitpid(script.pid, if (is_sync) 0 else std.os.W.NOHANG)) { + // .err => |err| { + // // script.readAllOutput(); + + // Output.prettyErrorln("<r><red>error<r>: Failed to run script <b>{s}<r> due to error <b>{d} {s}<r>", .{ name, err.errno, @tagName(err.getErrno()) }); + // Output.flush(); + // // ?! kill process? + // return true; + // }, + // .result => |result| { + // if (result.pid == 0) return false; + // if (std.os.W.IFEXITED(result.status)) { + // defer script.deinit(alloc); + + // // script.readAllOutput(); + + // const code = std.os.W.EXITSTATUS(result.status); + // if (code > 0) { + // if (code != 2) { + // Output.prettyErrorln("<r><red>error<r><d>:<r> script <b>\"{s}\"<r> exited with {any}<r>", .{ name, bun.SignalCode.from(code) }); + // Output.flush(); + // } + // Global.exit(code); + // } + // return true; + // } + // if (std.os.W.IFSIGNALED(result.status)) { + // const signal = std.os.W.TERMSIG(result.status); + + // // script.readAllOutput(); + + // Output.prettyErrorln("<r><red>error<r><d>:<r> script <b>\"{s}\"<r> exited with {any}<r>", .{ name, bun.SignalCode.from(signal) }); + // Output.flush(); + // Global.exit(1); + // } + // if (std.os.W.IFSTOPPED(result.status)) { + // const signal = std.os.W.STOPSIG(result.status); + + // // script.readAllOutput(); + + // Output.prettyErrorln("<r><red>error<r><d>:<r> script <b>\"{s}\"<r> was stopped by signal {any}<r>", .{ name, bun.SignalCode.from(signal) }); + // Output.flush(); + // Global.exit(1); + // } + // }, + // } + // } + // } + + pub fn runPackageScriptForeground( allocator: std.mem.Allocator, original_script: string, name: string, @@ -410,9 +598,86 @@ pub const RunCommand = struct { passthrough: []const string, silent: bool, ) !bool { - if (try spawnPackageScript(allocator, original_script, name, cwd, env, passthrough, silent)) |script| { - _ = waitForPackageScript(script, name, true, allocator); + const shell_bin = findShell(env.map.get("PATH") orelse "", cwd) orelse return error.MissingShell; + + var script = original_script; + var copy_script = try std.ArrayList(u8).initCapacity(allocator, script.len); + + // We're going to do this slowly. + // Find exact matches of yarn, pnpm, npm + + try replacePackageManagerRun(©_script, script); + + var combined_script: []u8 = copy_script.items; + + log("Script: \"{s}\"", .{combined_script}); + + if (passthrough.len > 0) { + var combined_script_len = script.len; + for (passthrough) |p| { + combined_script_len += p.len + 1; + } + var combined_script_buf = try allocator.alloc(u8, combined_script_len); + bun.copy(u8, combined_script_buf, script); + var remaining_script_buf = combined_script_buf[script.len..]; + for (passthrough) |part| { + var p = part; + remaining_script_buf[0] = ' '; + bun.copy(u8, remaining_script_buf[1..], p); + remaining_script_buf = remaining_script_buf[p.len + 1 ..]; + } + combined_script = combined_script_buf; + } + + var argv = [_]string{ shell_bin, "-c", combined_script }; + + if (!silent) { + Output.prettyErrorln("<r><d><magenta>$<r> <d><b>{s}<r>", .{combined_script}); + Output.flush(); } + + var child_process = std.ChildProcess.init(&argv, allocator); + var buf_map = try env.map.cloneToEnvMap(allocator); + + child_process.env_map = &buf_map; + child_process.cwd = cwd; + child_process.stderr_behavior = .Inherit; + child_process.stdin_behavior = .Inherit; + child_process.stdout_behavior = .Inherit; + + const result = child_process.spawnAndWait() catch |err| { + Output.prettyErrorln("<r><red>error<r>: Failed to run script <b>{s}<r> due to error <b>{s}<r>", .{ name, @errorName(err) }); + Output.flush(); + return true; + }; + + switch (result) { + .Exited => |code| { + if (code > 0) { + if (code != 2) { + Output.prettyErrorln("<r><red>error<r><d>:<r> script <b>\"{s}\"<r> exited with {any}<r>", .{ name, bun.SignalCode.from(code) }); + Output.flush(); + } + + Global.exit(code); + } + }, + .Signal => |signal| { + Output.prettyErrorln("<r><red>error<r><d>:<r> script <b>\"{s}\"<r> exited with {any}<r>", .{ name, bun.SignalCode.from(signal) }); + Output.flush(); + + Global.exit(1); + }, + .Stopped => |signal| { + Output.prettyErrorln("<r><red>error<r><d>:<r> script <b>\"{s}\"<r> was stopped by signal {any}<r>", .{ name, bun.SignalCode.from(signal) }); + Output.flush(); + + Global.exit(1); + }, + + else => {}, + } + return true; } @@ -1136,11 +1401,11 @@ pub const RunCommand = struct { else => { if (scripts.get(script_name_to_search)) |script_content| { // allocate enough to hold "post${scriptname}" - var temp_script_buffer = try std.fmt.allocPrint(ctx.allocator, "ppre{s}", .{script_name_to_search}); + defer ctx.allocator.free(temp_script_buffer); if (scripts.get(temp_script_buffer[1..])) |prescript| { - if (!try runPackageScript( + if (!try runPackageScriptForeground( ctx.allocator, prescript, temp_script_buffer[1..], @@ -1153,7 +1418,7 @@ pub const RunCommand = struct { } } - if (!try runPackageScript( + if (!try runPackageScriptForeground( ctx.allocator, script_content, script_name_to_search, @@ -1166,7 +1431,7 @@ pub const RunCommand = struct { temp_script_buffer[0.."post".len].* = "post".*; if (scripts.get(temp_script_buffer)) |postscript| { - if (!try runPackageScript( + if (!try runPackageScriptForeground( ctx.allocator, postscript, temp_script_buffer, diff --git a/src/deps/uws.zig b/src/deps/uws.zig index b8016a6b7..032e5f6e6 100644 --- a/src/deps/uws.zig +++ b/src/deps/uws.zig @@ -708,12 +708,11 @@ pub const SocketContext = opaque { pub fn deinit(this: *SocketContext, ssl: bool) void { this.close(ssl); //always deinit in next iteration - if (Loop.get()) |loop| { - if (ssl) { - loop.nextTick(*SocketContext, this, SocketContext._deinit_ssl); - } else { - loop.nextTick(*SocketContext, this, SocketContext._deinit); - } + const loop = Loop.get(); + if (ssl) { + loop.nextTick(*SocketContext, this, SocketContext._deinit_ssl); + } else { + loop.nextTick(*SocketContext, this, SocketContext._deinit); } } @@ -817,7 +816,9 @@ pub const Loop = extern struct { this.active -|= @as(u32, @intCast(count)); } - pub fn get() ?*Loop { + /// Lazily initializes a per-thread loop and returns it. + /// Will automatically free all initialized loops at exit. + pub fn get() *Loop { return uws_get_loop(); } @@ -891,7 +892,7 @@ pub const Loop = extern struct { extern fn uws_loop_defer(loop: *Loop, ctx: *anyopaque, cb: *const (fn (ctx: *anyopaque) callconv(.C) void)) void; - extern fn uws_get_loop() ?*Loop; + extern fn uws_get_loop() *Loop; extern fn us_create_loop( hint: ?*anyopaque, wakeup_cb: ?*const fn (*Loop) callconv(.C) void, @@ -902,7 +903,7 @@ pub const Loop = extern struct { extern fn us_loop_free(loop: ?*Loop) void; extern fn us_loop_ext(loop: ?*Loop) ?*anyopaque; extern fn us_loop_run(loop: ?*Loop) void; - extern fn us_loop_run_bun_tick(loop: ?*Loop, timouetMs: i64) void; + extern fn us_loop_run_bun_tick(loop: ?*Loop, timeoutMs: i64) void; extern fn us_wakeup_loop(loop: ?*Loop) void; extern fn us_loop_integrate(loop: ?*Loop) void; extern fn us_loop_iteration_number(loop: ?*Loop) c_longlong; diff --git a/src/install/install.zig b/src/install/install.zig index e5d886fe8..d455e5670 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -9,6 +9,8 @@ const stringZ = bun.stringZ; const default_allocator = bun.default_allocator; const C = bun.C; const std = @import("std"); +const uws = @import("../deps/uws.zig"); +const JSC = bun.JSC; const JSLexer = bun.js_lexer; const logger = bun.logger; @@ -1613,6 +1615,51 @@ pub const CacheLevel = struct { const AsyncIO = bun.AsyncIO; const Waker = AsyncIO.Waker; +const Waiter = struct { + onWait: *const fn (this: *anyopaque) AsyncIO.Errno!usize, + onWake: *const fn (this: *anyopaque) void, + ctx: *anyopaque, + + pub fn init( + ctx: anytype, + comptime onWait: *const fn (this: @TypeOf(ctx)) AsyncIO.Errno!usize, + comptime onWake: *const fn (this: @TypeOf(ctx)) void, + ) Waiter { + return Waiter{ + .ctx = @ptrCast(ctx), + .onWait = @alignCast(@ptrCast(@as(*const anyopaque, @ptrCast(onWait)))), + .onWake = @alignCast(@ptrCast(@as(*const anyopaque, @ptrCast(onWake)))), + }; + } + + pub fn wait(this: *Waiter) AsyncIO.Errno!usize { + return this.onWait(this.ctx); + } + + pub fn wake(this: *Waiter) void { + this.onWake(this.ctx); + } + + pub fn fromUWSLoop(loop: *uws.Loop) Waiter { + const Handlers = struct { + fn onWait(uws_loop: *uws.Loop) AsyncIO.Errno!usize { + uws_loop.run(); + return 0; + } + + fn onWake(uws_loop: *uws.Loop) void { + uws_loop.wakeup(); + } + }; + + return Waiter.init( + loop, + Handlers.onWait, + Handlers.onWake, + ); + } +}; + // We can't know all the packages we need until we've downloaded all the packages // The easy way would be: // 1. Download all packages, parsing their dependencies and enqueuing all dependencies for resolution @@ -1671,11 +1718,14 @@ pub const PackageManager = struct { global_link_dir: ?std.fs.IterableDir = null, global_dir: ?std.fs.IterableDir = null, global_link_dir_path: string = "", - waiter: Waker = undefined, + waiter: Waiter = undefined, wait_count: std.atomic.Atomic(usize) = std.atomic.Atomic(usize).init(0), onWake: WakeHandler = .{}, + uws_event_loop: *uws.Loop, + file_poll_store: JSC.FilePoll.Store, + const PreallocatedNetworkTasks = std.BoundedArray(NetworkTask, 1024); const NetworkTaskQueue = std.HashMapUnmanaged(u64, void, IdentityContext(u64), 80); pub var verbose_install = false; @@ -1728,7 +1778,7 @@ pub const PackageManager = struct { } _ = this.wait_count.fetchAdd(1, .Monotonic); - this.waiter.wake() catch {}; + this.waiter.wake(); } pub fn sleep(this: *PackageManager) void { @@ -3688,6 +3738,7 @@ pub const PackageManager = struct { return CacheDir{ .is_node_modules = true, .path = Fs.FileSystem.instance.abs(&fallback_parts) }; } + /// fn tick( pub fn runTasks( manager: *PackageManager, comptime ExtractCompletionContext: type, @@ -4308,6 +4359,8 @@ pub const PackageManager = struct { manager.drainDependencyList(); + manager.uws_event_loop.run(); + if (comptime log_level.showProgress()) { if (@hasField(@TypeOf(callbacks), "progress_bar") and callbacks.progress_bar == true) { const completed_items = manager.total_tasks - manager.pending_tasks; @@ -5295,8 +5348,10 @@ pub const PackageManager = struct { .resolve_tasks = TaskChannel.init(), .lockfile = undefined, .root_package_json_file = package_json_file, - .waiter = try Waker.init(ctx.allocator), + .waiter = Waiter.fromUWSLoop(uws.Loop.get()), // .progress + .uws_event_loop = uws.Loop.get(), + .file_poll_store = JSC.FilePoll.Store.init(ctx.allocator), }; manager.lockfile = try ctx.allocator.create(Lockfile); @@ -5372,7 +5427,9 @@ pub const PackageManager = struct { .resolve_tasks = TaskChannel.init(), .lockfile = undefined, .root_package_json_file = undefined, - .waiter = try Waker.init(allocator), + .waiter = Waiter.fromUWSLoop(uws.Loop.get()), + .uws_event_loop = uws.Loop.get(), + .file_poll_store = JSC.FilePoll.Store.init(allocator), }; manager.lockfile = try allocator.create(Lockfile); @@ -6856,7 +6913,7 @@ pub const PackageManager = struct { .posix, ); - scripts.enqueue(this.lockfile, buf, path_str); + scripts.enqueue(this.lockfile, buf, path_str, name); } else if (!scripts.filled) { var path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; const path_str = Path.joinAbsString( @@ -6871,6 +6928,7 @@ pub const PackageManager = struct { this.node_modules_folder.dir, destination_dir_subpath, path_str, + name, ) catch |err| { if (comptime log_level != .silent) { const fmt = "\n<r><red>error:<r> failed to parse life-cycle scripts for <b>{s}<r>: {s}\n"; @@ -7630,6 +7688,7 @@ pub const PackageManager = struct { manager.lockfile, lockfile.buffers.string_bytes.items, strings.withoutTrailingSlash(Fs.FileSystem.instance.top_level_dir), + maybe_root.name.slice(lockfile.buffers.string_bytes.items), ); } } @@ -7745,12 +7804,7 @@ pub const PackageManager = struct { try manager.setupGlobalDir(&ctx); } - // We don't always save the lockfile. - // This is for two reasons. - // 1. It's unnecessary work if there are no changes - // 2. There is a determinism issue in the file where alignment bytes might be garbage data - // This is a bug that needs to be fixed, however we can work around it for now - // by avoiding saving the lockfile + // It's unnecessary work to re-save the lockfile if there are no changes if (manager.options.do.save_lockfile and (did_meta_hash_change or manager.lockfile.isEmpty() or manager.options.enable.force_save_lockfile)) save: { @@ -7801,6 +7855,7 @@ pub const PackageManager = struct { manager.lockfile, manager.lockfile.buffers.string_bytes.items, strings.withoutTrailingSlash(Fs.FileSystem.instance.top_level_dir), + root.name.slice(manager.lockfile.buffers.string_bytes.items), ); } @@ -7945,15 +8000,15 @@ pub const PackageManager = struct { if (run_lifecycle_scripts and install_summary.fail == 0) { // 2. install // 3. postinstall - try manager.lockfile.scripts.run(manager.allocator, manager.env, log_level != .silent, "install"); - try manager.lockfile.scripts.runInParallel(manager.allocator, manager.env, log_level != .silent, "postinstall"); + try manager.lockfile.scripts.spawnAllPackageScripts(manager, log_level, log_level != .silent, "install"); + try manager.lockfile.scripts.spawnAllPackageScripts(manager, log_level, log_level != .silent, "postinstall"); // 4. preprepare // 5. prepare // 6. postprepare - try manager.lockfile.scripts.run(manager.allocator, manager.env, log_level != .silent, "preprepare"); - try manager.lockfile.scripts.run(manager.allocator, manager.env, log_level != .silent, "prepare"); - try manager.lockfile.scripts.run(manager.allocator, manager.env, log_level != .silent, "postprepare"); + try manager.lockfile.scripts.spawnAllPackageScripts(manager, log_level, log_level != .silent, "preprepare"); + try manager.lockfile.scripts.spawnAllPackageScripts(manager, log_level, log_level != .silent, "prepare"); + try manager.lockfile.scripts.spawnAllPackageScripts(manager, log_level, log_level != .silent, "postprepare"); } if (comptime log_level != .silent) { diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index 460992968..759c5be90 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -11,6 +11,7 @@ const stringZ = bun.stringZ; const default_allocator = bun.default_allocator; const C = bun.C; const JSAst = bun.JSAst; +const JSC = bun.JSC; const JSLexer = bun.js_lexer; const logger = bun.logger; @@ -122,6 +123,7 @@ pub const Scripts = struct { const Entry = struct { cwd: string, script: string, + package_name: string, }; const Entries = std.ArrayListUnmanaged(Entry); @@ -145,31 +147,25 @@ pub const Scripts = struct { pub fn run(this: *Scripts, allocator: Allocator, env: *DotEnv.Loader, silent: bool, comptime hook: []const u8) !void { for (@field(this, hook).items) |entry| { if (comptime Environment.allow_assert) std.debug.assert(Fs.FileSystem.instance_loaded); - _ = try RunCommand.runPackageScript(allocator, entry.script, hook, entry.cwd, env, &.{}, silent); + _ = try RunCommand.runPackageScriptForeground(allocator, entry.script, hook, entry.cwd, env, &.{}, silent); } } - pub fn runInParallel(this: *Scripts, allocator: Allocator, env: *DotEnv.Loader, silent: bool, comptime hook: []const u8) !void { + pub fn spawnAllPackageScripts(this: *Scripts, ctx: *PackageManager, comptime log_level: anytype, silent: bool, comptime hook: []const u8) !void { + _ = log_level; if (comptime Environment.allow_assert) std.debug.assert(Fs.FileSystem.instance_loaded); - var queue = Queue.init(allocator); - defer queue.deinit(); - for (@field(this, hook).items) |entry| { - if (try RunCommand.spawnPackageScript(allocator, entry.script, hook, entry.cwd, env, &.{}, silent)) |script| { - try queue.writeItem(script); - } + const items = @field(this, hook).items; + if (items.len > 0) { + ctx.pending_tasks = @truncate(items.len); - while (queue.readableLength() >= MAX_PARALLEL_PROCESSES) { - if (queue.readItem()) |script| { - if (!RunCommand.waitForPackageScript(script, hook, false, allocator)) { - try queue.writeItem(script); - } - } + for (items) |entry| { + try RunCommand.spawnPackageScript(ctx, entry.script, hook, entry.package_name, entry.cwd, &.{}, silent); } - } - while (queue.readItem()) |script| { - _ = RunCommand.waitForPackageScript(script, hook, true, allocator); + while (ctx.pending_tasks > 0) { + ctx.uws_event_loop.tick(); + } } } @@ -1883,13 +1879,14 @@ pub const Package = extern struct { return false; } - pub fn enqueue(this: *const Package.Scripts, lockfile: *Lockfile, buf: []const u8, cwd: string) void { + pub fn enqueue(this: *const Package.Scripts, lockfile: *Lockfile, buf: []const u8, cwd: string, package_name: string) void { inline for (Package.Scripts.Hooks) |hook| { const script = @field(this, hook); if (!script.isEmpty()) { @field(lockfile.scripts, hook).append(lockfile.allocator, .{ .cwd = lockfile.allocator.dupe(u8, cwd) catch unreachable, .script = lockfile.allocator.dupe(u8, script.slice(buf)) catch unreachable, + .package_name = package_name, }) catch unreachable; } } @@ -1930,6 +1927,7 @@ pub const Package = extern struct { node_modules: std.fs.Dir, subpath: [:0]const u8, cwd: string, + name: string, ) !void { var pkg_dir = try bun.openDir(node_modules, subpath); defer pkg_dir.close(); @@ -1954,7 +1952,7 @@ pub const Package = extern struct { try builder.allocate(); this.parseAlloc(lockfile.allocator, &builder, json); - this.enqueue(lockfile, tmp.buffers.string_bytes.items, cwd); + this.enqueue(lockfile, tmp.buffers.string_bytes.items, cwd, name); } }; @@ -4383,21 +4381,14 @@ const default_trusted_dependencies = brk: { // This file contains a list of dependencies that Bun runs `postinstall` on by default. const data = @embedFile("./default-trusted-dependencies.txt"); - comptime var line_start = 0; - comptime var i = 0; - @setEvalBranchQuota(123456); - while (i < data.len) { - while (i < data.len) : (i += 1) { - if (data[i] == '\n' or data[i] == '\r') break; - } - while (data[i] == '\r' or data[i] == '\n') i += 1; - const line_slice = data[line_start..i]; - if (line_slice.len == 0) break; + @setEvalBranchQuota(99999); + + var iter = std.mem.tokenizeAny(u8, data, " \n\t"); + while (iter.next()) |dep| { if (map.len == max_values) { @compileError("default-trusted-dependencies.txt is too large, please increase 'max_values' in lockfile.zig"); } - line_start = i; - map.putAssumeCapacity(line_slice, 0); + map.putAssumeCapacity(dep, 0); } break :brk ↦ diff --git a/src/io/io_darwin.zig b/src/io/io_darwin.zig index cb2d15afb..f789142ba 100644 --- a/src/io/io_darwin.zig +++ b/src/io/io_darwin.zig @@ -512,7 +512,7 @@ pub const Waker = struct { const zeroed = std.mem.zeroes([16]Kevent64); - pub fn wake(this: *Waker) !void { + pub fn wake(this: *Waker) void { bun.JSC.markBinding(@src()); if (io_darwin_schedule_wakeup(this.machport)) { diff --git a/src/io/io_linux.zig b/src/io/io_linux.zig index 8f054490b..b36c10e1d 100644 --- a/src/io/io_linux.zig +++ b/src/io/io_linux.zig @@ -1003,7 +1003,7 @@ pub const Waker = struct { return @as(u64, @intCast(bytes)); } - pub fn wake(this: *const Waker) !void { + pub fn wake(this: *const Waker) void { var bytes: usize = 1; _ = std.os.write( this.fd, diff --git a/src/network_thread.zig b/src/network_thread.zig index e1c2046a8..60d7d30f9 100644 --- a/src/network_thread.zig +++ b/src/network_thread.zig @@ -164,7 +164,7 @@ pub fn schedule(this: *@This(), batch: Batch) void { const one = @as([8]u8, @bitCast(@as(usize, batch.len))); _ = std.os.write(this.waker.fd, &one) catch @panic("Failed to write to eventfd"); } else { - this.waker.wake() catch @panic("Failed to wake"); + this.waker.wake(); } } |