From 5fa13625a1ca0ea1a3a1c5bb86d0880dcfac349f Mon Sep 17 00:00:00 2001 From: Dylan Conway <35280289+dylan-conway@users.noreply.github.com> Date: Wed, 21 Jun 2023 23:38:18 -0700 Subject: upgrade zig to `v0.11.0-dev.3737+9eb008717` (#3374) * progress * finish `@memset/@memcpy` update * Update build.zig * change `@enumToInt` to `@intFromEnum` and friends * update zig versions * it was 1 * add link to issue * add `compileError` reminder * fix merge * format * upgrade to llvm 16 * Revert "upgrade to llvm 16" This reverts commit cc930ceb1c5b4db9614a7638596948f704544ab8. --------- Co-authored-by: Jarred Sumner Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> --- src/install/install.zig | 86 ++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) (limited to 'src/install/install.zig') diff --git a/src/install/install.zig b/src/install/install.zig index 88c8f653f..32c24548c 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -146,12 +146,12 @@ pub fn ExternalSliceAligned(comptime Type: type, comptime alignment_: ?u29) type pub fn init(buf: []const Type, in: []const Type) Slice { // if (comptime Environment.allow_assert) { - // std.debug.assert(@ptrToInt(buf.ptr) <= @ptrToInt(in.ptr)); - // std.debug.assert((@ptrToInt(in.ptr) + in.len) <= (@ptrToInt(buf.ptr) + buf.len)); + // std.debug.assert(@intFromPtr(buf.ptr) <= @intFromPtr(in.ptr)); + // std.debug.assert((@intFromPtr(in.ptr) + in.len) <= (@intFromPtr(buf.ptr) + buf.len)); // } return Slice{ - .off = @truncate(u32, (@ptrToInt(in.ptr) - @ptrToInt(buf.ptr)) / @sizeOf(Type)), + .off = @truncate(u32, (@intFromPtr(in.ptr) - @intFromPtr(buf.ptr)) / @sizeOf(Type)), .len = @truncate(u32, in.len), }; } @@ -183,7 +183,7 @@ pub const Aligner = struct { } pub inline fn skipAmount(comptime Type: type, pos: usize) usize { - return std.mem.alignForward(pos, @alignOf(Type)) - pos; + return std.mem.alignForward(usize, pos, @alignOf(Type)) - pos; } }; @@ -351,7 +351,7 @@ const NetworkTask = struct { }, ); header_builder.header_count = 1; - header_builder.content = GlobalStringBuilder{ .ptr = @intToPtr([*]u8, @ptrToInt(bun.span(default_headers_buf).ptr)), .len = default_headers_buf.len, .cap = default_headers_buf.len }; + header_builder.content = GlobalStringBuilder{ .ptr = @ptrFromInt([*]u8, @intFromPtr(bun.span(default_headers_buf).ptr)), .len = default_headers_buf.len, .cap = default_headers_buf.len }; } this.response_buffer = try MutableString.init(allocator, 0); @@ -474,12 +474,12 @@ pub const Features = struct { pub fn behavior(this: Features) Behavior { var out: u8 = 0; - out |= @as(u8, @boolToInt(this.dependencies)) << 1; - out |= @as(u8, @boolToInt(this.optional_dependencies)) << 2; - out |= @as(u8, @boolToInt(this.dev_dependencies)) << 3; - out |= @as(u8, @boolToInt(this.peer_dependencies)) << 4; - out |= @as(u8, @boolToInt(this.workspaces)) << 5; - return @intToEnum(Behavior, out); + out |= @as(u8, @intFromBool(this.dependencies)) << 1; + out |= @as(u8, @intFromBool(this.optional_dependencies)) << 2; + out |= @as(u8, @intFromBool(this.dev_dependencies)) << 3; + out |= @as(u8, @intFromBool(this.peer_dependencies)) << 4; + out |= @as(u8, @intFromBool(this.workspaces)) << 5; + return @enumFromInt(Behavior, out); } pub const main = Features{ @@ -541,7 +541,7 @@ const Task = struct { /// An ID that lets us register a callback without keeping the same pointer around pub const Id = struct { pub fn forNPMPackage(package_name: string, package_version: Semver.Version) u64 { - var hasher = std.hash.Wyhash.init(0); + var hasher = bun.Wyhash.init(0); hasher.update(package_name); hasher.update("@"); hasher.update(std.mem.asBytes(&package_version)); @@ -549,28 +549,28 @@ const Task = struct { } pub fn forBinLink(package_id: PackageID) u64 { - const hash = std.hash.Wyhash.hash(0, std.mem.asBytes(&package_id)); + const hash = bun.Wyhash.hash(0, std.mem.asBytes(&package_id)); return @as(u64, 1 << 61) | @as(u64, @truncate(u61, hash)); } pub fn forManifest(name: string) u64 { - return @as(u64, 2 << 61) | @as(u64, @truncate(u61, std.hash.Wyhash.hash(0, name))); + return @as(u64, 2 << 61) | @as(u64, @truncate(u61, bun.Wyhash.hash(0, name))); } pub fn forTarball(url: string) u64 { - var hasher = std.hash.Wyhash.init(0); + var hasher = bun.Wyhash.init(0); hasher.update(url); return @as(u64, 3 << 61) | @as(u64, @truncate(u61, hasher.final())); } pub fn forGitClone(url: string) u64 { - var hasher = std.hash.Wyhash.init(0); + var hasher = bun.Wyhash.init(0); hasher.update(url); return @as(u64, 4 << 61) | @as(u64, @truncate(u61, hasher.final())); } pub fn forGitCheckout(url: string, resolved: string) u64 { - var hasher = std.hash.Wyhash.init(0); + var hasher = bun.Wyhash.init(0); hasher.update(url); hasher.update("@"); hasher.update(resolved); @@ -1079,10 +1079,10 @@ const PackageInstall = struct { var stackpath: [bun.MAX_PATH_BYTES]u8 = undefined; while (try walker.next()) |entry| { switch (entry.kind) { - .Directory => { + .directory => { std.os.mkdirat(destination_dir_.dir.fd, entry.path, 0o755) catch {}; }, - .File => { + .file => { bun.copy(u8, &stackpath, entry.path); stackpath[entry.path.len] = 0; var path: [:0]u8 = stackpath[0..entry.path.len :0]; @@ -1190,7 +1190,7 @@ const PackageInstall = struct { ) !u32 { var real_file_count: u32 = 0; while (try walker.next()) |entry| { - if (entry.kind != .File) continue; + if (entry.kind != .file) continue; real_file_count += 1; var outfile = destination_dir_.createFile(entry.path, .{}) catch brk: { @@ -1268,10 +1268,10 @@ const PackageInstall = struct { var real_file_count: u32 = 0; while (try walker.next()) |entry| { switch (entry.kind) { - .Directory => { + .directory => { std.os.mkdirat(destination_dir_.dir.fd, entry.path, 0o755) catch {}; }, - .File => { + .file => { try std.os.linkat(entry.dir.dir.fd, entry.basename, destination_dir_.dir.fd, entry.path, 0); real_file_count += 1; }, @@ -1351,16 +1351,16 @@ const PackageInstall = struct { while (try walker.next()) |entry| { switch (entry.kind) { // directories are created - .Directory => { + .directory => { std.os.mkdirat(dest_dir_fd, entry.path, 0o755) catch {}; }, // but each file in the directory is a symlink - .File => { - @memcpy(remain.ptr, entry.path.ptr, entry.path.len); + .file => { + @memcpy(remain[0..entry.path.len], entry.path); remain[entry.path.len] = 0; var from_path = buf[0 .. cache_dir_offset + entry.path.len :0]; - @memcpy(dest_remaining.ptr, entry.path.ptr, entry.path.len); + @memcpy(dest_remaining[0..entry.path.len], entry.path); dest_remaining[entry.path.len] = 0; var to_path = dest_buf[0 .. dest_dir_offset + entry.path.len :0]; @@ -1830,7 +1830,7 @@ pub const PackageManager = struct { const offset = this.preinstall_state.items.len; try this.preinstall_state.ensureTotalCapacity(this.allocator, count); this.preinstall_state.expandToCapacity(); - std.mem.set(PreinstallState, this.preinstall_state.items[offset..], PreinstallState.unknown); + @memset(this.preinstall_state.items[offset..], PreinstallState.unknown); } pub fn setPreinstallState(this: *PackageManager, package_id: PackageID, lockfile: *Lockfile, value: PreinstallState) void { @@ -2263,7 +2263,7 @@ pub const PackageManager = struct { var iter = dir.iterate(); while (try iter.next()) |entry| { - if (entry.kind != .Directory and entry.kind != .SymLink) continue; + if (entry.kind != .directory and entry.kind != .sym_link) continue; const name = entry.name; const sliced = SlicedString.init(name, name); const parsed = Semver.Version.parse(sliced, allocator); @@ -2305,7 +2305,7 @@ pub const PackageManager = struct { }; // TODO: make this fewer passes - std.sort.sort( + std.sort.block( Semver.Version, installed_versions.items, @as([]const u8, tags_buf.items), @@ -3752,7 +3752,7 @@ pub const PackageManager = struct { if (comptime log_level.isVerbose()) { Output.prettyError(" ", .{}); - Output.printElapsed(@intToFloat(f64, task.http.elapsed) / std.time.ns_per_ms); + Output.printElapsed(@floatFromInt(f64, task.http.elapsed) / std.time.ns_per_ms); Output.prettyError("\n Downloaded {s} versions\n", .{name.slice()}); Output.flush(); } @@ -3886,7 +3886,7 @@ pub const PackageManager = struct { if (comptime log_level.isVerbose()) { Output.prettyError(" ", .{}); - Output.printElapsed(@floatCast(f64, @intToFloat(f64, task.http.elapsed) / std.time.ns_per_ms)); + Output.printElapsed(@floatCast(f64, @floatFromInt(f64, task.http.elapsed) / std.time.ns_per_ms)); Output.prettyError(" Downloaded {s} tarball\n", .{extract.name.slice()}); Output.flush(); } @@ -4814,7 +4814,7 @@ pub const PackageManager = struct { var new_dependencies = try allocator.alloc(G.Property, dependencies.len + remaining - replacing); bun.copy(G.Property, new_dependencies, dependencies); - std.mem.set(G.Property, new_dependencies[dependencies.len..], G.Property{}); + @memset(new_dependencies[dependencies.len..], G.Property{}); outer: for (updates) |*update| { if (update.e_string != null) continue; @@ -6372,7 +6372,7 @@ pub const PackageManager = struct { var iter: std.fs.IterableDir.Iterator = node_modules_bin.iterate(); iterator: while (iter.next() catch null) |entry| { switch (entry.kind) { - std.fs.IterableDir.Entry.Kind.SymLink => { + std.fs.IterableDir.Entry.Kind.sym_link => { // any symlinks which we are unable to open are assumed to be dangling // note that using access won't work here, because access doesn't resolve symlinks @@ -6547,7 +6547,7 @@ pub const PackageManager = struct { if (folder.len == 0 or (folder.len == 1 and folder[0] == '.')) { installer.cache_dir_subpath = "."; } else { - @memcpy(&this.folder_path_buf, folder.ptr, folder.len); + @memcpy(this.folder_path_buf[0..folder.len], folder); this.folder_path_buf[folder.len] = 0; installer.cache_dir_subpath = this.folder_path_buf[0..folder.len :0]; } @@ -6567,7 +6567,7 @@ pub const PackageManager = struct { if (folder.len == 0 or (folder.len == 1 and folder[0] == '.')) { installer.cache_dir_subpath = "."; } else { - @memcpy(&this.folder_path_buf, folder.ptr, folder.len); + @memcpy(this.folder_path_buf[0..folder.len], folder); this.folder_path_buf[folder.len] = 0; installer.cache_dir_subpath = this.folder_path_buf[0..folder.len :0]; } @@ -6608,16 +6608,16 @@ pub const PackageManager = struct { const global_link_dir = this.manager.globalLinkDirPath() catch unreachable; var ptr = &this.folder_path_buf; var remain: []u8 = this.folder_path_buf[0..]; - @memcpy(ptr, global_link_dir.ptr, global_link_dir.len); + @memcpy(ptr[0..global_link_dir.len], global_link_dir); remain = remain[global_link_dir.len..]; if (global_link_dir[global_link_dir.len - 1] != std.fs.path.sep) { remain[0] = std.fs.path.sep; remain = remain[1..]; } - @memcpy(remain.ptr, folder.ptr, folder.len); + @memcpy(remain[0..folder.len], folder); remain = remain[folder.len..]; remain[0] = 0; - const len = @ptrToInt(remain.ptr) - @ptrToInt(ptr); + const len = @intFromPtr(remain.ptr) - @intFromPtr(ptr); installer.cache_dir_subpath = this.folder_path_buf[0..len :0]; installer.cache_dir = directory; } @@ -6626,7 +6626,7 @@ pub const PackageManager = struct { } const needs_install = this.force_install or this.skip_verify_installed_version_number or !installer.verify(resolution, buf); - this.summary.skipped += @as(u32, @boolToInt(!needs_install)); + this.summary.skipped += @as(u32, @intFromBool(!needs_install)); if (needs_install) { const result: PackageInstall.Result = switch (resolution.tag) { @@ -6637,7 +6637,7 @@ pub const PackageManager = struct { switch (result) { .success => { const is_duplicate = this.successfully_installed.isSet(package_id); - this.summary.success += @as(u32, @boolToInt(!is_duplicate)); + this.summary.success += @as(u32, @intFromBool(!is_duplicate)); this.successfully_installed.set(package_id); if (comptime log_level.showProgress()) { @@ -7390,7 +7390,7 @@ pub const PackageManager = struct { ); } var mapping = try manager.lockfile.allocator.alloc(PackageID, maybe_root.dependencies.len); - std.mem.set(PackageID, mapping, invalid_package_id); + @memset(mapping, invalid_package_id); manager.summary = try Package.Diff.generate( ctx.allocator, @@ -7443,8 +7443,8 @@ pub const PackageManager = struct { var resolutions = manager.lockfile.buffers.resolutions.items.ptr[off .. off + len]; // It is too easy to accidentally undefined memory - std.mem.set(PackageID, resolutions, invalid_package_id); - std.mem.set(Dependency, dependencies, Dependency{}); + @memset(resolutions, invalid_package_id); + @memset(dependencies, Dependency{}); manager.lockfile.buffers.dependencies.items = manager.lockfile.buffers.dependencies.items.ptr[0 .. off + len]; manager.lockfile.buffers.resolutions.items = manager.lockfile.buffers.resolutions.items.ptr[0 .. off + len]; -- cgit v1.2.3 From d8817c2d32a237440a7677622ba351aa95f47c22 Mon Sep 17 00:00:00 2001 From: Tiago Teixeira Date: Mon, 26 Jun 2023 01:43:39 +0200 Subject: Add support for install with --frozen-lockfile (#3365) * Add support for install with --frozen-lockfile * Add test * Add test for frozenLockfile in config file --- completions/bun.bash | 4 +-- completions/bun.zsh | 6 +++++ completions/spec.yaml | 3 +++ docs/cli/bun-install.md | 4 +++ docs/cli/install.md | 9 +++++++ docs/install/index.md | 9 +++++++ docs/runtime/configuration.md | 3 +++ src/api/demo/schema.d.ts | 1 + src/api/demo/schema.js | 4 +++ src/api/demo/schema.peechy | 1 + src/api/demo/schema.zig | 10 ++++++++ src/api/schema.d.ts | 1 + src/api/schema.js | 10 ++++++++ src/api/schema.peechy | 1 + src/api/schema.zig | 10 ++++++++ src/bunfig.zig | 6 +++++ src/install/install.zig | 13 ++++++++++ test/cli/install/bun-install.test.ts | 49 ++++++++++++++++++++++++++++++++++++ 18 files changed, 142 insertions(+), 2 deletions(-) (limited to 'src/install/install.zig') diff --git a/completions/bun.bash b/completions/bun.bash index 7eb83c48b..499adc470 100644 --- a/completions/bun.bash +++ b/completions/bun.bash @@ -92,10 +92,10 @@ _bun_completions() { PACKAGE_OPTIONS[REMOVE_OPTIONS_LONG]=""; PACKAGE_OPTIONS[REMOVE_OPTIONS_SHORT]=""; - PACKAGE_OPTIONS[SHARED_OPTIONS_LONG]="--config --yarn --production --no-save --dry-run --lockfile --force --cache-dir --no-cache --silent --verbose --global --cwd --backend --link-native-bins --help"; + PACKAGE_OPTIONS[SHARED_OPTIONS_LONG]="--config --yarn --production --frozen-lockfile --no-save --dry-run --lockfile --force --cache-dir --no-cache --silent --verbose --global --cwd --backend --link-native-bins --help"; PACKAGE_OPTIONS[SHARED_OPTIONS_SHORT]="-c -y -p -f -g"; - PM_OPTIONS[LONG_OPTIONS]="--config --yarn --production --no-save --dry-run --lockfile --force --cache-dir --no-cache --silent --verbose --no-progress --no-summary --no-verify --ignore-scripts --global --cwd --backend --link-native-bins --help" + PM_OPTIONS[LONG_OPTIONS]="--config --yarn --production --frozen-lockfile --no-save --dry-run --lockfile --force --cache-dir --no-cache --silent --verbose --no-progress --no-summary --no-verify --ignore-scripts --global --cwd --backend --link-native-bins --help" PM_OPTIONS[SHORT_OPTIONS]="-c -y -p -f -g" local cur_word="${COMP_WORDS[${COMP_CWORD}]}"; diff --git a/completions/bun.zsh b/completions/bun.zsh index 946445b64..a8f66b4fa 100644 --- a/completions/bun.zsh +++ b/completions/bun.zsh @@ -47,6 +47,7 @@ _bun() { '-g[Add a package globally]' \ '--global[Add a package globally]' \ '--production[Don'"'"'t install devDependencies]' \ + '--frozen-lockfile[Disallow changes to lockfile]' \ '--optional[Add dependency to optionalDependencies]' \ '--development[Add dependency to devDependencies]' \ '-d[Add dependency to devDependencies]' \ @@ -88,6 +89,7 @@ _bun() { '--yarn[Write a yarn.lock file (yarn v1)]' \ '--global[Add a package globally]' \ '--production[Don'"'"'t install devDependencies]' \ + '--frozen-lockfile[Disallow changes to lockfile]' \ '--optional[Add dependency to optionalDependencies]' \ '--development[Add dependency to devDependencies]' \ '-d[Add dependency to devDependencies]' \ @@ -123,6 +125,7 @@ _bun() { '--yarn[Write a yarn.lock file (yarn v1)]' \ '--global[Add a package globally]' \ '--production[Don'"'"'t install devDependencies]' \ + '--frozen-lockfile[Disallow changes to lockfile]' \ '--optional[Add dependency to optionalDependencies]' \ '--development[Add dependency to devDependencies]' \ '-d[Add dependency to devDependencies]' \ @@ -278,6 +281,7 @@ _bun() { '--yarn[Write a yarn.lock file (yarn v1)]' '-p[Do not install devDependencies]' '--production[Do not install devDependencies]' + '--frozen-lockfile[Disallow changes to lockfile]' \ '--no-save[Do not save a lockfile]' '--dry-run[Do not install anything]' '--lockfile[Store & load a lockfile at a specific filepath]' @@ -532,6 +536,7 @@ _bun() { '--yarn[Write a yarn.lock file (yarn v1)]' \ '--production[Don'"'"'t install devDependencies]' \ '-p[Don'"'"'t install devDependencies]' \ + '--frozen-lockfile[Disallow changes to lockfile]' \ '--no-save[]' \ '--dry-run[Don'"'"'t install anything]' \ '--force[Always request the latest versions from the registry & reinstall all dependenices]' \ @@ -565,6 +570,7 @@ _bun() { '--yarn[Write a yarn.lock file (yarn v1)]' \ '--production[Don'"'"'t install devDependencies]' \ '-p[Don'"'"'t install devDependencies]' \ + '--frozen-lockfile[Disallow changes to lockfile]' \ '--no-save[]' \ '--dry-run[Don'"'"'t install anything]' \ '-g[Remove a package globally]' \ diff --git a/completions/spec.yaml b/completions/spec.yaml index 9c37ae89e..c3391c192 100644 --- a/completions/spec.yaml +++ b/completions/spec.yaml @@ -115,6 +115,7 @@ subcommands: - yarn -- "Write a yarn.lock file (yarn v1)" - production -- "Don't install devDependencies" - p -- "Don't install devDependencies" + - frozen-lockfile -- "Disallow changes to lockfile" - no-save -- - dry-run -- "Don't install anything" - force -- "Always request the latest versions from the registry & reinstall all dependenices" @@ -152,6 +153,7 @@ subcommands: - development -- "Add dependency to devDependencies" - d -- "Add dependency to devDependencies" - p -- "Don't install devDependencies" + - frozen-lockfile -- "Disallow changes to lockfile" - no-save -- - dry-run -- "Don't install anything" - force -- "Always request the latest versions from the registry & reinstall all dependenices" @@ -192,6 +194,7 @@ subcommands: - yarn -- "Write a yarn.lock file (yarn v1)" - production -- "Don't install devDependencies" - p -- "Don't install devDependencies" + - frozen-lockfile -- "Disallow changes to lockfile" - no-save -- - dry-run -- "Don't install anything" - force -- "Always request the latest versions from the registry & reinstall all dependenices" diff --git a/docs/cli/bun-install.md b/docs/cli/bun-install.md index 11cf3ee81..8050070be 100644 --- a/docs/cli/bun-install.md +++ b/docs/cli/bun-install.md @@ -47,6 +47,9 @@ registry = "https://registry.yarnpkg.com/" # Install for production? This is the equivalent to the "--production" CLI argument production = false +# Disallow changes to lockfile? This is the equivalent to the "--fozen-lockfile" CLI argument +frozenLockfile = false + # Don't actually install dryRun = true @@ -108,6 +111,7 @@ export interface Install { scopes: Scopes; registry: Registry; production: boolean; + frozenLockfile: boolean; dryRun: boolean; optional: boolean; dev: boolean; diff --git a/docs/cli/install.md b/docs/cli/install.md index 695c975f9..4489a0d4a 100644 --- a/docs/cli/install.md +++ b/docs/cli/install.md @@ -49,6 +49,12 @@ To install in production mode (i.e. without `devDependencies`): $ bun install --production ``` +To install dependencies without allowing changes to lockfile (useful on CI): + +```bash +$ bun install --frozen-lockfile +``` + To perform a dry run (i.e. don't actually install anything): ```bash @@ -80,6 +86,9 @@ peer = false # equivalent to `--production` flag production = false +# equivalent to `--frozen-lockfile` flag +frozenLockfile = false + # equivalent to `--dry-run` flag dryRun = false ``` diff --git a/docs/install/index.md b/docs/install/index.md index 48e001275..162a4abac 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -49,6 +49,12 @@ To install in production mode (i.e. without `devDependencies`): $ bun install --production ``` +To install dependencies without allowing changes to lockfile (useful on CI): + +```bash +$ bun install --frozen-lockfile +``` + To perform a dry run (i.e. don't actually install anything): ```bash @@ -80,6 +86,9 @@ peer = false # equivalent to `--production` flag production = false +# equivalent to `--frozen-lockfile` flag +frozenLockfile = false + # equivalent to `--dry-run` flag dryRun = false ``` diff --git a/docs/runtime/configuration.md b/docs/runtime/configuration.md index e1572c990..2ae81713a 100644 --- a/docs/runtime/configuration.md +++ b/docs/runtime/configuration.md @@ -129,6 +129,9 @@ peer = false # equivalent to `--production` flag production = false +# equivalent to `--frozen-lockfile` flag +frozenLockfile = false + # equivalent to `--dry-run` flag dryRun = false ``` diff --git a/src/api/demo/schema.d.ts b/src/api/demo/schema.d.ts index 6f3949c77..e8a6994e7 100644 --- a/src/api/demo/schema.d.ts +++ b/src/api/demo/schema.d.ts @@ -681,6 +681,7 @@ export interface BunInstall { disable_manifest_cache?: boolean; global_dir?: string; global_bin_dir?: string; + frozen_lockfile?: boolean; } export declare function encodeStackFrame(message: StackFrame, bb: ByteBuffer): void; diff --git a/src/api/demo/schema.js b/src/api/demo/schema.js index 7bdd13b65..d23d64a14 100644 --- a/src/api/demo/schema.js +++ b/src/api/demo/schema.js @@ -2992,6 +2992,10 @@ function decodeBunInstall(bb) { result["global_bin_dir"] = bb.readString(); break; + case 19: + result["frozen-lockfile"] = !!bb.readByte(); + break; + default: throw new Error("Attempted to parse invalid message"); } diff --git a/src/api/demo/schema.peechy b/src/api/demo/schema.peechy index 09d3c1fac..e495bb9c0 100644 --- a/src/api/demo/schema.peechy +++ b/src/api/demo/schema.peechy @@ -550,4 +550,5 @@ message BunInstall { bool disable_manifest_cache = 16; string global_dir = 17; string global_bin_dir = 18; + string frozen_lockfile = 19; } diff --git a/src/api/demo/schema.zig b/src/api/demo/schema.zig index d57a5c725..a6de100de 100644 --- a/src/api/demo/schema.zig +++ b/src/api/demo/schema.zig @@ -2728,6 +2728,9 @@ pub const Api = struct { /// global_bin_dir global_bin_dir: ?[]const u8 = null, + /// frozen_lockfile + frozen_lockfile: ?bool = null, + pub fn decode(reader: anytype) anyerror!BunInstall { var this = std.mem.zeroes(BunInstall); @@ -2791,6 +2794,9 @@ pub const Api = struct { 18 => { this.global_bin_dir = try reader.readValue([]const u8); }, + 19 => { + this.frozen_lockfile = try reader.readValue(bool); + }, else => { return error.InvalidMessage; }, @@ -2872,6 +2878,10 @@ pub const Api = struct { try writer.writeFieldID(18); try writer.writeValue(@TypeOf(global_bin_dir), global_bin_dir); } + if (this.frozen_lockfile) |frozen_lockfile| { + try writer.writeFieldID(19); + try writer.writeInt(@as(u8, @boolToInt(frozen_lockfile))); + } try writer.endMessage(); } }; diff --git a/src/api/schema.d.ts b/src/api/schema.d.ts index 4114d951d..ac6183878 100644 --- a/src/api/schema.d.ts +++ b/src/api/schema.d.ts @@ -709,6 +709,7 @@ export interface BunInstall { disable_manifest_cache?: boolean; global_dir?: string; global_bin_dir?: string; + frozen_lockfile?: boolean; } export interface ClientServerModule { diff --git a/src/api/schema.js b/src/api/schema.js index c4f2400ed..270eb9a62 100644 --- a/src/api/schema.js +++ b/src/api/schema.js @@ -3044,6 +3044,10 @@ function decodeBunInstall(bb) { result["global_bin_dir"] = bb.readString(); break; + case 19: + result["frozen_lockfile"] = !!bb.readByte(); + break; + default: throw new Error("Attempted to parse invalid message"); } @@ -3164,6 +3168,12 @@ function encodeBunInstall(message, bb) { bb.writeByte(18); bb.writeString(value); } + + var value = message["frozen_lockfile"]; + if (value != null) { + bb.writeByte(19); + bb.writeByte(value); + } bb.writeByte(0); } diff --git a/src/api/schema.peechy b/src/api/schema.peechy index 71e85d68e..6d28381c4 100644 --- a/src/api/schema.peechy +++ b/src/api/schema.peechy @@ -590,6 +590,7 @@ message BunInstall { bool disable_manifest_cache = 16; string global_dir = 17; string global_bin_dir = 18; + bool frozen_lockfile = 19; } struct ClientServerModule { diff --git a/src/api/schema.zig b/src/api/schema.zig index 1012e6051..708d32ca0 100644 --- a/src/api/schema.zig +++ b/src/api/schema.zig @@ -2901,6 +2901,9 @@ pub const Api = struct { /// global_bin_dir global_bin_dir: ?[]const u8 = null, + /// frozen_lockfile + frozen_lockfile: ?bool = null, + pub fn decode(reader: anytype) anyerror!BunInstall { var this = std.mem.zeroes(BunInstall); @@ -2964,6 +2967,9 @@ pub const Api = struct { 18 => { this.global_bin_dir = try reader.readValue([]const u8); }, + 19 => { + this.frozen_lockfile = try reader.readValue(bool); + }, else => { return error.InvalidMessage; }, @@ -3045,6 +3051,10 @@ pub const Api = struct { try writer.writeFieldID(18); try writer.writeValue(@TypeOf(global_bin_dir), global_bin_dir); } + if (this.frozen_lockfile) |frozen_lockfile| { + try writer.writeFieldID(19); + try writer.writeInt(@as(u8, @boolToInt(frozen_lockfile))); + } try writer.endMessage(); } }; diff --git a/src/bunfig.zig b/src/bunfig.zig index 9df2978b0..597fb0985 100644 --- a/src/bunfig.zig +++ b/src/bunfig.zig @@ -322,6 +322,12 @@ pub const Bunfig = struct { } } + if (_bun.get("frozenLockfile")) |frozen_lockfile| { + if (frozen_lockfile.asBool()) |value| { + install.frozen_lockfile = value; + } + } + if (_bun.get("lockfile")) |lockfile_expr| { if (lockfile_expr.get("print")) |lockfile| { try this.expect(lockfile, .e_string); diff --git a/src/install/install.zig b/src/install/install.zig index 32c24548c..81e2a7bb8 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -4443,6 +4443,12 @@ pub const PackageManager = struct { } } + if (bun_install.frozen_lockfile) |frozen_lockfile| { + if (frozen_lockfile) { + this.enable.frozen_lockfile = true; + } + } + if (bun_install.save_optional) |save| { this.remote_package_features.optional_dependencies = save; this.local_package_features.optional_dependencies = save; @@ -4676,6 +4682,10 @@ pub const PackageManager = struct { this.enable.frozen_lockfile = true; } + if (cli.frozen_lockfile) { + this.enable.frozen_lockfile = true; + } + if (cli.force) { this.enable.manifest_cache_control = false; this.enable.force_install = true; @@ -5649,6 +5659,7 @@ pub const PackageManager = struct { clap.parseParam("--save Save to package.json") catch unreachable, clap.parseParam("--dry-run Don't install anything") catch unreachable, clap.parseParam("--lockfile Store & load a lockfile at a specific filepath") catch unreachable, + clap.parseParam("--frozen-lockfile Disallow changes to lockfile") catch unreachable, clap.parseParam("-f, --force Always request the latest versions from the registry & reinstall all dependencies") catch unreachable, clap.parseParam("--cache-dir Store & load cached data from a specific directory path") catch unreachable, clap.parseParam("--no-cache Ignore manifest cache entirely") catch unreachable, @@ -5709,6 +5720,7 @@ pub const PackageManager = struct { yarn: bool = false, production: bool = false, + frozen_lockfile: bool = false, no_save: bool = false, dry_run: bool = false, force: bool = false, @@ -5777,6 +5789,7 @@ pub const PackageManager = struct { var cli = CommandLineArguments{}; cli.yarn = args.flag("--yarn"); cli.production = args.flag("--production"); + cli.frozen_lockfile = args.flag("--frozen-lockfile"); cli.no_progress = args.flag("--no-progress"); cli.dry_run = args.flag("--dry-run"); cli.global = args.flag("--global"); diff --git a/test/cli/install/bun-install.test.ts b/test/cli/install/bun-install.test.ts index 4b3342e3a..6baee23a9 100644 --- a/test/cli/install/bun-install.test.ts +++ b/test/cli/install/bun-install.test.ts @@ -4241,6 +4241,55 @@ it("should handle --cwd", async () => { }); }); +it("should handle --frozen-lockfile", async () => { + await writeFile( + join(package_dir, "package.json"), + JSON.stringify({ name: "foo", version: "0.0.1", dependencies: { bar: "0.0.2" } }), + ); + + const { stderr, exited } = spawn({ + cmd: [bunExe(), "install", "--frozen-lockfile"], + cwd: package_dir, + stdout: null, + stdin: "pipe", + stderr: "pipe", + env, + }); + + expect(stderr).toBeDefined(); + const err = await new Response(stderr).text(); + expect(err).toContain("error: lockfile had changes, but lockfile is frozen"); + expect(await exited).toBe(1); +}); + +it("should handle frozenLockfile in config file", async () => { + await writeFile( + join(package_dir, "package.json"), + JSON.stringify({ name: "foo", version: "0.0.1", dependencies: { bar: "0.0.2" } }), + ); + await writeFile( + join(package_dir, "bunfig.toml"), + ` +[install] +frozenLockfile = true +`, + ); + + const { stderr, exited } = spawn({ + cmd: [bunExe(), "install"], + cwd: package_dir, + stdout: null, + stdin: "pipe", + stderr: "pipe", + env, + }); + + expect(stderr).toBeDefined(); + const err = await new Response(stderr).text(); + expect(err).toContain("error: lockfile had changes, but lockfile is frozen"); + expect(await exited).toBe(1); +}); + it("should perform bin-linking across multiple dependencies", async () => { const foo_package = JSON.stringify({ name: "foo", -- cgit v1.2.3 From 318879d1741035a01fc383821aea7a5ae972b40e Mon Sep 17 00:00:00 2001 From: "Alex Lam S.L" Date: Mon, 26 Jun 2023 21:55:26 +0300 Subject: [install] support trustedDependencies (#3288) * [install] support trustedDependencies closes #2073 * use `strings.indexOfChar()` * use hashes instead of strings * utilise `inline else` --- src/install/bin.zig | 2 +- src/install/extract_tarball.zig | 137 +++++++---------- src/install/install.zig | 284 ++++++++++++++++++----------------- src/install/integrity.zig | 2 +- src/install/lockfile.zig | 77 +++++++--- src/install/npm.zig | 8 +- src/install/repository.zig | 2 +- src/logger.zig | 15 +- test/cli/install/bun-install.test.ts | 74 ++++++++- 9 files changed, 342 insertions(+), 259 deletions(-) (limited to 'src/install/install.zig') diff --git a/src/install/bin.zig b/src/install/bin.zig index b0e988269..f8117c1e8 100644 --- a/src/install/bin.zig +++ b/src/install/bin.zig @@ -281,7 +281,7 @@ pub const Bin = extern struct { if (name[0] != '@') return name; var name_ = name; name_ = name[1..]; - return name_[(std.mem.indexOfScalar(u8, name_, '/') orelse return name) + 1 ..]; + return name_[(strings.indexOfChar(name_, '/') orelse return name) + 1 ..]; } fn setPermissions(folder: std.os.fd_t, target: [:0]const u8) void { diff --git a/src/install/extract_tarball.zig b/src/install/extract_tarball.zig index 0e5f8e188..3be00853f 100644 --- a/src/install/extract_tarball.zig +++ b/src/install/extract_tarball.zig @@ -157,13 +157,14 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD var tmpdir = this.temp_dir; var tmpname_buf: [256]u8 = undefined; const name = this.name.slice(); - - var basename = this.name.slice(); - if (basename[0] == '@') { - if (std.mem.indexOfScalar(u8, basename, '/')) |i| { - basename = basename[i + 1 ..]; + const basename = brk: { + if (name[0] == '@') { + if (strings.indexOfChar(name, '/')) |i| { + break :brk name[i + 1 ..]; + } } - } + break :brk name; + }; var resolved: string = ""; var tmpname = try FileSystem.instance.tmpname(basename[0..@min(basename.len, 32)], &tmpname_buf, tgz_bytes.len); @@ -216,8 +217,8 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD }; var dirname_reader = DirnameReader{ .outdirname = &resolved }; - _ = if (PackageManager.verbose_install) - try Archive.extractToDir( + switch (PackageManager.verbose_install) { + inline else => |log| _ = try Archive.extractToDir( zlib_pool.data.list.items, extract_destination, null, @@ -226,20 +227,9 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD // for GitHub tarballs, the root dir is always -- 1, true, - true, - ) - else - try Archive.extractToDir( - zlib_pool.data.list.items, - extract_destination, - null, - *DirnameReader, - &dirname_reader, - // for GitHub tarballs, the root dir is always -- - 1, - true, - false, - ); + log, + ), + } // This tag is used to know which version of the package was // installed from GitHub. package.json version becomes sort of @@ -252,31 +242,18 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD }; } }, - else => { - _ = if (PackageManager.verbose_install) - try Archive.extractToDir( - zlib_pool.data.list.items, - extract_destination, - null, - void, - {}, - // for npm packages, the root dir is always "package" - 1, - true, - true, - ) - else - try Archive.extractToDir( - zlib_pool.data.list.items, - extract_destination, - null, - void, - {}, - // for npm packages, the root dir is always "package" - 1, - true, - false, - ); + else => switch (PackageManager.verbose_install) { + inline else => |log| _ = try Archive.extractToDir( + zlib_pool.data.list.items, + extract_destination, + null, + void, + {}, + // for npm packages, the root dir is always "package" + 1, + true, + log, + ), }, } @@ -343,7 +320,7 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD }; // create an index storing each version of a package installed - if (std.mem.indexOfScalar(u8, basename, '/') == null) create_index: { + if (strings.indexOfChar(basename, '/') == null) create_index: { var index_dir = cache_dir.makeOpenPathIterable(name, .{}) catch break :create_index; defer index_dir.close(); index_dir.dir.symLink( @@ -361,39 +338,39 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !Install.ExtractD var json_path: []u8 = ""; var json_buf: []u8 = ""; var json_len: usize = 0; - switch (this.resolution.tag) { - .github, .local_tarball, .remote_tarball => { - const json_file = final_dir.openFileZ("package.json", .{ .mode = .read_only }) catch |err| { - this.package_manager.log.addErrorFmt( - null, - logger.Loc.Empty, - this.package_manager.allocator, - "\"package.json\" for \"{s}\" failed to open: {s}", - .{ name, @errorName(err) }, - ) catch unreachable; - return error.InstallFailed; - }; - defer json_file.close(); - const json_stat = try json_file.stat(); - json_buf = try this.package_manager.allocator.alloc(u8, json_stat.size + 64); - json_len = try json_file.preadAll(json_buf, 0); + if (switch (this.resolution.tag) { + // TODO remove extracted files not matching any globs under "files" + .github, .local_tarball, .remote_tarball => true, + else => this.package_manager.lockfile.trusted_dependencies.contains(@truncate(u32, Semver.String.Builder.stringHash(name))), + }) { + const json_file = final_dir.openFileZ("package.json", .{ .mode = .read_only }) catch |err| { + this.package_manager.log.addErrorFmt( + null, + logger.Loc.Empty, + this.package_manager.allocator, + "\"package.json\" for \"{s}\" failed to open: {s}", + .{ name, @errorName(err) }, + ) catch unreachable; + return error.InstallFailed; + }; + defer json_file.close(); + const json_stat = try json_file.stat(); + json_buf = try this.package_manager.allocator.alloc(u8, json_stat.size + 64); + json_len = try json_file.preadAll(json_buf, 0); - json_path = bun.getFdPath( - json_file.handle, - &json_path_buf, - ) catch |err| { - this.package_manager.log.addErrorFmt( - null, - logger.Loc.Empty, - this.package_manager.allocator, - "\"package.json\" for \"{s}\" failed to resolve: {s}", - .{ name, @errorName(err) }, - ) catch unreachable; - return error.InstallFailed; - }; - // TODO remove extracted files not matching any globs under "files" - }, - else => {}, + json_path = bun.getFdPath( + json_file.handle, + &json_path_buf, + ) catch |err| { + this.package_manager.log.addErrorFmt( + null, + logger.Loc.Empty, + this.package_manager.allocator, + "\"package.json\" for \"{s}\" failed to resolve: {s}", + .{ name, @errorName(err) }, + ) catch unreachable; + return error.InstallFailed; + }; } const ret_json_path = try FileSystem.instance.dirname_store.append(@TypeOf(json_path), json_path); diff --git a/src/install/install.zig b/src/install/install.zig index 81e2a7bb8..22068bbf3 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -166,8 +166,8 @@ pub const ExternalStringList = ExternalSlice(ExternalString); pub const VersionSlice = ExternalSlice(Semver.Version); pub const ExternalStringMap = extern struct { - name: ExternalStringList = ExternalStringList{}, - value: ExternalStringList = ExternalStringList{}, + name: ExternalStringList = .{}, + value: ExternalStringList = .{}, }; pub const PackageNameHash = u64; @@ -467,7 +467,7 @@ pub const Features = struct { is_main: bool = false, optional_dependencies: bool = false, peer_dependencies: bool = true, - scripts: bool = false, + trusted_dependencies: bool = false, workspaces: bool = false, check_for_duplicate_dependencies: bool = false, @@ -487,7 +487,7 @@ pub const Features = struct { .dev_dependencies = true, .is_main = true, .optional_dependencies = true, - .scripts = true, + .trusted_dependencies = true, .workspaces = true, }; @@ -499,7 +499,7 @@ pub const Features = struct { pub const workspace = Features{ .dev_dependencies = true, .optional_dependencies = true, - .scripts = true, + .trusted_dependencies = true, }; pub const link = Features{ @@ -3546,7 +3546,34 @@ pub const PackageManager = struct { return package; }, - else => {}, + else => if (data.json_len > 0) { + const package_json_source = logger.Source.initPathString( + data.json_path, + data.json_buf[0..data.json_len], + ); + initializeStore(); + const json = json_parser.ParseJSONUTF8( + &package_json_source, + manager.log, + manager.allocator, + ) catch |err| { + if (comptime log_level != .silent) { + const string_buf = manager.lockfile.buffers.string_bytes.items; + Output.prettyErrorln("error: expected package.json in {any} to be a JSON file: {s}\n", .{ + resolution.fmtURL(&manager.options, string_buf), + @errorName(err), + }); + } + Global.crash(); + }; + var builder = manager.lockfile.stringBuilder(); + Lockfile.Package.Scripts.parseCount(manager.allocator, &builder, json); + builder.allocate() catch unreachable; + if (comptime Environment.allow_assert) std.debug.assert(package_id.* != invalid_package_id); + var scripts = manager.lockfile.packages.items(.scripts)[package_id.*]; + scripts.parseAlloc(manager.allocator, &builder, json); + scripts.filled = true; + }, } return null; @@ -3910,10 +3937,10 @@ pub const PackageManager = struct { var task: Task = task_; if (task.log.msgs.items.len > 0) { - if (Output.enable_ansi_colors) { - try task.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try task.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + try task.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors); + }, } } @@ -5308,36 +5335,31 @@ pub const PackageManager = struct { // When using bun, we only do staleness checks once per day ) -| std.time.s_per_day; - manager.lockfile = brk: { + if (root_dir.entries.hasComptimeQuery("bun.lockb")) { var buf: [bun.MAX_PATH_BYTES]u8 = undefined; + var parts = [_]string{ + "./bun.lockb", + }; + var lockfile_path = Path.joinAbsStringBuf( + Fs.FileSystem.instance.top_level_dir, + &buf, + &parts, + .auto, + ); + buf[lockfile_path.len] = 0; + var lockfile_path_z = buf[0..lockfile_path.len :0]; - if (root_dir.entries.hasComptimeQuery("bun.lockb")) { - var parts = [_]string{ - "./bun.lockb", - }; - var lockfile_path = Path.joinAbsStringBuf( - Fs.FileSystem.instance.top_level_dir, - &buf, - &parts, - .auto, - ); - buf[lockfile_path.len] = 0; - var lockfile_path_z = buf[0..lockfile_path.len :0]; - - const result = manager.lockfile.loadFromDisk( - allocator, - log, - lockfile_path_z, - ); - - if (result == .ok) { - break :brk result.ok; - } + switch (manager.lockfile.loadFromDisk( + allocator, + log, + lockfile_path_z, + )) { + .ok => |lockfile| manager.lockfile = lockfile, + else => try manager.lockfile.initEmpty(allocator), } - + } else { try manager.lockfile.initEmpty(allocator); - break :brk manager.lockfile; - }; + } return manager; } @@ -5448,7 +5470,7 @@ pub const PackageManager = struct { // create scope if specified if (name[0] == '@') { - if (std.mem.indexOfScalar(u8, name, '/')) |i| { + if (strings.indexOfChar(name, '/')) |i| { node_modules.dir.makeDir(name[0..i]) catch |err| brk: { if (err == error.PathAlreadyExists) break :brk; if (manager.options.log_level != .silent) @@ -5513,11 +5535,7 @@ pub const PackageManager = struct { } else { // bun link lodash switch (manager.options.log_level) { - .default => try updatePackageJSONAndInstallWithManager(ctx, manager, .link, .default), - .verbose => try updatePackageJSONAndInstallWithManager(ctx, manager, .link, .verbose), - .silent => try updatePackageJSONAndInstallWithManager(ctx, manager, .link, .silent), - .default_no_progress => try updatePackageJSONAndInstallWithManager(ctx, manager, .link, .default_no_progress), - .verbose_no_progress => try updatePackageJSONAndInstallWithManager(ctx, manager, .link, .verbose_no_progress), + inline else => |log_level| try updatePackageJSONAndInstallWithManager(ctx, manager, .link, log_level), } } } @@ -6019,11 +6037,7 @@ pub const PackageManager = struct { } switch (manager.options.log_level) { - .default => try updatePackageJSONAndInstallWithManager(ctx, manager, op, .default), - .verbose => try updatePackageJSONAndInstallWithManager(ctx, manager, op, .verbose), - .silent => try updatePackageJSONAndInstallWithManager(ctx, manager, op, .silent), - .default_no_progress => try updatePackageJSONAndInstallWithManager(ctx, manager, op, .default_no_progress), - .verbose_no_progress => try updatePackageJSONAndInstallWithManager(ctx, manager, op, .verbose_no_progress), + inline else => |log_level| try updatePackageJSONAndInstallWithManager(ctx, manager, op, log_level), } } @@ -6153,13 +6167,12 @@ pub const PackageManager = struct { ) !void { if (ctx.log.errors > 0) { if (comptime log_level != .silent) { - if (Output.enable_ansi_colors) { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true) catch {}; - } else { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false) catch {}; + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; + }, } } - Global.crash(); } @@ -6183,10 +6196,10 @@ pub const PackageManager = struct { initializeStore(); var current_package_json = json_parser.ParseJSONUTF8(&package_json_source, ctx.log, manager.allocator) catch |err| { - if (Output.enable_ansi_colors) { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true) catch {}; - } else { - ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false) catch {}; + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + ctx.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; + }, } if (err == error.ParserError and ctx.log.errors > 0) { @@ -6313,7 +6326,7 @@ pub const PackageManager = struct { // haha unless defer if (auto_free) bun.default_allocator.free(old_ast_nodes); - try installWithManager(ctx, manager, new_package_json_source, log_level); + try manager.installWithManager(ctx, new_package_json_source, log_level); if (op == .update or op == .add or op == .link) { for (manager.package_json_updates) |update| { @@ -6435,11 +6448,7 @@ pub const PackageManager = struct { }; try switch (manager.options.log_level) { - .default => installWithManager(ctx, manager, package_json_contents, .default), - .verbose => installWithManager(ctx, manager, package_json_contents, .verbose), - .silent => installWithManager(ctx, manager, package_json_contents, .silent), - .default_no_progress => installWithManager(ctx, manager, package_json_contents, .default_no_progress), - .verbose_no_progress => installWithManager(ctx, manager, package_json_contents, .verbose_no_progress), + inline else => |log_level| manager.installWithManager(ctx, package_json_contents, log_level), }; } @@ -6593,10 +6602,10 @@ pub const PackageManager = struct { const args = .{ name, @errorName(err) }; if (comptime log_level.showProgress()) { - if (Output.enable_ansi_colors) { - this.progress.log(comptime Output.prettyFmt(fmt, true), args); - } else { - this.progress.log(comptime Output.prettyFmt(fmt, false), args); + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); + }, } } else { Output.prettyErrorln(fmt, args); @@ -6699,10 +6708,10 @@ pub const PackageManager = struct { const args = .{ alias, @errorName(err) }; if (comptime log_level.showProgress()) { - if (Output.enable_ansi_colors) { - this.progress.log(comptime Output.prettyFmt(fmt, true), args); - } else { - this.progress.log(comptime Output.prettyFmt(fmt, false), args); + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); + }, } } else { Output.prettyErrorln(fmt, args); @@ -6718,61 +6727,56 @@ pub const PackageManager = struct { } } - var scripts = this.lockfile.packages.items(.scripts)[package_id]; - if (scripts.hasAny()) { - var path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; - const path_str = Path.joinAbsString( - bun.getFdPath(this.node_modules_folder.dir.fd, &path_buf) catch unreachable, - &[_]string{destination_dir_subpath}, - .posix, - ); + if (resolution.tag == .workspace or this.lockfile.trusted_dependencies.contains(@truncate(u32, String.Builder.stringHash(name)))) { + var scripts = this.lockfile.packages.items(.scripts)[package_id]; + if (scripts.hasAny()) { + var path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + const path_str = Path.joinAbsString( + bun.getFdPath(this.node_modules_folder.dir.fd, &path_buf) catch unreachable, + &[_]string{destination_dir_subpath}, + .posix, + ); - scripts.enqueue(this.lockfile, buf, path_str); - } else if (!scripts.filled and switch (resolution.tag) { - .folder => Features.folder.scripts, - .npm => Features.npm.scripts, - .git, .github, .gitlab, .local_tarball, .remote_tarball => Features.tarball.scripts, - .symlink => Features.link.scripts, - .workspace => Features.workspace.scripts, - else => false, - }) { - var path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; - const path_str = Path.joinAbsString( - bun.getFdPath(this.node_modules_folder.dir.fd, &path_buf) catch unreachable, - &[_]string{destination_dir_subpath}, - .posix, - ); + scripts.enqueue(this.lockfile, buf, path_str); + } else if (!scripts.filled) { + var path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + const path_str = Path.joinAbsString( + bun.getFdPath(this.node_modules_folder.dir.fd, &path_buf) catch unreachable, + &[_]string{destination_dir_subpath}, + .posix, + ); - scripts.enqueueFromPackageJSON( - this.manager.log, - this.lockfile, - this.node_modules_folder.dir, - destination_dir_subpath, - path_str, - ) catch |err| { - if (comptime log_level != .silent) { - const fmt = "\nerror: failed to parse life-cycle scripts for {s}: {s}\n"; - const args = .{ name, @errorName(err) }; - - if (comptime log_level.showProgress()) { - if (Output.enable_ansi_colors) { - this.progress.log(comptime Output.prettyFmt(fmt, true), args); + scripts.enqueueFromPackageJSON( + this.manager.log, + this.lockfile, + this.node_modules_folder.dir, + destination_dir_subpath, + path_str, + ) catch |err| { + if (comptime log_level != .silent) { + const fmt = "\nerror: failed to parse life-cycle scripts for {s}: {s}\n"; + const args = .{ name, @errorName(err) }; + + if (comptime log_level.showProgress()) { + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); + }, + } } else { - this.progress.log(comptime Output.prettyFmt(fmt, false), args); + Output.prettyErrorln(fmt, args); } - } else { - Output.prettyErrorln(fmt, args); } - } - if (this.manager.options.enable.fail_early) { - Global.exit(1); - } + if (this.manager.options.enable.fail_early) { + Global.exit(1); + } - Output.flush(); - this.summary.fail += 1; - return; - }; + Output.flush(); + this.summary.fail += 1; + return; + }; + } } }, .fail => |cause| { @@ -7241,10 +7245,10 @@ pub const PackageManager = struct { const args = .{ name, @errorName(err) }; if (comptime log_level.showProgress()) { - if (Output.enable_ansi_colors) { - this.progress.log(comptime Output.prettyFmt(fmt, true), args); - } else { - this.progress.log(comptime Output.prettyFmt(fmt, false), args); + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); + }, } } else { Output.prettyErrorln(fmt, args); @@ -7262,10 +7266,10 @@ pub const PackageManager = struct { const args = .{lockfile.str(&names[package_id])}; if (comptime log_level.showProgress()) { - if (Output.enable_ansi_colors) { - this.progress.log(comptime Output.prettyFmt(fmt, true), args); - } else { - this.progress.log(comptime Output.prettyFmt(fmt, false), args); + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); + }, } } else { Output.prettyErrorln(fmt, args); @@ -7312,8 +7316,8 @@ pub const PackageManager = struct { } fn installWithManager( - ctx: Command.Context, manager: *PackageManager, + ctx: Command.Context, package_json_contents: string, comptime log_level: Options.LogLevel, ) !void { @@ -7326,7 +7330,7 @@ pub const PackageManager = struct { manager.options.lockfile_path, ) else - Lockfile.LoadFromDiskResult{ .not_found = {} }; + .{ .not_found = {} }; var root = Lockfile.Package{}; var needs_new_lockfile = load_lockfile_result != .ok or (load_lockfile_result.ok.buffers.dependencies.items.len == 0 and manager.package_json_updates.len > 0); // this defaults to false @@ -7360,10 +7364,10 @@ pub const PackageManager = struct { } if (ctx.log.errors > 0) { - if (Output.enable_ansi_colors) { - try manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + try manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors); + }, } } Output.flush(); @@ -7567,10 +7571,10 @@ pub const PackageManager = struct { } } - if (Output.enable_ansi_colors) { - try manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + try manager.log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors); + }, } if (manager.log.hasErrors()) Global.crash(); @@ -7731,10 +7735,10 @@ pub const PackageManager = struct { .successfully_installed = install_summary.successfully_installed, }; - if (Output.enable_ansi_colors) { - try Lockfile.Printer.Tree.print(&printer, Output.WriterType, Output.writer(), true); - } else { - try Lockfile.Printer.Tree.print(&printer, Output.WriterType, Output.writer(), false); + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + try Lockfile.Printer.Tree.print(&printer, Output.WriterType, Output.writer(), enable_ansi_colors); + }, } if (!did_meta_hash_change) { diff --git a/src/install/integrity.zig b/src/install/integrity.zig index 19e55c223..4634c2dfd 100644 --- a/src/install/integrity.zig +++ b/src/install/integrity.zig @@ -117,7 +117,7 @@ pub const Integrity = extern struct { pub fn parse(buf: []const u8) Tag { const Matcher = strings.ExactSizeMatcher(8); - const i = std.mem.indexOfScalar(u8, buf[0..@min(buf.len, 7)], '-') orelse return Tag.unknown; + const i = strings.indexOfChar(buf[0..@min(buf.len, 7)], '-') orelse return Tag.unknown; return switch (Matcher.match(buf[0..i])) { Matcher.case("sha1") => Tag.sha1, diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index b17691853..a51d2b2ee 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -85,6 +85,7 @@ const PackageJSON = @import("../resolver/package_json.zig").PackageJSON; const MetaHash = [std.crypto.hash.sha2.Sha512256.digest_length]u8; const zero_hash = std.mem.zeroes(MetaHash); const NameHashMap = std.ArrayHashMapUnmanaged(u32, String, ArrayIdentityContext, false); +const NameHashSet = std.ArrayHashMapUnmanaged(u32, void, ArrayIdentityContext, false); // Serialized data /// The version of the lockfile format, intended to prevent data corruption for format changes. @@ -103,6 +104,7 @@ allocator: Allocator, scratch: Scratch = .{}, scripts: Scripts = .{}, +trusted_dependencies: NameHashSet = .{}, workspace_paths: NameHashMap = .{}, const Stream = std.io.FixedBufferStream([]u8); @@ -113,15 +115,15 @@ pub const Scripts = struct { cwd: string, script: string, }; - const StringArrayList = std.ArrayListUnmanaged(Entry); + const Entries = std.ArrayListUnmanaged(Entry); const RunCommand = @import("../cli/run_command.zig").RunCommand; - preinstall: StringArrayList = .{}, - install: StringArrayList = .{}, - postinstall: StringArrayList = .{}, - preprepare: StringArrayList = .{}, - prepare: StringArrayList = .{}, - postprepare: StringArrayList = .{}, + preinstall: Entries = .{}, + install: Entries = .{}, + postinstall: Entries = .{}, + preprepare: Entries = .{}, + prepare: Entries = .{}, + postprepare: Entries = .{}, pub fn hasAny(this: *Scripts) bool { inline for (Package.Scripts.Hooks) |hook| { @@ -195,6 +197,7 @@ pub fn loadFromBytes(this: *Lockfile, buf: []u8, allocator: Allocator, log: *log this.format = FormatVersion.current; this.scripts = .{}; + this.trusted_dependencies = .{}; this.workspace_paths = .{}; Lockfile.Serializer.load(this, &stream, allocator, log) catch |err| { @@ -633,6 +636,7 @@ pub fn clean(old: *Lockfile, updates: []PackageManager.UpdateRequest) !*Lockfile } pub fn cleanWithLogger(old: *Lockfile, updates: []PackageManager.UpdateRequest, log: *logger.Log) !*Lockfile { + const old_trusted_dependencies = old.trusted_dependencies; const old_scripts = old.scripts; // We will only shrink the number of packages here. // never grow @@ -738,6 +742,7 @@ pub fn cleanWithLogger(old: *Lockfile, updates: []PackageManager.UpdateRequest, } } } + new.trusted_dependencies = old_trusted_dependencies; new.scripts = old_scripts; return new; } @@ -909,10 +914,10 @@ pub const Printer = struct { }), } if (log.errors > 0) { - if (Output.enable_ansi_colors) { - try log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true); - } else { - try log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false); + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + try log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors); + }, } } Global.crash(); @@ -1493,6 +1498,7 @@ pub fn initEmpty(this: *Lockfile, allocator: Allocator) !void { .allocator = allocator, .scratch = Scratch.init(allocator), .scripts = .{}, + .trusted_dependencies = .{}, .workspace_paths = .{}, }; } @@ -2438,12 +2444,11 @@ pub const Package = extern struct { initializeStore(); const json = json_parser.ParseJSONUTF8(&source, log, allocator) catch |err| { - if (Output.enable_ansi_colors) { - log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), true) catch {}; - } else { - log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), false) catch {}; + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + log.printForLogLevelWithEnableAnsiColors(Output.errorWriter(), enable_ansi_colors) catch {}; + }, } - Output.prettyErrorln("{s} parsing package.json in \"{s}\"", .{ @errorName(err), source.path.prettyDir() }); Global.crash(); }; @@ -2956,9 +2961,7 @@ pub const Package = extern struct { } } - if (comptime features.scripts) { - Package.Scripts.parseCount(allocator, &string_builder, json); - } + Package.Scripts.parseCount(allocator, &string_builder, json); if (comptime ResolverContext != void) { resolver.count(*Lockfile.StringBuilder, &string_builder, json); @@ -3113,6 +3116,37 @@ pub const Package = extern struct { } } + if (comptime features.trusted_dependencies) { + if (json.asProperty("trustedDependencies")) |q| { + switch (q.expr.data) { + .e_array => |arr| { + try lockfile.trusted_dependencies.ensureUnusedCapacity(allocator, arr.items.len); + for (arr.slice()) |item| { + const name = item.asString(allocator) orelse { + log.addErrorFmt(&source, q.loc, allocator, + \\trustedDependencies expects an array of strings, e.g. + \\"trustedDependencies": [ + \\ "package_name" + \\] + , .{}) catch {}; + return error.InvalidPackageJSON; + }; + lockfile.trusted_dependencies.putAssumeCapacity(@truncate(u32, String.Builder.stringHash(name)), {}); + } + }, + else => { + log.addErrorFmt(&source, q.loc, allocator, + \\trustedDependencies expects an array of strings, e.g. + \\"trustedDependencies": [ + \\ "package_name" + \\] + , .{}) catch {}; + return error.InvalidPackageJSON; + }, + } + } + } + try string_builder.allocate(); try lockfile.buffers.dependencies.ensureUnusedCapacity(lockfile.allocator, total_dependencies_count); try lockfile.buffers.resolutions.ensureUnusedCapacity(lockfile.allocator, total_dependencies_count); @@ -3233,9 +3267,7 @@ pub const Package = extern struct { } } - if (comptime features.scripts) { - package.scripts.parseAlloc(allocator, &string_builder, json); - } + package.scripts.parseAlloc(allocator, &string_builder, json); package.scripts.filled = true; // It is allowed for duplicate dependencies to exist in optionalDependencies and regular dependencies @@ -3511,6 +3543,7 @@ pub fn deinit(this: *Lockfile) void { this.packages.deinit(this.allocator); this.string_pool.deinit(); this.scripts.deinit(this.allocator); + this.trusted_dependencies.deinit(this.allocator); this.workspace_paths.deinit(this.allocator); } diff --git a/src/install/npm.zig b/src/install/npm.zig index 074041056..6edb6dcb4 100644 --- a/src/install/npm.zig +++ b/src/install/npm.zig @@ -80,14 +80,14 @@ pub const Registry = struct { url.path = pathname; } - while (std.mem.lastIndexOfScalar(u8, pathname, ':')) |colon| { + while (strings.lastIndexOfChar(pathname, ':')) |colon| { var segment = pathname[colon + 1 ..]; pathname = pathname[0..colon]; if (pathname.len > 1 and pathname[pathname.len - 1] == '/') { pathname = pathname[0 .. pathname.len - 1]; } - const eql_i = std.mem.indexOfScalar(u8, segment, '=') orelse continue; + const eql_i = strings.indexOfChar(segment, '=') orelse continue; var value = segment[eql_i + 1 ..]; segment = segment[0..eql_i]; @@ -847,11 +847,11 @@ pub const PackageManifest = struct { for (versions) |prop| { const version_name = prop.key.?.asString(allocator) orelse continue; - if (std.mem.indexOfScalar(u8, version_name, '-') != null) { + if (strings.indexOfChar(version_name, '-') != null) { pre_versions_len += 1; extern_string_count += 1; } else { - extern_string_count += @as(usize, @intFromBool(std.mem.indexOfScalar(u8, version_name, '+') != null)); + extern_string_count += @as(usize, @intFromBool(strings.indexOfChar(version_name, '+') != null)); release_versions_len += 1; } diff --git a/src/install/repository.zig b/src/install/repository.zig index c4b68d9be..6546481e9 100644 --- a/src/install/repository.zig +++ b/src/install/repository.zig @@ -94,7 +94,7 @@ pub const Repository = extern struct { if (!formatter.repository.resolved.isEmpty()) { try writer.writeAll("#"); var resolved = formatter.repository.resolved.slice(formatter.buf); - if (std.mem.lastIndexOfScalar(u8, resolved, '-')) |i| { + if (strings.lastIndexOfChar(resolved, '-')) |i| { resolved = resolved[i + 1 ..]; } try writer.writeAll(resolved); diff --git a/src/logger.zig b/src/logger.zig index 621e643b5..3279e9fd5 100644 --- a/src/logger.zig +++ b/src/logger.zig @@ -860,10 +860,9 @@ pub const Log = struct { } inline fn allocPrint(allocator: std.mem.Allocator, comptime fmt: string, args: anytype) !string { - return if (Output.enable_ansi_colors) - try std.fmt.allocPrint(allocator, Output.prettyFmt(fmt, true), args) - else - try std.fmt.allocPrint(allocator, Output.prettyFmt(fmt, false), args); + return try switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| std.fmt.allocPrint(allocator, Output.prettyFmt(fmt, enable_ansi_colors), args), + }; } inline fn _addResolveErrorWithLevel( @@ -1174,11 +1173,9 @@ pub const Log = struct { } pub fn printForLogLevel(self: *Log, to: anytype) !void { - if (Output.enable_ansi_colors) { - return self.printForLogLevelWithEnableAnsiColors(to, true); - } else { - return self.printForLogLevelWithEnableAnsiColors(to, false); - } + return switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| self.printForLogLevelWithEnableAnsiColors(to, enable_ansi_colors), + }; } pub fn printForLogLevelWithEnableAnsiColors(self: *Log, to: anytype, comptime enable_ansi_colors: bool) !void { diff --git a/test/cli/install/bun-install.test.ts b/test/cli/install/bun-install.test.ts index 6baee23a9..595e7bcdd 100644 --- a/test/cli/install/bun-install.test.ts +++ b/test/cli/install/bun-install.test.ts @@ -1,7 +1,7 @@ import { file, listen, Socket, spawn } from "bun"; import { afterAll, afterEach, beforeAll, beforeEach, expect, it } from "bun:test"; import { bunExe, bunEnv as env } from "harness"; -import { access, mkdir, readlink, rm, writeFile } from "fs/promises"; +import { access, mkdir, readlink, realpath, rm, writeFile } from "fs/promises"; import { join } from "path"; import { dummyAfterAll, @@ -4473,3 +4473,75 @@ cache = false expect(await file(join(package_dir, "package.json")).text()).toEqual(foo_package); expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([]); }, 20000); + +it("should handle trustedDependencies", async () => { + const scripts = { + preinstall: `${bunExe()} echo.js preinstall`, + install: `${bunExe()} echo.js install`, + postinstall: `${bunExe()} echo.js postinstall`, + preprepare: `${bunExe()} echo.js preprepare`, + prepare: `${bunExe()} echo.js prepare`, + postprepare: `${bunExe()} echo.js postprepare`, + }; + await writeFile( + join(package_dir, "package.json"), + JSON.stringify({ + name: "foo", + version: "0.1.0", + dependencies: { + bar: "file:./bar", + moo: "file:./moo", + }, + trustedDependencies: ["moo"], + }), + ); + await mkdir(join(package_dir, "bar")); + const bar_package = JSON.stringify({ + name: "bar", + version: "0.2.0", + scripts, + }); + await writeFile(join(package_dir, "bar", "package.json"), bar_package); + await writeFile(join(package_dir, "bar", "echo.js"), "console.log(`bar|${process.argv[2]}|${import.meta.dir}`);"); + await mkdir(join(package_dir, "moo")); + const moo_package = JSON.stringify({ + name: "moo", + version: "0.3.0", + scripts, + }); + await writeFile(join(package_dir, "moo", "package.json"), moo_package); + await writeFile(join(package_dir, "moo", "echo.js"), "console.log(`moo|${process.argv[2]}|${import.meta.dir}`);"); + const { stdout, stderr, exited } = spawn({ + cmd: [bunExe(), "install"], + cwd: package_dir, + stdout: null, + stdin: "pipe", + stderr: "pipe", + env, + }); + expect(stderr).toBeDefined(); + const err = await new Response(stderr).text(); + expect(err).toContain("Saved lockfile"); + expect(stdout).toBeDefined(); + const out = await new Response(stdout).text(); + const moo_dir = await realpath(join(package_dir, "node_modules", "moo")); + expect(out.replace(/\s*\[[0-9\.]+m?s\]\s*$/, "").split(/\r?\n/)).toEqual([ + `moo|preinstall|${moo_dir}`, + " + bar@bar", + " + moo@moo", + `moo|install|${moo_dir}`, + `moo|postinstall|${moo_dir}`, + `moo|preprepare|${moo_dir}`, + `moo|prepare|${moo_dir}`, + `moo|postprepare|${moo_dir}`, + "", + " 2 packages installed", + ]); + expect(await exited).toBe(0); + expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".cache", "bar", "moo"]); + expect(await readdirSorted(join(package_dir, "node_modules", "bar"))).toEqual(["echo.js", "package.json"]); + expect(await file(join(package_dir, "node_modules", "bar", "package.json")).text()).toEqual(bar_package); + expect(await readdirSorted(join(package_dir, "node_modules", "moo"))).toEqual(["echo.js", "package.json"]); + expect(await file(join(package_dir, "node_modules", "moo", "package.json")).text()).toEqual(moo_package); + await access(join(package_dir, "bun.lockb")); +}); -- cgit v1.2.3 From 28f27f733b3db072944156694301651b09b7696b Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 26 Jun 2023 15:51:57 -0700 Subject: [bun install] Implement `--exact` flag (#3409) * [bun install] Implement `--exact` flag * Rename to --save-exact * Rename --exact to --save-exact * Update bun-add.test.ts * We're going with --exact as the flag name --------- Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com> --- src/api/schema.d.ts | 1 + src/api/schema.js | 10 ++++++++ src/api/schema.peechy | 1 + src/api/schema.zig | 10 ++++++++ src/bunfig.zig | 8 ++++++ src/install/install.zig | 53 +++++++++++++++++++++++++++++++++----- src/install/lockfile.zig | 32 +++++++++++++++++------ test/cli/install/bun-add.test.ts | 55 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 155 insertions(+), 15 deletions(-) (limited to 'src/install/install.zig') diff --git a/src/api/schema.d.ts b/src/api/schema.d.ts index ac6183878..2a86340ad 100644 --- a/src/api/schema.d.ts +++ b/src/api/schema.d.ts @@ -710,6 +710,7 @@ export interface BunInstall { global_dir?: string; global_bin_dir?: string; frozen_lockfile?: boolean; + exact?: boolean; } export interface ClientServerModule { diff --git a/src/api/schema.js b/src/api/schema.js index 270eb9a62..f1e68031e 100644 --- a/src/api/schema.js +++ b/src/api/schema.js @@ -3048,6 +3048,10 @@ function decodeBunInstall(bb) { result["frozen_lockfile"] = !!bb.readByte(); break; + case 20: + result["exact"] = !!bb.readByte(); + break; + default: throw new Error("Attempted to parse invalid message"); } @@ -3174,6 +3178,12 @@ function encodeBunInstall(message, bb) { bb.writeByte(19); bb.writeByte(value); } + + var value = message["exact"]; + if (value != null) { + bb.writeByte(20); + bb.writeByte(value); + } bb.writeByte(0); } diff --git a/src/api/schema.peechy b/src/api/schema.peechy index 6d28381c4..a172606f7 100644 --- a/src/api/schema.peechy +++ b/src/api/schema.peechy @@ -591,6 +591,7 @@ message BunInstall { string global_dir = 17; string global_bin_dir = 18; bool frozen_lockfile = 19; + bool exact = 20; } struct ClientServerModule { diff --git a/src/api/schema.zig b/src/api/schema.zig index 2de80d42c..ec8efa9f6 100644 --- a/src/api/schema.zig +++ b/src/api/schema.zig @@ -2904,6 +2904,9 @@ pub const Api = struct { /// frozen_lockfile frozen_lockfile: ?bool = null, + /// exact + exact: ?bool = null, + pub fn decode(reader: anytype) anyerror!BunInstall { var this = std.mem.zeroes(BunInstall); @@ -2970,6 +2973,9 @@ pub const Api = struct { 19 => { this.frozen_lockfile = try reader.readValue(bool); }, + 20 => { + this.exact = try reader.readValue(bool); + }, else => { return error.InvalidMessage; }, @@ -3055,6 +3061,10 @@ pub const Api = struct { try writer.writeFieldID(19); try writer.writeInt(@as(u8, @intFromBool(frozen_lockfile))); } + if (this.exact) |exact| { + try writer.writeFieldID(20); + try writer.writeInt(@as(u8, @intFromBool(exact))); + } try writer.endMessage(); } }; diff --git a/src/bunfig.zig b/src/bunfig.zig index 597fb0985..1244f52b8 100644 --- a/src/bunfig.zig +++ b/src/bunfig.zig @@ -259,6 +259,14 @@ pub const Bunfig = struct { } } + if (json.get("exact")) |exact_install_expr| { + try this.expect(exact_install_expr, .e_boolean); + + if (exact_install_expr.asBool().?) { + install.exact = true; + } + } + if (json.get("prefer")) |prefer_expr| { try this.expect(prefer_expr, .e_string); diff --git a/src/install/install.zig b/src/install/install.zig index 22068bbf3..9465c4897 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -4461,6 +4461,10 @@ pub const PackageManager = struct { this.remote_package_features.peer_dependencies = save; } + if (bun_install.exact) |exact| { + this.enable.exact_versions = exact; + } + if (bun_install.production) |production| { if (production) { this.local_package_features.dev_dependencies = false; @@ -4569,6 +4573,10 @@ pub const PackageManager = struct { this.scope.url = URL.parse(cli.registry); } + if (cli.exact) { + this.enable.exact_versions = true; + } + if (cli.token.len > 0) { this.scope.token = cli.token; } @@ -4755,6 +4763,8 @@ pub const PackageManager = struct { force_save_lockfile: bool = false, force_install: bool = false, + + exact_versions: bool = false, }; }; @@ -4802,6 +4812,7 @@ pub const PackageManager = struct { updates: []UpdateRequest, current_package_json: *JSAst.Expr, dependency_list: string, + exact_versions: bool, ) !void { const G = JSAst.G; @@ -4989,9 +5000,14 @@ pub const PackageManager = struct { if (update.e_string) |e_string| { e_string.data = switch (update.resolution.tag) { .npm => if (update.version.tag == .dist_tag and update.version.literal.isEmpty()) - std.fmt.allocPrint(allocator, "^{}", .{ - update.resolution.value.npm.version.fmt(update.version_buf), - }) catch unreachable + switch (exact_versions) { + false => std.fmt.allocPrint(allocator, "^{}", .{ + update.resolution.value.npm.version.fmt(update.version_buf), + }) catch unreachable, + true => std.fmt.allocPrint(allocator, "{}", .{ + update.resolution.value.npm.version.fmt(update.version_buf), + }) catch unreachable, + } else null, .uninitialized => switch (update.version.tag) { @@ -5709,6 +5725,7 @@ pub const PackageManager = struct { const add_params = install_params_ ++ [_]ParamType{ clap.parseParam("-d, --development Add dependency to \"devDependencies\"") catch unreachable, clap.parseParam("--optional Add dependency to \"optionalDependencies\"") catch unreachable, + clap.parseParam("--exact Add the exact version instead of the ^range") catch unreachable, clap.parseParam(" ... \"name\" or \"name@version\" of packages to install") catch unreachable, }; @@ -5759,6 +5776,8 @@ pub const PackageManager = struct { no_optional: bool = false, omit: Omit = Omit{}, + exact: bool = false, + const Omit = struct { dev: bool = false, optional: bool = true, @@ -5837,6 +5856,7 @@ pub const PackageManager = struct { if (comptime subcommand == .add) { cli.development = args.flag("--development"); cli.optional = args.flag("--optional"); + cli.exact = args.flag("--exact"); } // for (args.options("--omit")) |omit| { @@ -6293,7 +6313,13 @@ pub const PackageManager = struct { manager.to_remove = updates; }, .link, .add, .update => { - try PackageJSONEditor.edit(ctx.allocator, updates, ¤t_package_json, dependency_list); + try PackageJSONEditor.edit( + ctx.allocator, + updates, + ¤t_package_json, + dependency_list, + manager.options.enable.exact_versions, + ); manager.package_json_updates = updates; }, else => {}, @@ -6346,7 +6372,13 @@ pub const PackageManager = struct { return; }; - try PackageJSONEditor.edit(ctx.allocator, updates, ¤t_package_json, dependency_list); + try PackageJSONEditor.edit( + ctx.allocator, + updates, + ¤t_package_json, + dependency_list, + manager.options.enable.exact_versions, + ); var buffer_writer_two = try JSPrinter.BufferWriter.init(ctx.allocator); try buffer_writer_two.buffer.list.ensureTotalCapacity(ctx.allocator, new_package_json_source.len + 1); buffer_writer_two.append_newline = @@ -7031,7 +7063,10 @@ pub const PackageManager = struct { ) !PackageInstall.Summary { var lockfile = lockfile_; if (!this.options.local_package_features.dev_dependencies) { - lockfile = try lockfile.maybeCloneFilteringRootPackages(this.options.local_package_features); + lockfile = try lockfile.maybeCloneFilteringRootPackages( + this.options.local_package_features, + this.options.enable.exact_versions, + ); } var root_node: *Progress.Node = undefined; @@ -7582,7 +7617,11 @@ pub const PackageManager = struct { const needs_clean_lockfile = had_any_diffs or needs_new_lockfile or manager.package_json_updates.len > 0; var did_meta_hash_change = needs_clean_lockfile; if (needs_clean_lockfile) { - manager.lockfile = try manager.lockfile.cleanWithLogger(manager.package_json_updates, manager.log); + manager.lockfile = try manager.lockfile.cleanWithLogger( + manager.package_json_updates, + manager.log, + manager.options.enable.exact_versions, + ); } if (manager.lockfile.packages.len > 0) { diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index a51d2b2ee..7d21860ef 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -538,6 +538,7 @@ pub const Tree = struct { pub fn maybeCloneFilteringRootPackages( old: *Lockfile, features: Features, + exact_versions: bool, ) !*Lockfile { const old_root_dependenices_list = old.packages.items(.dependencies)[0]; var old_root_resolutions = old.packages.items(.resolutions)[0]; @@ -555,10 +556,10 @@ pub fn maybeCloneFilteringRootPackages( if (!any_changes) return old; - return try old.clean(&.{}); + return try old.clean(&.{}, exact_versions); } -fn preprocessUpdateRequests(old: *Lockfile, updates: []PackageManager.UpdateRequest) !void { +fn preprocessUpdateRequests(old: *Lockfile, updates: []PackageManager.UpdateRequest, exact_versions: bool) !void { const root_deps_list: Lockfile.DependencySlice = old.packages.items(.dependencies)[0]; if (@as(usize, root_deps_list.off) < old.buffers.dependencies.items.len) { var string_builder = old.stringBuilder(); @@ -575,7 +576,10 @@ fn preprocessUpdateRequests(old: *Lockfile, updates: []PackageManager.UpdateRequ if (dep.name_hash == String.Builder.stringHash(update.name)) { if (old_resolution > old.packages.len) continue; const res = resolutions_of_yore[old_resolution]; - const len = std.fmt.count("^{}", .{res.value.npm.fmt(old.buffers.string_bytes.items)}); + const len = switch (exact_versions) { + false => std.fmt.count("^{}", .{res.value.npm.fmt(old.buffers.string_bytes.items)}), + true => std.fmt.count("{}", .{res.value.npm.fmt(old.buffers.string_bytes.items)}), + }; if (len >= String.max_inline_len) { string_builder.cap += len; } @@ -603,7 +607,10 @@ fn preprocessUpdateRequests(old: *Lockfile, updates: []PackageManager.UpdateRequ if (dep.name_hash == String.Builder.stringHash(update.name)) { if (old_resolution > old.packages.len) continue; const res = resolutions_of_yore[old_resolution]; - var buf = std.fmt.bufPrint(&temp_buf, "^{}", .{res.value.npm.fmt(old.buffers.string_bytes.items)}) catch break; + var buf = switch (exact_versions) { + false => std.fmt.bufPrint(&temp_buf, "^{}", .{res.value.npm.fmt(old.buffers.string_bytes.items)}) catch break, + true => std.fmt.bufPrint(&temp_buf, "{}", .{res.value.npm.fmt(old.buffers.string_bytes.items)}) catch break, + }; const external_version = string_builder.append(ExternalString, buf); const sliced = external_version.value.sliced(old.buffers.string_bytes.items); dep.version = Dependency.parse( @@ -622,7 +629,11 @@ fn preprocessUpdateRequests(old: *Lockfile, updates: []PackageManager.UpdateRequ } } } -pub fn clean(old: *Lockfile, updates: []PackageManager.UpdateRequest) !*Lockfile { +pub fn clean( + old: *Lockfile, + updates: []PackageManager.UpdateRequest, + exact_versions: bool, +) !*Lockfile { // This is wasteful, but we rarely log anything so it's fine. var log = logger.Log.init(bun.default_allocator); defer { @@ -632,17 +643,22 @@ pub fn clean(old: *Lockfile, updates: []PackageManager.UpdateRequest) !*Lockfile log.deinit(); } - return old.cleanWithLogger(updates, &log); + return old.cleanWithLogger(updates, &log, exact_versions); } -pub fn cleanWithLogger(old: *Lockfile, updates: []PackageManager.UpdateRequest, log: *logger.Log) !*Lockfile { +pub fn cleanWithLogger( + old: *Lockfile, + updates: []PackageManager.UpdateRequest, + log: *logger.Log, + exact_versions: bool, +) !*Lockfile { const old_trusted_dependencies = old.trusted_dependencies; const old_scripts = old.scripts; // We will only shrink the number of packages here. // never grow if (updates.len > 0) { - try old.preprocessUpdateRequests(updates); + try old.preprocessUpdateRequests(updates, exact_versions); } // Deduplication works like this diff --git a/test/cli/install/bun-add.test.ts b/test/cli/install/bun-add.test.ts index 79804f0e0..9dd38c8cd 100644 --- a/test/cli/install/bun-add.test.ts +++ b/test/cli/install/bun-add.test.ts @@ -303,6 +303,61 @@ it("should add dependency with capital letters", async () => { await access(join(package_dir, "bun.lockb")); }); +it("should add exact version", async () => { + const urls: string[] = []; + setHandler(dummyRegistry(urls)); + await writeFile( + join(package_dir, "package.json"), + JSON.stringify({ + name: "foo", + version: "0.0.1", + }), + ); + const { stdout, stderr, exited } = spawn({ + cmd: [bunExe(), "add", "--exact", "BaR"], + cwd: package_dir, + stdout: null, + stdin: "pipe", + stderr: "pipe", + env, + }); + expect(stderr).toBeDefined(); + const err = await new Response(stderr).text(); + expect(err).toContain("Saved lockfile"); + expect(stdout).toBeDefined(); + const out = await new Response(stdout).text(); + expect(out.replace(/\s*\[[0-9\.]+m?s\]\s*$/, "").split(/\r?\n/)).toEqual([ + "", + " installed BaR@0.0.2", + "", + "", + " 1 packages installed", + ]); + expect(await exited).toBe(0); + expect(urls.sort()).toEqual([`${root_url}/BaR`, `${root_url}/BaR-0.0.2.tgz`]); + expect(requested).toBe(2); + expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".cache", "BaR"]); + expect(await readdirSorted(join(package_dir, "node_modules", "BaR"))).toEqual(["package.json"]); + expect(await file(join(package_dir, "node_modules", "BaR", "package.json")).json()).toEqual({ + name: "bar", + version: "0.0.2", + }); + expect(await file(join(package_dir, "package.json")).text()).toEqual( + JSON.stringify( + { + name: "foo", + version: "0.0.1", + dependencies: { + BaR: "0.0.2", + }, + }, + null, + 2, + ), + ); + await access(join(package_dir, "bun.lockb")); +}); + it("should add dependency with specified semver", async () => { const urls: string[] = []; setHandler( -- cgit v1.2.3 From 0de5bb22af427dcad57731d7063f8678b13265e2 Mon Sep 17 00:00:00 2001 From: "Alex Lam S.L" Date: Wed, 28 Jun 2023 11:20:59 +0300 Subject: [install] workaround run-time module loading issue (#3432) --- src/bun.js/module_loader.zig | 6 --- src/install/install.zig | 40 ++++++++++++++--- src/resolver/resolver.zig | 27 ++++++----- test/cli/install/bun-run.test.ts | 96 ++++++++++++++++++++++++++++++++++++++++ test/cli/install/bunx.test.ts | 80 +-------------------------------- 5 files changed, 143 insertions(+), 106 deletions(-) create mode 100644 test/cli/install/bun-run.test.ts (limited to 'src/install/install.zig') diff --git a/src/bun.js/module_loader.zig b/src/bun.js/module_loader.zig index b25bb4b10..6fd4fef99 100644 --- a/src/bun.js/module_loader.zig +++ b/src/bun.js/module_loader.zig @@ -271,14 +271,8 @@ pub const ModuleLoader = struct { pub fn onPoll(this: *Queue) void { debug("onPoll", .{}); - var pm = this.vm().packageManager(); - this.runTasks(); - _ = pm.scheduleTasks(); - this.runTasks(); - this.pollModules(); - _ = pm.flushDependencyQueue(); } pub fn runTasks(this: *Queue) void { diff --git a/src/install/install.zig b/src/install/install.zig index 9465c4897..87f931291 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -1718,9 +1718,8 @@ pub const PackageManager = struct { } pub fn wake(this: *PackageManager) void { - if (this.onWake.context != null) { - this.onWake.getHandler()(this.onWake.context.?, this); - return; + if (this.onWake.context) |ctx| { + this.onWake.getHandler()(ctx, this); } _ = this.wait_count.fetchAdd(1, .Monotonic); @@ -1791,12 +1790,37 @@ pub const PackageManager = struct { return .{ .failure = err }; }; - const resolution_id = this.lockfile.buffers.resolutions.items[index]; + const resolution_id = switch (this.lockfile.buffers.resolutions.items[index]) { + invalid_package_id => brk: { + this.drainDependencyList(); + + switch (this.options.log_level) { + inline else => |log_level| { + if (log_level.showProgress()) this.startProgressBarIfNone(); + while (this.pending_tasks > 0) : (this.sleep()) { + this.runTasks( + void, + {}, + .{ + .onExtract = {}, + .onResolve = {}, + .onPackageManifestError = {}, + .onPackageDownloadError = {}, + }, + log_level, + ) catch |err| { + return .{ .failure = err }; + }; + } + }, + } - // check if we managed to synchronously resolve the dependency - if (resolution_id == invalid_package_id) return .{ .pending = index }; + break :brk this.lockfile.buffers.resolutions.items[index]; + }, + // we managed to synchronously resolve the dependency + else => |pkg_id| pkg_id, + }; - this.drainDependencyList(); return .{ .resolution = .{ .resolution = this.lockfile.packages.items(.resolution)[resolution_id], @@ -5310,6 +5334,8 @@ pub const PackageManager = struct { manager.progress.supports_ansi_escape_codes = Output.enable_ansi_colors_stderr; manager.root_progress_node = manager.progress.start("", 0); manager.root_download_node = manager.root_progress_node.start(ProgressStrings.download(), 0); + } else { + manager.options.log_level = .default_no_progress; } if (!manager.options.enable.cache) { diff --git a/src/resolver/resolver.zig b/src/resolver/resolver.zig index 40b106f3a..d9f4dc887 100644 --- a/src/resolver/resolver.zig +++ b/src/resolver/resolver.zig @@ -534,20 +534,19 @@ pub const Resolver = struct { dir_cache: *DirInfo.HashMap, pub fn getPackageManager(this: *Resolver) *PackageManager { - if (this.package_manager != null) { - return this.package_manager.?; - } - bun.HTTPThead.init() catch unreachable; - this.package_manager = PackageManager.initWithRuntime( - this.log, - this.opts.install, - this.allocator, - .{}, - this.env_loader.?, - ) catch @panic("Failed to initialize package manager"); - this.package_manager.?.onWake = this.onWakePackageManager; - - return this.package_manager.?; + return this.package_manager orelse brk: { + bun.HTTPThead.init() catch unreachable; + const pm = PackageManager.initWithRuntime( + this.log, + this.opts.install, + this.allocator, + .{}, + this.env_loader.?, + ) catch @panic("Failed to initialize package manager"); + pm.onWake = this.onWakePackageManager; + this.package_manager = pm; + break :brk pm; + }; } pub inline fn usePackageManager(self: *const ThisResolver) bool { diff --git a/test/cli/install/bun-run.test.ts b/test/cli/install/bun-run.test.ts new file mode 100644 index 000000000..fecbfc3d5 --- /dev/null +++ b/test/cli/install/bun-run.test.ts @@ -0,0 +1,96 @@ +import { file, spawn } from "bun"; +import { afterEach, beforeEach, expect, it } from "bun:test"; +import { bunExe, bunEnv as env } from "harness"; +import { mkdtemp, realpath, rm, writeFile } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path"; +import { readdirSorted } from "./dummy.registry"; + +let run_dir: string; + +beforeEach(async () => { + run_dir = await realpath(await mkdtemp(join(tmpdir(), "bun-run.test"))); +}); +afterEach(async () => { + await rm(run_dir, { force: true, recursive: true }); +}); + +it("should download dependency to run local file", async () => { + await writeFile( + join(run_dir, "test.js"), + ` + const { minify } = require("uglify-js@3.17.4"); + + console.log(minify("print(6 * 7)").code); + `, + ); + const { stdout, stderr, exited } = spawn({ + cmd: [bunExe(), "test.js"], + cwd: run_dir, + stdout: null, + stdin: "pipe", + stderr: "pipe", + env: { + ...env, + BUN_INSTALL_CACHE_DIR: join(run_dir, ".cache"), + }, + }); + expect(stderr).toBeDefined(); + const err = await new Response(stderr).text(); + expect(err).toBe(""); + expect(stdout).toBeDefined(); + const out = await new Response(stdout).text(); + expect(out.split(/\r?\n/)).toEqual(["print(42);", ""]); + expect(await exited).toBe(0); + expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); +}); + +it("should download dependencies to run local file", async () => { + await writeFile( + join(run_dir, "test.js"), + ` + import { file } from "bun"; + import decompress from "decompress@4.2.1"; + + const buffer = await file("${join(import.meta.dir, "baz-0.0.3.tgz")}").arrayBuffer(); + for (const entry of await decompress(Buffer.from(buffer))) { + console.log(\`\${entry.type}: \${entry.path}\`); + } + `, + ); + const { stdout, stderr, exited } = spawn({ + cmd: [bunExe(), "test.js"], + cwd: run_dir, + stdout: null, + stdin: "pipe", + stderr: "pipe", + env: { + ...env, + BUN_INSTALL_CACHE_DIR: join(run_dir, ".cache"), + }, + }); + expect(stderr).toBeDefined(); + const err = await new Response(stderr).text(); + expect(err).toBe(""); + expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); + expect(await readdirSorted(join(run_dir, ".cache"))).toContain("decompress"); + expect(await readdirSorted(join(run_dir, ".cache", "decompress"))).toEqual(["4.2.1"]); + expect(await readdirSorted(join(run_dir, ".cache", "decompress", "4.2.1"))).toEqual([ + "index.js", + "license", + "package.json", + "readme.md", + ]); + expect(await file(join(run_dir, ".cache", "decompress", "4.2.1", "index.js")).text()).toContain( + "\nmodule.exports = ", + ); + expect(stdout).toBeDefined(); + const out = await new Response(stdout).text(); + expect(out.split(/\r?\n/)).toEqual([ + "directory: package/", + "file: package/index.js", + "file: package/package.json", + "", + ]); + expect(await exited).toBe(0); +}); diff --git a/test/cli/install/bunx.test.ts b/test/cli/install/bunx.test.ts index 3605f5b6b..70d7aac29 100644 --- a/test/cli/install/bunx.test.ts +++ b/test/cli/install/bunx.test.ts @@ -1,4 +1,4 @@ -import { file, spawn } from "bun"; +import { spawn } from "bun"; import { afterEach, beforeEach, expect, it } from "bun:test"; import { bunExe, bunEnv as env } from "harness"; import { mkdtemp, realpath, rm, writeFile } from "fs/promises"; @@ -109,84 +109,6 @@ it("should work for @scoped packages", async () => { expect(await cached.exited).toBe(0); }); -it("should download dependency to run local file", async () => { - await writeFile( - join(x_dir, "test.js"), - ` -const { minify } = require("uglify-js@3.17.4"); - -console.log(minify("print(6 * 7)").code); -`, - ); - const { stdout, stderr, exited } = spawn({ - cmd: [bunExe(), "test.js"], - cwd: x_dir, - stdout: null, - stdin: "pipe", - stderr: "pipe", - env: { - ...env, - BUN_INSTALL_CACHE_DIR: join(x_dir, ".cache"), - }, - }); - expect(stderr).toBeDefined(); - const err = await new Response(stderr).text(); - expect(err).toBe(""); - expect(stdout).toBeDefined(); - const out = await new Response(stdout).text(); - expect(out.split(/\r?\n/)).toEqual(["print(42);", ""]); - expect(await exited).toBe(0); - expect(await readdirSorted(x_dir)).toEqual([".cache", "test.js"]); -}); - -it("should download dependencies to run local file", async () => { - await writeFile( - join(x_dir, "test.js"), - ` -import { file } from "bun"; -import decompress from "decompress@4.2.1"; - -const buffer = await file("${join(import.meta.dir, "baz-0.0.3.tgz")}").arrayBuffer(); -for (const entry of await decompress(Buffer.from(buffer))) { - console.log(\`\${entry.type}: \${entry.path}\`); -} -`, - ); - const { stdout, stderr, exited } = spawn({ - cmd: [bunExe(), "test.js"], - cwd: x_dir, - stdout: null, - stdin: "pipe", - stderr: "pipe", - env: { - ...env, - BUN_INSTALL_CACHE_DIR: join(x_dir, ".cache"), - }, - }); - expect(stderr).toBeDefined(); - const err = await new Response(stderr).text(); - expect(err).toBe(""); - expect(await readdirSorted(x_dir)).toEqual([".cache", "test.js"]); - expect(await readdirSorted(join(x_dir, ".cache"))).toContain("decompress"); - expect(await readdirSorted(join(x_dir, ".cache", "decompress"))).toEqual(["4.2.1"]); - expect(await readdirSorted(join(x_dir, ".cache", "decompress", "4.2.1"))).toEqual([ - "index.js", - "license", - "package.json", - "readme.md", - ]); - expect(await file(join(x_dir, ".cache", "decompress", "4.2.1", "index.js")).text()).toContain("\nmodule.exports = "); - expect(stdout).toBeDefined(); - const out = await new Response(stdout).text(); - expect(out.split(/\r?\n/)).toEqual([ - "directory: package/", - "file: package/index.js", - "file: package/package.json", - "", - ]); - expect(await exited).toBe(0); -}); - it("should execute from current working directory", async () => { await writeFile( join(x_dir, "test.js"), -- cgit v1.2.3 From 3d0ffc48cb2608ed696581e10cb082c68b56c6b9 Mon Sep 17 00:00:00 2001 From: "Alex Lam S.L" Date: Tue, 4 Jul 2023 12:09:58 +0300 Subject: [install] fix run-time module loading (#3510) - fix version buffer confusion - improve workaround to handle cached modules fixes #3507 --- src/install/install.zig | 84 +++++++++++++++--------------- src/js/out/modules/node/http.js | 2 + src/resolver/resolver.zig | 46 ++++++++--------- test/cli/install/bun-run.test.ts | 108 +++++++++++++++++++++++++++++++++------ 4 files changed, 154 insertions(+), 86 deletions(-) (limited to 'src/install/install.zig') diff --git a/src/install/install.zig b/src/install/install.zig index 87f931291..f6133afce 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -1742,55 +1742,51 @@ pub const PackageManager = struct { pub fn enqueueDependencyToRoot( this: *PackageManager, name: []const u8, - version_buf: []const u8, version: *const Dependency.Version, + version_buf: []const u8, behavior: Dependency.Behavior, ) DependencyToEnqueue { - const str_buf = this.lockfile.buffers.string_bytes.items; - for (this.lockfile.buffers.dependencies.items, 0..) |dependency, dependency_id| { - if (!strings.eqlLong(dependency.name.slice(str_buf), name, true)) continue; - if (!dependency.version.eql(version, str_buf, version_buf)) continue; - return switch (this.lockfile.buffers.resolutions.items[dependency_id]) { - invalid_package_id => .{ - .pending = @truncate(DependencyID, dependency_id), - }, - else => |resolution_id| .{ - .resolution = .{ - .resolution = this.lockfile.packages.items(.resolution)[resolution_id], - .package_id = resolution_id, - }, - }, + const dep_id = @truncate(DependencyID, brk: { + const str_buf = this.lockfile.buffers.string_bytes.items; + for (this.lockfile.buffers.dependencies.items, 0..) |dep, id| { + if (!strings.eqlLong(dep.name.slice(str_buf), name, true)) continue; + if (!dep.version.eql(version, str_buf, version_buf)) continue; + break :brk id; + } + + var builder = this.lockfile.stringBuilder(); + const dummy = Dependency{ + .name = String.init(name, name), + .name_hash = String.Builder.stringHash(name), + .version = version.*, + .behavior = behavior, }; - } + dummy.countWithDifferentBuffers(name, version_buf, @TypeOf(&builder), &builder); - var builder = this.lockfile.stringBuilder(); - const dependency = Dependency{ - .name = String.init(name, name), - .name_hash = String.Builder.stringHash(name), - .version = version.*, - .behavior = behavior, - }; - dependency.countWithDifferentBuffers(name, version_buf, @TypeOf(&builder), &builder); - - builder.allocate() catch |err| return .{ .failure = err }; - - const cloned_dependency = dependency.cloneWithDifferentBuffers(name, version_buf, @TypeOf(&builder), &builder) catch unreachable; - builder.clamp(); - const index = @truncate(DependencyID, this.lockfile.buffers.dependencies.items.len); - this.lockfile.buffers.dependencies.append(this.allocator, cloned_dependency) catch unreachable; - this.lockfile.buffers.resolutions.append(this.allocator, invalid_package_id) catch unreachable; - if (comptime Environment.allow_assert) std.debug.assert(this.lockfile.buffers.dependencies.items.len == this.lockfile.buffers.resolutions.items.len); - this.enqueueDependencyWithMainAndSuccessFn( - index, - &cloned_dependency, - invalid_package_id, - assignRootResolution, - failRootResolution, - ) catch |err| { - return .{ .failure = err }; - }; + builder.allocate() catch |err| return .{ .failure = err }; + + const dep = dummy.cloneWithDifferentBuffers(name, version_buf, @TypeOf(&builder), &builder) catch unreachable; + builder.clamp(); + const index = this.lockfile.buffers.dependencies.items.len; + this.lockfile.buffers.dependencies.append(this.allocator, dep) catch unreachable; + this.lockfile.buffers.resolutions.append(this.allocator, invalid_package_id) catch unreachable; + if (comptime Environment.allow_assert) std.debug.assert(this.lockfile.buffers.dependencies.items.len == this.lockfile.buffers.resolutions.items.len); + break :brk index; + }); + + if (this.lockfile.buffers.resolutions.items[dep_id] == invalid_package_id) { + this.enqueueDependencyWithMainAndSuccessFn( + dep_id, + &this.lockfile.buffers.dependencies.items[dep_id], + invalid_package_id, + assignRootResolution, + failRootResolution, + ) catch |err| { + return .{ .failure = err }; + }; + } - const resolution_id = switch (this.lockfile.buffers.resolutions.items[index]) { + const resolution_id = switch (this.lockfile.buffers.resolutions.items[dep_id]) { invalid_package_id => brk: { this.drainDependencyList(); @@ -1815,7 +1811,7 @@ pub const PackageManager = struct { }, } - break :brk this.lockfile.buffers.resolutions.items[index]; + break :brk this.lockfile.buffers.resolutions.items[dep_id]; }, // we managed to synchronously resolve the dependency else => |pkg_id| pkg_id, diff --git a/src/js/out/modules/node/http.js b/src/js/out/modules/node/http.js index 955c83642..f07dcc2e0 100644 --- a/src/js/out/modules/node/http.js +++ b/src/js/out/modules/node/http.js @@ -1085,6 +1085,8 @@ var tokenRegExp = /^[\^_`a-zA-Z\-0-9!#$%&'*+.|~]+$/, METHODS = [ debug(`${NODE_HTTP_WARNING}\n`, "setMaxIdleHTTPParsers() is a no-op"); }, globalAgent, + ClientRequest, + OutgoingMessage, [Symbol.for("CommonJS")]: 0 }, http_default = defaultObject; export { diff --git a/src/resolver/resolver.zig b/src/resolver/resolver.zig index e1e83ba4f..409df85af 100644 --- a/src/resolver/resolver.zig +++ b/src/resolver/resolver.zig @@ -1256,14 +1256,8 @@ pub const Resolver = struct { if (check_package) { if (r.opts.polyfill_node_globals) { - var import_path_without_node_prefix = import_path; - const had_node_prefix = import_path_without_node_prefix.len > "node:".len and - strings.eqlComptime(import_path_without_node_prefix[0.."node:".len], "node:"); - - import_path_without_node_prefix = if (had_node_prefix) - import_path_without_node_prefix["node:".len..] - else - import_path_without_node_prefix; + const had_node_prefix = strings.hasPrefixComptime(import_path, "node:"); + const import_path_without_node_prefix = if (had_node_prefix) import_path["node:".len..] else import_path; if (NodeFallbackModules.Map.get(import_path_without_node_prefix)) |*fallback_module| { result.path_pair.primary = fallback_module.path; @@ -1278,7 +1272,7 @@ pub const Resolver = struct { } else if (had_node_prefix or (strings.hasPrefixComptime(import_path_without_node_prefix, "fs") and (import_path_without_node_prefix.len == 2 or - import_path_without_node_prefix[3] == '/'))) + import_path_without_node_prefix[2] == '/'))) { result.path_pair.primary.namespace = "node"; result.path_pair.primary.text = import_path_without_node_prefix; @@ -1698,8 +1692,9 @@ pub const Resolver = struct { // If the source directory doesn't have a node_modules directory, we can // check the global cache directory for a package.json file. var manager = r.getPackageManager(); - var dependency_version: Dependency.Version = .{}; + var dependency_version = Dependency.Version{}; var dependency_behavior = @enumFromInt(Dependency.Behavior, Dependency.Behavior.normal); + var string_buf = esm.version; // const initial_pending_tasks = manager.pending_tasks; var resolved_package_id: Install.PackageID = brk: { @@ -1707,7 +1702,6 @@ pub const Resolver = struct { // and try to look up the dependency from there if (dir_info.package_json_for_dependencies) |package_json| { var dependencies_list: []const Dependency = &[_]Dependency{}; - var string_buf: []const u8 = ""; const resolve_from_lockfile = package_json.package_manager_package_id != Install.invalid_package_id; if (resolve_from_lockfile) { @@ -1723,24 +1717,21 @@ pub const Resolver = struct { } for (dependencies_list, 0..) |dependency, dependency_id| { - const dep_name = dependency.name.slice(string_buf); - if (dep_name.len == esm.name.len) { - if (!strings.eqlLong(dep_name, esm.name, false)) { - continue; - } + if (!strings.eqlLong(dependency.name.slice(string_buf), esm.name, true)) { + continue; + } - dependency_version = dependency.version; - dependency_behavior = dependency.behavior; + dependency_version = dependency.version; + dependency_behavior = dependency.behavior; - if (resolve_from_lockfile) { - const resolutions = &manager.lockfile.packages.items(.resolutions)[package_json.package_manager_package_id]; + if (resolve_from_lockfile) { + const resolutions = &manager.lockfile.packages.items(.resolutions)[package_json.package_manager_package_id]; - // found it! - break :brk resolutions.get(manager.lockfile.buffers.resolutions.items)[dependency_id]; - } - - break; + // found it! + break :brk resolutions.get(manager.lockfile.buffers.resolutions.items)[dependency_id]; } + + break; } } @@ -1770,6 +1761,7 @@ pub const Resolver = struct { if (esm_.?.version.len > 0 and dir_info.enclosing_package_json != null and global_cache.allowVersionSpecifier()) { return .{ .failure = error.VersionSpecifierNotAllowedHere }; } + string_buf = esm.version; dependency_version = Dependency.parse( r.allocator, Semver.String.init(esm.name, esm.name), @@ -1795,6 +1787,7 @@ pub const Resolver = struct { dependency_behavior, &resolved_package_id, dependency_version, + string_buf, )) { .resolution => |res| break :brk res, .pending => |pending| return .{ .pending = pending }, @@ -2073,6 +2066,7 @@ pub const Resolver = struct { behavior: Dependency.Behavior, input_package_id_: *Install.PackageID, version: Dependency.Version, + version_buf: []const u8, ) DependencyToResolve { if (r.debug_logs) |*debug| { debug.addNoteFmt("Enqueueing pending dependency \"{s}@{s}\"", .{ esm.name, esm.version }); @@ -2135,7 +2129,7 @@ pub const Resolver = struct { // All packages are enqueued to the root // because we download all the npm package dependencies - switch (pm.enqueueDependencyToRoot(esm.name, esm.version, &version, behavior)) { + switch (pm.enqueueDependencyToRoot(esm.name, &version, version_buf, behavior)) { .resolution => |result| { input_package_id_.* = result.package_id; return .{ .resolution = result.resolution }; diff --git a/test/cli/install/bun-run.test.ts b/test/cli/install/bun-run.test.ts index 9ab094f08..95f33ebb8 100644 --- a/test/cli/install/bun-run.test.ts +++ b/test/cli/install/bun-run.test.ts @@ -24,7 +24,37 @@ const { minify } = require("uglify-js@3.17.4"); console.log(minify("print(6 * 7)").code); `, ); - const { stdout, stderr, exited } = spawn({ + const { + stdout: stdout1, + stderr: stderr1, + exited: exited1, + } = spawn({ + cmd: [bunExe(), "run", "test.js"], + cwd: run_dir, + stdout: null, + stdin: "pipe", + stderr: "pipe", + env: { + ...env, + BUN_INSTALL_CACHE_DIR: join(run_dir, ".cache"), + }, + }); + expect(stderr1).toBeDefined(); + const err1 = await new Response(stderr1).text(); + expect(err1).toBe(""); + expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); + expect(await readdirSorted(join(run_dir, ".cache"))).toContain("uglify-js"); + expect(await readdirSorted(join(run_dir, ".cache", "uglify-js"))).toEqual(["3.17.4"]); + expect(stdout1).toBeDefined(); + const out1 = await new Response(stdout1).text(); + expect(out1.split(/\r?\n/)).toEqual(["print(42);", ""]); + expect(await exited1).toBe(0); + // Perform `bun test.js` with cached dependencies + const { + stdout: stdout2, + stderr: stderr2, + exited: exited2, + } = spawn({ cmd: [bunExe(), "test.js"], cwd: run_dir, stdout: null, @@ -35,14 +65,16 @@ console.log(minify("print(6 * 7)").code); BUN_INSTALL_CACHE_DIR: join(run_dir, ".cache"), }, }); - expect(stderr).toBeDefined(); - const err = await new Response(stderr).text(); - expect(err).toBe(""); - expect(stdout).toBeDefined(); - const out = await new Response(stdout).text(); - expect(out.split(/\r?\n/)).toEqual(["print(42);", ""]); - expect(await exited).toBe(0); + expect(stderr2).toBeDefined(); + const err2 = await new Response(stderr2).text(); + expect(err2).toBe(""); expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); + expect(await readdirSorted(join(run_dir, ".cache"))).toContain("uglify-js"); + expect(await readdirSorted(join(run_dir, ".cache", "uglify-js"))).toEqual(["3.17.4"]); + expect(stdout2).toBeDefined(); + const out2 = await new Response(stdout2).text(); + expect(out2.split(/\r?\n/)).toEqual(["print(42);", ""]); + expect(await exited2).toBe(0); }); it("should download dependencies to run local file", async () => { @@ -58,7 +90,11 @@ for (const entry of await decompress(Buffer.from(buffer))) { } `, ); - const { stdout, stderr, exited } = spawn({ + const { + stdout: stdout1, + stderr: stderr1, + exited: exited1, + } = spawn({ cmd: [bunExe(), "test.js"], cwd: run_dir, stdout: null, @@ -69,9 +105,49 @@ for (const entry of await decompress(Buffer.from(buffer))) { BUN_INSTALL_CACHE_DIR: join(run_dir, ".cache"), }, }); - expect(stderr).toBeDefined(); - const err = await new Response(stderr).text(); - expect(err).toBe(""); + expect(stderr1).toBeDefined(); + const err1 = await new Response(stderr1).text(); + expect(err1).toBe(""); + expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); + expect(await readdirSorted(join(run_dir, ".cache"))).toContain("decompress"); + expect(await readdirSorted(join(run_dir, ".cache", "decompress"))).toEqual(["4.2.1"]); + expect(await readdirSorted(join(run_dir, ".cache", "decompress", "4.2.1"))).toEqual([ + "index.js", + "license", + "package.json", + "readme.md", + ]); + expect(await file(join(run_dir, ".cache", "decompress", "4.2.1", "index.js")).text()).toContain( + "\nmodule.exports = ", + ); + expect(stdout1).toBeDefined(); + const out1 = await new Response(stdout1).text(); + expect(out1.split(/\r?\n/)).toEqual([ + "directory: package/", + "file: package/index.js", + "file: package/package.json", + "", + ]); + expect(await exited1).toBe(0); + // Perform `bun run test.js` with cached dependencies + const { + stdout: stdout2, + stderr: stderr2, + exited: exited2, + } = spawn({ + cmd: [bunExe(), "run", "test.js"], + cwd: run_dir, + stdout: null, + stdin: "pipe", + stderr: "pipe", + env: { + ...env, + BUN_INSTALL_CACHE_DIR: join(run_dir, ".cache"), + }, + }); + expect(stderr2).toBeDefined(); + const err2 = await new Response(stderr2).text(); + expect(err2).toBe(""); expect(await readdirSorted(run_dir)).toEqual([".cache", "test.js"]); expect(await readdirSorted(join(run_dir, ".cache"))).toContain("decompress"); expect(await readdirSorted(join(run_dir, ".cache", "decompress"))).toEqual(["4.2.1"]); @@ -84,13 +160,13 @@ for (const entry of await decompress(Buffer.from(buffer))) { expect(await file(join(run_dir, ".cache", "decompress", "4.2.1", "index.js")).text()).toContain( "\nmodule.exports = ", ); - expect(stdout).toBeDefined(); - const out = await new Response(stdout).text(); - expect(out.split(/\r?\n/)).toEqual([ + expect(stdout2).toBeDefined(); + const out2 = await new Response(stdout2).text(); + expect(out2.split(/\r?\n/)).toEqual([ "directory: package/", "file: package/index.js", "file: package/package.json", "", ]); - expect(await exited).toBe(0); + expect(await exited2).toBe(0); }); -- cgit v1.2.3 From a1fb289c96ca41fc3fc50ccb76f6e63e70b6a772 Mon Sep 17 00:00:00 2001 From: dave caruso Date: Tue, 11 Jul 2023 18:50:34 -0700 Subject: typo (#3610) --- src/install/install.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/install/install.zig') diff --git a/src/install/install.zig b/src/install/install.zig index f6133afce..65f4a35e9 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -5554,7 +5554,7 @@ pub const PackageManager = struct { // Done if (manager.options.log_level != .silent) Output.prettyln( - \\Success! Registered \"{[name]s}\" + \\Success! Registered "{[name]s}" \\ \\To use {[name]s} in a project, run: \\ bun link {[name]s} -- cgit v1.2.3