diff options
author | 2023-10-11 02:27:07 -0700 | |
---|---|---|
committer | 2023-10-11 02:27:07 -0700 | |
commit | 1bf28e0d77a8b2261befbdb708cefd03e0126960 (patch) | |
tree | 1d1120922f2fd935be47ab4255ac57455a0bede8 /src | |
parent | 6a17ebe6696ebdf5c20de9de3281d308959c13b5 (diff) | |
download | bun-1bf28e0d77a8b2261befbdb708cefd03e0126960.tar.gz bun-1bf28e0d77a8b2261befbdb708cefd03e0126960.tar.zst bun-1bf28e0d77a8b2261befbdb708cefd03e0126960.zip |
feat(install): automatically migrate package-lock.json to bun.lockb (#6352)bun-v1.0.5
* work so far
* stuff
* a
* basics work
* stuff
* yoo
* build lockfile
* correct
* f
* a
* install fixture havent tested
* i made it worse
* lol
* be more reasonable
* make the test easier to pass because bun install doesn't handle obscure lockfile edge cases :/
* a
* works now
* ok
* a
* a
* cool
* nah
* fix stuff
* l
* a
* idfk
* LAME
* prettier errors
* does this fix tests?
* Add more safety checks to Integrity
* Add another check
* More careful lifetime handling
* Fix linux debugger issue
* a
* tmp dir and snapshot test
---------
Co-authored-by: Jarred SUmner <jarred@jarredsumner.com>
Diffstat (limited to 'src')
-rw-r--r-- | src/bun.js/bindings/JSCUSocketsLoopIntegration.cpp | 20 | ||||
-rw-r--r-- | src/cli/package_manager_command.zig | 27 | ||||
-rw-r--r-- | src/install/dependency.zig | 158 | ||||
-rw-r--r-- | src/install/install.zig | 231 | ||||
-rw-r--r-- | src/install/integrity.zig | 48 | ||||
-rw-r--r-- | src/install/lockfile.zig | 462 | ||||
-rw-r--r-- | src/install/migration.zig | 947 | ||||
-rw-r--r-- | src/install/npm.zig | 14 | ||||
-rw-r--r-- | src/install/resolution.zig | 47 | ||||
-rw-r--r-- | src/resolver/resolver.zig | 2 |
10 files changed, 1710 insertions, 246 deletions
diff --git a/src/bun.js/bindings/JSCUSocketsLoopIntegration.cpp b/src/bun.js/bindings/JSCUSocketsLoopIntegration.cpp index f607d586a..4db97d116 100644 --- a/src/bun.js/bindings/JSCUSocketsLoopIntegration.cpp +++ b/src/bun.js/bindings/JSCUSocketsLoopIntegration.cpp @@ -1,12 +1,30 @@ #include "root.h" #include "JavaScriptCore/VM.h" +// On Linux, signals are used to suspend/resume threads in JavaScriptCore +// When `.acquireAccess` is called, the signal might be raised. +// This causes issues with LLDB which might catch the signal. +// So we want to avoid that, we really only want this code to be executed when the debugger is attached +// But it's pretty hard to tell if LLDB is attached or not, so we just disable this code on Linux when in debug mode +#ifndef ACQUIRE_RELEASE_HEAP_ACCESS +#if OS(DARWIN) +#define ACQUIRE_RELEASE_HEAP_ACCESS 1 +#else +#ifndef BUN_DEBUG +#define ACQUIRE_RELEASE_HEAP_ACCESS 1 +#endif +#endif +#endif + extern "C" void bun_on_tick_before(JSC::VM* vm) { - // Let the GC do some work while we are idle +#if ACQUIRE_RELEASE_HEAP_ACCESS vm->heap.releaseAccess(); +#endif } extern "C" void bun_on_tick_after(JSC::VM* vm) { +#if ACQUIRE_RELEASE_HEAP_ACCESS vm->heap.acquireAccess(); +#endif }
\ No newline at end of file diff --git a/src/cli/package_manager_command.zig b/src/cli/package_manager_command.zig index aef7c28a9..00326bc97 100644 --- a/src/cli/package_manager_command.zig +++ b/src/cli/package_manager_command.zig @@ -244,6 +244,32 @@ pub const PackageManagerCommand = struct { } Global.exit(0); + } else if (strings.eqlComptime(subcommand, "migrate")) { + if (!pm.options.enable.force_save_lockfile) try_load_bun: { + std.fs.cwd().accessZ("bun.lockb", .{ .mode = .read_only }) catch break :try_load_bun; + + Output.prettyErrorln( + \\<r><red>error<r>: bun.lockb already exists + \\run with --force to overwrite + , .{}); + Global.exit(1); + } + const load_lockfile = @import("../install/migration.zig").detectAndLoadOtherLockfile( + pm.lockfile, + ctx.allocator, + pm.log, + pm.options.lockfile_path, + ); + if (load_lockfile == .not_found) { + Output.prettyErrorln( + \\<r><red>error<r>: could not find any other lockfile + , .{}); + Global.exit(1); + } + handleLoadLockfileErrors(load_lockfile, pm); + const lockfile = load_lockfile.ok; + lockfile.saveToDisk(pm.options.lockfile_path); + Global.exit(0); } Output.prettyln( @@ -258,6 +284,7 @@ pub const PackageManagerCommand = struct { \\ bun pm <b>hash-print<r> print the hash stored in the current lockfile \\ bun pm <b>cache<r> print the path to the cache folder \\ bun pm <b>cache rm<r> clear the cache + \\ bun pm <b>migrate<r> migrate another package manager's lockfile without installing anything \\ \\Learn more about these at <magenta>https://bun.sh/docs/install/utilities<r> \\ diff --git a/src/install/dependency.zig b/src/install/dependency.zig index b621dfae9..ca0d702aa 100644 --- a/src/install/dependency.zig +++ b/src/install/dependency.zig @@ -49,7 +49,7 @@ version: Dependency.Version = .{}, /// - `peerDependencies` /// Technically, having the same package name specified under multiple fields is invalid /// But we don't want to allocate extra arrays for them. So we use a bitfield instead. -behavior: Behavior = .uninitialized, +behavior: Behavior = Behavior.uninitialized, /// Sorting order for dependencies is: /// 1. [ `peerDependencies`, `optionalDependencies`, `devDependencies`, `dependencies` ] @@ -147,7 +147,7 @@ pub fn toDependency( return Dependency{ .name = name, .name_hash = @as(u64, @bitCast(this[8..16].*)), - .behavior = @as(Dependency.Behavior, @enumFromInt(this[16])), + .behavior = @bitCast(this[16]), .version = Dependency.Version.toVersion(name, this[17..this.len].*, ctx), }; } @@ -156,7 +156,7 @@ pub fn toExternal(this: Dependency) External { var bytes: External = undefined; bytes[0..this.name.bytes.len].* = this.name.bytes; bytes[8..16].* = @as([8]u8, @bitCast(this.name_hash)); - bytes[16] = @intFromEnum(this.behavior); + bytes[16] = @bitCast(this.behavior); bytes[17..bytes.len].* = this.version.toExternal(); return bytes; } @@ -221,12 +221,16 @@ pub inline fn isGitHubRepoPath(dependency: string) bool { return hash_index != dependency.len - 1 and first_slash_index > 0 and first_slash_index != dependency.len - 1; } -// Github allows for the following format of URL: -// https://github.com/<org>/<repo>/tarball/<ref> -// This is a legacy (but still supported) method of retrieving a tarball of an -// entire source tree at some git reference. (ref = branch, tag, etc. Note: branch -// can have arbitrary number of slashes) +/// Github allows for the following format of URL: +/// https://github.com/<org>/<repo>/tarball/<ref> +/// This is a legacy (but still supported) method of retrieving a tarball of an +/// entire source tree at some git reference. (ref = branch, tag, etc. Note: branch +/// can have arbitrary number of slashes) +/// +/// This also checks for a github url that ends with ".tar.gz" pub inline fn isGitHubTarballPath(dependency: string) bool { + if (isTarball(dependency)) return true; + var parts = strings.split(dependency, "/"); var n_parts: usize = 0; @@ -248,7 +252,7 @@ pub inline fn isTarball(dependency: string) bool { } pub const Version = struct { - tag: Dependency.Version.Tag = .uninitialized, + tag: Tag = .uninitialized, literal: String = .{}, value: Value = .{ .uninitialized = {} }, @@ -610,7 +614,7 @@ pub const Version = struct { } }; - const NpmInfo = struct { + pub const NpmInfo = struct { name: String, version: Semver.Query.Group, @@ -619,7 +623,7 @@ pub const Version = struct { } }; - const TagInfo = struct { + pub const TagInfo = struct { name: String, tag: String, @@ -628,7 +632,7 @@ pub const Version = struct { } }; - const TarballInfo = struct { + pub const TarballInfo = struct { uri: URI, package_name: String = .{}, @@ -670,7 +674,8 @@ pub inline fn parse( sliced: *const SlicedString, log: ?*logger.Log, ) ?Version { - return parseWithOptionalTag(allocator, alias, dependency, null, sliced, log); + const dep = std.mem.trimLeft(u8, dependency, " \t\n\r"); + return parseWithTag(allocator, alias, dep, Version.Tag.infer(dep), sliced, log); } pub fn parseWithOptionalTag( @@ -888,6 +893,12 @@ pub fn parseWithTag( .literal = sliced.value(), .value = .{ .tarball = .{ .uri = .{ .local = sliced.sub(dependency[7..]).value() } } }, }; + } else if (strings.hasPrefixComptime(dependency, "file:")) { + return .{ + .tag = .tarball, + .literal = sliced.value(), + .value = .{ .tarball = .{ .uri = .{ .local = sliced.sub(dependency[5..]).value() } } }, + }; } else if (strings.contains(dependency, "://")) { if (log_) |log| log.addErrorFmt(null, logger.Loc.Empty, allocator, "invalid or unsupported dependency \"{s}\"", .{dependency}) catch unreachable; return null; @@ -950,78 +961,83 @@ pub fn parseWithTag( } } -pub const Behavior = enum(u8) { - uninitialized = 0, - _, +pub const Behavior = packed struct(u8) { + pub const uninitialized: Behavior = .{}; + + // these padding fields are to have compatibility + // with older versions of lockfile v2 + _unused_1: u1 = 0, + + normal: bool = false, + optional: bool = false, + dev: bool = false, + peer: bool = false, + workspace: bool = false, + + _unused_2: u2 = 0, - pub const normal: u8 = 1 << 1; - pub const optional: u8 = 1 << 2; - pub const dev: u8 = 1 << 3; - pub const peer: u8 = 1 << 4; - pub const workspace: u8 = 1 << 5; + pub const normal = Behavior{ .normal = true }; + pub const optional = Behavior{ .optional = true }; + pub const dev = Behavior{ .dev = true }; + pub const peer = Behavior{ .peer = true }; + pub const workspace = Behavior{ .workspace = true }; pub inline fn isNormal(this: Behavior) bool { - return (@intFromEnum(this) & Behavior.normal) != 0; + return this.normal; } pub inline fn isOptional(this: Behavior) bool { - return (@intFromEnum(this) & Behavior.optional) != 0 and !this.isPeer(); + return this.optional and !this.isPeer(); } pub inline fn isDev(this: Behavior) bool { - return (@intFromEnum(this) & Behavior.dev) != 0; + return this.dev; } pub inline fn isPeer(this: Behavior) bool { - return (@intFromEnum(this) & Behavior.peer) != 0; + return this.peer; } pub inline fn isWorkspace(this: Behavior) bool { - return (@intFromEnum(this) & Behavior.workspace) != 0; + return this.workspace; } pub inline fn setNormal(this: Behavior, value: bool) Behavior { - if (value) { - return @as(Behavior, @enumFromInt(@intFromEnum(this) | Behavior.normal)); - } else { - return @as(Behavior, @enumFromInt(@intFromEnum(this) & ~Behavior.normal)); - } + var b = this; + b.normal = value; + return b; } pub inline fn setOptional(this: Behavior, value: bool) Behavior { - if (value) { - return @as(Behavior, @enumFromInt(@intFromEnum(this) | Behavior.optional)); - } else { - return @as(Behavior, @enumFromInt(@intFromEnum(this) & ~Behavior.optional)); - } + var b = this; + b.optional = value; + return b; } pub inline fn setDev(this: Behavior, value: bool) Behavior { - if (value) { - return @as(Behavior, @enumFromInt(@intFromEnum(this) | Behavior.dev)); - } else { - return @as(Behavior, @enumFromInt(@intFromEnum(this) & ~Behavior.dev)); - } + var b = this; + b.dev = value; + return b; } pub inline fn setPeer(this: Behavior, value: bool) Behavior { - if (value) { - return @as(Behavior, @enumFromInt(@intFromEnum(this) | Behavior.peer)); - } else { - return @as(Behavior, @enumFromInt(@intFromEnum(this) & ~Behavior.peer)); - } + var b = this; + b.peer = value; + return b; } pub inline fn setWorkspace(this: Behavior, value: bool) Behavior { - if (value) { - return @as(Behavior, @enumFromInt(@intFromEnum(this) | Behavior.workspace)); - } else { - return @as(Behavior, @enumFromInt(@intFromEnum(this) & ~Behavior.workspace)); - } + var b = this; + b.workspace = value; + return b; + } + + pub inline fn eq(lhs: Behavior, rhs: Behavior) bool { + return @as(u8, @bitCast(lhs)) == @as(u8, @bitCast(rhs)); } pub inline fn cmp(lhs: Behavior, rhs: Behavior) std.math.Order { - if (@intFromEnum(lhs) == @intFromEnum(rhs)) { + if (eq(lhs, rhs)) { return .eq; } @@ -1074,4 +1090,42 @@ pub const Behavior = enum(u8) { (features.peer_dependencies and this.isPeer()) or this.isWorkspace(); } + + pub fn format(self: Behavior, comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void { + const fields = std.meta.fields(Behavior); + var num_fields: u8 = 0; + inline for (fields) |f| { + if (f.type == bool and @field(self, f.name)) { + num_fields += 1; + } + } + switch (num_fields) { + 0 => try writer.writeAll("Behavior.uninitialized"), + 1 => { + inline for (fields) |f| { + if (f.type == bool and @field(self, f.name)) { + try writer.writeAll("Behavior." ++ f.name); + break; + } + } + }, + else => { + try writer.writeAll("Behavior{"); + inline for (fields) |f| { + if (f.type == bool and @field(self, f.name)) { + try writer.writeAll(" " ++ f.name); + } + } + try writer.writeAll(" }"); + }, + } + } + + comptime { + std.debug.assert(@as(u8, @bitCast(Behavior.normal)) == (1 << 1)); + std.debug.assert(@as(u8, @bitCast(Behavior.optional)) == (1 << 2)); + std.debug.assert(@as(u8, @bitCast(Behavior.dev)) == (1 << 3)); + std.debug.assert(@as(u8, @bitCast(Behavior.peer)) == (1 << 4)); + std.debug.assert(@as(u8, @bitCast(Behavior.workspace)) == (1 << 5)); + } }; diff --git a/src/install/install.zig b/src/install/install.zig index 218dace5c..378401b8a 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -123,8 +123,8 @@ pub fn ExternalSlice(comptime Type: type) type { pub fn ExternalSliceAligned(comptime Type: type, comptime alignment_: ?u29) type { return extern struct { - const alignment = alignment_ orelse @alignOf(*Type); - const Slice = @This(); + pub const alignment = alignment_ orelse @alignOf(*Type); + pub const Slice = @This(); pub const Child: type = Type; @@ -170,7 +170,7 @@ pub const ExternalStringMap = extern struct { value: ExternalStringList = .{}, }; -pub const PackageNameHash = u64; +pub const PackageNameHash = u64; // Use String.Builder.stringHash to compute this pub const Aligner = struct { pub fn write(comptime Type: type, comptime Writer: type, writer: Writer, pos: usize) !usize { @@ -2618,7 +2618,7 @@ pub const PackageManager = struct { if (comptime Environment.allow_assert) { std.debug.assert(dependency_id < buffers.resolutions.items.len); std.debug.assert(package_id < this.lockfile.packages.len); - std.debug.assert(buffers.resolutions.items[dependency_id] == invalid_package_id); + // std.debug.assert(buffers.resolutions.items[dependency_id] == invalid_package_id); } buffers.resolutions.items[dependency_id] = package_id; const string_buf = buffers.string_bytes.items; @@ -4430,6 +4430,9 @@ pub const PackageManager = struct { manager.setPreinstallState(package_id, manager.lockfile, .done); if (comptime @TypeOf(callbacks.onExtract) != void) { + if (ExtractCompletionContext == *PackageInstaller) { + extract_ctx.fixCachedLockfilePackageSlices(); + } callbacks.onExtract(extract_ctx, dependency_id, task.data.extract, comptime log_level); } @@ -6844,6 +6847,7 @@ pub const PackageManager = struct { folder_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined, install_count: usize = 0, successfully_installed: Bitset, + tree_iterator: *Lockfile.Tree.Iterator, // For linking native binaries, we only want to link after we've installed the companion dependencies // We don't want to introduce dependent callbacks like that for every single package @@ -6855,6 +6859,16 @@ pub const PackageManager = struct { node_modules_folder: std.fs.IterableDir, }; + /// Call when you mutate the length of `lockfile.packages` + pub fn fixCachedLockfilePackageSlices(this: *PackageInstaller) void { + var packages = this.lockfile.packages.slice(); + this.metas = packages.items(.meta); + this.names = packages.items(.name); + this.bins = packages.items(.bin); + this.resolutions = packages.items(.resolution); + this.tree_iterator.reload(this.lockfile); + } + /// Install versions of a package which are waiting on a network request pub fn installEnqueuedPackages( this: *PackageInstaller, @@ -7463,38 +7477,38 @@ pub const PackageManager = struct { var summary = PackageInstall.Summary{}; { - var parts = lockfile.packages.slice(); - var metas = parts.items(.meta); - var names = parts.items(.name); - var dependencies = lockfile.buffers.dependencies.items; - const resolutions_buffer: []const PackageID = lockfile.buffers.resolutions.items; - const resolution_lists: []const Lockfile.PackageIDSlice = parts.items(.resolutions); - var resolutions = parts.items(.resolution); - var iterator = Lockfile.Tree.Iterator.init(lockfile); - var installer = PackageInstaller{ - .manager = this, - .options = &this.options, - .metas = metas, - .bins = parts.items(.bin), - .root_node_modules_folder = node_modules_folder, - .names = names, - .resolutions = resolutions, - .lockfile = lockfile, - .node = &install_node, - .node_modules_folder = node_modules_folder, - .progress = progress, - .skip_verify_installed_version_number = skip_verify_installed_version_number, - .skip_delete = skip_delete, - .summary = &summary, - .global_bin_dir = this.options.global_bin_dir, - .force_install = force_install, - .install_count = lockfile.buffers.hoisted_dependencies.items.len, - .successfully_installed = try Bitset.initEmpty( - this.allocator, - lockfile.packages.len, - ), + var installer: PackageInstaller = brk: { + // These slices potentially get resized during iteration + // so we want to make sure they're not accessible to the rest of this function + // to make mistakes harder + var parts = lockfile.packages.slice(); + + break :brk PackageInstaller{ + .manager = this, + .options = &this.options, + .metas = parts.items(.meta), + .bins = parts.items(.bin), + .root_node_modules_folder = node_modules_folder, + .names = parts.items(.name), + .resolutions = parts.items(.resolution), + .lockfile = lockfile, + .node = &install_node, + .node_modules_folder = node_modules_folder, + .progress = progress, + .skip_verify_installed_version_number = skip_verify_installed_version_number, + .skip_delete = skip_delete, + .summary = &summary, + .global_bin_dir = this.options.global_bin_dir, + .force_install = force_install, + .install_count = lockfile.buffers.hoisted_dependencies.items.len, + .successfully_installed = try Bitset.initEmpty( + this.allocator, + lockfile.packages.len, + ), + .tree_iterator = &iterator, + }; }; while (iterator.nextNodeModulesFolder()) |node_modules| { @@ -7587,87 +7601,95 @@ pub const PackageManager = struct { if (!installer.options.do.install_packages) return error.InstallFailed; summary.successfully_installed = installer.successfully_installed; - outer: for (installer.platform_binlinks.items) |deferred| { - const dependency_id = deferred.dependency_id; - const package_id = resolutions_buffer[dependency_id]; - const folder = deferred.node_modules_folder; - - const package_resolutions: []const PackageID = resolution_lists[package_id].get(resolutions_buffer); - const original_bin: Bin = installer.bins[package_id]; - - for (package_resolutions) |resolved_id| { - if (resolved_id >= names.len) continue; - const meta: Lockfile.Package.Meta = metas[resolved_id]; - - // This is specifically for platform-specific binaries - if (meta.os == .all and meta.arch == .all) continue; - - // Don't attempt to link incompatible binaries - if (meta.isDisabled()) continue; - - const name = lockfile.str(&dependencies[dependency_id].name); - - if (!installer.has_created_bin) { - if (!this.options.global) { - if (comptime Environment.isWindows) { - std.os.mkdiratW(node_modules_folder.dir.fd, bun.strings.w(".bin"), 0) catch {}; - } else { - node_modules_folder.dir.makeDirZ(".bin") catch {}; + { + var parts = lockfile.packages.slice(); + var metas = parts.items(.meta); + var names = parts.items(.name); + var dependencies = lockfile.buffers.dependencies.items; + const resolutions_buffer: []const PackageID = lockfile.buffers.resolutions.items; + const resolution_lists: []const Lockfile.PackageIDSlice = parts.items(.resolutions); + outer: for (installer.platform_binlinks.items) |deferred| { + const dependency_id = deferred.dependency_id; + const package_id = resolutions_buffer[dependency_id]; + const folder = deferred.node_modules_folder; + + const package_resolutions: []const PackageID = resolution_lists[package_id].get(resolutions_buffer); + const original_bin: Bin = installer.bins[package_id]; + + for (package_resolutions) |resolved_id| { + if (resolved_id >= names.len) continue; + const meta: Lockfile.Package.Meta = metas[resolved_id]; + + // This is specifically for platform-specific binaries + if (meta.os == .all and meta.arch == .all) continue; + + // Don't attempt to link incompatible binaries + if (meta.isDisabled()) continue; + + const name = lockfile.str(&dependencies[dependency_id].name); + + if (!installer.has_created_bin) { + if (!this.options.global) { + if (comptime Environment.isWindows) { + std.os.mkdiratW(node_modules_folder.dir.fd, bun.strings.w(".bin"), 0) catch {}; + } else { + node_modules_folder.dir.makeDirZ(".bin") catch {}; + } } + if (comptime Environment.isPosix) + Bin.Linker.umask = C.umask(0); + installer.has_created_bin = true; } - if (comptime Environment.isPosix) - Bin.Linker.umask = C.umask(0); - installer.has_created_bin = true; - } - var bin_linker = Bin.Linker{ - .bin = original_bin, - .package_installed_node_modules = bun.toFD(folder.dir.fd), - .root_node_modules_folder = bun.toFD(node_modules_folder.dir.fd), - .global_bin_path = this.options.bin_path, - .global_bin_dir = this.options.global_bin_dir.dir, + var bin_linker = Bin.Linker{ + .bin = original_bin, + .package_installed_node_modules = bun.toFD(folder.dir.fd), + .root_node_modules_folder = bun.toFD(node_modules_folder.dir.fd), + .global_bin_path = this.options.bin_path, + .global_bin_dir = this.options.global_bin_dir.dir, - .package_name = strings.StringOrTinyString.init(name), - .string_buf = lockfile.buffers.string_bytes.items, - .extern_string_buf = lockfile.buffers.extern_strings.items, - }; + .package_name = strings.StringOrTinyString.init(name), + .string_buf = lockfile.buffers.string_bytes.items, + .extern_string_buf = lockfile.buffers.extern_strings.items, + }; - bin_linker.link(this.options.global); + bin_linker.link(this.options.global); - if (bin_linker.err) |err| { - if (comptime log_level != .silent) { - const fmt = "\n<r><red>error:<r> linking <b>{s}<r>: {s}\n"; - const args = .{ name, @errorName(err) }; + if (bin_linker.err) |err| { + if (comptime log_level != .silent) { + const fmt = "\n<r><red>error:<r> linking <b>{s}<r>: {s}\n"; + const args = .{ name, @errorName(err) }; - if (comptime log_level.showProgress()) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); - }, + if (comptime log_level.showProgress()) { + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); + }, + } + } else { + Output.prettyErrorln(fmt, args); } - } else { - Output.prettyErrorln(fmt, args); } + + if (this.options.enable.fail_early) Global.crash(); } - if (this.options.enable.fail_early) Global.crash(); + continue :outer; } - continue :outer; - } - - if (comptime log_level != .silent) { - const fmt = "\n<r><yellow>warn:<r> no compatible binaries found for <b>{s}<r>\n"; - const args = .{lockfile.str(&names[package_id])}; + if (comptime log_level != .silent) { + const fmt = "\n<r><yellow>warn:<r> no compatible binaries found for <b>{s}<r>\n"; + const args = .{lockfile.str(&names[package_id])}; - if (comptime log_level.showProgress()) { - switch (Output.enable_ansi_colors) { - inline else => |enable_ansi_colors| { - this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); - }, + if (comptime log_level.showProgress()) { + switch (Output.enable_ansi_colors) { + inline else => |enable_ansi_colors| { + this.progress.log(comptime Output.prettyFmt(fmt, enable_ansi_colors), args); + }, + } + } else { + Output.prettyErrorln(fmt, args); } - } else { - Output.prettyErrorln(fmt, args); } } } @@ -7726,15 +7748,17 @@ pub const PackageManager = struct { ) else .{ .not_found = {} }; + var root = Lockfile.Package{}; - var needs_new_lockfile = load_lockfile_result != .ok or (load_lockfile_result.ok.buffers.dependencies.items.len == 0 and manager.package_json_updates.len > 0); + var needs_new_lockfile = load_lockfile_result != .ok or + (load_lockfile_result.ok.buffers.dependencies.items.len == 0 and manager.package_json_updates.len > 0); + // this defaults to false // but we force allowing updates to the lockfile when you do bun add var had_any_diffs = false; manager.progress = .{}; // Step 2. Parse the package.json file - // var package_json_source = logger.Source.initPathString(package_json_cwd, package_json_contents); switch (load_lockfile_result) { @@ -7750,6 +7774,9 @@ pub const PackageManager = struct { .read_file => Output.prettyError("<r><red>error<r> reading lockfile:<r> {s}\n<r>", .{ @errorName(cause.value), }), + .migrating => Output.prettyError("<r><red>error<r> migrating lockfile:<r> {s}\n<r>", .{ + @errorName(cause.value), + }), } if (manager.options.enable.fail_early) { diff --git a/src/install/integrity.zig b/src/install/integrity.zig index dd11140de..fb9a55509 100644 --- a/src/install/integrity.zig +++ b/src/install/integrity.zig @@ -3,34 +3,27 @@ const strings = @import("../string_immutable.zig"); const Crypto = @import("../sha.zig").Hashers; pub const Integrity = extern struct { + // this is zeroed like this to work around a comptime issue. + const empty_digest_buf: [Integrity.digest_buf_len]u8 = [_]u8{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + tag: Tag = Tag.unknown, /// Possibly a [Subresource Integrity](https://developer.mozilla.org/en-US/docs/Web/Security/Subresource_Integrity) value initially /// We transform it though. - value: [digest_buf_len]u8 = undefined, + value: [digest_buf_len]u8 = empty_digest_buf, const Base64 = std.base64.standard_no_pad; - pub const digest_buf_len: usize = brk: { - const values = [_]usize{ - std.crypto.hash.Sha1.digest_length, - std.crypto.hash.sha2.Sha512.digest_length, - std.crypto.hash.sha2.Sha256.digest_length, - std.crypto.hash.sha2.Sha384.digest_length, - }; - - var value: usize = 0; - for (values) |val| { - value = @max(val, value); - } - - break :brk value; - }; + pub const digest_buf_len: usize = @max( + std.crypto.hash.Sha1.digest_length, + std.crypto.hash.sha2.Sha512.digest_length, + std.crypto.hash.sha2.Sha256.digest_length, + std.crypto.hash.sha2.Sha384.digest_length, + ); pub fn parseSHASum(buf: []const u8) !Integrity { if (buf.len == 0) { return Integrity{ .tag = Tag.unknown, - .value = undefined, }; } @@ -40,8 +33,11 @@ pub const Integrity = extern struct { var out_i: usize = 0; var i: usize = 0; - { - @memset(&integrity.value, 0); + // initializer should zero it out + if (comptime @import("root").bun.Environment.allow_assert) { + for (integrity.value) |c| { + std.debug.assert(c == 0); + } } while (i < end) { @@ -74,23 +70,20 @@ pub const Integrity = extern struct { if (buf.len < "sha256-".len) { return Integrity{ .tag = Tag.unknown, - .value = undefined, }; } - var out: [digest_buf_len]u8 = undefined; + var out: [digest_buf_len]u8 = empty_digest_buf; const tag = Tag.parse(buf); if (tag == Tag.unknown) { return Integrity{ .tag = Tag.unknown, - .value = undefined, }; } Base64.Decoder.decode(&out, std.mem.trimRight(u8, buf["sha256-".len..], "=")) catch { return Integrity{ .tag = Tag.unknown, - .value = undefined, }; }; @@ -203,4 +196,13 @@ pub const Integrity = extern struct { unreachable; } + + comptime { + var integrity = Integrity{ .tag = Tag.sha1 }; + for (integrity.value) |c| { + if (c != 0) { + @compileError("Integrity buffer is not zeroed"); + } + } + } }; diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index 165a7159c..d9e459b3c 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -21,6 +21,7 @@ const json_parser = bun.JSON; const JSPrinter = bun.js_printer; const linker = @import("../linker.zig"); +const migration = @import("./migration.zig"); const sync = @import("../sync.zig"); const Api = @import("../api/schema.zig").Api; @@ -92,7 +93,7 @@ const assertNoUninitializedPadding = @import("./padding_checker.zig").assertNoUn // Serialized data /// The version of the lockfile format, intended to prevent data corruption for format changes. -format: FormatVersion = .v1, +format: FormatVersion = FormatVersion.current, meta_hash: MetaHash = zero_hash, @@ -159,7 +160,7 @@ pub fn isEmpty(this: *const Lockfile) bool { return this.packages.len == 0 or this.packages.len == 1 or this.packages.get(0).resolutions.len == 0; } -pub const LoadFromDiskResult = union(Tag) { +pub const LoadFromDiskResult = union(enum) { not_found: void, err: struct { step: Step, @@ -167,26 +168,30 @@ pub const LoadFromDiskResult = union(Tag) { }, ok: *Lockfile, - pub const Step = enum { open_file, read_file, parse_file }; - - pub const Tag = enum { - not_found, - err, - ok, - }; + pub const Step = enum { open_file, read_file, parse_file, migrating }; }; pub fn loadFromDisk(this: *Lockfile, allocator: Allocator, log: *logger.Log, filename: stringZ) LoadFromDiskResult { if (comptime Environment.allow_assert) std.debug.assert(FileSystem.instance_loaded); - var file = std.io.getStdIn(); - if (filename.len > 0) - file = std.fs.cwd().openFileZ(filename, .{ .mode = .read_only }) catch |err| { + var file = if (filename.len > 0) + std.fs.cwd().openFileZ(filename, .{ .mode = .read_only }) catch |err| { return switch (err) { - error.FileNotFound, error.AccessDenied, error.BadPathName => LoadFromDiskResult{ .not_found = {} }, + error.FileNotFound => { + // Attempt to load from "package-lock.json", "yarn.lock", etc. + return migration.detectAndLoadOtherLockfile( + this, + allocator, + log, + filename, + ); + }, + error.AccessDenied, error.BadPathName => LoadFromDiskResult{ .not_found = {} }, else => LoadFromDiskResult{ .err = .{ .step = .open_file, .value = err } }, }; - }; + } + else + std.io.getStdIn(); defer file.close(); var buf = file.readToEndAlloc(allocator, std.math.maxInt(usize)) catch |err| { @@ -209,6 +214,10 @@ pub fn loadFromBytes(this: *Lockfile, buf: []u8, allocator: Allocator, log: *log return LoadFromDiskResult{ .err = .{ .step = .parse_file, .value = err } }; }; + if (Environment.allow_assert) { + this.verifyData() catch @panic("lockfile data is corrupt"); + } + return LoadFromDiskResult{ .ok = this }; } @@ -289,6 +298,14 @@ pub const Tree = struct { }; } + pub fn reload(this: *Iterator, lockfile: *const Lockfile) void { + this.trees = lockfile.buffers.trees.items; + this.dependency_ids = lockfile.buffers.hoisted_dependencies.items; + this.dependencies = lockfile.buffers.dependencies.items; + this.resolutions = lockfile.buffers.resolutions.items; + this.string_buf = lockfile.buffers.string_bytes.items; + } + pub fn nextNodeModulesFolder(this: *Iterator) ?NodeModulesFolder { if (this.tree_id >= this.trees.len) return null; @@ -997,6 +1014,9 @@ pub const Printer = struct { .read_file => Output.prettyErrorln("<r><red>error<r> reading lockfile:<r> {s}", .{ @errorName(cause.value), }), + .migrating => Output.prettyErrorln("<r><red>error<r> while migrating lockfile:<r> {s}", .{ + @errorName(cause.value), + }), } if (log.errors > 0) { switch (Output.enable_ansi_colors) { @@ -1260,6 +1280,24 @@ pub const Printer = struct { comptime Writer: type, writer: Writer, ) !void { + // internal for debugging, print the lockfile as custom json + // limited to debug because we don't want people to rely on this format. + if (Environment.isDebug) { + if (std.os.getenv("JSON")) |_| { + try std.json.stringify( + this.lockfile, + .{ + .whitespace = .indent_2, + .emit_null_optional_fields = true, + .emit_nonportable_numbers_as_strings = true, + }, + writer, + ); + try writer.writeAll("\n"); + return; + } + } + try writer.writeAll( \\# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. \\# yarn lockfile v1 @@ -1416,7 +1454,7 @@ pub const Printer = struct { var behavior = Behavior.uninitialized; var dependency_behavior_change_count: u8 = 0; for (dependencies) |dep| { - if (dep.behavior != behavior) { + if (!dep.behavior.eq(behavior)) { if (dep.behavior.isOptional()) { try writer.writeAll(" optionalDependencies:\n"); if (comptime Environment.allow_assert) dependency_behavior_change_count += 1; @@ -1458,20 +1496,18 @@ pub const Printer = struct { pub fn verifyData(this: *Lockfile) !void { std.debug.assert(this.format == Lockfile.FormatVersion.current); - { - var i: usize = 0; - while (i < this.packages.len) : (i += 1) { - const package: Lockfile.Package = this.packages.get(i); - std.debug.assert(this.str(&package.name).len == @as(usize, package.name.len())); - std.debug.assert(String.Builder.stringHash(this.str(&package.name)) == @as(usize, package.name_hash)); - std.debug.assert(package.dependencies.get(this.buffers.dependencies.items).len == @as(usize, package.dependencies.len)); - std.debug.assert(package.resolutions.get(this.buffers.resolutions.items).len == @as(usize, package.resolutions.len)); - std.debug.assert(package.resolutions.get(this.buffers.resolutions.items).len == @as(usize, package.dependencies.len)); - const dependencies = package.dependencies.get(this.buffers.dependencies.items); - for (dependencies) |dependency| { - std.debug.assert(this.str(&dependency.name).len == @as(usize, dependency.name.len())); - std.debug.assert(String.Builder.stringHash(this.str(&dependency.name)) == dependency.name_hash); - } + var i: usize = 0; + while (i < this.packages.len) : (i += 1) { + const package: Lockfile.Package = this.packages.get(i); + std.debug.assert(this.str(&package.name).len == @as(usize, package.name.len())); + std.debug.assert(String.Builder.stringHash(this.str(&package.name)) == @as(usize, package.name_hash)); + std.debug.assert(package.dependencies.get(this.buffers.dependencies.items).len == @as(usize, package.dependencies.len)); + std.debug.assert(package.resolutions.get(this.buffers.resolutions.items).len == @as(usize, package.resolutions.len)); + std.debug.assert(package.resolutions.get(this.buffers.resolutions.items).len == @as(usize, package.dependencies.len)); + const dependencies = package.dependencies.get(this.buffers.dependencies.items); + for (dependencies) |dependency| { + std.debug.assert(this.str(&dependency.name).len == @as(usize, dependency.name.len())); + std.debug.assert(String.Builder.stringHash(this.str(&dependency.name)) == dependency.name_hash); } } } @@ -1688,7 +1724,7 @@ pub fn appendPackage(this: *Lockfile, package_: Lockfile.Package) !Lockfile.Pack fn appendPackageWithID(this: *Lockfile, package_: Lockfile.Package, id: PackageID) !Lockfile.Package { defer { - if (comptime Environment.isDebug) { + if (comptime Environment.allow_assert) { std.debug.assert(this.getPackageID(package_.name_hash, null, &package_.resolution) != null); } } @@ -1850,12 +1886,13 @@ pub const PackageIndex = struct { }; pub const FormatVersion = enum(u32) { - v0, + v0 = 0, // bun v0.0.x - bun v0.1.6 - v1, + v1 = 1, // bun v0.1.7+ // This change added tarball URLs to npm-resolved packages - v2, + v2 = 2, + _, pub const current = FormatVersion.v2; }; @@ -1875,7 +1912,7 @@ pub const Package = extern struct { name: String = .{}, name_hash: PackageNameHash = 0, - /// How a package has been resolved + /// How this package has been resolved /// When .tag is uninitialized, that means the package is not resolved yet. resolution: Resolution = .{}, @@ -1884,8 +1921,18 @@ pub const Package = extern struct { /// if resolutions[i] is an invalid package ID, then dependencies[i] is not resolved dependencies: DependencySlice = .{}, - /// The resolved package IDs for the dependencies - resolutions: DependencyIDSlice = .{}, + /// The resolved package IDs for this package's dependencies. Instead of storing this + /// on the `Dependency` struct within `.dependencies`, it is stored on the package itself + /// so we can access it faster. + /// + /// Each index in this array corresponds to the same index in dependencies. + /// Each value in this array corresponds to the resolved package ID for that dependency. + /// + /// So this is how you say "what package ID for lodash does this package actually resolve to?" + /// + /// By default, the underlying buffer is filled with "invalid_id" to indicate this package ID + /// was not resolved + resolutions: PackageIDSlice = .{}, meta: Meta = .{}, bin: Bin = .{}, @@ -2023,11 +2070,11 @@ pub const Package = extern struct { field: string, behavior: Behavior, - pub const dependencies = DependencyGroup{ .prop = "dependencies", .field = "dependencies", .behavior = @as(Behavior, @enumFromInt(Behavior.normal)) }; - pub const dev = DependencyGroup{ .prop = "devDependencies", .field = "dev_dependencies", .behavior = @as(Behavior, @enumFromInt(Behavior.dev)) }; - pub const optional = DependencyGroup{ .prop = "optionalDependencies", .field = "optional_dependencies", .behavior = @as(Behavior, @enumFromInt(Behavior.optional)) }; - pub const peer = DependencyGroup{ .prop = "peerDependencies", .field = "peer_dependencies", .behavior = @as(Behavior, @enumFromInt(Behavior.peer)) }; - pub const workspaces = DependencyGroup{ .prop = "workspaces", .field = "workspaces", .behavior = @as(Behavior, @enumFromInt(Behavior.workspace)) }; + pub const dependencies = DependencyGroup{ .prop = "dependencies", .field = "dependencies", .behavior = Behavior.normal }; + pub const dev = DependencyGroup{ .prop = "devDependencies", .field = "dev_dependencies", .behavior = Behavior.dev }; + pub const optional = DependencyGroup{ .prop = "optionalDependencies", .field = "optional_dependencies", .behavior = Behavior.optional }; + pub const peer = DependencyGroup{ .prop = "peerDependencies", .field = "peer_dependencies", .behavior = Behavior.peer }; + pub const workspaces = DependencyGroup{ .prop = "workspaces", .field = "workspaces", .behavior = Behavior.workspace }; }; pub inline fn isDisabled(this: *const Lockfile.Package) bool { @@ -3425,15 +3472,7 @@ pub const Package = extern struct { return error.InvalidPackageJSON; } for (obj.properties.slice()) |item| { - const key = item.key.?.asString(allocator) orelse { - log.addErrorFmt(&source, item.key.?.loc, allocator, - \\{0s} expects a map of specifiers, e.g. - \\"{0s}": {{ - \\ "bun": "latest" - \\}} - , .{group.prop}) catch {}; - return error.InvalidPackageJSON; - }; + const key = item.key.?.asString(allocator).?; const value = item.value.?.asString(allocator) orelse { log.addErrorFmt(&source, item.value.?.loc, allocator, \\{0s} expects a map of specifiers, e.g. @@ -3732,15 +3771,19 @@ pub const Package = extern struct { string_builder.clamp(); } - pub const List = std.MultiArrayList(Lockfile.Package); + pub const List = bun.MultiArrayList(Lockfile.Package); pub const Meta = extern struct { + // TODO: when we bump the lockfile version, we should reorder this to: + // id(32), arch(16), os(16), id(8), man_dir(8), integrity(72 align 8) + // should allow us to remove padding bytes + + // TODO: remove origin. it doesnt do anything and can be inferred from the resolution origin: Origin = Origin.npm, _padding_origin: u8 = 0, arch: Npm.Architecture = Npm.Architecture.all, os: Npm.OperatingSystem = Npm.OperatingSystem.all, - _padding_os: u16 = 0, id: PackageID = invalid_package_id, @@ -3759,11 +3802,14 @@ pub const Package = extern struct { } pub fn clone(this: *const Meta, id: PackageID, buf: []const u8, comptime StringBuilderType: type, builder: StringBuilderType) Meta { - var new = this.*; - new.id = id; - new.man_dir = builder.append(String, this.man_dir.slice(buf)); - - return new; + return Meta{ + .id = id, + .man_dir = builder.append(String, this.man_dir.slice(buf)), + .integrity = this.integrity, + .arch = this.arch, + .os = this.os, + .origin = this.origin, + }; } }; @@ -3840,6 +3886,8 @@ pub const Package = extern struct { inline for (FieldsEnum.fields) |field| { const value = sliced.items(@field(Lockfile.Package.List.Field, field.name)); + if (comptime Environment.allow_assert) + debug("save(\"{s}\") = {d} bytes", .{ field.name, std.mem.sliceAsBytes(value).len }); comptime assertNoUninitializedPadding(@TypeOf(value)); try writer.writeAll(std.mem.sliceAsBytes(value)); @@ -3926,11 +3974,14 @@ pub fn deinit(this: *Lockfile) void { const Buffers = struct { trees: Tree.List = .{}, hoisted_dependencies: DependencyIDList = .{}, + /// This is the underlying buffer used for the `resolutions` external slices inside of `Package` + /// Should be the same length as `dependencies` resolutions: PackageIDList = .{}, + /// This is the underlying buffer used for the `dependencies` external slices inside of `Package` dependencies: DependencyList = .{}, + /// This is the underlying buffer used for any `Semver.ExternalString` instance in the lockfile extern_strings: ExternalStringBuffer = .{}, - // node_modules_folders: NodeModulesFolderList = NodeModulesFolderList{}, - // node_modules_package_ids: PackageIDList = PackageIDList{}, + /// This is where all non-inlinable `Semver.String`s are stored. string_bytes: StringBuffer = .{}, pub fn deinit(this: *Buffers, allocator: Allocator) void { @@ -4470,7 +4521,7 @@ pub fn hasMetaHashChanged(this: *Lockfile, print_name_version_string: bool) !boo this.meta_hash = try this.generateMetaHash(print_name_version_string); return !strings.eqlLong(&previous_meta_hash, &this.meta_hash, false); } -fn generateMetaHash(this: *Lockfile, print_name_version_string: bool) !MetaHash { +pub fn generateMetaHash(this: *Lockfile, print_name_version_string: bool) !MetaHash { if (this.packages.len <= 1) return zero_hash; @@ -4600,3 +4651,294 @@ pub fn resolve(this: *Lockfile, package_name: []const u8, version: Dependency.Ve return null; } + +pub fn jsonStringifyDependency(this: *const Lockfile, w: anytype, dep: Dependency, res: ?PackageID) !void { + const sb = this.buffers.string_bytes.items; + var buf: [2048]u8 = undefined; + + try w.beginObject(); + defer w.endObject() catch {}; + + try w.objectField("literal"); + try w.write(dep.version.literal.slice(sb)); + + try w.objectField(@tagName(dep.version.tag)); + switch (dep.version.tag) { + .uninitialized => try w.write(null), + .npm => { + try w.beginObject(); + defer w.endObject() catch {}; + + const info: Dependency.Version.NpmInfo = dep.version.value.npm; + + try w.objectField("name"); + try w.write(info.name.slice(sb)); + + try w.objectField("version"); + try w.write(try std.fmt.bufPrint(&buf, "{}", .{info.version})); + }, + .dist_tag => { + try w.beginObject(); + defer w.endObject() catch {}; + + const info: Dependency.Version.TagInfo = dep.version.value.dist_tag; + + try w.objectField("name"); + try w.write(info.name.slice(sb)); + + try w.objectField("tag"); + try w.write(info.name.slice(sb)); + }, + .tarball => { + try w.beginObject(); + defer w.endObject() catch {}; + + const info: Dependency.Version.TarballInfo = dep.version.value.tarball; + try w.objectField(@tagName(info.uri)); + try w.write(switch (info.uri) { + inline else => |s| s.slice(sb), + }); + + try w.objectField("package_name"); + try w.write(info.package_name.slice(sb)); + }, + .folder => { + try w.write(dep.version.value.folder.slice(sb)); + }, + .symlink => { + try w.write(dep.version.value.symlink.slice(sb)); + }, + .workspace => { + try w.write(dep.version.value.workspace.slice(sb)); + }, + .git => { + try w.beginObject(); + defer w.endObject() catch {}; + + const info: Repository = dep.version.value.git; + + try w.objectField("owner"); + try w.write(info.owner.slice(sb)); + try w.objectField("repo"); + try w.write(info.repo.slice(sb)); + try w.objectField("committish"); + try w.write(info.committish.slice(sb)); + try w.objectField("resolved"); + try w.write(info.resolved.slice(sb)); + try w.objectField("package_name"); + try w.write(info.package_name.slice(sb)); + }, + .github => { + try w.beginObject(); + defer w.endObject() catch {}; + + const info: Repository = dep.version.value.github; + + try w.objectField("owner"); + try w.write(info.owner.slice(sb)); + try w.objectField("repo"); + try w.write(info.repo.slice(sb)); + try w.objectField("committish"); + try w.write(info.committish.slice(sb)); + try w.objectField("resolved"); + try w.write(info.resolved.slice(sb)); + try w.objectField("package_name"); + try w.write(info.package_name.slice(sb)); + }, + } + + try w.objectField("resolved_id"); + try w.write(if (res) |r| if (r == invalid_package_id) null else r else null); + + const behavior = try std.fmt.bufPrint(&buf, "{}", .{dep.behavior}); + try w.objectField("behavior"); + try w.write(behavior); +} + +pub fn jsonStringify(this: *const Lockfile, w: anytype) !void { + var buf: [2048]u8 = undefined; + const sb = this.buffers.string_bytes.items; + try w.beginObject(); + defer w.endObject() catch {}; + + try w.objectField("format"); + try w.write(@tagName(this.format)); + try w.objectField("meta_hash"); + try w.write(std.fmt.bytesToHex(this.meta_hash, .lower)); + + { + try w.objectField("package_index"); + try w.beginObject(); + defer w.endObject() catch {}; + + var iter = this.package_index.iterator(); + while (iter.next()) |it| { + const entry: PackageIndex.Entry = it.value_ptr.*; + const first_id = switch (entry) { + .PackageID => |id| id, + .PackageIDMultiple => |ids| ids.items[0], + }; + const name = this.packages.items(.name)[first_id].slice(sb); + try w.objectField(name); + switch (entry) { + .PackageID => |id| try w.write(id), + .PackageIDMultiple => |ids| { + try w.beginArray(); + for (ids.items) |id| { + try w.write(id); + } + try w.endArray(); + }, + } + } + } + { + try w.objectField("packages"); + try w.beginArray(); + defer w.endArray() catch {}; + + for (0..this.packages.len) |i| { + const pkg: Package = this.packages.get(i); + try w.beginObject(); + defer w.endObject() catch {}; + + try w.objectField("id"); + try w.write(i); + + try w.objectField("name"); + try w.write(pkg.name.slice(sb)); + + try w.objectField("name_hash"); + try w.write(pkg.name_hash); + + try w.objectField("resolution"); + if (pkg.resolution.tag == .uninitialized) { + try w.write(null); + } else { + const b = try std.fmt.bufPrint(&buf, "{s} {s}", .{ @tagName(pkg.resolution.tag), pkg.resolution.fmt(sb) }); + try w.write(b); + } + + try w.objectField("dependencies"); + { + try w.beginObject(); + defer w.endObject() catch {}; + + for (pkg.dependencies.get(this.buffers.dependencies.items), pkg.resolutions.get(this.buffers.resolutions.items)) |dep_, res| { + const dep: Dependency = dep_; + try w.objectField(dep.name.slice(sb)); + try this.jsonStringifyDependency(w, dep, res); + } + } + + if (@as(u16, @intFromEnum(pkg.meta.arch)) != Npm.Architecture.all_value) { + try w.objectField("arch"); + try w.beginArray(); + defer w.endArray() catch {}; + + for (Npm.Architecture.NameMap.kvs) |kv| { + if (pkg.meta.arch.has(kv.value)) { + try w.write(kv.key); + } + } + } + + if (@as(u16, @intFromEnum(pkg.meta.os)) != Npm.OperatingSystem.all_value) { + try w.objectField("os"); + try w.beginArray(); + defer w.endArray() catch {}; + + for (Npm.OperatingSystem.NameMap.kvs) |kv| { + if (pkg.meta.os.has(kv.value)) { + try w.write(kv.key); + } + } + } + + try w.objectField("integrity"); + if (pkg.meta.integrity.tag != .unknown) { + try w.write(try std.fmt.bufPrint(&buf, "{}", .{pkg.meta.integrity})); + } else { + try w.write(null); + } + + try w.objectField("man_dir"); + try w.write(pkg.meta.man_dir.slice(sb)); + + try w.objectField("origin"); + try w.write(@tagName(pkg.meta.origin)); + + try w.objectField("bin"); + switch (pkg.bin.tag) { + .none => try w.write(null), + .file => { + try w.beginObject(); + defer w.endObject() catch {}; + + try w.objectField("file"); + try w.write(pkg.bin.value.file.slice(sb)); + }, + .named_file => { + try w.beginObject(); + defer w.endObject() catch {}; + + try w.objectField("name"); + try w.write(pkg.bin.value.named_file[0].slice(sb)); + + try w.objectField("file"); + try w.write(pkg.bin.value.named_file[1].slice(sb)); + }, + .dir => { + try w.objectField("dir"); + try w.write(pkg.bin.value.dir.slice(sb)); + }, + .map => { + try w.beginObject(); + defer w.endObject() catch {}; + + const data: []const ExternalString = pkg.bin.value.map.get(this.buffers.extern_strings.items); + var bin_i: usize = 0; + while (bin_i < data.len) : (bin_i += 2) { + try w.objectField(data[bin_i].slice(sb)); + try w.write(data[bin_i + 1].slice(sb)); + } + }, + } + + { + try w.objectField("scripts"); + try w.beginObject(); + defer w.endObject() catch {}; + + inline for (comptime std.meta.fieldNames(Lockfile.Scripts)) |field_name| { + var script = @field(pkg.scripts, field_name).slice(sb); + if (script.len > 0) { + try w.objectField(field_name); + try w.write(script); + } + } + } + } + } + + try w.objectField("workspace_paths"); + { + try w.beginObject(); + defer w.endObject() catch {}; + + for (this.workspace_paths.keys(), this.workspace_paths.values()) |k, v| { + try w.objectField(try std.fmt.bufPrint(&buf, "{d}", .{k})); + try w.write(v.slice(sb)); + } + } + try w.objectField("workspace_versions"); + { + try w.beginObject(); + defer w.endObject() catch {}; + + for (this.workspace_versions.keys(), this.workspace_versions.values()) |k, v| { + try w.objectField(try std.fmt.bufPrint(&buf, "{d}", .{k})); + try w.write(try std.fmt.bufPrint(&buf, "{}", .{v.fmt(sb)})); + } + } +} diff --git a/src/install/migration.zig b/src/install/migration.zig new file mode 100644 index 000000000..d74be7265 --- /dev/null +++ b/src/install/migration.zig @@ -0,0 +1,947 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const bun = @import("root").bun; +const string = bun.string; +const Output = bun.Output; +const Global = bun.Global; +const Environment = bun.Environment; +const strings = bun.strings; +const MutableString = bun.MutableString; +const stringZ = bun.stringZ; +const logger = bun.logger; + +const Install = @import("./install.zig"); +const Resolution = @import("./resolution.zig").Resolution; +const Dependency = @import("./dependency.zig"); +const VersionedURL = @import("./versioned_url.zig"); +const Npm = @import("./npm.zig"); +const Integrity = @import("./integrity.zig").Integrity; +const Bin = @import("./bin.zig").Bin; + +const Semver = @import("./semver.zig"); +const String = Semver.String; +const ExternalString = Semver.ExternalString; +const stringHash = String.Builder.stringHash; + +const Lockfile = @import("./lockfile.zig"); +const LoadFromDiskResult = Lockfile.LoadFromDiskResult; + +const JSAst = bun.JSAst; +const Expr = JSAst.Expr; +const B = JSAst.B; +const E = JSAst.E; +const G = JSAst.G; +const S = JSAst.S; + +const debug = Output.scoped(.migrate, false); + +pub fn detectAndLoadOtherLockfile(this: *Lockfile, allocator: Allocator, log: *logger.Log, bun_lockfile_path: stringZ) LoadFromDiskResult { + const dirname = bun_lockfile_path[0 .. strings.lastIndexOfChar(bun_lockfile_path, '/') orelse 0]; + // check for package-lock.json, yarn.lock, etc... + // if it exists, do an in-memory migration + var buf: [bun.MAX_PATH_BYTES]u8 = undefined; + @memcpy(buf[0..dirname.len], dirname); + + const cwd = std.fs.cwd(); + + npm: { + const npm_lockfile_name = "package-lock.json"; + @memcpy(buf[dirname.len .. dirname.len + npm_lockfile_name.len], npm_lockfile_name); + buf[dirname.len + npm_lockfile_name.len] = 0; + const lockfile_path = buf[0 .. dirname.len + npm_lockfile_name.len :0]; + var timer = std.time.Timer.start() catch unreachable; + const file = cwd.openFileZ(lockfile_path, .{ .mode = .read_only }) catch break :npm; + defer file.close(); + var data = file.readToEndAlloc(allocator, std.math.maxInt(usize)) catch |err| { + return LoadFromDiskResult{ .err = .{ .step = .migrating, .value = err } }; + }; + const lockfile = migrateNPMLockfile(this, allocator, log, data, lockfile_path) catch |err| { + if (err == error.NPMLockfileVersionMismatch) { + Output.prettyErrorln( + \\<red><b>error<r><d>:<r> Please upgrade package-lock.json to lockfileVersion 3 + \\ + \\Run 'npm i --lockfile-version 3 --frozen-lockfile' to upgrade your lockfile without changing dependencies. + , .{}); + Global.exit(1); + } + if (Environment.allow_assert) { + const maybe_trace = @errorReturnTrace(); + Output.prettyErrorln("Error: {s}", .{@errorName(err)}); + log.printForLogLevel(Output.errorWriter()) catch {}; + if (maybe_trace) |trace| { + std.debug.dumpStackTrace(trace.*); + } + Output.prettyErrorln("Invalid NPM package-lock.json\nIn a release build, this would ignore and do a fresh install.\nAborting", .{}); + Global.exit(1); + } + return LoadFromDiskResult{ .err = .{ .step = .migrating, .value = err } }; + }; + + if (lockfile == .ok) { + Output.printElapsed(@as(f64, @floatFromInt(timer.read())) / std.time.ns_per_ms); + Output.prettyError(" ", .{}); + Output.prettyErrorln("<d>migrated lockfile from <r><green>package-lock.json<r>", .{}); + Output.flush(); + } + + return lockfile; + } + + return LoadFromDiskResult{ .not_found = {} }; +} + +const IdMap = std.StringHashMapUnmanaged(IdMapValue); +const IdMapValue = struct { + /// index into the old package-lock.json package entries. + old_json_index: u32, + /// this is the new package id for the bun lockfile + /// + /// - if this new_package_id is set to `package_id_is_link`, it means it's a link + /// and to get the actual package id, you need to lookup `.resolved` in the hashmap. + /// - if it is `package_id_is_bundled`, it means it's a bundled dependency that was not + /// marked by npm, which can happen to some transitive dependencies. + new_package_id: u32, +}; +const package_id_is_link = std.math.maxInt(u32); +const package_id_is_bundled = std.math.maxInt(u32) - 1; + +const unset_package_id = Install.invalid_package_id - 1; + +const dependency_keys = .{ + .dependencies, + .devDependencies, + .peerDependencies, + .optionalDependencies, +}; + +pub fn migrateNPMLockfile(this: *Lockfile, allocator: Allocator, log: *logger.Log, data: string, path: string) !LoadFromDiskResult { + debug("begin lockfile migration", .{}); + + try this.initEmpty(allocator); + Install.initializeStore(); + + const json_src = logger.Source.initPathString(path, data); + const json = bun.JSON.ParseJSONUTF8(&json_src, log, allocator) catch return error.InvalidNPMLockfile; + + if (json.data != .e_object) { + return error.InvalidNPMLockfile; + } + if (json.get("lockfileVersion")) |version| { + if (!(version.data == .e_number and version.data.e_number.value == 3)) { + return error.NPMLockfileVersionMismatch; + } + } else { + return error.InvalidNPMLockfile; + } + + // Count pass + var builder_ = this.stringBuilder(); + var builder = &builder_; + const name = (if (json.get("name")) |expr| expr.asString(allocator) else null) orelse ""; + builder.count(name); + + var root_package: *E.Object = undefined; + var packages_properties = brk: { + const obj = json.get("packages") orelse return error.InvalidNPMLockfile; + if (obj.data != .e_object) return error.InvalidNPMLockfile; + if (obj.data.e_object.properties.len == 0) return error.InvalidNPMLockfile; + const prop1 = obj.data.e_object.properties.at(0); + if (prop1.key) |k| { + if (k.data != .e_string) return error.InvalidNPMLockfile; + // first key must be the "", self reference + if (k.data.e_string.data.len != 0) return error.InvalidNPMLockfile; + if (prop1.value.?.data != .e_object) return error.InvalidNPMLockfile; + root_package = prop1.value.?.data.e_object; + } else return error.InvalidNPMLockfile; + break :brk obj.data.e_object.properties; + }; + + var num_deps: u32 = 0; + + const workspace_map: ?Lockfile.Package.WorkspaceMap = workspace_map: { + if (root_package.get("workspaces")) |wksp| { + var workspaces = Lockfile.Package.WorkspaceMap.init(allocator); + + const json_array = switch (wksp.data) { + .e_array => |arr| arr, + .e_object => |obj| if (obj.get("packages")) |packages| switch (packages.data) { + .e_array => |arr| arr, + else => return error.InvalidNPMLockfile, + } else return error.InvalidNPMLockfile, + else => return error.InvalidNPMLockfile, + }; + + const workspace_packages_count = try Lockfile.Package.processWorkspaceNamesArray( + &workspaces, + allocator, + log, + json_array, + &json_src, + wksp.loc, + builder, + ); + debug("found {d} workspace packages", .{workspace_packages_count}); + num_deps += workspace_packages_count; + break :workspace_map workspaces; + } + break :workspace_map null; + }; + + // Counting Phase + // This "IdMap" is used to make object key lookups faster for the `packages` object + // it also lets us resolve linked and bundled packages. + var id_map = IdMap{}; + try id_map.ensureTotalCapacity(allocator, packages_properties.len); + var num_extern_strings: u32 = 0; + var package_idx: u32 = 0; + for (packages_properties.slice(), 0..) |entry, i| { + const pkg_path = entry.key.?.asString(allocator).?; + if (entry.value.?.data != .e_object) + return error.InvalidNPMLockfile; + + const pkg = entry.value.?.data.e_object; + + if (pkg.get("link") != null) { + id_map.putAssumeCapacity( + pkg_path, + IdMapValue{ + .old_json_index = @truncate(i), + .new_package_id = package_id_is_link, + }, + ); + continue; + } + if (pkg.get("inBundle")) |x| if (x.data == .e_boolean and x.data.e_boolean.value) { + id_map.putAssumeCapacity( + pkg_path, + IdMapValue{ + .old_json_index = @truncate(i), + .new_package_id = package_id_is_bundled, + }, + ); + continue; + }; + if (pkg.get("extraneous")) |x| if (x.data == .e_boolean and x.data.e_boolean.value) { + continue; + }; + + id_map.putAssumeCapacity( + pkg_path, + IdMapValue{ + .old_json_index = @truncate(i), + .new_package_id = package_idx, + }, + ); + package_idx += 1; + + inline for (dependency_keys) |dep_key| { + if (pkg.get(@tagName(dep_key))) |deps| { + if (deps.data != .e_object) { + return error.InvalidNPMLockfile; + } + num_deps +|= @as(u32, deps.data.e_object.properties.len); + + for (deps.data.e_object.properties.slice()) |dep| { + const dep_name = dep.key.?.asString(allocator).?; + const version_string = dep.value.?.asString(allocator) orelse return error.InvalidNPMLockfile; + + builder.count(dep_name); + builder.count(version_string); + + // If it's a folder or workspace, pessimistically assume we will need a maximum path + switch (Dependency.Version.Tag.infer(version_string)) { + .folder, .workspace => builder.cap += bun.MAX_PATH_BYTES, + else => {}, + } + } + } + } + + if (pkg.get("bin")) |bin| { + if (bin.data != .e_object) return error.InvalidNPMLockfile; + switch (bin.data.e_object.properties.len) { + 0 => return error.InvalidNPMLockfile, + 1 => { + const first_bin = bin.data.e_object.properties.at(0); + const key = first_bin.key.?.asString(allocator).?; + + const workspace_entry = if (workspace_map) |map| map.map.get(pkg_path) else null; + const is_workspace = workspace_entry != null; + + const pkg_name = if (is_workspace) + workspace_entry.?.name + else if (entry.value.?.get("name")) |set_name| + (set_name.asString(this.allocator) orelse return error.InvalidNPMLockfile) + else + packageNameFromPath(pkg_path); + + if (!strings.eql(key, pkg_name)) { + builder.count(key); + } + builder.count(first_bin.value.?.asString(allocator) orelse return error.InvalidNPMLockfile); + }, + else => { + for (bin.data.e_object.properties.slice()) |bin_entry| { + builder.count(bin_entry.key.?.asString(allocator).?); + builder.count(bin_entry.value.?.asString(allocator) orelse return error.InvalidNPMLockfile); + } + num_extern_strings += @truncate(bin.data.e_object.properties.len * 2); + }, + } + } + + if (pkg.get("resolved")) |resolved_expr| { + const resolved = resolved_expr.asString(allocator) orelse return error.InvalidNPMLockfile; + if (strings.hasPrefixComptime(resolved, "file:")) { + builder.count(resolved[5..]); + } else if (strings.hasPrefixComptime(resolved, "git+")) { + builder.count(resolved[4..]); + } else { + builder.count(resolved); + + // this is over-counting but whatever. it would be too hard to determine if the case here + // is an `npm`/`dist_tag` version (the only times this is actually used) + if (pkg.get("version")) |v| if (v.asString(allocator)) |s| { + builder.count(s); + }; + } + } else { + builder.count(pkg_path); + } + } + if (num_deps == std.math.maxInt(u32)) return error.InvalidNPMLockfile; // lol + + debug("counted {d} dependencies", .{num_deps}); + debug("counted {d} extern strings", .{num_extern_strings}); + debug("counted {d} packages", .{package_idx}); + + try this.buffers.dependencies.ensureTotalCapacity(allocator, num_deps); + try this.buffers.resolutions.ensureTotalCapacity(allocator, num_deps); + try this.buffers.extern_strings.ensureTotalCapacity(allocator, num_extern_strings); + try this.packages.ensureTotalCapacity(allocator, package_idx); + // The package index is overallocated, but we know the upper bound + try this.package_index.ensureTotalCapacity(package_idx); + try builder.allocate(); + + if (workspace_map) |wksp| { + try this.workspace_paths.ensureTotalCapacity(allocator, wksp.map.unmanaged.entries.len); + try this.workspace_versions.ensureTotalCapacity(allocator, wksp.map.unmanaged.entries.len); + + for (wksp.map.keys(), wksp.map.values()) |k, v| { + const name_hash = stringHash(v.name); + this.workspace_paths.putAssumeCapacity(name_hash, builder.append(String, k)); + if (v.version) |version| this.workspace_versions.putAssumeCapacity(name_hash, version); + } + } + + // Package Building Phase + // This initializes every package and sets the resolution to uninitialized + for (packages_properties.slice()) |entry| { + // this pass is allowed to make more assumptions because we already checked things during + // the counting pass + const pkg = entry.value.?.data.e_object; + + if (pkg.get("link") != null or if (pkg.get("inBundle") orelse pkg.get("extraneous")) |x| x.data == .e_boolean and x.data.e_boolean.value else false) continue; + + const pkg_path = entry.key.?.asString(allocator).?; + + const workspace_entry = if (workspace_map) |map| map.map.get(pkg_path) else null; + const is_workspace = workspace_entry != null; + + const pkg_name = if (is_workspace) + workspace_entry.?.name + else if (pkg.get("name")) |set_name| + (set_name.asString(this.allocator) orelse unreachable) + else + packageNameFromPath(pkg_path); + + const name_hash = stringHash(pkg_name); + + const package_id: Install.PackageID = @intCast(this.packages.len); + if (Environment.allow_assert) { + // If this is false, then it means we wrote wrong resolved ids + // During counting phase we assign all the packages an id. + std.debug.assert(package_id == id_map.get(pkg_path).?.new_package_id); + } + + // Instead of calling this.appendPackage, manually append + // the other function has some checks that will fail since we have not set resolution+dependencies yet. + this.packages.appendAssumeCapacity(Lockfile.Package{ + .name = builder.appendWithHash(String, pkg_name, name_hash), + .name_hash = name_hash, + + // For non workspace packages these are set to .uninitialized, then in the third phase + // they are resolved. This is because the resolution uses the dependant's version + // specifier as a "hint" to resolve the dependency. + .resolution = if (is_workspace) Resolution.init(.{ + // This string is counted by `processWorkspaceNamesArray` + .workspace = builder.append(String, pkg_path), + }) else Resolution{}, + + // we fill this data in later + .dependencies = undefined, + .resolutions = undefined, + + .meta = .{ + .id = package_id, + + .origin = if (package_id == 0) .local else .npm, + + .arch = if (pkg.get("cpu")) |cpu_array| arch: { + if (cpu_array.data != .e_array) return error.InvalidNPMLockfile; + var arch: Npm.Architecture = .none; + for (cpu_array.data.e_array.items.slice()) |item| { + if (item.data != .e_string) return error.InvalidNPMLockfile; + arch = arch.apply(item.data.e_string.data); + } + break :arch arch; + } else .all, + + .os = if (pkg.get("os")) |cpu_array| arch: { + if (cpu_array.data != .e_array) return error.InvalidNPMLockfile; + var os: Npm.OperatingSystem = .none; + for (cpu_array.data.e_array.items.slice()) |item| { + if (item.data != .e_string) return error.InvalidNPMLockfile; + os = os.apply(item.data.e_string.data); + } + break :arch os; + } else .all, + + .man_dir = String{}, + + .integrity = if (pkg.get("integrity")) |integrity| + try Integrity.parse( + integrity.asString(this.allocator) orelse + return error.InvalidNPMLockfile, + ) + else + Integrity{}, + }, + .bin = if (pkg.get("bin")) |bin| bin: { + // we already check these conditions during counting + std.debug.assert(bin.data == .e_object); + std.debug.assert(bin.data.e_object.properties.len > 0); + + // in npm lockfile, the bin is always an object, even if it is only a single one + // we need to detect if it's a single entry and lower it to a file. + if (bin.data.e_object.properties.len == 1) { + const prop = bin.data.e_object.properties.at(0); + const key = prop.key.?.asString(this.allocator) orelse return error.InvalidNPMLockfile; + const script_value = prop.value.?.asString(this.allocator) orelse return error.InvalidNPMLockfile; + + if (strings.eql(key, pkg_name)) { + break :bin .{ + .tag = .file, + .value = Bin.Value.init(.{ + .file = builder.append(String, script_value), + }), + }; + } + + break :bin .{ + .tag = .named_file, + .value = Bin.Value.init(.{ + .named_file = .{ + builder.append(String, key), + builder.append(String, script_value), + }, + }), + }; + } + + const view: Install.ExternalStringList = .{ + .off = @truncate(this.buffers.extern_strings.items.len), + .len = @intCast(bin.data.e_object.properties.len * 2), + }; + + for (bin.data.e_object.properties.slice()) |bin_entry| { + const key = bin_entry.key.?.asString(this.allocator) orelse return error.InvalidNPMLockfile; + const script_value = bin_entry.value.?.asString(this.allocator) orelse return error.InvalidNPMLockfile; + this.buffers.extern_strings.appendAssumeCapacity(builder.append(ExternalString, key)); + this.buffers.extern_strings.appendAssumeCapacity(builder.append(ExternalString, script_value)); + } + + if (Environment.allow_assert) { + std.debug.assert(this.buffers.extern_strings.items.len == view.off + view.len); + std.debug.assert(this.buffers.extern_strings.items.len <= this.buffers.extern_strings.capacity); + } + + break :bin .{ + .tag = .map, + .value = Bin.Value.init(.{ + .map = view, + }), + }; + } else Bin.init(), + + .scripts = .{}, + }); + + if (is_workspace) { + std.debug.assert(package_id != 0); // root package should not be in it's own workspace + + // we defer doing getOrPutID for non-workspace packages because it depends on the resolution being set. + try this.getOrPutID(package_id, name_hash); + } + } + + if (Environment.allow_assert) { + std.debug.assert(this.packages.len == package_idx); + } + + // ignoring length check because we pre-allocated it. the length may shrink later + // so it's faster if we ignore the underlying length buffer and just assign it at the very end. + var dependencies_buf = this.buffers.dependencies.items.ptr[0..num_deps]; + var resolutions_buf = this.buffers.resolutions.items.ptr[0..num_deps]; + + // pre-initialize the dependencies and resolutions to `unset_package_id` + if (Environment.allow_assert) { + @memset(dependencies_buf, Dependency{}); + @memset(resolutions_buf, unset_package_id); + } + + var resolutions = this.packages.items(.resolution); + var metas = this.packages.items(.meta); + var dependencies_list = this.packages.items(.dependencies); + var resolution_list = this.packages.items(.resolutions); + + if (Environment.allow_assert) { + for (resolutions) |r| { + std.debug.assert(r.tag == .uninitialized or r.tag == .workspace); + } + } + + // Root resolution isn't hit through dependency tracing. + resolutions[0] = Resolution.init(.{ .root = {} }); + metas[0].origin = .local; + try this.getOrPutID(0, this.packages.items(.name_hash)[0]); + + // made it longer than max path just in case something stupid happens + var name_checking_buf: [bun.MAX_PATH_BYTES * 2]u8 = undefined; + + // Dependency Linking Phase + package_idx = 0; + var is_first = true; + for (packages_properties.slice()) |entry| { + // this pass is allowed to make more assumptions because we already checked things during + // the counting pass + const pkg = entry.value.?.data.e_object; + + if (pkg.get("link") != null or if (pkg.get("inBundle") orelse pkg.get("extraneous")) |x| x.data == .e_boolean and x.data.e_boolean.value else false) continue; + + const pkg_path = entry.key.?.asString(allocator).?; + + const dependencies_start = dependencies_buf.ptr; + const resolutions_start = resolutions_buf.ptr; + + // this is in a defer because there are two places we end this loop iteration at. + defer { + if (dependencies_start == dependencies_buf.ptr) { + dependencies_list[package_idx] = .{ .len = 0 }; + resolution_list[package_idx] = .{ .len = 0 }; + } else { + // Calculate the offset + length by pointer arithmetic + const len: u32 = @truncate((@intFromPtr(resolutions_buf.ptr) - @intFromPtr(resolutions_start)) / @sizeOf(Install.PackageID)); + if (Environment.allow_assert) { + std.debug.assert(len > 0); + std.debug.assert(len == ((@intFromPtr(dependencies_buf.ptr) - @intFromPtr(dependencies_start)) / @sizeOf(Dependency))); + } + dependencies_list[package_idx] = .{ + .off = @truncate((@intFromPtr(dependencies_start) - @intFromPtr(this.buffers.dependencies.items.ptr)) / @sizeOf(Dependency)), + .len = len, + }; + resolution_list[package_idx] = .{ + .off = @truncate((@intFromPtr(resolutions_start) - @intFromPtr(this.buffers.resolutions.items.ptr)) / @sizeOf(Install.PackageID)), + .len = len, + }; + } + + package_idx += 1; + } + + // a feature no one has heard about: https://docs.npmjs.com/cli/v10/configuring-npm/package-json#bundledependencies + const bundled_dependencies = if (pkg.get("bundleDependencies") orelse pkg.get("bundledDependencies")) |expr| deps: { + if (expr.data == .e_boolean) { + if (expr.data.e_boolean.value) continue; + break :deps null; + } + if (expr.data != .e_array) return error.InvalidNPMLockfile; + const arr: *E.Array = expr.data.e_array; + var map = std.StringArrayHashMapUnmanaged(void){}; + try map.ensureTotalCapacity(allocator, arr.items.len); + for (arr.items.slice()) |item| { + map.putAssumeCapacity(item.asString(allocator) orelse return error.InvalidNPMLockfile, {}); + } + break :deps map; + } else null; + + if (is_first) { + is_first = false; + if (workspace_map) |wksp| { + for (wksp.keys(), wksp.values()) |key, value| { + const entry1 = id_map.get(key) orelse return error.InvalidNPMLockfile; + const name_hash = stringHash(value.name); + const wksp_name = builder.append(String, value.name); + const wksp_path = builder.append(String, key); + dependencies_buf[0] = Dependency{ + .name = wksp_name, + .name_hash = name_hash, + .version = .{ + .tag = .workspace, + .literal = wksp_path, + .value = .{ + .workspace = wksp_path, + }, + }, + .behavior = .{ + .workspace = true, + }, + }; + resolutions_buf[0] = entry1.new_package_id; + + dependencies_buf = dependencies_buf[1..]; + resolutions_buf = resolutions_buf[1..]; + } + } + } + + inline for (dependency_keys) |dep_key| { + if (pkg.get(@tagName(dep_key))) |deps| { + // fetch the peerDependenciesMeta if it exists + // this is only done for peerDependencies, obviously + const peer_dep_meta = if (dep_key == .peerDependencies) + if (pkg.get("peerDependenciesMeta")) |expr| peer_dep_meta: { + if (expr.data != .e_object) return error.InvalidNPMLockfile; + break :peer_dep_meta expr.data.e_object; + } else null + else + void{}; + + if (deps.data != .e_object) return error.InvalidNPMLockfile; + const properties = deps.data.e_object.properties; + + dep_loop: for (properties.slice()) |prop| { + const name_bytes = prop.key.?.asString(this.allocator).?; + if (bundled_dependencies != null and bundled_dependencies.?.getIndex(name_bytes) != null) continue :dep_loop; + + const version_bytes = prop.value.?.asString(this.allocator) orelse return error.InvalidNPMLockfile; + const name_hash = stringHash(name_bytes); + const dep_name = builder.appendWithHash(String, name_bytes, name_hash); + + const dep_version = builder.append(String, version_bytes); + const sliced = dep_version.sliced(this.buffers.string_bytes.items); + + debug("parsing {s}, {s}\n", .{ name_bytes, version_bytes }); + const version = Dependency.parse( + this.allocator, + dep_name, + sliced.slice, + &sliced, + log, + ) orelse { + return error.InvalidNPMLockfile; + }; + debug("-> {s}, {}\n", .{ @tagName(version.tag), version.value }); + + if (Environment.allow_assert) { + std.debug.assert(version.tag != .uninitialized); + } + + const str_node_modules = if (pkg_path.len == 0) "node_modules/" else "/node_modules/"; + const suffix_len = str_node_modules.len + name_bytes.len; + + var buf_len: u32 = @as(u32, @intCast(pkg_path.len + suffix_len)); + if (buf_len > name_checking_buf.len) { + return error.PathTooLong; + } + + bun.copy(u8, name_checking_buf[0..pkg_path.len], pkg_path); + bun.copy(u8, name_checking_buf[pkg_path.len .. pkg_path.len + str_node_modules.len], str_node_modules); + bun.copy(u8, name_checking_buf[pkg_path.len + str_node_modules.len .. pkg_path.len + suffix_len], name_bytes); + + while (true) { + debug("checking {s}", .{name_checking_buf[0..buf_len]}); + if (id_map.get(name_checking_buf[0..buf_len])) |found_| { + var found = found_; + if (found.new_package_id == package_id_is_link) { + // it is a workspace package, resolve from the "link": true entry to the real entry. + const ref_pkg = packages_properties.at(found.old_json_index).value.?.data.e_object; + // the `else` here is technically possible to hit + const resolved_v = ref_pkg.get("resolved") orelse return error.LockfileWorkspaceMissingResolved; + const resolved = resolved_v.asString(this.allocator) orelse return error.InvalidNPMLockfile; + found = (id_map.get(resolved) orelse return error.InvalidNPMLockfile); + } else if (found.new_package_id == package_id_is_bundled) { + debug("skipping bundled dependency {s}", .{name_bytes}); + continue :dep_loop; + } + + const id = found.new_package_id; + + var is_workspace = resolutions[id].tag == .workspace; + + dependencies_buf[0] = Dependency{ + .name = dep_name, + .name_hash = name_hash, + .version = version, + .behavior = .{ + .normal = dep_key == .dependencies, + .optional = dep_key == .optionalDependencies, + .dev = dep_key == .devDependencies, + .peer = dep_key == .peerDependencies, + .workspace = is_workspace, + }, + }; + resolutions_buf[0] = id; + + dependencies_buf = dependencies_buf[1..]; + resolutions_buf = resolutions_buf[1..]; + + // If the package resolution is not set, resolve the target package + // using the information we have from the dependency declaration. + if (resolutions[id].tag == .uninitialized) { + debug("resolving '{s}'", .{name_bytes}); + + const res = resolved: { + const dep_pkg = packages_properties.at(found.old_json_index).value.?.data.e_object; + const npm_resolution = dep_pkg.get("resolved") orelse { + break :resolved Resolution.init(.{ + .folder = builder.append( + String, + packages_properties.at(found.old_json_index).key.?.asString(allocator).?, + ), + }); + }; + const dep_resolved = npm_resolution.asString(this.allocator) orelse return error.InvalidNPMLockfile; + + break :resolved switch (version.tag) { + .uninitialized => std.debug.panic("Version string {s} resolved to `.uninitialized`", .{version_bytes}), + .npm, .dist_tag => res: { + // It is theoretically possible to hit this in a case where the resolved dependency is NOT + // an npm dependency, but that case is so convoluted that it is not worth handling. + // + // Deleting 'package-lock.json' would completely break the installation of the project. + // + // We assume that the given URL is to *some* npm registry, or the resolution is to a workspace package. + // If it is a workspace package, then this branch will not be hit as the resolution was already set earlier. + const dep_actual_version = (dep_pkg.get("version") orelse return error.InvalidNPMLockfile) + .asString(this.allocator) orelse return error.InvalidNPMLockfile; + + const dep_actual_version_str = builder.append(String, dep_actual_version); + const dep_actual_version_sliced = dep_actual_version_str.sliced(this.buffers.string_bytes.items); + + break :res Resolution.init(.{ + .npm = .{ + .url = builder.append(String, dep_resolved), + .version = Semver.Version.parse(dep_actual_version_sliced).version.fill(), + }, + }); + }, + .tarball => if (strings.hasPrefixComptime(dep_resolved, "file:")) + Resolution.init(.{ .local_tarball = builder.append(String, dep_resolved[5..]) }) + else + Resolution.init(.{ .remote_tarball = builder.append(String, dep_resolved) }), + .folder => Resolution.init(.{ .folder = builder.append(String, dep_resolved) }), + // not sure if this is possible to hit + .symlink => Resolution.init(.{ .folder = builder.append(String, dep_resolved) }), + .workspace => workspace: { + var input = builder.append(String, dep_resolved).sliced(this.buffers.string_bytes.items); + if (strings.hasPrefixComptime(input.slice, "workspace:")) { + input = input.sub(input.slice["workspace:".len..]); + } + break :workspace Resolution.init(.{ + .workspace = input.value(), + }); + }, + .git => res: { + const str = (if (strings.hasPrefixComptime(dep_resolved, "git+")) + builder.append(String, dep_resolved[4..]) + else + builder.append(String, dep_resolved)) + .sliced(this.buffers.string_bytes.items); + + const hash_index = strings.lastIndexOfChar(str.slice, '#') orelse return error.InvalidNPMLockfile; + + const commit = str.sub(str.slice[hash_index + 1 ..]).value(); + break :res Resolution.init(.{ + .git = .{ + .owner = version.value.git.owner, + .repo = str.sub(str.slice[0..hash_index]).value(), + .committish = commit, + .resolved = commit, + .package_name = dep_name, + }, + }); + }, + .github => res: { + const str = (if (strings.hasPrefixComptime(dep_resolved, "git+")) + builder.append(String, dep_resolved[4..]) + else + builder.append(String, dep_resolved)) + .sliced(this.buffers.string_bytes.items); + + const hash_index = strings.lastIndexOfChar(str.slice, '#') orelse return error.InvalidNPMLockfile; + + const commit = str.sub(str.slice[hash_index + 1 ..]).value(); + break :res Resolution.init(.{ + .git = .{ + .owner = version.value.github.owner, + .repo = str.sub(str.slice[0..hash_index]).value(), + .committish = commit, + .resolved = commit, + .package_name = dep_name, + }, + }); + }, + }; + }; + debug("-> {}", .{res.fmtForDebug(this.buffers.string_bytes.items)}); + + resolutions[id] = res; + metas[id].origin = switch (res.tag) { + // This works? + .root => .local, + else => .npm, + }; + + try this.getOrPutID(id, this.packages.items(.name_hash)[id]); + } + + continue :dep_loop; + } + // step + if (strings.lastIndexOf(name_checking_buf[0..buf_len -| ("node_modules/".len + name_bytes.len)], "node_modules/")) |idx| { + debug("found 'node_modules/' at {d}", .{idx}); + buf_len = @intCast(idx + "node_modules/".len + name_bytes.len); + bun.copy(u8, name_checking_buf[idx + "node_modules/".len .. idx + "node_modules/".len + name_bytes.len], name_bytes); + } else if (!strings.hasPrefixComptime(name_checking_buf[0..buf_len], "node_modules/")) { + // this is hit if you start from `packages/etc`, from `packages/etc/node_modules/xyz` + // we need to hit the root node_modules + buf_len = @intCast("node_modules/".len + name_bytes.len); + bun.copy(u8, name_checking_buf[0..buf_len], "node_modules/"); + bun.copy(u8, name_checking_buf[buf_len - name_bytes.len .. buf_len], name_bytes); + } else { + // optional peer dependencies can be ... optional + if (dep_key == .peerDependencies) { + if (peer_dep_meta) |o| if (o.get(name_bytes)) |meta| { + if (meta.data != .e_object) return error.InvalidNPMLockfile; + if (meta.data.e_object.get("optional")) |optional| { + if (optional.data != .e_boolean) return error.InvalidNPMLockfile; + if (optional.data.e_boolean.value) { + dependencies_buf[0] = Dependency{ + .name = dep_name, + .name_hash = name_hash, + .version = version, + .behavior = .{ + .normal = dep_key == .dependencies, + .optional = true, + .dev = dep_key == .devDependencies, + .peer = dep_key == .peerDependencies, + .workspace = false, + }, + }; + resolutions_buf[0] = Install.invalid_package_id; + dependencies_buf = dependencies_buf[1..]; + resolutions_buf = resolutions_buf[1..]; + continue :dep_loop; + } + } + }; + } + + // it is technically possible to get a package-lock.json without a dependency. + // it's very unlikely, but possible. when NPM sees this, it essentially doesnt install the package, and treats it like it doesn't exist. + // in test/cli/install/migrate-fixture, you can observe this for `iconv-lite` + debug("could not find package '{s}' in '{s}'", .{ name_bytes, pkg_path }); + continue :dep_loop; + } + } + } + } + } + } + + this.buffers.resolutions.items.len = (@intFromPtr(resolutions_buf.ptr) - @intFromPtr(this.buffers.resolutions.items.ptr)) / @sizeOf(Install.PackageID); + this.buffers.dependencies.items.len = this.buffers.resolutions.items.len; + + // In allow_assert, we prefill this buffer with uninitialized values that we can detect later + // It is our fault if we hit an error here, making it safe to disable in release. + if (Environment.allow_assert) { + std.debug.assert(this.buffers.dependencies.items.len == (@intFromPtr(dependencies_buf.ptr) - @intFromPtr(this.buffers.dependencies.items.ptr)) / @sizeOf(Dependency)); + std.debug.assert(this.buffers.dependencies.items.len <= num_deps); + var crash = false; + for (this.buffers.dependencies.items, 0..) |r, i| { + // 'if behavior is uninitialized' + if (r.behavior.eq(.{})) { + debug("dependency index '{d}' was not set", .{i}); + crash = true; + } + } + for (this.buffers.resolutions.items, 0..) |r, i| { + if (r == unset_package_id) { + debug("resolution index '{d}' was not set", .{i}); + crash = true; + } + } + if (crash) { + std.debug.panic("Assertion failure, see above", .{}); + } + } + + // A package not having a resolution, however, is not our fault. + // This can be triggered by a bad lockfile with extra packages. NPM should trim packages out automatically. + var is_missing_resolutions = false; + for (resolutions, 0..) |r, i| { + if (r.tag == .uninitialized) { + Output.printErrorln("Could not resolve package '{s}' in lockfile.", .{this.packages.items(.name)[i].slice(this.buffers.string_bytes.items)}); + is_missing_resolutions = true; + } else if (Environment.allow_assert) { + // Assertion from appendPackage. If we do this too early it will always fail as we dont have the resolution written + // but after we write all the data, there is no excuse for this to fail. + // + // If this is hit, it means getOrPutID was not called on this package id. Look for where 'resolution[i]' is set + std.debug.assert(this.getPackageID(this.packages.items(.name_hash)[i], null, &r) != null); + } + } + if (is_missing_resolutions) { + return error.NotAllPackagesGotResolved; + } + + // if (Environment.isDebug) { + // const dump_file = try std.fs.cwd().createFileZ("before-clean.json", .{}); + // defer dump_file.close(); + // try std.json.stringify(this, .{ .whitespace = .indent_2 }, dump_file.writer()); + // } + + // This is definitely a memory leak, but it's fine because there is no install api, so this can only be leaked once per process. + // This operation is neccecary because callers of `loadFromDisk` assume the data is written into the passed `this`. + // You'll find that not cleaning the lockfile will cause `bun install` to not actually install anything since it doesnt have any hoisted trees. + this.* = (try this.cleanWithLogger(&[_]Install.PackageManager.UpdateRequest{}, log, false)).*; + + // if (Environment.isDebug) { + // const dump_file = try std.fs.cwd().createFileZ("after-clean.json", .{}); + // defer dump_file.close(); + // try std.json.stringify(this, .{ .whitespace = .indent_2 }, dump_file.writer()); + // } + + if (Environment.allow_assert) { + try this.verifyData(); + } + + this.meta_hash = try this.generateMetaHash(false); + + return LoadFromDiskResult{ .ok = this }; +} + +fn packageNameFromPath(pkg_path: []const u8) []const u8 { + if (pkg_path.len == 0) return ""; + + const pkg_name_start: usize = if (strings.lastIndexOf(pkg_path, "/node_modules/")) |last_index| + last_index + "/node_modules/".len + else if (strings.hasPrefixComptime(pkg_path, "node_modules/")) + "node_modules/".len + else + strings.lastIndexOf(pkg_path, "/") orelse 0; + + return pkg_path[pkg_name_start..]; +} diff --git a/src/install/npm.zig b/src/install/npm.zig index 78d0f6061..75945ba74 100644 --- a/src/install/npm.zig +++ b/src/install/npm.zig @@ -327,12 +327,18 @@ pub const OperatingSystem = enum(u16) { return (@intFromEnum(this) & linux) != 0; } else if (comptime Environment.isMac) { return (@intFromEnum(this) & darwin) != 0; + } else if (comptime Environment.isWindows) { + return (@intFromEnum(this) & win32) != 0; } else { return false; } } - const NameMap = ComptimeStringMap(u16, .{ + pub inline fn has(this: OperatingSystem, other: u16) bool { + return (@intFromEnum(this) & other) != 0; + } + + pub const NameMap = ComptimeStringMap(u16, .{ .{ "aix", aix }, .{ "darwin", darwin }, .{ "freebsd", freebsd }, @@ -383,7 +389,7 @@ pub const Architecture = enum(u16) { pub const all_value: u16 = arm | arm64 | ia32 | mips | mipsel | ppc | ppc64 | s390 | s390x | x32 | x64; - const NameMap = ComptimeStringMap(u16, .{ + pub const NameMap = ComptimeStringMap(u16, .{ .{ "arm", arm }, .{ "arm64", arm64 }, .{ "ia32", ia32 }, @@ -397,6 +403,10 @@ pub const Architecture = enum(u16) { .{ "x64", x64 }, }); + pub inline fn has(this: Architecture, other: u16) bool { + return (@intFromEnum(this) & other) != 0; + } + pub fn isMatch(this: Architecture) bool { if (comptime Environment.isAarch64) { return (@intFromEnum(this) & arm64) != 0; diff --git a/src/install/resolution.zig b/src/install/resolution.zig index 23f3bc5c0..84d43ff1a 100644 --- a/src/install/resolution.zig +++ b/src/install/resolution.zig @@ -15,6 +15,14 @@ pub const Resolution = extern struct { _padding: [7]u8 = .{0} ** 7, value: Value = .{ .uninitialized = {} }, + /// Use like Resolution.init(.{ .npm = VersionedURL{ ... } }) + pub inline fn init(value: anytype) Resolution { + return Resolution{ + .tag = @field(Tag, @typeInfo(@TypeOf(value)).Struct.fields[0].name), + .value = Value.init(value), + }; + } + pub fn order( lhs: *const Resolution, rhs: *const Resolution, @@ -107,18 +115,22 @@ pub const Resolution = extern struct { }), .root => Value.init(.{ .root = {} }), else => { - std.debug.panic("Internal error: unexpected resolution tag:,) {}", .{this.tag}); + std.debug.panic("Internal error: unexpected resolution tag: {}", .{this.tag}); }, }, }; } - pub fn fmt(this: *const Resolution, buf: []const u8) Formatter { - return Formatter{ .resolution = this, .buf = buf }; + pub fn fmt(this: *const Resolution, string_bytes: []const u8) Formatter { + return Formatter{ .resolution = this, .buf = string_bytes }; } - pub fn fmtURL(this: *const Resolution, options: *const PackageManager.Options, buf: []const u8) URLFormatter { - return URLFormatter{ .resolution = this, .buf = buf, .options = options }; + pub fn fmtURL(this: *const Resolution, options: *const PackageManager.Options, string_bytes: []const u8) URLFormatter { + return URLFormatter{ .resolution = this, .buf = string_bytes, .options = options }; + } + + pub fn fmtForDebug(this: *const Resolution, string_bytes: []const u8) DebugFormatter { + return DebugFormatter{ .resolution = this, .buf = string_bytes }; } pub fn eql( @@ -225,6 +237,31 @@ pub const Resolution = extern struct { } }; + pub const DebugFormatter = struct { + resolution: *const Resolution, + buf: []const u8, + + pub fn format(formatter: DebugFormatter, comptime layout: []const u8, opts: std.fmt.FormatOptions, writer: anytype) !void { + try writer.writeAll("Resolution{ ."); + try writer.writeAll(std.enums.tagName(Tag, formatter.resolution.tag) orelse "invalid"); + try writer.writeAll(" = "); + switch (formatter.resolution.tag) { + .npm => try formatter.resolution.value.npm.version.fmt(formatter.buf).format(layout, opts, writer), + .local_tarball => try writer.writeAll(formatter.resolution.value.local_tarball.slice(formatter.buf)), + .folder => try writer.writeAll(formatter.resolution.value.folder.slice(formatter.buf)), + .remote_tarball => try writer.writeAll(formatter.resolution.value.remote_tarball.slice(formatter.buf)), + .git => try formatter.resolution.value.git.formatAs("git+", formatter.buf, layout, opts, writer), + .github => try formatter.resolution.value.github.formatAs("github:", formatter.buf, layout, opts, writer), + .gitlab => try formatter.resolution.value.gitlab.formatAs("gitlab:", formatter.buf, layout, opts, writer), + .workspace => try std.fmt.format(writer, "workspace:{s}", .{formatter.resolution.value.workspace.slice(formatter.buf)}), + .symlink => try std.fmt.format(writer, "link:{s}", .{formatter.resolution.value.symlink.slice(formatter.buf)}), + .single_file_module => try std.fmt.format(writer, "module:{s}", .{formatter.resolution.value.single_file_module.slice(formatter.buf)}), + else => try writer.writeAll("{}"), + } + try writer.writeAll(" }"); + } + }; + pub const Value = extern union { uninitialized: void, root: void, diff --git a/src/resolver/resolver.zig b/src/resolver/resolver.zig index 8835e57ce..4aeda410e 100644 --- a/src/resolver/resolver.zig +++ b/src/resolver/resolver.zig @@ -1679,7 +1679,7 @@ pub const Resolver = struct { // check the global cache directory for a package.json file. var manager = r.getPackageManager(); var dependency_version = Dependency.Version{}; - var dependency_behavior = @as(Dependency.Behavior, @enumFromInt(Dependency.Behavior.normal)); + var dependency_behavior = Dependency.Behavior.normal; var string_buf = esm.version; // const initial_pending_tasks = manager.pending_tasks; |