diff options
author | 2023-10-03 02:17:21 -0700 | |
---|---|---|
committer | 2023-10-03 02:17:21 -0700 | |
commit | 745b6b94ee56cad24d475799690cc9a89957d15b (patch) | |
tree | 634bd13b4fdb1c331538e968258143764a414449 | |
parent | 47651f321ae5eb82686da5d759e9b1b5b12340ad (diff) | |
download | bun-745b6b94ee56cad24d475799690cc9a89957d15b.tar.gz bun-745b6b94ee56cad24d475799690cc9a89957d15b.tar.zst bun-745b6b94ee56cad24d475799690cc9a89957d15b.zip |
Store workspace package versions (#6258)bun-v1.0.4
* Store workspace package versions in the lockfile
* more logging
* wip
* keep information from workspace name array and cache
* hash key
* remove cache, compare workspaces with initially loaded
* uncomment sort
* remove comments
* remove allocation
* package json
* test `bun add <package>` without workspace prefix
* Update test/cli/install/bun-install.test.ts
Co-authored-by: Markus Staab <maggus.staab@googlemail.com>
---------
Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>
Co-authored-by: Markus Staab <maggus.staab@googlemail.com>
-rw-r--r-- | src/identity_context.zig | 10 | ||||
-rw-r--r-- | src/install/install.zig | 41 | ||||
-rw-r--r-- | src/install/lockfile.zig | 238 | ||||
-rw-r--r-- | test/cli/install/bun-install.test.ts | 79 |
4 files changed, 346 insertions, 22 deletions
diff --git a/src/identity_context.zig b/src/identity_context.zig index 7bcfa3dbc..5e0cfe987 100644 --- a/src/identity_context.zig +++ b/src/identity_context.zig @@ -20,4 +20,14 @@ pub const ArrayIdentityContext = struct { pub fn eql(_: @This(), a: u32, b: u32, _: usize) bool { return a == b; } + + pub const U64 = struct { + pub fn hash(_: @This(), key: u64) u32 { + return @truncate(key); + } + + pub fn eql(_: @This(), a: u64, b: u64, _: usize) bool { + return a == b; + } + }; }; diff --git a/src/install/install.zig b/src/install/install.zig index 2663e49a5..65cc00d12 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -1658,6 +1658,9 @@ pub const PackageManager = struct { cpu_count: u32 = 0, package_json_updates: []UpdateRequest = &[_]UpdateRequest{}, + // used for looking up workspaces that aren't loaded into Lockfile.workspace_paths + workspaces: std.StringArrayHashMap(?Semver.Version), + // progress bar stuff when not stack allocated root_progress_node: *std.Progress.Node = undefined, root_download_node: std.Progress.Node = undefined, @@ -2646,6 +2649,27 @@ pub const PackageManager = struct { switch (version.tag) { .npm, .dist_tag => { + if (version.tag == .npm) { + if (this.lockfile.workspace_versions.count() > 0) resolve_from_workspace: { + if (this.lockfile.workspace_versions.get(@truncate(name_hash))) |workspace_version| { + if (version.value.npm.version.satisfies(workspace_version)) { + const root_package = this.lockfile.rootPackage() orelse break :resolve_from_workspace; + const root_dependencies = root_package.dependencies.get(this.lockfile.buffers.dependencies.items); + const root_resolutions = root_package.resolutions.get(this.lockfile.buffers.resolutions.items); + + for (root_dependencies, root_resolutions) |root_dep, workspace_package_id| { + if (workspace_package_id != invalid_package_id and root_dep.version.tag == .workspace and root_dep.name_hash == name_hash) { + return .{ + .package = this.lockfile.packages.get(workspace_package_id), + .is_first_time = false, + }; + } + } + } + } + } + } + // Resolve the version from the loaded NPM manifest const manifest = this.manifests.getPtr(name_hash) orelse return null; // manifest might still be downloading. This feels unreliable. const find_result: Npm.PackageManifest.FindResult = switch (version.tag) { @@ -5284,6 +5308,8 @@ pub const PackageManager = struct { bun.copy(u8, &cwd_buf, original_cwd); + var workspace_names = Package.WorkspaceMap.init(ctx.allocator); + // Step 1. Find the nearest package.json directory // // We will walk up from the cwd, trying to find the nearest package.json file. @@ -5332,8 +5358,6 @@ pub const PackageManager = struct { initializeStore(); const json = try json_parser.ParseJSONUTF8(&json_source, ctx.log, ctx.allocator); if (json.asProperty("workspaces")) |prop| { - var workspace_names = Package.WorkspaceMap.init(ctx.allocator); - defer workspace_names.deinit(); const json_array = switch (prop.expr.data) { .e_array => |arr| arr, .e_object => |obj| if (obj.get("packages")) |packages| switch (packages.data) { @@ -5344,7 +5368,7 @@ pub const PackageManager = struct { }; var log = logger.Log.init(ctx.allocator); defer log.deinit(); - _ = Package.processWorkspaceNamesArray( + const workspace_packages_count = Package.processWorkspaceNamesArray( &workspace_names, ctx.allocator, &log, @@ -5353,6 +5377,7 @@ pub const PackageManager = struct { prop.loc, null, ) catch break; + _ = workspace_packages_count; for (workspace_names.keys()) |path| { if (strings.eql(child_cwd, path)) { fs.top_level_dir = parent; @@ -5415,6 +5440,13 @@ pub const PackageManager = struct { } else |_| {} } + var workspaces = std.StringArrayHashMap(?Semver.Version).init(ctx.allocator); + for (workspace_names.values()) |entry| { + try workspaces.put(entry.name, entry.version); + } + + workspace_names.map.deinit(); + var manager = &instance; // var progress = Progress{}; // var node = progress.start(name: []const u8, estimated_total_items: usize) @@ -5433,6 +5465,7 @@ pub const PackageManager = struct { .lockfile = undefined, .root_package_json_file = package_json_file, .waiter = try Waker.init(ctx.allocator), + .workspaces = workspaces, // .progress }; manager.lockfile = try ctx.allocator.create(Lockfile); @@ -5511,6 +5544,7 @@ pub const PackageManager = struct { .lockfile = undefined, .root_package_json_file = undefined, .waiter = try Waker.init(allocator), + .workspaces = std.StringArrayHashMap(?Semver.Version).init(allocator), }; manager.lockfile = try allocator.create(Lockfile); @@ -7753,6 +7787,7 @@ pub const PackageManager = struct { _ = manager.getCacheDirectory(); _ = manager.getTemporaryDirectory(); + while (counter_i < changes) : (counter_i += 1) { if (mapping[counter_i] == invalid_package_id) { const dependency_i = counter_i + off; diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index 47bae0735..acd40b42e 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -84,9 +84,9 @@ const PackageJSON = @import("../resolver/package_json.zig").PackageJSON; const MetaHash = [std.crypto.hash.sha2.Sha512256.digest_length]u8; const zero_hash = std.mem.zeroes(MetaHash); -const NameHashMap = std.ArrayHashMapUnmanaged(u32, String, ArrayIdentityContext, false); -const NameHashSet = std.ArrayHashMapUnmanaged(u32, void, ArrayIdentityContext, false); -const VersionHashMap = std.ArrayHashMapUnmanaged(u32, Semver.Version, ArrayIdentityContext, false); +pub const NameHashMap = std.ArrayHashMapUnmanaged(PackageNameHash, String, ArrayIdentityContext.U64, false); +pub const NameHashSet = std.ArrayHashMapUnmanaged(u32, void, ArrayIdentityContext, false); +pub const VersionHashMap = std.ArrayHashMapUnmanaged(PackageNameHash, Semver.Version, ArrayIdentityContext.U64, false); const assertNoUninitializedPadding = @import("./padding_checker.zig").assertNoUninitializedPadding; @@ -703,7 +703,7 @@ pub fn cleanWithLogger( // } // } - var new = try old.allocator.create(Lockfile); + var new: *Lockfile = try old.allocator.create(Lockfile); try new.initEmpty( old.allocator, ); @@ -730,12 +730,73 @@ pub fn cleanWithLogger( .clone_queue = clone_queue_, .log = log, }; + // try clone_queue.ensureUnusedCapacity(root.dependencies.len); _ = try root.clone(old, new, package_id_mapping, &cloner); + // Clone workspace_paths and workspace_versions at the end. + if (old.workspace_paths.count() > 0 or old.workspace_versions.count() > 0) { + try new.workspace_paths.ensureTotalCapacity(z_allocator, old.workspace_paths.count()); + try new.workspace_versions.ensureTotalCapacity(z_allocator, old.workspace_versions.count()); + + var workspace_paths_builder = new.stringBuilder(); + + const WorkspacePathSorter = struct { + string_buf: []const u8, + entries: NameHashMap.DataList, + + pub fn lessThan(sorter: @This(), a: usize, b: usize) bool { + const left = sorter.entries.items(.value)[a]; + const right = sorter.entries.items(.value)[b]; + return strings.order(left.slice(sorter.string_buf), right.slice(sorter.string_buf)) == .lt; + } + }; + + // Sort by name for determinism + old.workspace_paths.sort(WorkspacePathSorter{ + .entries = old.workspace_paths.entries, + .string_buf = old.buffers.string_bytes.items, + }); + + for (old.workspace_paths.values()) |*path| { + workspace_paths_builder.count(old.str(path)); + } + const versions: []const Semver.Version = old.workspace_versions.values(); + for (versions) |version| { + version.count(old.buffers.string_bytes.items, @TypeOf(&workspace_paths_builder), &workspace_paths_builder); + } + + try workspace_paths_builder.allocate(); + + new.workspace_paths.entries.len = old.workspace_paths.entries.len; + + for (old.workspace_paths.values(), new.workspace_paths.values()) |*src, *dest| { + dest.* = workspace_paths_builder.append(String, old.str(src)); + } + @memcpy( + new.workspace_paths.keys(), + old.workspace_paths.keys(), + ); + + try new.workspace_versions.ensureTotalCapacity(z_allocator, old.workspace_versions.count()); + new.workspace_versions.entries.len = old.workspace_versions.entries.len; + for (versions, new.workspace_versions.values()) |src, *dest| { + dest.* = src.clone(old.buffers.string_bytes.items, @TypeOf(&workspace_paths_builder), &workspace_paths_builder); + } + + @memcpy( + new.workspace_versions.keys(), + old.workspace_versions.keys(), + ); + + workspace_paths_builder.clamp(); + + try new.workspace_versions.reIndex(z_allocator); + try new.workspace_paths.reIndex(z_allocator); + } + // When you run `"bun add react" // This is where we update it in the lockfile from "latest" to "^17.0.2" - try cloner.flush(); // Don't allow invalid memory to happen @@ -2427,6 +2488,7 @@ pub const Package = extern struct { const from_deps = from.dependencies.get(from_lockfile.buffers.dependencies.items); const from_resolutions = from.resolutions.get(from_lockfile.buffers.resolutions.items); var to_i: usize = 0; + var skipped_workspaces: usize = 0; for (from_deps, 0..) |*from_dep, i| { found: { @@ -2445,6 +2507,11 @@ pub const Package = extern struct { if (from_dep.name_hash == to_deps[to_i].name_hash) break :found; } + if (PackageManager.instance.workspaces.contains(from_lockfile.str(&from_dep.name))) { + skipped_workspaces += 1; + continue; + } + // We found a removed dependency! // We don't need to remove it // It will be cleaned up later @@ -2470,7 +2537,7 @@ pub const Package = extern struct { if (id_mapping) |mapping| { const version = to_deps[to_i].version; if (switch (version.tag) { - .workspace => if (to_lockfile.workspace_paths.getPtr(@truncate(from_dep.name_hash))) |path_ptr| brk: { + .workspace => if (to_lockfile.workspace_paths.getPtr(from_dep.name_hash)) |path_ptr| brk: { const path = to_lockfile.str(path_ptr); var file = std.fs.cwd().openFile(Path.join( &[_]string{ path, "package.json" }, @@ -2512,7 +2579,7 @@ pub const Package = extern struct { summary.update += 1; } - summary.add = @truncate(to_deps.len - (from_deps.len - summary.remove)); + summary.add = @truncate((to_deps.len + skipped_workspaces) - (from_deps.len - summary.remove)); inline for (Package.Scripts.Hooks) |hook| { if (!@field(to.scripts, hook).eql( @@ -2558,7 +2625,6 @@ pub const Package = extern struct { comptime features: Features, ) !void { initializeStore(); - const json = json_parser.ParseJSONUTF8(&source, log, allocator) catch |err| { switch (Output.enable_ansi_colors) { inline else => |enable_ansi_colors| { @@ -2628,8 +2694,20 @@ pub const Package = extern struct { } else external_alias.hash, else => external_alias.hash, }; - const workspace_path = if (comptime tag == null) lockfile.workspace_paths.get(@truncate(name_hash)) else null; - const workspace_version = if (comptime tag == null) lockfile.workspace_versions.get(@truncate(name_hash)) else workspace_ver; + + var workspace_path: ?String = null; + var workspace_version = workspace_ver; + if (comptime tag == null) { + workspace_path = lockfile.workspace_paths.get(name_hash); + workspace_version = lockfile.workspace_versions.get(name_hash); + + if (workspace_path == null or workspace_version == null) { + if (PackageManager.instance.workspaces.get(lockfile.str(&external_alias.value))) |_workspace_version| { + workspace_path = external_alias.value; + workspace_version = _workspace_version; + } + } + } switch (dependency_version.tag) { .folder => { @@ -2707,7 +2785,7 @@ pub const Package = extern struct { dependency_version.literal = path; dependency_version.value.workspace = path; - var workspace_entry = try lockfile.workspace_paths.getOrPut(allocator, @truncate(name_hash)); + var workspace_entry = try lockfile.workspace_paths.getOrPut(allocator, name_hash); if (workspace_entry.found_existing) { if (strings.eqlComptime(workspace, "*")) return null; @@ -2724,7 +2802,7 @@ pub const Package = extern struct { workspace_entry.value_ptr.* = path; if (workspace_version) |ver| { - try lockfile.workspace_versions.put(allocator, @truncate(name_hash), ver); + try lockfile.workspace_versions.put(allocator, name_hash, ver); for (package_dependencies[0..dependencies_count]) |*package_dep| { if (switch (package_dep.version.tag) { @@ -3172,7 +3250,7 @@ pub const Package = extern struct { a: usize, b: usize, ) bool { - return std.mem.order(u8, self.values[a].name, self.values[b].name) == .lt; + return strings.order(self.values[a].name, self.values[b].name) == .lt; } }{ .values = workspace_names.values(), @@ -3587,6 +3665,11 @@ pub const Package = extern struct { )) |dep| { package_dependencies[total_dependencies_count] = dep; total_dependencies_count += 1; + + try lockfile.workspace_paths.put(allocator, external_name.hash, dep.version.value.workspace); + if (entry.version) |v| { + try lockfile.workspace_versions.put(allocator, external_name.hash, v); + } } } } else { @@ -4137,6 +4220,8 @@ pub const Serializer = struct { pub const version = "bun-lockfile-format-v0\n"; const header_bytes: string = "#!/usr/bin/env bun\n" ++ version; + const has_workspace_package_ids_tag: u64 = @bitCast([_]u8{ 'w', 'O', 'r', 'K', 's', 'P', 'a', 'C' }); + pub fn save(this: *Lockfile, comptime StreamType: type, stream: StreamType) !void { var old_package_list = this.packages; this.packages = try this.packages.clone(z_allocator); @@ -4154,6 +4239,49 @@ pub const Serializer = struct { try Lockfile.Package.Serializer.save(this.packages, StreamType, stream, @TypeOf(&writer), &writer); try Lockfile.Buffers.save(this.buffers, z_allocator, StreamType, stream, @TypeOf(&writer), &writer); try writer.writeIntLittle(u64, 0); + + // < Bun v1.0.4 stopped right here when reading the lockfile + // So we add an extra 8 byte tag to say "hey, there's more data here" + if (this.workspace_versions.count() > 0) { + try writer.writeAll(std.mem.asBytes(&has_workspace_package_ids_tag)); + + // We need to track the "version" field in "package.json" of workspace member packages + // We do not necessarily have that in the Resolution struct. So we store it here. + try Lockfile.Buffers.writeArray( + StreamType, + stream, + @TypeOf(&writer), + &writer, + []PackageNameHash, + this.workspace_versions.keys(), + ); + try Lockfile.Buffers.writeArray( + StreamType, + stream, + @TypeOf(&writer), + &writer, + []Semver.Version, + this.workspace_versions.values(), + ); + + try Lockfile.Buffers.writeArray( + StreamType, + stream, + @TypeOf(&writer), + &writer, + []PackageNameHash, + this.workspace_paths.keys(), + ); + try Lockfile.Buffers.writeArray( + StreamType, + stream, + @TypeOf(&writer), + &writer, + []String, + this.workspace_paths.values(), + ); + } + const end = try stream.getPos(); try writer.writeAll(&alignment_bytes_to_repeat_buffer); @@ -4200,29 +4328,101 @@ pub const Serializer = struct { return error.@"Lockfile is malformed (expected 0 at the end)"; } - if (comptime Environment.allow_assert) std.debug.assert(stream.pos == total_buffer_size); + var has_workspace_name_hashes = false; + // < Bun v1.0.4 stopped right here when reading the lockfile + // So we add an extra 8 byte tag to say "hey, there's more data here" + { + const remaining_in_buffer = total_buffer_size -| stream.pos; + + if (remaining_in_buffer > 8 and total_buffer_size <= stream.buffer.len) { + const next_num = try reader.readIntLittle(u64); + if (next_num == has_workspace_package_ids_tag) { + { + var workspace_package_name_hashes = try Lockfile.Buffers.readArray( + stream, + allocator, + std.ArrayListUnmanaged(PackageNameHash), + ); + defer workspace_package_name_hashes.deinit(allocator); + + var workspace_versions_list = try Lockfile.Buffers.readArray( + stream, + allocator, + std.ArrayListUnmanaged(Semver.Version), + ); + comptime { + if (PackageNameHash != @TypeOf((VersionHashMap.KV{ .key = undefined, .value = undefined }).key)) { + @compileError("VersionHashMap must be in sync with serialization"); + } + if (Semver.Version != @TypeOf((VersionHashMap.KV{ .key = undefined, .value = undefined }).value)) { + @compileError("VersionHashMap must be in sync with serialization"); + } + } + defer workspace_versions_list.deinit(allocator); + try lockfile.workspace_versions.ensureTotalCapacity(allocator, workspace_versions_list.items.len); + lockfile.workspace_versions.entries.len = workspace_versions_list.items.len; + @memcpy(lockfile.workspace_versions.keys(), workspace_package_name_hashes.items); + @memcpy(lockfile.workspace_versions.values(), workspace_versions_list.items); + try lockfile.workspace_versions.reIndex(allocator); + } + + { + var workspace_paths_hashes = try Lockfile.Buffers.readArray( + stream, + allocator, + std.ArrayListUnmanaged(PackageNameHash), + ); + defer workspace_paths_hashes.deinit(allocator); + var workspace_paths_strings = try Lockfile.Buffers.readArray( + stream, + allocator, + std.ArrayListUnmanaged(String), + ); + defer workspace_paths_strings.deinit(allocator); + + try lockfile.workspace_paths.ensureTotalCapacity(allocator, workspace_paths_strings.items.len); + + lockfile.workspace_paths.entries.len = workspace_paths_strings.items.len; + @memcpy(lockfile.workspace_paths.keys(), workspace_paths_hashes.items); + @memcpy(lockfile.workspace_paths.values(), workspace_paths_strings.items); + try lockfile.workspace_paths.reIndex(allocator); + } + } else { + stream.pos -= 8; + } + } + } lockfile.scratch = Lockfile.Scratch.init(allocator); + lockfile.package_index = PackageIndex.Map.initContext(allocator, .{}); + lockfile.string_pool = StringPool.initContext(allocator, .{}); + try lockfile.package_index.ensureTotalCapacity(@as(u32, @truncate(lockfile.packages.len))); - { - lockfile.package_index = PackageIndex.Map.initContext(allocator, .{}); - lockfile.string_pool = StringPool.initContext(allocator, .{}); - try lockfile.package_index.ensureTotalCapacity(@as(u32, @truncate(lockfile.packages.len))); + if (!has_workspace_name_hashes) { const slice = lockfile.packages.slice(); const name_hashes = slice.items(.name_hash); const resolutions = slice.items(.resolution); for (name_hashes, resolutions, 0..) |name_hash, resolution, id| { try lockfile.getOrPutID(@as(PackageID, @truncate(id)), name_hash); + // compatibility with < Bun v1.0.4 switch (resolution.tag) { .workspace => { - try lockfile.workspace_paths.put(allocator, @as(u32, @truncate(name_hash)), resolution.value.workspace); + try lockfile.workspace_paths.put(allocator, name_hash, resolution.value.workspace); }, else => {}, } } + } else { + const slice = lockfile.packages.slice(); + const name_hashes = slice.items(.name_hash); + for (name_hashes, 0..) |name_hash, id| { + try lockfile.getOrPutID(@as(PackageID, @truncate(id)), name_hash); + } } + if (comptime Environment.allow_assert) std.debug.assert(stream.pos == total_buffer_size); + // const end = try reader.readIntLittle(u64); } }; diff --git a/test/cli/install/bun-install.test.ts b/test/cli/install/bun-install.test.ts index 19a5851a1..ce61268a0 100644 --- a/test/cli/install/bun-install.test.ts +++ b/test/cli/install/bun-install.test.ts @@ -5550,6 +5550,85 @@ it("should handle installing packages from inside a workspace with `*`", async ( await access(join(package_dir, "bun.lockb")); }); +it("should handle installing packages from inside a workspace without prefix", async () => { + await writeFile( + join(package_dir, "package.json"), + JSON.stringify({ + name: "main", + workspaces: ["packages/*"], + private: true, + }), + ); + await mkdir(join(package_dir, "packages", "p1"), { recursive: true }); + const p1_package = JSON.stringify({ + name: "p1", + version: "0.0.1", + dependencies: { + p2: "0.1.0", + }, + }); + await writeFile(join(package_dir, "packages", "p1", "package.json"), p1_package); + + await mkdir(join(package_dir, "packages", "p2")); + const p2_package = JSON.stringify({ + name: "p2", + version: "0.1.0", + }); + await writeFile(join(package_dir, "packages", "p2", "package.json"), p2_package); + + const urls: string[] = []; + setHandler(dummyRegistry(urls)); + + const { + stdout: stdout1, + stderr: stderr1, + exited: exited1, + } = spawn({ + cmd: [bunExe(), "install"], + cwd: join(package_dir, "packages", "p1"), + stdout: null, + stdin: "pipe", + stderr: "pipe", + env, + }); + expect(stderr1).toBeDefined(); + const err1 = await new Response(stderr1).text(); + expect(err1).toContain("Saved lockfile"); + expect(stdout1).toBeDefined(); + const out1 = await new Response(stdout1).text(); + expect(out1.replace(/\s*\[[0-9\.]+m?s\]\s*$/, "").split(/\r?\n/)).toEqual([ + " + p1@workspace:packages/p1", + " + p2@workspace:packages/p2", + "", + " 2 packages installed", + ]); + expect(await exited1).toBe(0); + expect(requested).toBe(0); + await access(join(package_dir, "bun.lockb")); + + const { + stdout: stdout2, + stderr: stderr2, + exited: exited2, + } = spawn({ + cmd: [bunExe(), "install", "bar"], + cwd: join(package_dir, "packages", "p1"), + stdout: null, + stdin: "pipe", + stderr: "pipe", + env, + }); + expect(stderr1).toBeDefined(); + const err2 = await new Response(stderr2).text(); + expect(err2).toContain("Saved lockfile"); + expect(stdout2).toBeDefined(); + const out2 = await new Response(stdout2).text(); + expect(out2).toContain("installed bar"); + expect(await exited2).toBe(0); + expect(urls.sort()).toEqual([`${root_url}/bar`, `${root_url}/bar-0.0.2.tgz`]); + await access(join(package_dir, "bun.lockb")); +}); + it("should handle installing packages inside workspaces with difference versions", async () => { let package_jsons = [ JSON.stringify({ |