diff options
Diffstat (limited to 'src/install/lockfile.zig')
| -rw-r--r-- | src/install/lockfile.zig | 398 |
1 files changed, 397 insertions, 1 deletions
diff --git a/src/install/lockfile.zig b/src/install/lockfile.zig index d9e459b3c..87d19742b 100644 --- a/src/install/lockfile.zig +++ b/src/install/lockfile.zig @@ -112,6 +112,8 @@ trusted_dependencies: NameHashSet = .{}, workspace_paths: NameHashMap = .{}, workspace_versions: VersionHashMap = .{}, +overrides: OverrideMap = .{}, + const Stream = std.io.FixedBufferStream([]u8); pub const default_filename = "bun.lockb"; @@ -209,6 +211,7 @@ pub fn loadFromBytes(this: *Lockfile, buf: []u8, allocator: Allocator, log: *log this.trusted_dependencies = .{}; this.workspace_paths = .{}; this.workspace_versions = .{}; + this.overrides = .{}; Lockfile.Serializer.load(this, &stream, allocator, log) catch |err| { return LoadFromDiskResult{ .err = .{ .step = .parse_file, .value = err } }; @@ -731,6 +734,13 @@ pub fn cleanWithLogger( old.scratch.dependency_list_queue.head = 0; + { + var builder = new.stringBuilder(); + old.overrides.count(old, &builder); + try builder.allocate(); + new.overrides = try old.overrides.clone(old, new, &builder); + } + // Step 1. Recreate the lockfile with only the packages that are still alive const root = old.rootPackage() orelse return error.NoPackage; @@ -843,6 +853,7 @@ pub fn cleanWithLogger( } new.trusted_dependencies = old_trusted_dependencies; new.scripts = old_scripts; + return new; } @@ -1885,6 +1896,299 @@ pub const PackageIndex = struct { }; }; +pub const OverrideMap = struct { + const debug = Output.scoped(.OverrideMap, false); + + map: std.ArrayHashMapUnmanaged(PackageNameHash, Dependency, ArrayIdentityContext.U64, false) = .{}, + + /// In the future, this `get` function should handle multi-level resolutions. This is difficult right + /// now because given a Dependency ID, there is no fast way to trace it to it's package. + /// + /// A potential approach is to add another buffer to the lockfile that maps Dependency ID to Package ID, + /// and from there `OverrideMap.map` can have a union as the value, where the union is between "override all" + /// and "here is a list of overrides depending on the package that imported" similar to PackageIndex above. + pub fn get(this: *const OverrideMap, name_hash: PackageNameHash) ?Dependency.Version { + debug("looking up override for {x}", .{name_hash}); + return if (this.map.get(name_hash)) |dep| + dep.version + else + null; + } + + pub fn deinit(this: *OverrideMap, allocator: Allocator) void { + this.map.deinit(allocator); + } + + pub fn count(this: *OverrideMap, lockfile: *Lockfile, builder: *Lockfile.StringBuilder) void { + for (this.map.values()) |dep| { + dep.count(lockfile.buffers.string_bytes.items, @TypeOf(builder), builder); + } + } + + pub fn clone(this: *OverrideMap, old_lockfile: *Lockfile, new_lockfile: *Lockfile, new_builder: *Lockfile.StringBuilder) !OverrideMap { + var new = OverrideMap{}; + try new.map.ensureTotalCapacity(new_lockfile.allocator, this.map.entries.len); + + for (this.map.keys(), this.map.values()) |k, v| { + new.map.putAssumeCapacity( + k, + try v.clone(old_lockfile.buffers.string_bytes.items, @TypeOf(new_builder), new_builder), + ); + } + + return new; + } + + // the rest of this struct is expression parsing code: + + pub fn parseCount( + _: *OverrideMap, + lockfile: *Lockfile, + expr: Expr, + builder: *Lockfile.StringBuilder, + ) void { + if (expr.asProperty("overrides")) |overrides| { + if (overrides.expr.data != .e_object) + return; + + for (overrides.expr.data.e_object.properties.slice()) |entry| { + builder.count(entry.key.?.asString(lockfile.allocator).?); + switch (entry.value.?.data) { + .e_string => |s| { + builder.count(s.slice(lockfile.allocator)); + }, + .e_object => { + if (entry.value.?.asProperty(".")) |dot| { + if (dot.expr.asString(lockfile.allocator)) |s| { + builder.count(s); + } + } + }, + else => {}, + } + } + } else if (expr.asProperty("resolutions")) |resolutions| { + if (resolutions.expr.data != .e_object) + return; + + for (resolutions.expr.data.e_object.properties.slice()) |entry| { + builder.count(entry.key.?.asString(lockfile.allocator).?); + builder.count(entry.value.?.asString(lockfile.allocator) orelse continue); + } + } + } + + /// Given a package json expression, detect and parse override configuration into the given override map. + /// It is assumed the input map is uninitialized (zero entries) + pub fn parseAppend( + this: *OverrideMap, + lockfile: *Lockfile, + root_package: *Lockfile.Package, + log: *logger.Log, + json_source: logger.Source, + expr: Expr, + builder: *Lockfile.StringBuilder, + ) !void { + if (Environment.allow_assert) { + std.debug.assert(this.map.entries.len == 0); // only call parse once + } + if (expr.asProperty("overrides")) |overrides| { + try this.parseFromOverrides(lockfile, root_package, json_source, log, overrides.expr, builder); + } else if (expr.asProperty("resolutions")) |resolutions| { + try this.parseFromResolutions(lockfile, root_package, json_source, log, resolutions.expr, builder); + } + debug("parsed {d} overrides", .{this.map.entries.len}); + } + + /// https://docs.npmjs.com/cli/v9/configuring-npm/package-json#overrides + pub fn parseFromOverrides( + this: *OverrideMap, + lockfile: *Lockfile, + root_package: *Lockfile.Package, + source: logger.Source, + log: *logger.Log, + expr: Expr, + builder: *Lockfile.StringBuilder, + ) !void { + if (expr.data != .e_object) { + try log.addWarningFmt(&source, expr.loc, lockfile.allocator, "\"overrides\" must be an object", .{}); + return error.Invalid; + } + + try this.map.ensureUnusedCapacity(lockfile.allocator, expr.data.e_object.properties.len); + + for (expr.data.e_object.properties.slice()) |prop| { + const key = prop.key.?; + var k = key.asString(lockfile.allocator).?; + if (k.len == 0) { + try log.addWarningFmt(&source, key.loc, lockfile.allocator, "Missing overridden package name", .{}); + continue; + } + + const name_hash = String.Builder.stringHash(k); + + const value = value: { + // for one level deep, we will only support a string and { ".": value } + const value_expr = prop.value.?; + if (value_expr.data == .e_string) { + break :value value_expr; + } else if (value_expr.data == .e_object) { + if (value_expr.asProperty(".")) |dot| { + if (dot.expr.data == .e_string) { + if (value_expr.data.e_object.properties.len > 1) { + try log.addWarningFmt(&source, value_expr.loc, lockfile.allocator, "Bun currently does not support nested \"overrides\"", .{}); + } + break :value dot.expr; + } else { + try log.addWarningFmt(&source, value_expr.loc, lockfile.allocator, "Invalid override value for \"{s}\"", .{k}); + continue; + } + } else { + try log.addWarningFmt(&source, value_expr.loc, lockfile.allocator, "Bun currently does not support nested \"overrides\"", .{}); + continue; + } + } + try log.addWarningFmt(&source, value_expr.loc, lockfile.allocator, "Invalid override value for \"{s}\"", .{k}); + continue; + }; + + if (try parseOverrideValue( + "override", + lockfile, + root_package, + source, + value.loc, + log, + k, + value.data.e_string.slice(lockfile.allocator), + builder, + )) |version| { + this.map.putAssumeCapacity(name_hash, version); + } + } + } + + /// yarn classic: https://classic.yarnpkg.com/lang/en/docs/selective-version-resolutions/ + /// yarn berry: https://yarnpkg.com/configuration/manifest#resolutions + pub fn parseFromResolutions( + this: *OverrideMap, + lockfile: *Lockfile, + root_package: *Lockfile.Package, + source: logger.Source, + log: *logger.Log, + expr: Expr, + builder: *Lockfile.StringBuilder, + ) !void { + if (expr.data != .e_object) { + try log.addWarningFmt(&source, expr.loc, lockfile.allocator, "\"resolutions\" must be an object with string values", .{}); + return; + } + try this.map.ensureUnusedCapacity(lockfile.allocator, expr.data.e_object.properties.len); + for (expr.data.e_object.properties.slice()) |prop| { + const key = prop.key.?; + var k = key.asString(lockfile.allocator).?; + if (strings.hasPrefixComptime(k, "**/")) + k = k[3..]; + if (k.len == 0) { + try log.addWarningFmt(&source, key.loc, lockfile.allocator, "Missing resolution package name", .{}); + continue; + } + const value = prop.value.?; + if (value.data != .e_string) { + try log.addWarningFmt(&source, key.loc, lockfile.allocator, "Expected string value for resolution \"{s}\"", .{k}); + continue; + } + // currently we only support one level deep, so we should error if there are more than one + // - "foo/bar": + // - "@namespace/hello/world" + if (k[0] == '@') { + const first_slash = strings.indexOfChar(k, '/') orelse { + try log.addWarningFmt(&source, key.loc, lockfile.allocator, "Invalid package name \"{s}\"", .{k}); + continue; + }; + if (strings.indexOfChar(k[first_slash + 1 ..], '/') != null) { + try log.addWarningFmt(&source, key.loc, lockfile.allocator, "Bun currently does not support nested \"resolutions\"", .{}); + continue; + } + } else if (strings.indexOfChar(k, '/') != null) { + try log.addWarningFmt(&source, key.loc, lockfile.allocator, "Bun currently does not support nested \"resolutions\"", .{}); + continue; + } + + if (try parseOverrideValue( + "resolution", + lockfile, + root_package, + source, + value.loc, + log, + k, + value.data.e_string.data, + builder, + )) |version| { + const name_hash = String.Builder.stringHash(k); + this.map.putAssumeCapacity(name_hash, version); + } + } + } + + pub fn parseOverrideValue( + comptime field: []const u8, + lockfile: *Lockfile, + root_package: *Lockfile.Package, + source: logger.Source, + loc: logger.Loc, + log: *logger.Log, + key: []const u8, + value: []const u8, + builder: *Lockfile.StringBuilder, + ) !?Dependency { + if (value.len == 0) { + try log.addWarningFmt(&source, loc, lockfile.allocator, "Missing " ++ field ++ " value", .{}); + return null; + } + + // "Overrides may also be defined as a reference to a spec for a direct dependency + // by prefixing the name of the package you wish the version to match with a `$`" + // https://docs.npmjs.com/cli/v9/configuring-npm/package-json#overrides + // This is why a `*Lockfile.Package` is needed here. + if (value[0] == '$') { + const ref_name = value[1..]; + // This is fine for this string to not share the string pool, because it's only used for .eql() + const ref_name_str = String.init(ref_name, ref_name); + const pkg_deps: []const Dependency = root_package.dependencies.get(lockfile.buffers.dependencies.items); + for (pkg_deps) |dep| { + if (dep.name.eql(ref_name_str, lockfile.buffers.string_bytes.items, ref_name)) { + return dep; + } + } + try log.addWarningFmt(&source, loc, lockfile.allocator, "Could not resolve " ++ field ++ " \"{s}\" (you need \"{s}\" in your dependencies)", .{ value, ref_name }); + return null; + } + + const literalString = builder.append(String, value); + const literalSliced = literalString.sliced(lockfile.buffers.string_bytes.items); + + const name_hash = String.Builder.stringHash(key); + const name = builder.appendWithHash(String, key, name_hash); + + return Dependency{ + .name = name, + .name_hash = name_hash, + .version = Dependency.parse( + lockfile.allocator, + name, + literalSliced.slice, + &literalSliced, + log, + ) orelse { + try log.addWarningFmt(&source, loc, lockfile.allocator, "Invalid " ++ field ++ " value \"{s}\"", .{value}); + return null; + }, + }; + } +}; + pub const FormatVersion = enum(u32) { v0 = 0, // bun v0.0.x - bun v0.1.6 @@ -2508,6 +2812,7 @@ pub const Package = extern struct { add: u32 = 0, remove: u32 = 0, update: u32 = 0, + overrides_changed: bool = false, pub inline fn sum(this: *Summary, that: Summary) void { this.add += that.add; @@ -2516,7 +2821,7 @@ pub const Package = extern struct { } pub inline fn hasDiffs(this: Summary) bool { - return this.add > 0 or this.remove > 0 or this.update > 0; + return this.add > 0 or this.remove > 0 or this.update > 0 or this.overrides_changed; } }; @@ -2537,6 +2842,22 @@ pub const Package = extern struct { var to_i: usize = 0; var skipped_workspaces: usize = 0; + if (from_lockfile.overrides.map.count() != to_lockfile.overrides.map.count()) { + summary.overrides_changed = true; + } else { + for ( + from_lockfile.overrides.map.keys(), + from_lockfile.overrides.map.values(), + to_lockfile.overrides.map.keys(), + to_lockfile.overrides.map.values(), + ) |from_k, *from_override, to_k, *to_override| { + if ((from_k != to_k) or (!from_override.eql(to_override, from_lockfile.buffers.string_bytes.items, to_lockfile.buffers.string_bytes.items))) { + summary.overrides_changed = true; + break; + } + } + } + for (from_deps, 0..) |*from_dep, i| { found: { const prev_i = to_i; @@ -3547,6 +3868,10 @@ pub const Package = extern struct { } } + if (comptime features.is_main) { + lockfile.overrides.parseCount(lockfile, json, &string_builder); + } + try string_builder.allocate(); try lockfile.buffers.dependencies.ensureUnusedCapacity(lockfile.allocator, total_dependencies_count); try lockfile.buffers.resolutions.ensureUnusedCapacity(lockfile.allocator, total_dependencies_count); @@ -3768,6 +4093,11 @@ pub const Package = extern struct { lockfile.buffers.dependencies.items = lockfile.buffers.dependencies.items.ptr[0..new_len]; lockfile.buffers.resolutions.items = lockfile.buffers.resolutions.items.ptr[0..new_len]; + // This function depends on package.dependencies being set, so it is done at the very end. + if (comptime features.is_main) { + try lockfile.overrides.parseAppend(lockfile, package, log, source, json, &string_builder); + } + string_builder.clamp(); } @@ -3969,6 +4299,7 @@ pub fn deinit(this: *Lockfile) void { this.trusted_dependencies.deinit(this.allocator); this.workspace_paths.deinit(this.allocator); this.workspace_versions.deinit(this.allocator); + this.overrides.deinit(this.allocator); } const Buffers = struct { @@ -4273,6 +4604,7 @@ pub const Serializer = struct { const has_workspace_package_ids_tag: u64 = @bitCast([_]u8{ 'w', 'O', 'r', 'K', 's', 'P', 'a', 'C' }); const has_trusted_dependencies_tag: u64 = @bitCast([_]u8{ 't', 'R', 'u', 'S', 't', 'E', 'D', 'd' }); + const has_overrides_tag: u64 = @bitCast([_]u8{ 'o', 'V', 'e', 'R', 'r', 'i', 'D', 's' }); pub fn save(this: *Lockfile, comptime StreamType: type, stream: StreamType) !void { var old_package_list = this.packages; @@ -4347,6 +4679,34 @@ pub const Serializer = struct { ); } + if (this.overrides.map.count() > 0) { + try writer.writeAll(std.mem.asBytes(&has_overrides_tag)); + + try Lockfile.Buffers.writeArray( + StreamType, + stream, + @TypeOf(&writer), + &writer, + []PackageNameHash, + this.overrides.map.keys(), + ); + var external_overrides = try std.ArrayListUnmanaged(Dependency.External).initCapacity(z_allocator, this.overrides.map.count()); + defer external_overrides.deinit(z_allocator); + external_overrides.items.len = this.overrides.map.count(); + for (external_overrides.items, this.overrides.map.values()) |*dest, src| { + dest.* = src.toExternal(); + } + + try Lockfile.Buffers.writeArray( + StreamType, + stream, + @TypeOf(&writer), + &writer, + []Dependency.External, + external_overrides.items, + ); + } + const end = try stream.getPos(); try writer.writeAll(&alignment_bytes_to_repeat_buffer); @@ -4482,6 +4842,42 @@ pub const Serializer = struct { } } + { + const remaining_in_buffer = total_buffer_size -| stream.pos; + + if (remaining_in_buffer > 8 and total_buffer_size <= stream.buffer.len) { + const next_num = try reader.readIntLittle(u64); + if (next_num == has_overrides_tag) { + var overrides_name_hashes = try Lockfile.Buffers.readArray( + stream, + allocator, + std.ArrayListUnmanaged(PackageNameHash), + ); + defer overrides_name_hashes.deinit(allocator); + + var map = lockfile.overrides.map; + defer lockfile.overrides.map = map; + + try map.ensureTotalCapacity(allocator, overrides_name_hashes.items.len); + var override_versions_external = try Lockfile.Buffers.readArray( + stream, + allocator, + std.ArrayListUnmanaged(Dependency.External), + ); + const context: Dependency.Context = .{ + .allocator = allocator, + .log = log, + .buffer = lockfile.buffers.string_bytes.items, + }; + for (overrides_name_hashes.items, override_versions_external.items) |name, value| { + map.putAssumeCapacity(name, Dependency.toDependency(value, context)); + } + } else { + stream.pos -= 8; + } + } + } + lockfile.scratch = Lockfile.Scratch.init(allocator); lockfile.package_index = PackageIndex.Map.initContext(allocator, .{}); lockfile.string_pool = StringPool.initContext(allocator, .{}); |