diff options
Diffstat (limited to 'src/install/install.zig')
| -rw-r--r-- | src/install/install.zig | 434 |
1 files changed, 384 insertions, 50 deletions
diff --git a/src/install/install.zig b/src/install/install.zig index cd915a900..3221605cd 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -183,6 +183,7 @@ const NetworkTask = struct { }, extract: ExtractTarball, binlink: void, + git_clone: void, }, pub fn notify(this: *NetworkTask, _: anytype) void { @@ -469,6 +470,8 @@ pub const Features = struct { .optional_dependencies = true, }; + pub const git = npm; + pub const tarball = npm; pub const npm_manifest = Features{ @@ -506,6 +509,14 @@ const Task = struct { return @as(u64, @truncate(u63, hasher.final())) | @as(u64, 1 << 63); } + pub fn forGitHubPackage(repo: string, owner: string) u64 { + var hasher = std.hash.Wyhash.init(0); + hasher.update(owner); + hasher.update("/~~"); + hasher.update(repo); + return @as(u64, @truncate(u63, hasher.final())) | @as(u64, 1 << 63); + } + pub fn forBinLink(package_id: PackageID) u64 { const hash = std.hash.Wyhash.hash(0, std.mem.asBytes(&package_id)); return @as(u64, @truncate(u62, hash)) | @as(u64, 1 << 62) | @as(u64, 1 << 63); @@ -528,6 +539,95 @@ const Task = struct { defer this.package_manager.wake(); switch (this.tag) { + .git_clone => { + const allocator = this.package_manager.allocator; + + const PATH = this.package_manager.env_loader.get("PATH") orelse ""; + + var git_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + + if (which(&git_path_buf, PATH, ".", "git")) |git| { + const lockfile = this.package_manager.lockfile; + const git_path = std.mem.span(git); + + const git_repo = this.request.git_clone.repository; + + const repo_name = lockfile.str(git_repo.repo); + + var url_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + const url = if (this.request.git_clone.version.tag == .github) git_repo.getGitHubURL(lockfile, &url_buf) else git_repo.getURL(lockfile, &url_buf); + + var temp_dir_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + const temp_dir = std.os.getFdPath(this.package_manager.getTemporaryDirectory().dir.fd, &temp_dir_path_buf) catch unreachable; + + var destination_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + std.mem.copy(u8, &destination_buf, temp_dir); + destination_buf[temp_dir.len] = '/'; + std.mem.copy(u8, destination_buf[temp_dir.len + 1 ..], repo_name); + const destination = destination_buf[0 .. temp_dir.len + repo_name.len + 1]; + + const args = [_]string{ + git_path, + "clone", + "-q", + url, + destination, + }; + + var process = std.ChildProcess.init(&args, allocator); + process.stdout_behavior = .Close; + process.stderr_behavior = .Close; + process.stdin_behavior = .Close; + + const term = process.spawnAndWait() catch { + this.log.addErrorFmt(null, logger.Loc.Empty, allocator, "Failed to spawn git process to clone github dependency \"{s}\"", .{repo_name}) catch unreachable; + this.status = .fail; + this.package_manager.resolve_tasks.writeItem(this.*) catch unreachable; + return; + }; + switch (term) { + else => {}, + } + + // get package.json bytes, send pointer back to main thread + var package_json_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + std.mem.copy(u8, &package_json_path_buf, destination); + std.mem.copy(u8, package_json_path_buf[destination.len..], "/package.json"); + + const package_json_path = package_json_path_buf[0 .. destination.len + "/package.json".len]; + + const package_json_file = std.fs.openFileAbsolute(package_json_path, .{}) catch { + this.status = .fail; + this.log.addErrorFmt(null, logger.Loc.Empty, allocator, "Failed to find package.json for github dependency \"{s}\"", .{repo_name}) catch unreachable; + this.package_manager.resolve_tasks.writeItem(this.*) catch unreachable; + return; + }; + + const package_json_file_stat = package_json_file.stat() catch unreachable; + const package_json_file_size = package_json_file_stat.size; + + const package_json_source = allocator.alloc(u8, package_json_file_size) catch unreachable; + _ = package_json_file.preadAll(package_json_source, 0) catch unreachable; + + if (package_json_file_size < "{\"name\":\"\",\"version\":\"\"}".len + repo_name.len + "0.0.0".len) { + // package.json smaller than minimum possible + this.status = .fail; + this.log.addErrorFmt(null, logger.Loc.Empty, allocator, "Invalid package.json for github dependency \"{s}\"", .{repo_name}) catch unreachable; + this.package_manager.resolve_tasks.writeItem(this.*) catch unreachable; + return; + } + + this.status = .success; + this.data = .{ .package_json = package_json_source }; + this.package_manager.resolve_tasks.writeItem(this.*) catch unreachable; + return; + } + + this.status = .fail; + this.log.addErrorFmt(null, logger.Loc.Empty, allocator, "Failed to find git executable", .{}) catch unreachable; + this.package_manager.resolve_tasks.writeItem(this.*) catch unreachable; + return; + }, .package_manifest => { var allocator = bun.default_allocator; const package_manifest = Npm.Registry.getPackageMetadata( @@ -595,10 +695,11 @@ const Task = struct { } } - pub const Tag = enum(u2) { + pub const Tag = enum(u3) { package_manifest = 1, extract = 2, binlink = 3, + git_clone = 4, // install = 3, }; @@ -612,6 +713,7 @@ const Task = struct { package_manifest: Npm.PackageManifest, extract: string, binlink: bool, + package_json: string, }; pub const Request = union { @@ -626,12 +728,18 @@ const Task = struct { tarball: ExtractTarball, }, binlink: Bin.Linker, + git_clone: struct { + repository: Repository, + version: Dependency.Version, + dependency_id: u32, + }, // install: PackageInstall, }; }; const PackageInstall = struct { cache_dir: std.fs.IterableDir, + git_cache_dir: std.fs.IterableDir, destination_dir: std.fs.IterableDir, cache_dir_subpath: stringZ = "", destination_dir_subpath: stringZ = "", @@ -688,6 +796,7 @@ const PackageInstall = struct { this.package_install = PackageInstall{ .cache_dir = undefined, + .git_cache_dir = undefined, .cache_dir_subpath = undefined, .progress = ctx.progress, @@ -1436,6 +1545,7 @@ const Waker = AsyncIO.Waker; // 2. pub const PackageManager = struct { cache_directory_: ?std.fs.IterableDir = null, + git_cache_directory_: ?std.fs.IterableDir = null, temp_dir_: ?std.fs.IterableDir = null, root_dir: *Fs.FileSystem.DirEntry, env_loader: *DotEnv.Loader, @@ -1760,6 +1870,17 @@ pub const PackageManager = struct { }; } + pub noinline fn getGitCacheDirectory(this: *PackageManager) std.fs.IterableDir { + return this.git_cache_directory_ orelse brk: { + const cache_dir = this.getCacheDirectory(); + this.git_cache_directory_ = cache_dir.dir.openIterableDir("../git", .{}) catch { + this.git_cache_directory_ = cache_dir.dir.makeOpenPathIterable("../git", .{}) catch unreachable; + break :brk this.git_cache_directory_.?; + }; + break :brk this.git_cache_directory_.?; + }; + } + pub inline fn getTemporaryDirectory(this: *PackageManager) std.fs.IterableDir { return this.temp_dir_ orelse brk: { this.temp_dir_ = this.ensureTemporaryDirectory(); @@ -2022,8 +2143,8 @@ pub const PackageManager = struct { } pub fn resolveFromDiskCache(this: *PackageManager, package_name: []const u8, version: Dependency.Version) ?PackageID { - if (version.tag != .npm) { - // only npm supported right now + if (version.tag != .npm and version.tag != .github and version.tag != .git) { + // only npm, git, and github supported right now // tags are more ambiguous return null; } @@ -2047,32 +2168,38 @@ pub const PackageManager = struct { Semver.Version.sortGt, ); for (installed_versions.items) |installed_version| { - if (version.value.npm.satisfies(installed_version)) { - var buf: [bun.MAX_PATH_BYTES]u8 = undefined; - var npm_package_path = this.pathForCachedNPMPath(&buf, package_name, installed_version) catch |err| { - Output.debug("error getting path for cached npm path: {s}", .{std.mem.span(@errorName(err))}); - return null; - }; - const dependency = Dependency.Version{ - .tag = .npm, - .value = .{ - .npm = Semver.Query.Group.from(installed_version), - }, - }; - switch (FolderResolution.getOrPut(.{ .cache_folder = npm_package_path }, dependency, ".", this)) { - .new_package_id => |id| { - this.enqueueDependencyList(this.lockfile.packages.items(.dependencies)[id], false); - return id; - }, - .package_id => |id| { - this.enqueueDependencyList(this.lockfile.packages.items(.dependencies)[id], false); - return id; - }, - .err => |err| { - Output.debug("error getting or putting folder resolution: {s}", .{std.mem.span(@errorName(err))}); - return null; - }, - } + switch (version.tag) { + .npm => { + if (version.value.npm.satisfies(installed_version)) { + var buf: [bun.MAX_PATH_BYTES]u8 = undefined; + var npm_package_path = this.pathForCachedNPMPath(&buf, package_name, installed_version) catch |err| { + Output.debug("error getting path for cached npm path: {s}", .{std.mem.span(@errorName(err))}); + return null; + }; + const dependency = Dependency.Version{ + .tag = .npm, + .value = .{ + .npm = Semver.Query.Group.from(installed_version), + }, + }; + switch (FolderResolution.getOrPut(.{ .cache_folder = npm_package_path }, dependency, ".", this)) { + .new_package_id => |id| { + this.enqueueDependencyList(this.lockfile.packages.items(.dependencies)[id], false); + return id; + }, + .package_id => |id| { + this.enqueueDependencyList(this.lockfile.packages.items(.dependencies)[id], false); + return id; + }, + .err => |err| { + Output.debug("error getting or putting folder resolution: {s}", .{std.mem.span(@errorName(err))}); + return null; + }, + } + } + }, + // TODO: handle git and github + else => {}, } } @@ -2347,6 +2474,49 @@ pub const PackageManager = struct { ); }, + .github => { + var cache_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + const cache_path = try version.value.github.getCachePathForGitHub(this, &cache_path_buf); + + const res = FolderResolution.getOrPut(.{ .git_cache_folder = cache_path }, version, cache_path, this); + + switch (res) { + .err => |err| return err, + .package_id => |package_id| { + successFn(this, dependency_id, package_id); + return ResolvedPackageResult{ .package = this.lockfile.packages.get(package_id) }; + }, + .new_package_id => |package_id| { + successFn(this, dependency_id, package_id); + return ResolvedPackageResult{ .package = this.lockfile.packages.get(package_id), .is_first_time = true }; + }, + } + + return null; + }, + + .git => { + var cache_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + + const cache_path = try version.value.git.getCachePath(this, &cache_path_buf); + + const res = FolderResolution.getOrPut(.{ .git_cache_folder = cache_path }, version, cache_path, this); + + switch (res) { + .err => |err| return err, + .package_id => |package_id| { + successFn(this, dependency_id, package_id); + return ResolvedPackageResult{ .package = this.lockfile.packages.get(package_id) }; + }, + .new_package_id => |package_id| { + successFn(this, dependency_id, package_id); + return ResolvedPackageResult{ .package = this.lockfile.packages.get(package_id), .is_first_time = true }; + }, + } + + return null; + }, + .folder => { // relative to cwd const res = FolderResolution.getOrPut(.{ .relative = void{} }, version, version.value.folder.slice(this.lockfile.buffers.string_bytes.items), this); @@ -2408,6 +2578,32 @@ pub const PackageManager = struct { return &task.threadpool_task; } + fn enqueueCloneGitPackage( + this: *PackageManager, + task_id: u64, + repository: Repository, + dependency_id: u32, + dep_version: Dependency.Version, + ) *ThreadPool.Task { + var task = this.allocator.create(Task) catch unreachable; + task.* = Task{ + .package_manager = &PackageManager.instance, + .log = logger.Log.init(this.allocator), + .tag = Task.Tag.git_clone, + .request = .{ + .git_clone = .{ + .repository = repository, + .version = dep_version, + .dependency_id = dependency_id, + }, + }, + .id = task_id, + .data = undefined, + }; + + return &task.threadpool_task; + } + fn enqueueExtractNPMPackage( this: *PackageManager, tarball: ExtractTarball, @@ -2543,13 +2739,60 @@ pub const PackageManager = struct { if (!this.isRootDependency(id)) if (!dependency.behavior.isEnabled(switch (dependency.version.tag) { .folder => this.options.remote_package_features, - .dist_tag, .npm => this.options.remote_package_features, + .dist_tag, .npm, .git, .github => this.options.remote_package_features, else => Features{}, })) return; } switch (dependency.version.tag) { + .github, .git => { + var resolve_result = this.getOrPutResolvedPackage( + name_hash, + name, + version, + dependency.behavior, + id, + resolution, + successFn, + ) catch |err| brk: { + if (err == error.MissingPackageJSON) { + break :brk @as(?ResolvedPackageResult, null); + } + + return err; + }; + + if (resolve_result == null) { + const lockfile = this.lockfile; + + const repo = if (version.tag == .github) version.value.github else version.value.git; + + const task_id = Task.Id.forGitHubPackage(lockfile.str(repo.repo), lockfile.str(repo.owner)); + const network_id = try this.network_dedupe_map.getOrPutContext(this.allocator, task_id, .{}); + if (!network_id.found_existing) { + var batch = ThreadPool.Batch{}; + batch.push(ThreadPool.Batch.from(this.enqueueCloneGitPackage( + task_id, + repo, + id, + dependency.version, + ))); + + const count = batch.len; + this.pending_tasks += @truncate(u32, count); + this.total_tasks += @truncate(u32, count); + this.thread_pool.schedule(batch); + } + var task_queue = try this.task_queue.getOrPutContext(this.allocator, task_id, .{}); + if (!task_queue.found_existing) { + task_queue.value_ptr.* = TaskCallbackList{}; + } + + const callback_tag = comptime if (successFn == assignRootResolution) "root_dependency" else "dependency"; + try task_queue.value_ptr.append(this.allocator, @unionInit(TaskCallbackContext, callback_tag, id)); + } + }, .folder, .npm, .dist_tag => { retry_from_manifests_ptr: while (true) { var resolve_result_ = this.getOrPutResolvedPackage( @@ -3293,6 +3536,7 @@ pub const PackageManager = struct { batch.push(ThreadPool.Batch.from(manager.enqueueExtractNPMPackage(extract, task))); }, .binlink => {}, + .git_clone => {}, } } @@ -3363,6 +3607,69 @@ pub const PackageManager = struct { } } }, + .git_clone => { + if (task.status == .fail) { + continue; + } + + const allocator = manager.allocator; + const package_json = task.data.package_json; + defer allocator.free(package_json); + const git_repo = task.request.git_clone.repository; + var package_name = manager.lockfile.str(git_repo.repo); + + package_name = manager.lockfile.str(task.request.git_clone.repository.repo); + + const temp_dir = manager.getTemporaryDirectory().dir; + var temp_dir_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + _ = try std.os.getFdPath(temp_dir.fd, &temp_dir_buf); + const git_cache_dir = manager.getGitCacheDirectory().dir; + git_cache_dir.deleteTree(package_name) catch unreachable; + + var destination_name_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + const destination_name = try git_repo.getCacheDirectoryForGitHub(manager, &destination_name_buf); + std.fs.rename(temp_dir, package_name, git_cache_dir, destination_name) catch unreachable; + temp_dir.deleteTree(package_name) catch unreachable; + + var cache_path_buf: [bun.MAX_PATH_BYTES]u8 = undefined; + const cache_path = try git_repo.getCachePathForGitHub(manager, &cache_path_buf); + const res = FolderResolution.getOrPutWithPackageJSONBytes( + .{ .git_cache_folder = cache_path }, + task.request.git_clone.version, + cache_path, + manager, + package_json, + ); + + const dependency_id = task.request.git_clone.dependency_id; + const pkg_id = brk: { + switch (res) { + .err => |err| return err, + .package_id => |package_id| { + manager.assignResolution(dependency_id, package_id); + break :brk package_id; + }, + .new_package_id => |package_id| { + manager.assignResolution(dependency_id, package_id); + break :brk package_id; + }, + } + }; + + const dependencies = manager.lockfile.packages.items(.dependencies)[pkg_id]; + + var dependency_list_entry = manager.task_queue.getEntry(task.id).?; + var dependency_list = dependency_list_entry.value_ptr.*; + dependency_list_entry.value_ptr.* = .{}; + + const end = dependencies.off + dependencies.len; + var i = dependencies.off; + while (i < end) : (i += 1) { + dependency_list.append(allocator, @unionInit(TaskCallbackContext, "dependency", i)) catch unreachable; + } + + try manager.processDependencyList(dependency_list, ExtractCompletionContext, extract_ctx, callbacks); + }, .extract => { if (task.status == .fail) { const err = task.err orelse error.TarballFailedToExtract; @@ -5041,8 +5348,8 @@ pub const PackageManager = struct { if (unscoped_name.len > i + 1) request.version_buf = unscoped_name[i + 1 ..]; } - if (strings.hasPrefix("http://", request.name) or - strings.hasPrefix("https://", request.name)) + if (strings.hasPrefixComptime(request.name, "http://") or + strings.hasPrefixComptime(request.name, "https://")) { if (Output.isEmojiEnabled()) { Output.prettyErrorln("<r>😢 <red>error<r><d>:<r> bun {s} http://url is not implemented yet.", .{ @@ -5068,8 +5375,8 @@ pub const PackageManager = struct { request.version_buf = std.mem.trim(u8, request.version_buf, "\n\r\t"); // https://github.com/npm/npm-package-arg/blob/fbaf2fd0b72a0f38e7c24260fd4504f4724c9466/npa.js#L330 - if (strings.hasPrefix("https://", request.version_buf) or - strings.hasPrefix("http://", request.version_buf)) + if (strings.hasPrefixComptime(request.version_buf, "https://") or + strings.hasPrefixComptime(request.version_buf, "http://")) { if (Output.isEmojiEnabled()) { Output.prettyErrorln("<r>😢 <red>error<r><d>:<r> bun {s} http://url is not implemented yet.", .{ @@ -5636,6 +5943,7 @@ pub const PackageManager = struct { var installer = PackageInstall{ .progress = this.progress, .cache_dir = undefined, + .git_cache_dir = undefined, .cache_dir_subpath = undefined, .destination_dir = this.node_modules_folder, .destination_dir_subpath = destination_dir_subpath, @@ -6370,6 +6678,7 @@ pub const PackageManager = struct { const changes = @truncate(PackageID, mapping.len); _ = manager.getCacheDirectory(); + _ = manager.getGitCacheDirectory(); _ = manager.getTemporaryDirectory(); var counter_i: PackageID = 0; while (counter_i < changes) : (counter_i += 1) { @@ -6424,6 +6733,7 @@ pub const PackageManager = struct { if (root.dependencies.len > 0) { _ = manager.getCacheDirectory(); + _ = manager.getGitCacheDirectory(); _ = manager.getTemporaryDirectory(); } manager.enqueueDependencyList( @@ -6440,6 +6750,7 @@ pub const PackageManager = struct { if (manager.pending_tasks > 0) { if (root.dependencies.len > 0) { _ = manager.getCacheDirectory(); + _ = manager.getGitCacheDirectory(); _ = manager.getTemporaryDirectory(); } @@ -6649,13 +6960,12 @@ pub const PackageManager = struct { if (install_summary.success > 0) { // it's confusing when it shows 3 packages and says it installed 1 - Output.pretty("\n <green>{d}<r> packages<r> installed ", .{@max( - install_summary.success, - @truncate( - u32, - manager.package_json_updates.len, - ), - )}); + const count = @max(install_summary.success, @truncate(u32, manager.package_json_updates.len)); + if (count == 1) { + Output.pretty("\n <green>1<r> package<r> installed ", .{}); + } else { + Output.pretty("\n <green>{d}<r> packages<r> installed ", .{count}); + } Output.printStartEndStdout(ctx.start_time, std.time.nanoTimestamp()); printed_timestamp = true; Output.pretty("<r>\n", .{}); @@ -6670,7 +6980,11 @@ pub const PackageManager = struct { } } - Output.pretty("\n <r><b>{d}<r> packages removed ", .{manager.summary.remove}); + if (manager.summary.remove == 1) { + Output.pretty("\n <r><b>1<r> package removed ", .{}); + } else { + Output.pretty("\n <r><b>{d}<r> packages removed ", .{manager.summary.remove}); + } Output.printStartEndStdout(ctx.start_time, std.time.nanoTimestamp()); printed_timestamp = true; Output.pretty("<r>\n", .{}); @@ -6679,17 +6993,33 @@ pub const PackageManager = struct { const count = @truncate(PackageID, manager.lockfile.packages.len); if (count != install_summary.skipped) { - Output.pretty("Checked <green>{d} installs<r> across {d} packages <d>(no changes)<r> ", .{ - install_summary.skipped, - count, - }); + if (install_summary.skipped == 1 and count == 1) { + Output.pretty("Checked <green>1 install<r> across 1 package <d>(no changes)<r> ", .{}); + } else if (install_summary.skipped == 1) { + Output.pretty("Checked <green>1 install<r> across {d} packages <d>(no changes)<r> ", .{ + count, + }); + } else if (count == 1) { + Output.pretty("Checked <green>{d} installs<r> across 1 package <d>(no changes)<r> ", .{ + install_summary.skipped, + }); + } else { + Output.pretty("Checked <green>{d} installs<r> across {d} packages <d>(no changes)<r> ", .{ + install_summary.skipped, + count, + }); + } Output.printStartEndStdout(ctx.start_time, std.time.nanoTimestamp()); printed_timestamp = true; Output.pretty("<r>\n", .{}); } else { - Output.pretty("<r> <green>Done<r>! Checked {d} packages<r> <d>(no changes)<r> ", .{ - install_summary.skipped, - }); + if (install_summary.skipped == 1) { + Output.pretty("<r> <green>Done<r>! Checked 1 package<r> <d>(no changes)<r> ", .{}); + } else { + Output.pretty("<r> <green>Done<r>! Checked {d} packages<r> <d>(no changes)<r> ", .{ + install_summary.skipped, + }); + } Output.printStartEndStdout(ctx.start_time, std.time.nanoTimestamp()); printed_timestamp = true; Output.pretty("<r>\n", .{}); @@ -6697,7 +7027,11 @@ pub const PackageManager = struct { } if (install_summary.fail > 0) { - Output.prettyln("<r>Failed to install <red><b>{d}<r> packages\n", .{install_summary.fail}); + if (install_summary.fail == 1) { + Output.prettyln("<r>Failed to install <red><b>1<r> package\n", .{}); + } else { + Output.prettyln("<r>Failed to install <red><b>{d}<r> packages\n", .{install_summary.fail}); + } Output.flush(); } } |