diff options
| -rw-r--r-- | src/fs.zig | 16 | ||||
| -rw-r--r-- | src/install/extract_tarball.zig | 136 | ||||
| -rw-r--r-- | src/install/install.zig | 174 | ||||
| -rw-r--r-- | src/install/npm.zig | 4 |
4 files changed, 220 insertions, 110 deletions
diff --git a/src/fs.zig b/src/fs.zig index 19c085f15..2f29ed2f1 100644 --- a/src/fs.zig +++ b/src/fs.zig @@ -82,11 +82,16 @@ pub const FileSystem = struct { dirname_store: *DirnameStore, filename_store: *FilenameStore, + inside_home_dir: bool = false, _tmpdir: ?std.fs.Dir = null, threadlocal var tmpdir_handle: ?std.fs.Dir = null; + pub fn setInsideHomeDir(this: *FileSystem, home_dir: string) void { + this.inside_home_dir = strings.hasPrefix(this.top_level_dir, home_dir); + } + pub fn tmpdir(fs: *FileSystem) std.fs.Dir { if (tmpdir_handle == null) { tmpdir_handle = fs.fs.openTmpDir() catch unreachable; @@ -97,7 +102,7 @@ pub const FileSystem = struct { pub fn tmpname(_: *const FileSystem, extname: string, buf: []u8, hash: u64) ![*:0]u8 { // PRNG was...not so random - return try std.fmt.bufPrintZ(buf, "{x}{s}", .{ @truncate(u64, @intCast(u128, hash) * @intCast(u128, std.time.nanoTimestamp())), extname }); + return try std.fmt.bufPrintZ(buf, ".{x}{s}", .{ @truncate(u64, @intCast(u128, hash) * @intCast(u128, std.time.nanoTimestamp())), extname }); } pub var max_fd: FileDescriptorType = 0; @@ -514,6 +519,15 @@ pub const FileSystem = struct { return try std.fs.openDirAbsolute(tmpdir_path, .{ .access_sub_paths = true, .iterate = true }); } + pub fn getDefaultTempDir() string { + return std.os.getenvZ("BUN_TMPDIR") orelse std.os.getenvZ("TMPDIR") orelse PLATFORM_TMP_DIR; + } + + pub fn setTempdir(path: ?string) void { + tmpdir_path = path orelse getDefaultTempDir(); + tmpdir_path_set = true; + } + pub fn fetchCacheFile(fs: *RealFS, basename: string) !std.fs.File { const file = try fs._fetchCacheFile(basename); if (comptime FeatureFlags.store_file_descriptors) { diff --git a/src/install/extract_tarball.zig b/src/install/extract_tarball.zig index 27c8a811b..5594ff0b1 100644 --- a/src/install/extract_tarball.zig +++ b/src/install/extract_tarball.zig @@ -16,7 +16,8 @@ const Global = @import("../global.zig").Global; name: strings.StringOrTinyString, resolution: Resolution, registry: string, -cache_dir: string, +cache_dir: std.fs.Dir, +temp_dir: std.fs.Dir, package_id: PackageID, extracted_file_count: usize = 0, skip_verify: bool = false, @@ -144,8 +145,8 @@ threadlocal var abs_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; threadlocal var abs_buf2: [std.fs.MAX_PATH_BYTES]u8 = undefined; fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !string { - var tmpdir = FileSystem.instance.tmpdir(); - var tmpname_buf: [128]u8 = undefined; + var tmpdir = this.temp_dir; + var tmpname_buf: [64]u8 = undefined; const name = this.name.slice(); var basename = this.name.slice(); @@ -156,85 +157,84 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !string { } var tmpname = try FileSystem.instance.tmpname(basename, &tmpname_buf, tgz_bytes.len); + { + var extract_destination = tmpdir.makeOpenPath(std.mem.span(tmpname), .{ .iterate = true }) catch |err| { + Output.panic("err: {s} when create temporary directory named {s} (while extracting {s})", .{ @errorName(err), tmpname, name }); + }; - var cache_dir = tmpdir.makeOpenPath(std.mem.span(tmpname), .{ .iterate = true }) catch |err| { - Output.panic("err: {s} when create temporary directory named {s} (while extracting {s})", .{ @errorName(err), tmpname, name }); - }; - var temp_destination = std.os.getFdPath(cache_dir.fd, &abs_buf) catch |err| { - Output.panic("err: {s} when resolve path for temporary directory named {s} (while extracting {s})", .{ @errorName(err), tmpname, name }); - }; - cache_dir.close(); + defer extract_destination.close(); - if (PackageManager.verbose_install) { - Output.prettyErrorln("[{s}] Start extracting {s}<r>", .{ name, tmpname }); - Output.flush(); - } + if (PackageManager.verbose_install) { + Output.prettyErrorln("[{s}] Start extracting {s}<r>", .{ name, tmpname }); + Output.flush(); + } - const Archive = @import("../libarchive/libarchive.zig").Archive; - const Zlib = @import("../zlib.zig"); - var zlib_pool = Npm.Registry.BodyPool.get(default_allocator); - zlib_pool.data.reset(); - defer Npm.Registry.BodyPool.release(zlib_pool); + const Archive = @import("../libarchive/libarchive.zig").Archive; + const Zlib = @import("../zlib.zig"); + var zlib_pool = Npm.Registry.BodyPool.get(default_allocator); + zlib_pool.data.reset(); + defer Npm.Registry.BodyPool.release(zlib_pool); - var zlib_entry = try Zlib.ZlibReaderArrayList.init(tgz_bytes, &zlib_pool.data.list, default_allocator); - zlib_entry.readAll() catch |err| { - Output.prettyErrorln( - "<r><red>Error {s}<r> decompressing {s}", - .{ - @errorName(err), - name, - }, - ); - Output.flush(); - Global.crash(); - }; - _ = if (PackageManager.verbose_install) - try Archive.extractToDisk( - zlib_pool.data.list.items, - temp_destination, - null, - void, - void{}, - // for npm packages, the root dir is always "package" - 1, - true, - true, - ) - else - try Archive.extractToDisk( - zlib_pool.data.list.items, - temp_destination, - null, - void, - void{}, - // for npm packages, the root dir is always "package" - 1, - true, - false, - ); + var zlib_entry = try Zlib.ZlibReaderArrayList.init(tgz_bytes, &zlib_pool.data.list, default_allocator); + zlib_entry.readAll() catch |err| { + Output.prettyErrorln( + "<r><red>Error {s}<r> decompressing {s}", + .{ + @errorName(err), + name, + }, + ); + Output.flush(); + Global.crash(); + }; + _ = if (PackageManager.verbose_install) + try Archive.extractToDir( + zlib_pool.data.list.items, + extract_destination, + null, + void, + void{}, + // for npm packages, the root dir is always "package" + 1, + true, + true, + ) + else + try Archive.extractToDir( + zlib_pool.data.list.items, + extract_destination, + null, + void, + void{}, + // for npm packages, the root dir is always "package" + 1, + true, + false, + ); - if (PackageManager.verbose_install) { - Output.prettyErrorln( - "[{s}] Extracted<r>", - .{ - name, - }, - ); - Output.flush(); + if (PackageManager.verbose_install) { + Output.prettyErrorln( + "[{s}] Extracted<r>", + .{ + name, + }, + ); + Output.flush(); + } } - var folder_name = PackageManager.cachedNPMPackageFolderNamePrint(&abs_buf2, name, this.resolution.value.npm); if (folder_name.len == 0 or (folder_name.len == 1 and folder_name[0] == '/')) @panic("Tried to delete root and stopped it"); - PackageManager.instance.cache_directory.deleteTree(folder_name) catch {}; + var cache_dir = this.cache_dir; + cache_dir.deleteTree(folder_name) catch {}; // e.g. @next // if it's a namespace package, we need to make sure the @name folder exists if (basename.len != name.len) { - PackageManager.instance.cache_directory.makeDir(std.mem.trim(u8, name[0 .. name.len - basename.len], "/")) catch {}; + cache_dir.makeDir(std.mem.trim(u8, name[0 .. name.len - basename.len], "/")) catch {}; } // Now that we've extracted the archive, we rename. - std.os.renameatZ(tmpdir.fd, tmpname, PackageManager.instance.cache_directory.fd, folder_name) catch |err| { + std.os.renameatZ(tmpdir.fd, tmpname, cache_dir.fd, folder_name) catch |err| { Output.prettyErrorln( "<r><red>Error {s}<r> moving {s} to cache dir:\n From: {s} To: {s}", .{ @@ -250,7 +250,7 @@ fn extract(this: *const ExtractTarball, tgz_bytes: []const u8) !string { // We return a resolved absolute absolute file path to the cache dir. // To get that directory, we open the directory again. - var final_dir = PackageManager.instance.cache_directory.openDirZ(folder_name, .{ .iterate = true }) catch |err| { + var final_dir = cache_dir.openDirZ(folder_name, .{ .iterate = true }) catch |err| { Output.prettyErrorln( "<r><red>Error {s}<r> failed to verify cache dir for {s}", .{ diff --git a/src/install/install.zig b/src/install/install.zig index 4eaf7cc67..94ade6abd 100644 --- a/src/install/install.zig +++ b/src/install/install.zig @@ -3587,8 +3587,8 @@ pub const CacheLevel = struct { // 1. Download all packages, parsing their dependencies and enqueuing all dependnecies for resolution // 2. pub const PackageManager = struct { - cache_directory_path: string = "", - cache_directory: std.fs.Dir = undefined, + cache_directory_: ?std.fs.Dir = null, + temp_dir_: ?std.fs.Dir = null, root_dir: *Fs.FileSystem.DirEntry, env_loader: *DotEnv.Loader, allocator: std.mem.Allocator, @@ -3669,6 +3669,114 @@ pub const PackageManager = struct { var cached_package_folder_name_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined; + pub inline fn getCacheDirectory(this: *PackageManager) std.fs.Dir { + return this.cache_directory_ orelse brk: { + this.cache_directory_ = this.ensureCacheDirectory(); + break :brk this.cache_directory_.?; + }; + } + + pub inline fn getTemporaryDirectory(this: *PackageManager) std.fs.Dir { + return this.temp_dir_ orelse brk: { + this.temp_dir_ = this.ensureTemporaryDirectory(); + break :brk this.temp_dir_.?; + }; + } + + noinline fn ensureCacheDirectory(this: *PackageManager) std.fs.Dir { + loop: while (true) { + if (this.options.enable.cache) { + const cache_dir = fetchCacheDirectoryPath(this.env_loader); + return std.fs.cwd().makeOpenPath(cache_dir.path, .{ .iterate = true }) catch { + this.options.enable.cache = false; + continue :loop; + }; + } + + return std.fs.cwd().makeOpenPath("node_modules/.cache", .{ .iterate = true }) catch |err| { + Output.prettyErrorln("<r><red>error<r>: Bun is unable to write files: {s}", .{@errorName(err)}); + Output.flush(); + Global.crash(); + }; + } + unreachable; + } + + // We need a temporary directory that can be rename() + // This is important for extracting files. + // + // However, we want it to be reused! Otherwise a cache is silly. + // Error RenameAcrossMountPoints moving react-is to cache dir: + noinline fn ensureTemporaryDirectory(this: *PackageManager) std.fs.Dir { + var cache_directory = this.getCacheDirectory(); + // The chosen tempdir must be on the same filesystem as the cache directory + // This makes renameat() work + const default_tempdir = Fs.FileSystem.RealFS.getDefaultTempDir(); + var tried_dot_tmp = false; + var tempdir: std.fs.Dir = std.fs.cwd().makeOpenPath(default_tempdir, .{ .iterate = true }) catch brk: { + tried_dot_tmp = true; + break :brk cache_directory.makeOpenPath(".tmp", .{ .iterate = true }) catch |err| { + Output.prettyErrorln("<r><red>error<r>: Bun is unable to access tempdir: {s}", .{@errorName(err)}); + Output.flush(); + Global.crash(); + }; + }; + var tmpbuf: ["18446744073709551615".len + 8]u8 = undefined; + const tmpname = Fs.FileSystem.instance.tmpname("hm", &tmpbuf, 999) catch unreachable; + var timer: std.time.Timer = if (this.options.log_level != .silent) std.time.Timer.start() catch unreachable else undefined; + brk: while (true) { + _ = tempdir.createFileZ(tmpname, .{ .truncate = true }) catch |err2| { + if (!tried_dot_tmp) { + tried_dot_tmp = true; + + tempdir = cache_directory.makeOpenPath(".tmp", .{ .iterate = true }) catch |err| { + Output.prettyErrorln("<r><red>error<r>: Bun is unable to access tempdir: {s}", .{@errorName(err)}); + Output.flush(); + Global.crash(); + }; + continue :brk; + } + Output.prettyErrorln("<r><red>error<r>: {s} accessing temporary directory. Please set <b>$BUN_TMPDIR<r> or <b>$BUN_INSTALL_DIR<r>", .{ + @errorName(err2), + }); + Output.flush(); + Global.crash(); + }; + + std.os.renameatZ(tempdir.fd, tmpname, cache_directory.fd, tmpname) catch |err| { + if (!tried_dot_tmp) { + tried_dot_tmp = true; + tempdir = cache_directory.makeOpenPath(".tmp", .{ .iterate = true }) catch |err2| { + Output.prettyErrorln("<r><red>error<r>: Bun is unable to write files to tempdir: {s}", .{@errorName(err2)}); + Output.flush(); + Global.crash(); + }; + continue :brk; + } + + Output.prettyErrorln("<r><red>error<r>: {s} accessing temporary directory. Please set <b>$BUN_TMPDIR<r> or <b>$BUN_INSTALL_DIR<r>", .{ + @errorName(err), + }); + Output.flush(); + Global.crash(); + }; + cache_directory.deleteFileZ(tmpname) catch {}; + break; + } + if (this.options.log_level != .silent) { + const elapsed = timer.read(); + if (elapsed > std.time.ns_per_ms * 10) { + var cache_dir_path = std.os.getFdPath(cache_directory.fd, &path_buf) catch "it's"; + Output.prettyErrorln( + "<r><yellow>warn<r>: Slow filesystem detected. If {s} is a network drive, consider setting $BUN_INSTALL_CACHE_DIR to a local folder.", + .{cache_dir_path}, + ); + } + } + + return tempdir; + } + pub var instance: PackageManager = undefined; pub fn getNetworkTask(this: *PackageManager) *NetworkTask { @@ -3717,7 +3825,7 @@ pub const PackageManager = struct { pub fn isFolderInCache(this: *PackageManager, folder_path: stringZ) bool { // TODO: is this slow? - var dir = this.cache_directory.openDirZ(folder_path, .{ .iterate = true }) catch return false; + var dir = this.getCacheDirectory().openDirZ(folder_path, .{ .iterate = true }) catch return false; dir.close(); return true; } @@ -3837,7 +3945,8 @@ pub const PackageManager = struct { strings.StringOrTinyString.init(this.lockfile.str(package.name)), .resolution = package.resolution, - .cache_dir = this.cache_directory_path, + .cache_dir = this.getCacheDirectory(), + .temp_dir = this.getTemporaryDirectory(), .registry = this.registry.url.href, .package_id = package.meta.id, .extracted_file_count = package.meta.file_count, @@ -4152,7 +4261,7 @@ pub const PackageManager = struct { var network_entry = try this.network_dedupe_map.getOrPutContext(this.allocator, task_id, .{}); if (!network_entry.found_existing) { if (this.options.enable.manifest_cache) { - if (Npm.PackageManifest.Serializer.load(this.allocator, this.cache_directory, name_str) catch null) |manifest_| { + if (Npm.PackageManifest.Serializer.load(this.allocator, this.getCacheDirectory(), name_str) catch null) |manifest_| { const manifest: Npm.PackageManifest = manifest_; loaded_manifest = manifest; @@ -4185,6 +4294,11 @@ pub const PackageManager = struct { _ = this.network_dedupe_map.remove(task_id); continue :retry_from_manifests_ptr; } + + // We want to make sure the temporary directory & cache directory are loaded on the main thread + // so that we don't run into weird threading issues + // the call to getCacheDirectory() above handles the cache dir + _ = this.getTemporaryDirectory(); } } @@ -4303,36 +4417,34 @@ pub const PackageManager = struct { this.network_resolve_batch = .{}; } + const CacheDir = struct { path: string, is_node_modules: bool }; pub fn fetchCacheDirectoryPath( - _: std.mem.Allocator, env_loader: *DotEnv.Loader, - _: *Fs.FileSystem.DirEntry, - ) ?string { + ) CacheDir { if (env_loader.map.get("BUN_INSTALL_CACHE_DIR")) |dir| { - return dir; + return CacheDir{ .path = dir, .is_node_modules = false }; } - if (env_loader.map.get("BUN_INSTALL")) |dir| { - var parts = [_]string{ dir, "install/", "cache/" }; - return Fs.FileSystem.instance.abs(&parts); + if (env_loader.map.get("HOME")) |dir| { + FileSystem.instance.setInsideHomeDir(dir); + if (FileSystem.instance.inside_home_dir) { + var parts = [_]string{ dir, ".bun/", "install/", "cache/" }; + return CacheDir{ .path = Fs.FileSystem.instance.abs(&parts), .is_node_modules = false }; + } } - if (env_loader.map.get("HOME")) |dir| { - var parts = [_]string{ dir, ".bun/", "install/", "cache/" }; - return Fs.FileSystem.instance.abs(&parts); + if (env_loader.map.get("BUN_INSTALL")) |dir| { + var parts = [_]string{ dir, "install/", "cache/" }; + return CacheDir{ .path = Fs.FileSystem.instance.abs(&parts), .is_node_modules = false }; } if (env_loader.map.get("XDG_CACHE_HOME")) |dir| { var parts = [_]string{ dir, ".bun/", "install/", "cache/" }; - return Fs.FileSystem.instance.abs(&parts); - } - - if (env_loader.map.get("TMPDIR")) |dir| { - var parts = [_]string{ dir, ".bun-cache" }; - return Fs.FileSystem.instance.abs(&parts); + return CacheDir{ .path = Fs.FileSystem.instance.abs(&parts), .is_node_modules = false }; } - return null; + var fallback_parts = [_]string{"node_modules/.bun-cache"}; + return CacheDir{ .is_node_modules = true, .path = Fs.FileSystem.instance.abs(&fallback_parts) }; } fn runTasks( @@ -4399,8 +4511,7 @@ pub const PackageManager = struct { entry.value_ptr.* = manifest; entry.value_ptr.*.pkg.public_max_age = @truncate(u32, @intCast(u64, @maximum(0, std.time.timestamp()))) + 300; { - var tmpdir = Fs.FileSystem.instance.tmpdir(); - Npm.PackageManifest.Serializer.save(entry.value_ptr, tmpdir, PackageManager.instance.cache_directory) catch {}; + Npm.PackageManifest.Serializer.save(entry.value_ptr, PackageManager.instance.getTemporaryDirectory(), PackageManager.instance.getCacheDirectory()) catch {}; } var dependency_list_entry = manager.task_queue.getEntry(task.task_id).?; @@ -5142,7 +5253,6 @@ pub const PackageManager = struct { var entries_option = try fs.fs.readDirectory(fs.top_level_dir, null); var options = Options{}; - var cache_directory: std.fs.Dir = undefined; var env_loader: *DotEnv.Loader = brk: { var map = try ctx.allocator.create(DotEnv.Map); @@ -5160,17 +5270,6 @@ pub const PackageManager = struct { PackageManager.verbose_install = true; } - if (PackageManager.fetchCacheDirectoryPath(ctx.allocator, env_loader, &entries_option.entries)) |cache_dir_path| { - options.cache_directory = try fs.dirname_store.append(@TypeOf(cache_dir_path), cache_dir_path); - cache_directory = std.fs.cwd().makeOpenPath(options.cache_directory, .{ .iterate = true }) catch |err| brk: { - options.enable.cache = false; - options.enable.manifest_cache = false; - options.enable.manifest_cache_control = false; - Output.prettyErrorln("Cache is disabled due to error: {s}", .{@errorName(err)}); - break :brk undefined; - }; - } else {} - if (PackageManager.verbose_install) { Output.prettyErrorln("Cache Dir: {s}", .{options.cache_directory}); Output.flush(); @@ -5190,7 +5289,6 @@ pub const PackageManager = struct { manager.* = PackageManager{ .options = options, .network_task_fifo = NetworkQueue.init(), - .cache_directory = cache_directory, .env_loader = env_loader, .allocator = ctx.allocator, .log = ctx.log, @@ -6016,7 +6114,7 @@ pub const PackageManager = struct { switch (resolution.tag) { .npm => { var installer = PackageInstall{ - .cache_dir = this.manager.cache_directory, + .cache_dir = this.manager.getCacheDirectory(), .progress = this.progress, .cache_dir_subpath = PackageManager.cachedNPMPackageFolderName(name, resolution.value.npm), .destination_dir = this.node_modules_folder, diff --git a/src/install/npm.zig b/src/install/npm.zig index fb1ff523a..8ca08fc4b 100644 --- a/src/install/npm.zig +++ b/src/install/npm.zig @@ -124,9 +124,7 @@ pub const Registry = struct { @truncate(u32, @intCast(u64, @maximum(0, std.time.timestamp()))) + 300, )) |package| { if (PackageManager.instance.options.enable.manifest_cache) { - var tmpdir = FileSystem.instance.tmpdir(); - - PackageManifest.Serializer.save(&package, tmpdir, PackageManager.instance.cache_directory) catch {}; + PackageManifest.Serializer.save(&package, PackageManager.instance.getTemporaryDirectory(), PackageManager.instance.getCacheDirectory()) catch {}; } return PackageVersionResponse{ .fresh = package }; |