[bun install] Support linking binaries & native binaries

4 files changed, 405 insertions, 6 deletions

diff --git a/src/install/bin.zig b/src/install/bin.zig
index db94a6432..fb343a16d 100644
--- a/src/install/bin.zig
+++ b/src/install/bin.zig
@@ -3,7 +3,10 @@ const Semver = @import("./semver.zig");
 const ExternalString = Semver.ExternalString;
 const String = Semver.String;
 const std = @import("std");
-
+const strings = @import("strings");
+const Environment = @import("../env.zig");
+const Path = @import("../resolver/resolve_path.zig");
+const C = @import("../c.zig");
 /// Normalized `bin` field in [package.json](https://docs.npmjs.com/cli/v8/configuring-npm/package-json#bin)
 /// Can be a:
 /// - file path (relative to the package root)
@@ -117,4 +120,168 @@ pub const Bin = extern struct {
         ///```
         map = 4,
     };
+
+    pub const Linker = struct {
+        bin: Bin,
+
+        package_installed_node_modules: std.os.fd_t = std.math.maxInt(std.os.fd_t),
+        root_node_modules_folder: std.os.fd_t = std.math.maxInt(std.os.fd_t),
+
+        /// Used for generating relative paths
+        package_name: strings.StringOrTinyString,
+
+        string_buf: []const u8,
+
+        err: ?anyerror = null,
+
+        pub var umask: std.os.mode_t = 0;
+
+        pub const Error = error{
+            NotImplementedYet,
+        } || std.os.SymLinkError || std.os.OpenError || std.os.RealPathError;
+
+        // It is important that we use symlinkat(2) with relative paths instead of symlink()
+        // That way, if you move your node_modules folder around, the symlinks in .bin still work
+        // If we used absolute paths for the symlinks, you'd end up with broken symlinks
+        pub fn link(this: *Linker) void {
+            var from_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
+            var path_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
+
+            from_buf[0..".bin/".len].* = ".bin/".*;
+            var from_remain: []u8 = from_buf[".bin/".len..];
+            path_buf[0.."../".len].* = "../".*;
+
+            var remain: []u8 = path_buf["../".len..];
+            const name = this.package_name.slice();
+            std.mem.copy(u8, remain, name);
+            remain = remain[name.len..];
+            remain[0] = std.fs.path.sep;
+            remain = remain[1..];
+            const base_len = @ptrToInt(remain.ptr) - @ptrToInt(&path_buf);
+
+            if (comptime Environment.isWindows) {
+                @compileError("Bin.Linker.link() needs to be updated to generate .cmd files on Windows");
+            }
+
+            switch (this.bin.tag) {
+                .none => {
+                    if (comptime Environment.isDebug) {
+                        unreachable;
+                    }
+                },
+                .file => {
+                    var target = this.bin.value.file.slice(this.string_buf);
+                    if (strings.hasPrefix(target, "./")) {
+                        target = target[2..];
+                    }
+                    std.mem.copy(u8, remain, target);
+                    remain = remain[target.len..];
+                    remain[0] = 0;
+                    const target_len = @ptrToInt(remain.ptr) - @ptrToInt(&path_buf);
+                    remain = remain[1..];
+
+                    var target_path: [:0]u8 = path_buf[0..target_len :0];
+                    std.mem.copy(u8, from_remain, name);
+                    from_remain = from_remain[name.len..];
+                    from_remain[0] = 0;
+                    var dest_path: [:0]u8 = from_buf[0 .. @ptrToInt(from_remain.ptr) - @ptrToInt(&from_buf) :0];
+
+                    _ = C.chmod(target_path, umask & 0o777);
+                    std.os.symlinkatZ(target_path, this.root_node_modules_folder, dest_path) catch |err| {
+                        // Silently ignore PathAlreadyExists
+                        // Most likely, the symlink was already created by another package
+                        if (err == error.PathAlreadyExists) return;
+
+                        this.err = err;
+                    };
+                },
+                .named_file => {
+                    var target = this.bin.value.named_file[1].slice(this.string_buf);
+                    if (strings.hasPrefix(target, "./")) {
+                        target = target[2..];
+                    }
+                    std.mem.copy(u8, remain, target);
+                    remain = remain[target.len..];
+                    remain[0] = 0;
+                    const target_len = @ptrToInt(remain.ptr) - @ptrToInt(&path_buf);
+                    remain = remain[1..];
+
+                    var target_path: [:0]u8 = path_buf[0..target_len :0];
+                    var name_to_use = this.bin.value.named_file[0].slice(this.string_buf);
+                    std.mem.copy(u8, from_remain, name_to_use);
+                    from_remain = from_remain[name_to_use.len..];
+                    from_remain[0] = 0;
+                    var dest_path: [:0]u8 = from_buf[0 .. @ptrToInt(from_remain.ptr) - @ptrToInt(&from_buf) :0];
+
+                    _ = C.chmod(target_path, umask & 0o777);
+                    std.os.symlinkatZ(target_path, this.root_node_modules_folder, dest_path) catch |err| {
+                        // Silently ignore PathAlreadyExists
+                        // Most likely, the symlink was already created by another package
+                        if (err == error.PathAlreadyExists) return;
+
+                        this.err = err;
+                    };
+                },
+                .dir => {
+                    var target = this.bin.value.dir.slice(this.string_buf);
+                    var parts = [_][]const u8{ name, target };
+                    if (strings.hasPrefix(target, "./")) {
+                        target = target[2..];
+                    }
+                    std.mem.copy(u8, remain, target);
+                    remain = remain[target.len..];
+                    remain[0] = 0;
+                    var dir = std.fs.Dir{ .fd = this.package_installed_node_modules };
+
+                    var joined = Path.joinStringBuf(&from_buf, &parts, .auto);
+                    from_buf[joined.len] = 0;
+                    var joined_: [:0]u8 = from_buf[0..joined.len :0];
+                    var child_dir = dir.openDirZ(joined_, .{ .iterate = true }) catch |err| {
+                        this.err = err;
+                        return;
+                    };
+                    defer child_dir.close();
+
+                    var iter = child_dir.iterate();
+
+                    var basedir_path = std.os.getFdPath(child_dir.fd, &from_buf) catch |err| {
+                        this.err = err;
+                        return;
+                    };
+                    from_buf[basedir_path.len] = std.fs.path.sep;
+                    var from_buf_remain = from_buf[basedir_path.len + 1 ..];
+
+                    while (iter.next() catch null) |entry_| {
+                        const entry: std.fs.Dir.Entry = entry_;
+                        switch (entry.kind) {
+                            std.fs.Dir.Entry.Kind.SymLink, std.fs.Dir.Entry.Kind.File => {
+                                std.mem.copy(u8, from_buf_remain, entry.name);
+                                from_buf_remain = from_buf_remain[entry.name.len..];
+                                from_buf_remain[0] = 0;
+                                var from_path: [:0]u8 = from_buf[0 .. @ptrToInt(from_buf_remain.ptr) - @ptrToInt(&from_buf) :0];
+                                var to_path = std.fmt.bufPrintZ(&path_buf, ".bin/{s}", .{entry.name}) catch unreachable;
+                                _ = C.chmod(from_path, umask & 0o777);
+                                std.os.symlinkatZ(
+                                    from_path,
+                                    this.root_node_modules_folder,
+                                    to_path,
+                                ) catch |err| {
+                                    // Silently ignore PathAlreadyExists
+                                    // Most likely, the symlink was already created by another package
+                                    if (err == error.PathAlreadyExists) return;
+
+                                    this.err = err;
+                                    return;
+                                };
+                            },
+                            else => {},
+                        }
+                    }
+                },
+                .map => {
+                    this.err = error.NotImplementedYet;
+                },
+            }
+        }
+    };
 };
diff --git a/src/install/dependency.zig b/src/install/dependency.zig
index 665d427ae..d41cfe8d4 100644
--- a/src/install/dependency.zig
+++ b/src/install/dependency.zig
@@ -549,7 +549,7 @@ pub const Behavior = enum(u8) {
     pub const peer: u8 = 1 << 4;
 
     pub inline fn isOptional(this: Behavior) bool {
-        return (@enumToInt(this) & Behavior.optional) != 0 and (@enumToInt(this) & Behavior.peer) == 0;
+        return (@enumToInt(this) & Behavior.optional) != 0 and !this.isPeer();
     }
 
     pub inline fn isDev(this: Behavior) bool {
diff --git a/src/install/install-scripts-allowlist.txt b/src/install/install-scripts-allowlist.txt
new file mode 100644
index 000000000..fdd5d1c06
--- /dev/null
+++ b/src/install/install-scripts-allowlist.txt
@@ -0,0 +1,4 @@
+static-ffmpeg
+canvas
+better-sqlite3
+node-sass
\ No newline at end of file
diff --git a/src/install/install.zig b/src/install/install.zig
index ab06ff930..9c6fa7948 100644
--- a/src/install/install.zig
+++ b/src/install/install.zig
@@ -201,6 +201,7 @@ const NetworkTask = struct {
             name: strings.StringOrTinyString,
         },
         extract: ExtractTarball,
+        binlink: void,
     },
 
     pub fn notify(http: *AsyncHTTP, sender: *AsyncHTTP.HTTPSender) void {
@@ -1972,6 +1973,8 @@ pub const Lockfile = struct {
                         string_builder.count(version_strings[i].slice(string_buf));
                     }
                 }
+
+                package_version.bin.count(string_buf, @TypeOf(&string_builder), &string_builder);
             }
 
             try string_builder.allocate();
@@ -2071,6 +2074,8 @@ pub const Lockfile = struct {
                     }
                 }
 
+                package.bin = package_version.bin.clone(string_buf, @TypeOf(&string_builder), &string_builder);
+
                 package.meta.arch = package_version.cpu;
                 package.meta.os = package_version.os;
                 package.meta.unpacked_size = package_version.unpacked_size;
@@ -2179,6 +2184,13 @@ pub const Lockfile = struct {
         pub fn determinePreinstallState(this: *Lockfile.Package, lockfile: *Lockfile, manager: *PackageManager) PreinstallState {
             switch (this.meta.preinstall_state) {
                 .unknown => {
+                    // Do not automatically start downloading packages which are disabled
+                    // i.e. don't download all of esbuild's versions or SWCs
+                    if (this.isDisabled()) {
+                        this.meta.preinstall_state = .done;
+                        return .done;
+                    }
+
                     const folder_path = PackageManager.cachedNPMPackageFolderName(this.name.slice(lockfile.buffers.string_bytes.items), this.resolution.value.npm);
                     if (manager.isFolderInCache(folder_path)) {
                         this.meta.preinstall_state = .done;
@@ -2888,6 +2900,11 @@ const Task = struct {
             return @as(u64, @truncate(u63, hasher.final())) | @as(u64, 1 << 63);
         }
 
+        pub fn forBinLink(package_id: PackageID) u64 {
+            const hash = std.hash.Wyhash.hash(0, std.mem.asBytes(&package_id));
+            return @as(u64, @truncate(u62, hash)) | @as(u64, 1 << 62) | @as(u64, 1 << 63);
+        }
+
         pub fn forManifest(
             tag: Task.Tag,
             name: string,
@@ -2952,12 +2969,14 @@ const Task = struct {
                 this.status = Status.success;
                 PackageManager.instance.resolve_tasks.writeItem(this.*) catch unreachable;
             },
+            .binlink => {},
         }
     }
 
     pub const Tag = enum(u2) {
         package_manifest = 1,
         extract = 2,
+        binlink = 3,
         // install = 3,
     };
 
@@ -2970,6 +2989,7 @@ const Task = struct {
     pub const Data = union {
         package_manifest: Npm.PackageManifest,
         extract: string,
+        binlink: bool,
     };
 
     pub const Request = union {
@@ -2983,6 +3003,7 @@ const Task = struct {
             network: *NetworkTask,
             tarball: ExtractTarball,
         },
+        binlink: Bin.Linker,
         // install: PackageInstall,
     };
 };
@@ -4475,6 +4496,7 @@ pub const PackageManager = struct {
 
                     batch.push(ThreadPool.Batch.from(manager.enqueueExtractNPMPackage(extract, task)));
                 },
+                .binlink => {},
             }
         }
 
@@ -4551,6 +4573,7 @@ pub const PackageManager = struct {
                         }
                     }
                 },
+                .binlink => {},
             }
         }
 
@@ -4602,6 +4625,36 @@ pub const PackageManager = struct {
         dry_run: bool = false,
         omit: CommandLineArguments.Omit = .{},
 
+        allowed_install_scripts: []const PackageNameHash = &default_allowed_install_scripts,
+
+        // The idea here is:
+        // 1. package has a platform-specific binary to install
+        // 2. To prevent downloading & installing incompatible versions, they stick the "real" one in optionalDependencies
+        // 3. The real one we want to link is in another package
+        // 4. Therefore, we remap the "bin" specified in the real package
+        //    to the target package which is the one which is:
+        //      1. In optionalDepenencies
+        //      2. Has a platform and/or os specified, which evaluates to not disabled
+        native_bin_link_allowlist: []const PackageNameHash = &default_native_bin_link_allowlist,
+
+        const default_native_bin_link_allowlist = [_]PackageNameHash{
+            String.Builder.stringHash("esbuild"),
+            String.Builder.stringHash("turbo"),
+        };
+
+        const install_scripts_package_count = 5;
+        const default_allowed_install_scripts: [install_scripts_package_count]PackageNameHash = brk: {
+            const names = std.mem.span(@embedFile("install-scripts-allowlist.txt"));
+            var hashes: [install_scripts_package_count]PackageNameHash = undefined;
+            var splitter = std.mem.split(u8, names, "\n");
+            var i: usize = 0;
+            while (splitter.next()) |item| {
+                hashes[i] = String.Builder.stringHash(item);
+                i += 1;
+            }
+            break :brk hashes;
+        };
+
         pub const LogLevel = enum {
             default,
             verbose,
@@ -4711,6 +4764,26 @@ pub const PackageManager = struct {
                 PackageInstall.supported_method = .copyfile;
             }
 
+            if (env_loader.map.get("BUN_CONFIG_YARN_LOCKFILE") != null) {
+                this.do.save_yarn_lock = true;
+            }
+
+            if (env_loader.map.get("BUN_CONFIG_LINK_NATIVE_BINS")) |native_packages| {
+                const len = std.mem.count(u8, native_packages, " ");
+                if (len > 0) {
+                    var all = try allocator.alloc(PackageNameHash, this.native_bin_link_allowlist.len + len);
+                    std.mem.copy(PackageNameHash, all, this.native_bin_link_allowlist);
+                    var remain = all[this.native_bin_link_allowlist.len..];
+                    var splitter = std.mem.split(u8, native_packages, " ");
+                    var i: usize = 0;
+                    while (splitter.next()) |name| {
+                        remain[i] = String.Builder.stringHash(name);
+                        i += 1;
+                    }
+                    this.native_bin_link_allowlist = all;
+                }
+            }
+
             // if (env_loader.map.get("BUN_CONFIG_NO_DEDUPLICATE") != null) {
             //     this.enable.deduplicate_packages = false;
             // }
@@ -4754,6 +4827,16 @@ pub const PackageManager = struct {
                     this.do.save_yarn_lock = true;
                 }
 
+                if (cli.link_native_bins.len > 0) {
+                    var all = try allocator.alloc(PackageNameHash, this.native_bin_link_allowlist.len + cli.link_native_bins.len);
+                    std.mem.copy(PackageNameHash, all, this.native_bin_link_allowlist);
+                    var remain = all[this.native_bin_link_allowlist.len..];
+                    for (cli.link_native_bins) |name, i| {
+                        remain[i] = String.Builder.stringHash(name);
+                    }
+                    this.native_bin_link_allowlist = all;
+                }
+
                 if (cli.backend) |backend| {
                     PackageInstall.supported_method = backend;
                 }
@@ -5175,6 +5258,9 @@ pub const PackageManager = struct {
         clap.parseParam("--cwd <STR>                       Set a specific cwd") catch unreachable,
         clap.parseParam("--backend <STR>                   Platform-specific optimizations for installing dependencies. For macOS, \"clonefile\" (default), \"copyfile\"") catch unreachable,
         clap.parseParam("--omit <STR>...                   Skip installing dependencies of a certain type. \"dev\", \"optional\", or \"peer\"") catch unreachable,
+
+        clap.parseParam("--link-native-bins <STR>...       Link \"bin\" from a matching platform-specific \"optionalDependencies\" instead. Default: esbuild, turbo") catch unreachable,
+
         clap.parseParam("--help                            Print this help menu") catch unreachable,
     };
 
@@ -5212,6 +5298,8 @@ pub const PackageManager = struct {
         silent: bool = false,
         verbose: bool = false,
 
+        link_native_bins: []const string = &[_]string{},
+
         development: bool = false,
         optional: bool = false,
 
@@ -5268,6 +5356,8 @@ pub const PackageManager = struct {
             cli.silent = args.flag("--silent");
             cli.verbose = args.flag("--verbose");
 
+            cli.link_native_bins = args.options("--link-native-bins");
+
             if (comptime params.len == add_params.len) {
                 cli.development = args.flag("--development");
                 cli.optional = args.flag("--optional");
@@ -5724,15 +5814,28 @@ pub const PackageManager = struct {
         skip_verify: bool,
         skip_delete: bool,
         force_install: bool,
+        root_node_modules_folder: std.fs.Dir,
         summary: *PackageInstall.Summary,
-        metas: []Lockfile.Package.Meta,
-        names: []String,
+        options: *const PackageManager.Options,
+        metas: []const Lockfile.Package.Meta,
+        names: []const String,
+        bins: []const Bin,
         resolutions: []Resolution,
-        trees: []const Lockfile.Tree,
         node: *Progress.Node,
+        has_created_bin: bool = false,
         destination_dir_subpath_buf: [std.fs.MAX_PATH_BYTES]u8 = undefined,
         install_count: usize = 0,
 
+        // For linking native binaries, we only want to link after we've installed the companion dependencies
+        // We don't want to introduce dependent callbacks like that for every single package
+        // Since this will only be a handful, it's fine to just say "run this at the end"
+        platform_binlinks: std.ArrayListUnmanaged(DeferredBinLink) = std.ArrayListUnmanaged(DeferredBinLink){},
+
+        pub const DeferredBinLink = struct {
+            package_id: PackageID,
+            node_modules_folder: std.fs.Dir,
+        };
+
         pub fn installEnqueuedPackages(
             this: *PackageInstaller,
             package_id: PackageID,
@@ -5797,6 +5900,56 @@ pub const PackageManager = struct {
                                 if (comptime log_level.showProgress()) {
                                     this.node.completeOne();
                                 }
+
+                                const bin = this.bins[package_id];
+                                if (bin.tag != .none) {
+                                    if (!this.has_created_bin) {
+                                        this.node_modules_folder.makeDirZ(".bin") catch {};
+                                        Bin.Linker.umask = C.umask(0);
+                                        this.has_created_bin = true;
+                                    }
+
+                                    const bin_task_id = Task.Id.forBinLink(package_id);
+                                    var task_queue = this.manager.task_queue.getOrPut(this.manager.allocator, bin_task_id) catch unreachable;
+                                    if (!task_queue.found_existing) {
+                                        run_bin_link: {
+                                            if (std.mem.indexOfScalar(PackageNameHash, this.options.native_bin_link_allowlist, String.Builder.stringHash(name)) != null) {
+                                                this.platform_binlinks.append(this.lockfile.allocator, .{
+                                                    .package_id = package_id,
+                                                    .node_modules_folder = this.node_modules_folder,
+                                                }) catch unreachable;
+                                                break :run_bin_link;
+                                            }
+
+                                            var bin_linker = Bin.Linker{
+                                                .bin = bin,
+                                                .package_installed_node_modules = this.node_modules_folder.fd,
+                                                .root_node_modules_folder = this.root_node_modules_folder.fd,
+                                                .package_name = strings.StringOrTinyString.init(name),
+                                                .string_buf = buf,
+                                            };
+
+                                            bin_linker.link();
+
+                                            if (comptime log_level != .silent) {
+                                                if (bin_linker.err) |err| {
+                                                    const fmt = "\n<r><red>error:<r> linking <b>{s}<r>: {s}\n";
+                                                    const args = .{ name, @errorName(err) };
+
+                                                    if (comptime log_level.showProgress()) {
+                                                        if (Output.enable_ansi_colors) {
+                                                            this.progress.log(comptime Output.prettyFmt(fmt, true), args);
+                                                        } else {
+                                                            this.progress.log(comptime Output.prettyFmt(fmt, false), args);
+                                                        }
+                                                    } else {
+                                                        Output.prettyErrorln(fmt, args);
+                                                    }
+                                                }
+                                            }
+                                        }
+                                    }
+                                }
                             },
                             .fail => |cause| {
                                 if (cause.isPackageMissingFromCache()) {
@@ -5937,7 +6090,10 @@ pub const PackageManager = struct {
 
             var installer = PackageInstaller{
                 .manager = this,
+                .options = &this.options,
                 .metas = metas,
+                .bins = parts.items(.bin),
+                .root_node_modules_folder = node_modules_folder,
                 .names = names,
                 .resolutions = resolutions,
                 .lockfile = lockfile,
@@ -5949,7 +6105,6 @@ pub const PackageManager = struct {
                 .summary = &summary,
                 .force_install = force_install,
                 .install_count = lockfile.buffers.hoisted_packages.items.len,
-                .trees = lockfile.buffers.trees.items,
             };
 
             // When it's a Good Idea, run the install in single-threaded
@@ -6071,6 +6226,79 @@ pub const PackageManager = struct {
                 //     std.atomic.spinLoopHint();
             }
             // }
+
+            outer: for (installer.platform_binlinks.items) |deferred| {
+                const package_id = deferred.package_id;
+                const folder = deferred.node_modules_folder;
+
+                const package_dependencies: []const Dependency = dependency_lists[package_id].get(dependencies);
+                const package_resolutions: []const PackageID = resolution_lists[package_id].get(resolutions_buffer);
+                const original_bin: Bin = installer.bins[package_id];
+
+                for (package_dependencies) |dependency, i| {
+                    const resolved_id = package_resolutions[i];
+                    if (resolved_id >= names.len) continue;
+                    const meta: Lockfile.Package.Meta = metas[resolved_id];
+
+                    // This is specifically for platform-specific binaries
+                    if (meta.os == .all and meta.arch == .all) continue;
+
+                    // Don't attempt to link incompatible binaries
+                    if (meta.isDisabled()) continue;
+
+                    const name: string = installer.names[resolved_id].slice(lockfile.buffers.string_bytes.items);
+
+                    if (!installer.has_created_bin) {
+                        node_modules_folder.makeDirZ(".bin") catch {};
+                        Bin.Linker.umask = C.umask(0);
+                        installer.has_created_bin = true;
+                    }
+
+                    var bin_linker = Bin.Linker{
+                        .bin = original_bin,
+                        .package_installed_node_modules = folder.fd,
+                        .root_node_modules_folder = node_modules_folder.fd,
+                        .package_name = strings.StringOrTinyString.init(name),
+                        .string_buf = lockfile.buffers.string_bytes.items,
+                    };
+
+                    bin_linker.link();
+
+                    if (comptime log_level != .silent) {
+                        if (bin_linker.err) |err| {
+                            const fmt = "\n<r><red>error:<r> linking <b>{s}<r>: {s}\n";
+                            const args = .{ name, @errorName(err) };
+
+                            if (comptime log_level.showProgress()) {
+                                if (Output.enable_ansi_colors) {
+                                    this.progress.log(comptime Output.prettyFmt(fmt, true), args);
+                                } else {
+                                    this.progress.log(comptime Output.prettyFmt(fmt, false), args);
+                                }
+                            } else {
+                                Output.prettyErrorln(fmt, args);
+                            }
+                        }
+                    }
+
+                    continue :outer;
+                }
+
+                if (comptime log_level != .silent) {
+                    const fmt = "\n<r><yellow>warn:<r> no compatible binaries found for <b>{s}<r>\n";
+                    const args = .{names[package_id]};
+
+                    if (comptime log_level.showProgress()) {
+                        if (Output.enable_ansi_colors) {
+                            this.progress.log(comptime Output.prettyFmt(fmt, true), args);
+                        } else {
+                            this.progress.log(comptime Output.prettyFmt(fmt, false), args);
+                        }
+                    } else {
+                        Output.prettyErrorln(fmt, args);
+                    }
+                }
+            }
         }
 
         return summary;