[install] fix re-run of tarball URL from lockfile (#2507)

3 files changed, 371 insertions, 30 deletions

diff --git a/src/install/install.zig b/src/install/install.zig
index b6d61e00a..a133b7b48 100644
--- a/src/install/install.zig
+++ b/src/install/install.zig
@@ -2705,7 +2705,7 @@ pub const PackageManager = struct {
         this: *PackageManager,
         task_id: u64,
         dependency_id: DependencyID,
-        name: String,
+        name: string,
         path: string,
         resolution: Resolution,
     ) *ThreadPool.Task {
@@ -2719,7 +2719,7 @@ pub const PackageManager = struct {
                     .tarball = .{
                         .package_manager = &PackageManager.instance, // https://github.com/ziglang/zig/issues/14005
                         .name = strings.StringOrTinyString.initAppendIfNeeded(
-                            this.lockfile.str(&name),
+                            name,
                             *FileSystem.FilenameStore,
                             &FileSystem.FilenameStore.instance,
                         ) catch unreachable,
@@ -3230,7 +3230,7 @@ pub const PackageManager = struct {
                         this.task_batch.push(ThreadPool.Batch.from(this.enqueueLocalTarball(
                             task_id,
                             id,
-                            dependency.name,
+                            this.lockfile.str(&dependency.name),
                             url,
                             res,
                         )));
@@ -6341,6 +6341,8 @@ pub const PackageManager = struct {
             const task_id = switch (resolution.tag) {
                 .git => Task.Id.forGitCheckout(data.url, data.resolved),
                 .github => Task.Id.forTarball(data.url),
+                .local_tarball => Task.Id.forTarball(this.lockfile.str(&resolution.value.local_tarball)),
+                .remote_tarball => Task.Id.forTarball(this.lockfile.str(&resolution.value.remote_tarball)),
                 .npm => Task.Id.forNPMPackage(name, resolution.value.npm.version),
                 else => unreachable,
             };
@@ -6575,7 +6577,24 @@ pub const PackageManager = struct {
                                     this.manager.enqueueTarballForDownload(
                                         dependency_id,
                                         package_id,
-                                        &resolution.value.github,
+                                        this.manager.allocGitHubURL(&resolution.value.github) catch unreachable,
+                                        .{ .node_modules_folder = this.node_modules_folder.dir.fd },
+                                    );
+                                },
+                                .local_tarball => {
+                                    this.manager.enqueueTarballForReading(
+                                        dependency_id,
+                                        package_id,
+                                        alias,
+                                        resolution.value.local_tarball.slice(buf),
+                                        .{ .node_modules_folder = this.node_modules_folder.dir.fd },
+                                    );
+                                },
+                                .remote_tarball => {
+                                    this.manager.enqueueTarballForDownload(
+                                        dependency_id,
+                                        package_id,
+                                        resolution.value.remote_tarball.slice(buf),
                                         .{ .node_modules_folder = this.node_modules_folder.dir.fd },
                                     );
                                 },
@@ -6659,17 +6678,17 @@ pub const PackageManager = struct {
             task_context,
         ) catch unreachable;
 
-        if (!task_queue.found_existing) {
-            if (this.generateNetworkTaskForTarball(
-                task_id,
-                url,
-                dependency_id,
-                this.lockfile.packages.get(package_id),
-            ) catch unreachable) |task| {
-                task.schedule(&this.network_tarball_batch);
-                if (this.network_tarball_batch.len > 0) {
-                    _ = this.scheduleTasks();
-                }
+        if (task_queue.found_existing) return;
+
+        if (this.generateNetworkTaskForTarball(
+            task_id,
+            url,
+            dependency_id,
+            this.lockfile.packages.get(package_id),
+        ) catch unreachable) |task| {
+            task.schedule(&this.network_tarball_batch);
+            if (this.network_tarball_batch.len > 0) {
+                _ = this.scheduleTasks();
             }
         }
     }
@@ -6678,10 +6697,9 @@ pub const PackageManager = struct {
         this: *PackageManager,
         dependency_id: PackageID,
         package_id: PackageID,
-        repository: *const Repository,
+        url: string,
         task_context: TaskCallbackContext,
     ) void {
-        const url = this.allocGitHubURL(repository) catch unreachable;
         const task_id = Task.Id.forTarball(url);
         var task_queue = this.task_queue.getOrPut(this.allocator, task_id) catch unreachable;
         if (!task_queue.found_existing) {
@@ -6693,21 +6711,51 @@ pub const PackageManager = struct {
             task_context,
         ) catch unreachable;
 
-        if (!task_queue.found_existing) {
-            if (this.generateNetworkTaskForTarball(
-                task_id,
-                url,
-                dependency_id,
-                this.lockfile.packages.get(package_id),
-            ) catch unreachable) |task| {
-                task.schedule(&this.network_tarball_batch);
-                if (this.network_tarball_batch.len > 0) {
-                    _ = this.scheduleTasks();
-                }
+        if (task_queue.found_existing) return;
+
+        if (this.generateNetworkTaskForTarball(
+            task_id,
+            url,
+            dependency_id,
+            this.lockfile.packages.get(package_id),
+        ) catch unreachable) |task| {
+            task.schedule(&this.network_tarball_batch);
+            if (this.network_tarball_batch.len > 0) {
+                _ = this.scheduleTasks();
             }
         }
     }
 
+    pub fn enqueueTarballForReading(
+        this: *PackageManager,
+        dependency_id: PackageID,
+        package_id: PackageID,
+        alias: string,
+        path: string,
+        task_context: TaskCallbackContext,
+    ) void {
+        const task_id = Task.Id.forTarball(path);
+        var task_queue = this.task_queue.getOrPut(this.allocator, task_id) catch unreachable;
+        if (!task_queue.found_existing) {
+            task_queue.value_ptr.* = .{};
+        }
+
+        task_queue.value_ptr.append(
+            this.allocator,
+            task_context,
+        ) catch unreachable;
+
+        if (task_queue.found_existing) return;
+
+        this.task_batch.push(ThreadPool.Batch.from(this.enqueueLocalTarball(
+            task_id,
+            dependency_id,
+            alias,
+            path,
+            this.lockfile.packages.items(.resolution)[package_id],
+        )));
+    }
+
     pub fn installPackages(
         this: *PackageManager,
         lockfile_: *Lockfile,
diff --git a/test/cli/install/bun-install.test.ts b/test/cli/install/bun-install.test.ts
index f9a0fe132..aa40b9220 100644
--- a/test/cli/install/bun-install.test.ts
+++ b/test/cli/install/bun-install.test.ts
@@ -1,7 +1,7 @@
 import { file, listen, Socket, spawn } from "bun";
 import { afterAll, afterEach, beforeAll, beforeEach, expect, it } from "bun:test";
 import { bunExe, bunEnv as env } from "harness";
-import { access, mkdir, readlink, writeFile } from "fs/promises";
+import { access, mkdir, readlink, rm, writeFile } from "fs/promises";
 import { join } from "path";
 import {
   dummyAfterAll,
@@ -1509,14 +1509,23 @@ it("should handle unscoped alias on scoped dependency", async () => {
   expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".cache", "@barn", "moo"]);
   expect(await readdirSorted(join(package_dir, "node_modules", "@barn"))).toEqual(["moo"]);
   expect(await readdirSorted(join(package_dir, "node_modules", "@barn", "moo"))).toEqual(["package.json"]);
-  expect(await file(join(package_dir, "node_modules", "moo", "package.json")).json()).toEqual({
+  expect(await file(join(package_dir, "node_modules", "@barn", "moo", "package.json")).json()).toEqual({
     name: "@barn/moo",
     version: "0.1.0",
+    // not installed as these are absent from manifest above
+    dependencies: {
+      bar: "0.0.2",
+      baz: "latest",
+    },
   });
   expect(await readdirSorted(join(package_dir, "node_modules", "moo"))).toEqual(["package.json"]);
   expect(await file(join(package_dir, "node_modules", "moo", "package.json")).json()).toEqual({
     name: "@barn/moo",
     version: "0.1.0",
+    dependencies: {
+      bar: "0.0.2",
+      baz: "latest",
+    },
   });
   await access(join(package_dir, "bun.lockb"));
 });
@@ -2990,3 +2999,287 @@ it("should handle tarball path with aliasing", async () => {
   });
   await access(join(package_dir, "bun.lockb"));
 });
+
+it("should handle tarball URL with existing lockfile", async () => {
+  const urls: string[] = [];
+  setHandler(
+    dummyRegistry(urls, {
+      "0.0.2": {},
+      "0.0.3": {
+        bin: {
+          "baz-run": "index.js",
+        },
+      },
+    }),
+  );
+  await writeFile(
+    join(package_dir, "package.json"),
+    JSON.stringify({
+      name: "foo",
+      version: "0.0.1",
+      dependencies: {
+        "@barn/moo": `${root_url}/moo-0.1.0.tgz`,
+      },
+    }),
+  );
+  const {
+    stdout: stdout1,
+    stderr: stderr1,
+    exited: exited1,
+  } = spawn({
+    cmd: [bunExe(), "install"],
+    cwd: package_dir,
+    stdout: null,
+    stdin: "pipe",
+    stderr: "pipe",
+    env,
+  });
+  expect(stderr1).toBeDefined();
+  const err1 = await new Response(stderr1).text();
+  expect(err1).toContain("Saved lockfile");
+  expect(stdout1).toBeDefined();
+  const out1 = await new Response(stdout1).text();
+  expect(out1.replace(/\s*\[[0-9\.]+m?s\]\s*$/, "").split(/\r?\n/)).toEqual([
+    ` + @barn/moo@${root_url}/moo-0.1.0.tgz`,
+    "",
+    " 3 packages installed",
+  ]);
+  expect(await exited1).toBe(0);
+  expect(urls.sort()).toEqual([
+    `${root_url}/bar`,
+    `${root_url}/bar-0.0.2.tgz`,
+    `${root_url}/baz`,
+    `${root_url}/baz-0.0.3.tgz`,
+    `${root_url}/moo-0.1.0.tgz`,
+  ]);
+  expect(requested).toBe(5);
+  expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "@barn", "bar", "baz"]);
+  expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toEqual(["baz-run"]);
+  expect(await readlink(join(package_dir, "node_modules", ".bin", "baz-run"))).toBe(join("..", "baz", "index.js"));
+  expect(await readdirSorted(join(package_dir, "node_modules", "@barn"))).toEqual(["moo"]);
+  expect(await readdirSorted(join(package_dir, "node_modules", "@barn", "moo"))).toEqual(["package.json"]);
+  expect(await file(join(package_dir, "node_modules", "@barn", "moo", "package.json")).json()).toEqual({
+    name: "@barn/moo",
+    version: "0.1.0",
+    dependencies: {
+      bar: "0.0.2",
+      baz: "latest",
+    },
+  });
+  expect(await readdirSorted(join(package_dir, "node_modules", "bar"))).toEqual(["package.json"]);
+  expect(await file(join(package_dir, "node_modules", "bar", "package.json")).json()).toEqual({
+    name: "bar",
+    version: "0.0.2",
+  });
+  expect(await readdirSorted(join(package_dir, "node_modules", "baz"))).toEqual(["index.js", "package.json"]);
+  expect(await file(join(package_dir, "node_modules", "baz", "package.json")).json()).toEqual({
+    name: "baz",
+    version: "0.0.3",
+    bin: {
+      "baz-run": "index.js",
+    },
+  });
+  await access(join(package_dir, "bun.lockb"));
+  // Perform `bun install` again but with lockfile from before
+  await rm(join(package_dir, "node_modules"), { force: true, recursive: true });
+  urls.length = 0;
+  const {
+    stdout: stdout2,
+    stderr: stderr2,
+    exited: exited2,
+  } = spawn({
+    cmd: [bunExe(), "install"],
+    cwd: package_dir,
+    stdout: null,
+    stdin: "pipe",
+    stderr: "pipe",
+    env,
+  });
+  expect(stderr2).toBeDefined();
+  const err2 = await new Response(stderr2).text();
+  expect(err2).not.toContain("Saved lockfile");
+  expect(stdout2).toBeDefined();
+  const out2 = await new Response(stdout2).text();
+  expect(out2.replace(/\s*\[[0-9\.]+m?s\]\s*$/, "").split(/\r?\n/)).toEqual([
+    ` + @barn/moo@${root_url}/moo-0.1.0.tgz`,
+    "",
+    " 3 packages installed",
+  ]);
+  expect(await exited2).toBe(0);
+  expect(urls.sort()).toEqual([
+    `${root_url}/bar`,
+    `${root_url}/bar-0.0.2.tgz`,
+    `${root_url}/baz`,
+    `${root_url}/baz-0.0.3.tgz`,
+    `${root_url}/moo-0.1.0.tgz`,
+  ]);
+  expect(requested).toBe(10);
+  expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "@barn", "bar", "baz"]);
+  expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toEqual(["baz-run"]);
+  expect(await readlink(join(package_dir, "node_modules", ".bin", "baz-run"))).toBe(join("..", "baz", "index.js"));
+  expect(await readdirSorted(join(package_dir, "node_modules", "@barn"))).toEqual(["moo"]);
+  expect(await readdirSorted(join(package_dir, "node_modules", "@barn", "moo"))).toEqual(["package.json"]);
+  expect(await file(join(package_dir, "node_modules", "@barn", "moo", "package.json")).json()).toEqual({
+    name: "@barn/moo",
+    version: "0.1.0",
+    dependencies: {
+      bar: "0.0.2",
+      baz: "latest",
+    },
+  });
+  expect(await readdirSorted(join(package_dir, "node_modules", "bar"))).toEqual(["package.json"]);
+  expect(await file(join(package_dir, "node_modules", "bar", "package.json")).json()).toEqual({
+    name: "bar",
+    version: "0.0.2",
+  });
+  expect(await readdirSorted(join(package_dir, "node_modules", "baz"))).toEqual(["index.js", "package.json"]);
+  expect(await file(join(package_dir, "node_modules", "baz", "package.json")).json()).toEqual({
+    name: "baz",
+    version: "0.0.3",
+    bin: {
+      "baz-run": "index.js",
+    },
+  });
+  await access(join(package_dir, "bun.lockb"));
+});
+
+it("should handle tarball path with existing lockfile", async () => {
+  const urls: string[] = [];
+  setHandler(
+    dummyRegistry(urls, {
+      "0.0.2": {},
+      "0.0.3": {
+        bin: {
+          "baz-run": "index.js",
+        },
+      },
+    }),
+  );
+  await writeFile(
+    join(package_dir, "package.json"),
+    JSON.stringify({
+      name: "foo",
+      version: "0.0.1",
+      dependencies: {
+        "@barn/moo": join(import.meta.dir, "moo-0.1.0.tgz"),
+      },
+    }),
+  );
+  const {
+    stdout: stdout1,
+    stderr: stderr1,
+    exited: exited1,
+  } = spawn({
+    cmd: [bunExe(), "install"],
+    cwd: package_dir,
+    stdout: null,
+    stdin: "pipe",
+    stderr: "pipe",
+    env,
+  });
+  expect(stderr1).toBeDefined();
+  const err1 = await new Response(stderr1).text();
+  expect(err1).toContain("Saved lockfile");
+  expect(stdout1).toBeDefined();
+  const out1 = await new Response(stdout1).text();
+  expect(out1.replace(/\s*\[[0-9\.]+m?s\]\s*$/, "").split(/\r?\n/)).toEqual([
+    ` + @barn/moo@${join(import.meta.dir, "moo-0.1.0.tgz")}`,
+    "",
+    " 3 packages installed",
+  ]);
+  expect(await exited1).toBe(0);
+  expect(urls.sort()).toEqual([
+    `${root_url}/bar`,
+    `${root_url}/bar-0.0.2.tgz`,
+    `${root_url}/baz`,
+    `${root_url}/baz-0.0.3.tgz`,
+  ]);
+  expect(requested).toBe(4);
+  expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "@barn", "bar", "baz"]);
+  expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toEqual(["baz-run"]);
+  expect(await readlink(join(package_dir, "node_modules", ".bin", "baz-run"))).toBe(join("..", "baz", "index.js"));
+  expect(await readdirSorted(join(package_dir, "node_modules", "@barn"))).toEqual(["moo"]);
+  expect(await readdirSorted(join(package_dir, "node_modules", "@barn", "moo"))).toEqual(["package.json"]);
+  expect(await file(join(package_dir, "node_modules", "@barn", "moo", "package.json")).json()).toEqual({
+    name: "@barn/moo",
+    version: "0.1.0",
+    dependencies: {
+      bar: "0.0.2",
+      baz: "latest",
+    },
+  });
+  expect(await readdirSorted(join(package_dir, "node_modules", "bar"))).toEqual(["package.json"]);
+  expect(await file(join(package_dir, "node_modules", "bar", "package.json")).json()).toEqual({
+    name: "bar",
+    version: "0.0.2",
+  });
+  expect(await readdirSorted(join(package_dir, "node_modules", "baz"))).toEqual(["index.js", "package.json"]);
+  expect(await file(join(package_dir, "node_modules", "baz", "package.json")).json()).toEqual({
+    name: "baz",
+    version: "0.0.3",
+    bin: {
+      "baz-run": "index.js",
+    },
+  });
+  await access(join(package_dir, "bun.lockb"));
+  // Perform `bun install` again but with lockfile from before
+  await rm(join(package_dir, "node_modules"), { force: true, recursive: true });
+  urls.length = 0;
+  const {
+    stdout: stdout2,
+    stderr: stderr2,
+    exited: exited2,
+  } = spawn({
+    cmd: [bunExe(), "install"],
+    cwd: package_dir,
+    stdout: null,
+    stdin: "pipe",
+    stderr: "pipe",
+    env,
+  });
+  expect(stderr2).toBeDefined();
+  const err2 = await new Response(stderr2).text();
+  expect(err2).not.toContain("Saved lockfile");
+  expect(stdout2).toBeDefined();
+  const out2 = await new Response(stdout2).text();
+  expect(out2.replace(/\s*\[[0-9\.]+m?s\]\s*$/, "").split(/\r?\n/)).toEqual([
+    ` + @barn/moo@${join(import.meta.dir, "moo-0.1.0.tgz")}`,
+    "",
+    " 3 packages installed",
+  ]);
+  expect(await exited2).toBe(0);
+  expect(urls.sort()).toEqual([
+    `${root_url}/bar`,
+    `${root_url}/bar-0.0.2.tgz`,
+    `${root_url}/baz`,
+    `${root_url}/baz-0.0.3.tgz`,
+  ]);
+  expect(requested).toBe(8);
+  expect(await readdirSorted(join(package_dir, "node_modules"))).toEqual([".bin", ".cache", "@barn", "bar", "baz"]);
+  expect(await readdirSorted(join(package_dir, "node_modules", ".bin"))).toEqual(["baz-run"]);
+  expect(await readlink(join(package_dir, "node_modules", ".bin", "baz-run"))).toBe(join("..", "baz", "index.js"));
+  expect(await readdirSorted(join(package_dir, "node_modules", "@barn"))).toEqual(["moo"]);
+  expect(await readdirSorted(join(package_dir, "node_modules", "@barn", "moo"))).toEqual(["package.json"]);
+  expect(await file(join(package_dir, "node_modules", "@barn", "moo", "package.json")).json()).toEqual({
+    name: "@barn/moo",
+    version: "0.1.0",
+    dependencies: {
+      bar: "0.0.2",
+      baz: "latest",
+    },
+  });
+  expect(await readdirSorted(join(package_dir, "node_modules", "bar"))).toEqual(["package.json"]);
+  expect(await file(join(package_dir, "node_modules", "bar", "package.json")).json()).toEqual({
+    name: "bar",
+    version: "0.0.2",
+  });
+  expect(await readdirSorted(join(package_dir, "node_modules", "baz"))).toEqual(["index.js", "package.json"]);
+  expect(await file(join(package_dir, "node_modules", "baz", "package.json")).json()).toEqual({
+    name: "baz",
+    version: "0.0.3",
+    bin: {
+      "baz-run": "index.js",
+    },
+  });
+  await access(join(package_dir, "bun.lockb"));
+});
diff --git a/test/cli/install/moo-0.1.0.tgz b/test/cli/install/moo-0.1.0.tgz
index 72efff27b..f93b9f46c 100644
--- a/test/cli/install/moo-0.1.0.tgz
+++ b/test/cli/install/moo-0.1.0.tgz