feat(bun/test): Implement "bail" option for "bun test" (#3253)

* Implement bun test --bail

* Fixes

* move printSummary() (more readable)

* Fixes 2

* idk why it got deleted

* Fixes 3

* fmt this better

* Update test_command.zig

* Fix "0 files"

* track number of files so bailing out early prints the right number

---------

Co-authored-by: dave caruso <me@paperdave.net>

5 files changed, 134 insertions, 4 deletions

diff --git a/src/bun.js/test/jest.zig b/src/bun.js/test/jest.zig
index 55600ded8..4c97b5c77 100644
--- a/src/bun.js/test/jest.zig
+++ b/src/bun.js/test/jest.zig
@@ -68,6 +68,7 @@ pub const TestRunner = struct {
     only: bool = false,
     run_todo: bool = false,
     last_file: u64 = 0,
+    bail: u32 = 0,
 
     allocator: std.mem.Allocator,
     callback: *Callback = undefined,
diff --git a/src/cli.zig b/src/cli.zig
index 3d0fb5b37..f0cc47918 100644
--- a/src/cli.zig
+++ b/src/cli.zig
@@ -217,6 +217,7 @@ pub const Arguments = struct {
         clap.parseParam("--rerun-each <NUMBER>            Re-run each test file <NUMBER> times, helps catch certain bugs") catch unreachable,
         clap.parseParam("--only                           Only run tests that are marked with \"test.only()\"") catch unreachable,
         clap.parseParam("--todo                           Include tests that are marked with \"test.todo()\"") catch unreachable,
+        clap.parseParam("--bail <NUMBER>?                 Exit the test suite after <NUMBER> failures. If you do not specify a number, it defaults to 1.") catch unreachable,
     };
 
     const build_params_public = public_params ++ build_only_params;
@@ -383,7 +384,21 @@ pub const Arguments = struct {
                     };
                 }
             }
-            ctx.test_options.update_snapshots = args.flag("--update-snapshots");
+            if (args.option("--bail")) |bail| {
+                if (bail.len > 0) {
+                    ctx.test_options.bail = std.fmt.parseInt(u32, bail, 10) catch |e| {
+                        Output.prettyErrorln("--bail expects a number: {s}", .{@errorName(e)});
+                        Global.exit(1);
+                    };
+
+                    if (ctx.test_options.bail == 0) {
+                        Output.prettyErrorln("--bail expects a number greater than 0", .{});
+                        Global.exit(1);
+                    }
+                } else {
+                    ctx.test_options.bail = 1;
+                }
+            }
             if (args.option("--rerun-each")) |repeat_count| {
                 if (repeat_count.len > 0) {
                     ctx.test_options.repeat_count = std.fmt.parseInt(u32, repeat_count, 10) catch |e| {
@@ -392,6 +407,7 @@ pub const Arguments = struct {
                     };
                 }
             }
+            ctx.test_options.update_snapshots = args.flag("--update-snapshots");
             ctx.test_options.run_todo = args.flag("--todo");
             ctx.test_options.only = args.flag("--only");
         }
@@ -929,6 +945,7 @@ pub const Command = struct {
         repeat_count: u32 = 0,
         run_todo: bool = false,
         only: bool = false,
+        bail: u32 = 0,
     };
 
     pub const Context = struct {
diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig
index dc67c6d98..3c6db7fc2 100644
--- a/src/cli/test_command.zig
+++ b/src/cli/test_command.zig
@@ -92,6 +92,7 @@ pub const CommandLineReporter = struct {
         skip: u32 = 0,
         todo: u32 = 0,
         fail: u32 = 0,
+        files: u32 = 0,
     };
 
     const DotColorMap = std.EnumMap(TestRunner.Test.Status, string);
@@ -200,6 +201,12 @@ pub const CommandLineReporter = struct {
         this.summary.fail += 1;
         this.summary.expectations += expectations;
         this.jest.tests.items(.status)[id] = TestRunner.Test.Status.fail;
+
+        if (this.jest.bail == this.summary.fail) {
+            this.printSummary();
+            Output.prettyError("\nBailed out after {d} failures<r>\n", .{this.jest.bail});
+            Global.exit(1);
+        }
     }
 
     pub fn handleTestSkip(cb: *TestRunner.Callback, id: Test.ID, _: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void {
@@ -246,6 +253,14 @@ pub const CommandLineReporter = struct {
         this.summary.expectations += expectations;
         this.jest.tests.items(.status)[id] = TestRunner.Test.Status.todo;
     }
+
+    pub fn printSummary(this: *CommandLineReporter) void {
+        const tests = this.summary.fail + this.summary.pass + this.summary.skip + this.summary.todo;
+        const files = this.summary.files;
+
+        Output.prettyError("Ran {d} tests across {d} files. ", .{ tests, files });
+        Output.printStartEnd(bun.start_time, std.time.nanoTimestamp());
+    }
 };
 
 const Scanner = struct {
@@ -441,6 +456,7 @@ pub const TestCommand = struct {
                 .default_timeout_ms = ctx.test_options.default_timeout_ms,
                 .run_todo = ctx.test_options.run_todo,
                 .only = ctx.test_options.only,
+                .bail = ctx.test_options.bail,
                 .snapshots = Snapshots{
                     .allocator = ctx.allocator,
                     .update_snapshots = ctx.test_options.update_snapshots,
@@ -650,9 +666,7 @@ pub const TestCommand = struct {
                 Output.prettyError(" {d:5>} expect() calls\n", .{reporter.summary.expectations});
             }
 
-            const total_tests = reporter.summary.fail + reporter.summary.pass + reporter.summary.skip + reporter.summary.todo;
-            Output.prettyError("Ran {d} tests across {d} files. <d>{d} total<r> ", .{ reporter.summary.fail + reporter.summary.pass, test_files.len, total_tests });
-            Output.printStartEnd(ctx.start_time, std.time.nanoTimestamp());
+            reporter.printSummary();
         }
 
         Output.prettyError("\n", .{});
@@ -763,12 +777,20 @@ pub const TestCommand = struct {
             Output.flush();
 
             var promise = try vm.loadEntryPoint(file_path);
+            reporter.summary.files += 1;
 
             switch (promise.status(vm.global.vm())) {
                 .Rejected => {
                     var result = promise.result(vm.global.vm());
                     vm.runErrorHandler(result, null);
                     reporter.summary.fail += 1;
+
+                    if (reporter.jest.bail == reporter.summary.fail) {
+                        reporter.printSummary();
+                        Output.prettyError("\nBailed out after {d} failures<r>\n", .{reporter.jest.bail});
+                        Global.exit(1);
+                    }
+
                     return;
                 },
                 else => {},
diff --git a/test/cli/test/bun-test.test.ts b/test/cli/test/bun-test.test.ts
index 534c17513..d26912b7c 100644
--- a/test/cli/test/bun-test.test.ts
+++ b/test/cli/test/bun-test.test.ts
@@ -179,6 +179,68 @@ describe("bun test", () => {
       expect(stderr.match(/reachable/g)).toHaveLength(4);
     });
   });
+  describe("--bail", () => {
+    test("must provide a number bail", () => {
+      const stderr = runTest({
+        args: ["--bail", "foo"],
+      });
+      expect(stderr).toContain("expects a number");
+    });
+
+    test("must provide non-negative bail", () => {
+      const stderr = runTest({
+        args: ["--bail", "-1"],
+      });
+      expect(stderr).toContain("expects a number");
+    });
+
+    test("should not be 0", () => {
+      const stderr = runTest({
+        args: ["--bail", "0"],
+      });
+      expect(stderr).toContain("expects a number");
+    });
+
+    test("bail should be 1 by default", () => {
+      const stderr = runTest({
+        args: ["--bail"],
+        input: `
+          import { test, expect } from "bun:test";
+          test("test #1", () => {
+            expect(true).toBe(false);
+          });
+          test("test #2", () => {
+            expect(true).toBe(true);
+          });
+        `,
+      });
+      expect(stderr).toContain("Bailed out after 1 failures");
+      expect(stderr).not.toContain("test #2");
+    });
+
+    test("should bail out after 3 failures", () => {
+      const stderr = runTest({
+        args: ["--bail", "3"],
+        input: `
+          import { test, expect } from "bun:test";
+          test("test #1", () => {
+            expect(true).toBe(false);
+          });
+          test("test #2", () => {
+            expect(true).toBe(false);
+          });
+          test("test #3", () => {
+            expect(true).toBe(false);
+          });
+          test("test #4", () => {
+            expect(true).toBe(true);
+          });
+        `,
+      });
+      expect(stderr).toContain("Bailed out after 3 failures");
+      expect(stderr).not.toContain("test #4");
+    });
+  });
   describe("--timeout", () => {
     test("must provide a number timeout", () => {
       const stderr = runTest({
diff --git a/test/js/bun/test/test-test.test.ts b/test/js/bun/test/test-test.test.ts
index 5f732bb82..2f824a9d0 100644
--- a/test/js/bun/test/test-test.test.ts
+++ b/test/js/bun/test/test-test.test.ts
@@ -301,6 +301,34 @@ it("should return non-zero exit code for invalid syntax", async () => {
   }
 });
 
+it("invalid syntax counts towards bail", async () => {
+  const test_dir = realpathSync(await mkdtemp(join(tmpdir(), "test")));
+  try {
+    await writeFile(join(test_dir, "bad1.test.js"), "!!!");
+    await writeFile(join(test_dir, "bad2.test.js"), "!!!");
+    await writeFile(join(test_dir, "bad3.test.js"), "!!!");
+    await writeFile(join(test_dir, "bad4.test.js"), "!!!");
+    await writeFile(join(test_dir, "bad5.test.js"), "!!!");
+    const { stdout, stderr, exited } = spawn({
+      cmd: [bunExe(), "test", "--bail", "3"],
+      cwd: test_dir,
+      stdout: null,
+      stdin: "pipe",
+      stderr: "pipe",
+      env: bunEnv,
+    });
+    const err = await new Response(stderr).text();
+    expect(err).toContain("Bailed out after 3 failures");
+    expect(err).not.toContain("DO NOT RUN ME");
+    expect(err).toContain("Ran 3 tests across 3 files");
+    expect(stdout).toBeDefined();
+    expect(await new Response(stdout).text()).toBe("");
+    expect(await exited).toBe(1);
+  } finally {
+    // await rm(test_dir, { force: true, recursive: true });
+  }
+});
+
 describe("skip test inner", () => {
   it("should pass", () => {
     expect(2 + 2).toBe(4);