Code coverage for `bun test` (#3975)

* WIP code coverage initial commit

* almost works

* one approach

* Code Coverage

* Update WebKit

* it works but is not yet accurate

* skip double ascii check

* wrapper

* it works but i'm not sure what to do about blocks

* hide blocks for now

* Update ZigSourceProvider.cpp

* Create coverage.md

* Update nav.ts

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>

1 files changed, 157 insertions, 3 deletions

diff --git a/src/cli/test_command.zig b/src/cli/test_command.zig
index 819aceb19..c4d78f4d5 100644
--- a/src/cli/test_command.zig
+++ b/src/cli/test_command.zig
@@ -261,6 +261,103 @@ pub const CommandLineReporter = struct {
         Output.prettyError("Ran {d} tests across {d} files. ", .{ tests, files });
         Output.printStartEnd(bun.start_time, std.time.nanoTimestamp());
     }
+
+    pub fn printCodeCoverage(this: *CommandLineReporter, vm: *JSC.VirtualMachine, opts: *TestCommand.CodeCoverageOptions, comptime enable_ansi_colors: bool) !void {
+        const trace = bun.tracy.traceNamed(@src(), "TestCommand.printCodeCoverage");
+        defer trace.end();
+
+        _ = this;
+        var map = bun.sourcemap.ByteRangeMapping.map orelse return;
+        var iter = map.valueIterator();
+        var max_filepath_length: usize = "All files".len;
+        const relative_dir = vm.bundler.fs.top_level_dir;
+
+        var byte_ranges = try std.ArrayList(bun.sourcemap.ByteRangeMapping).initCapacity(bun.default_allocator, map.count());
+
+        while (iter.next()) |entry| {
+            const value: bun.sourcemap.ByteRangeMapping = entry.*;
+            var utf8 = value.source_url.slice();
+            byte_ranges.appendAssumeCapacity(value);
+            max_filepath_length = @max(bun.path.relative(relative_dir, utf8).len, max_filepath_length);
+        }
+
+        if (byte_ranges.items.len == 0) {
+            return;
+        }
+
+        std.sort.block(bun.sourcemap.ByteRangeMapping, byte_ranges.items, void{}, bun.sourcemap.ByteRangeMapping.isLessThan);
+
+        iter = map.valueIterator();
+        var writer = Output.errorWriter();
+        var base_fraction = opts.fractions;
+        var failing = false;
+
+        writer.writeAll(Output.prettyFmt("<r><d>", enable_ansi_colors)) catch return;
+        writer.writeByteNTimes('-', max_filepath_length + 2) catch return;
+        writer.writeAll(Output.prettyFmt("|---------|---------|-------------------<r>\n", enable_ansi_colors)) catch return;
+        writer.writeAll("File") catch return;
+        writer.writeByteNTimes(' ', max_filepath_length - "File".len + 1) catch return;
+        // writer.writeAll(Output.prettyFmt(" <d>|<r> % Funcs <d>|<r> % Blocks <d>|<r> % Lines <d>|<r> Uncovered Line #s\n", enable_ansi_colors)) catch return;
+        writer.writeAll(Output.prettyFmt(" <d>|<r> % Funcs <d>|<r> % Lines <d>|<r> Uncovered Line #s\n", enable_ansi_colors)) catch return;
+        writer.writeAll(Output.prettyFmt("<d>", enable_ansi_colors)) catch return;
+        writer.writeByteNTimes('-', max_filepath_length + 2) catch return;
+        writer.writeAll(Output.prettyFmt("|---------|---------|-------------------<r>\n", enable_ansi_colors)) catch return;
+
+        var coverage_buffer = bun.MutableString.initEmpty(bun.default_allocator);
+        var coverage_buffer_buffer = coverage_buffer.bufferedWriter();
+        var coverage_writer = coverage_buffer_buffer.writer();
+
+        var avg = bun.sourcemap.CoverageFraction{
+            .functions = 0.0,
+            .lines = 0.0,
+            .stmts = 0.0,
+        };
+        var avg_count: f64 = 0;
+
+        for (byte_ranges.items) |*entry| {
+            var report = bun.sourcemap.CodeCoverageReport.generate(vm.global, bun.default_allocator, entry, opts.ignore_sourcemap) orelse continue;
+            defer report.deinit(bun.default_allocator);
+            var fraction = base_fraction;
+            report.writeFormat(max_filepath_length, &fraction, relative_dir, coverage_writer, enable_ansi_colors) catch continue;
+            avg.functions += fraction.functions;
+            avg.lines += fraction.lines;
+            avg.stmts += fraction.stmts;
+            avg_count += 1.0;
+            if (fraction.failing) {
+                failing = true;
+            }
+
+            coverage_writer.writeAll("\n") catch continue;
+        }
+
+        {
+            avg.functions /= avg_count;
+            avg.lines /= avg_count;
+            avg.stmts /= avg_count;
+
+            try bun.sourcemap.CodeCoverageReport.writeFormatWithValues(
+                "All files",
+                max_filepath_length,
+                avg,
+                base_fraction,
+                failing,
+                writer,
+                false,
+                enable_ansi_colors,
+            );
+
+            try writer.writeAll(Output.prettyFmt("<r><d> |<r>\n", enable_ansi_colors));
+        }
+
+        coverage_buffer_buffer.flush() catch return;
+        try writer.writeAll(coverage_buffer.list.items);
+        try writer.writeAll(Output.prettyFmt("<r><d>", enable_ansi_colors));
+        writer.writeByteNTimes('-', max_filepath_length + 2) catch return;
+        writer.writeAll(Output.prettyFmt("|---------|---------|-------------------<r>\n", enable_ansi_colors)) catch return;
+
+        opts.fractions.failing = failing;
+        Output.flush();
+    }
 };
 
 const Scanner = struct {
@@ -334,6 +431,42 @@ const Scanner = struct {
         "_spec",
     };
 
+    export fn BunTest__shouldGenerateCodeCoverage(test_name_str: bun.String) callconv(.C) bool {
+        var zig_slice: bun.JSC.ZigString.Slice = .{};
+        defer zig_slice.deinit();
+
+        // In this particular case, we don't actually care about non-ascii latin1 characters.
+        // so we skip the ascii check
+        const slice = if (test_name_str.is8Bit()) test_name_str.latin1() else brk: {
+            zig_slice = test_name_str.toUTF8(bun.default_allocator);
+            break :brk zig_slice.slice();
+        };
+
+        // always ignore node_modules.
+        if (strings.contains(slice, "/" ++ "node_modules" ++ "/")) {
+            return false;
+        }
+
+        const ext = std.fs.path.extension(slice);
+        const loader_by_ext = JSC.VirtualMachine.get().bundler.options.loader(ext);
+
+        // allow file loader just incase they use a custom loader with a non-standard extension
+        if (!(loader_by_ext.isJavaScriptLike() or loader_by_ext == .file)) {
+            return false;
+        }
+
+        if (jest.Jest.runner.?.test_options.coverage.skip_test_files) {
+            const name_without_extension = slice[0 .. slice.len - ext.len];
+            inline for (test_name_suffixes) |suffix| {
+                if (strings.endsWithComptime(name_without_extension, suffix)) {
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
+
     pub fn couldBeTestFile(this: *Scanner, name: string) bool {
         const extname = std.fs.path.extension(name);
         if (!this.options.loader(extname).isJavaScriptLike()) return false;
@@ -419,6 +552,13 @@ pub const TestCommand = struct {
     pub const name = "test";
     pub const old_name = "wiptest";
 
+    pub const CodeCoverageOptions = struct {
+        skip_test_files: bool = !Environment.allow_assert,
+        fractions: bun.sourcemap.CoverageFraction = .{},
+        ignore_sourcemap: bool = false,
+        enabled: bool = false,
+    };
+
     pub fn exec(ctx: Command.Context) !void {
         if (comptime is_bindgen) unreachable;
 
@@ -480,7 +620,7 @@ pub const TestCommand = struct {
         reporter.repeat_count = @max(ctx.test_options.repeat_count, 1);
         reporter.jest.callback = &reporter.callback;
         jest.Jest.runner = &reporter.jest;
-
+        reporter.jest.test_options = &ctx.test_options;
         js_ast.Expr.Data.Store.create(default_allocator);
         js_ast.Stmt.Data.Store.create(default_allocator);
         var vm = try JSC.VirtualMachine.init(
@@ -507,6 +647,14 @@ pub const TestCommand = struct {
         vm.is_main_thread = true;
         JSC.VirtualMachine.is_main_thread_vm = true;
 
+        if (ctx.test_options.coverage.enabled) {
+            vm.bundler.options.code_coverage = true;
+            vm.bundler.options.minify_syntax = false;
+            vm.bundler.options.minify_identifiers = false;
+            vm.bundler.options.minify_whitespace = false;
+            vm.global.vm().setControlFlowProfiler(true);
+        }
+
         // For tests, we default to UTC time zone
         // unless the user inputs TZ="", in which case we use local time zone
         var TZ_NAME: string =
@@ -538,7 +686,6 @@ pub const TestCommand = struct {
 
         scanner.scan(dir_to_scan);
         scanner.dirs_to_scan.deinit();
-
         const test_files = try scanner.results.toOwnedSlice();
         if (test_files.len > 0) {
             vm.hot_reload = ctx.debug.hot_reload;
@@ -549,6 +696,7 @@ pub const TestCommand = struct {
         }
 
         try jest.Jest.runner.?.snapshots.writeSnapshotFile();
+        var coverage = ctx.test_options.coverage;
 
         if (reporter.summary.pass > 20) {
             if (reporter.summary.skip > 0) {
@@ -604,6 +752,12 @@ pub const TestCommand = struct {
         } else {
             Output.prettyError("\n", .{});
 
+            if (coverage.enabled) {
+                switch (Output.enable_ansi_colors_stderr) {
+                    inline else => |colors| reporter.printCodeCoverage(vm, &coverage, colors) catch {},
+                }
+            }
+
             if (reporter.summary.pass > 0) {
                 Output.prettyError("<r><green>", .{});
             }
@@ -688,7 +842,7 @@ pub const TestCommand = struct {
             }
         }
 
-        if (reporter.summary.fail > 0) {
+        if (reporter.summary.fail > 0 or (coverage.enabled and coverage.fractions.failing)) {
             Global.exit(1);
         }
     }