Merge pull request #2917 from newsboat/bugfix/2912-ci-coveralls

Migrate to source-based code coverage

4 files changed, 54 insertions, 48 deletions

diff --git a/.github/workflows/coveralls.yaml b/.github/workflows/coveralls.yaml
index 74359a7a..4f1a7591 100644
--- a/.github/workflows/coveralls.yaml
+++ b/.github/workflows/coveralls.yaml
@@ -8,24 +8,22 @@ jobs:
     runs-on: ubuntu-24.04
 
     env:
-      # We use Rust Nightly, which builds upon LLVM 18. To ensure best
-      # compatibility, we use a matching C++ compiler.
+      # We use Rust 1.81, which builds upon LLVM 18. We have to use a matching
+      # C++ compiler, otherwise grcov won't be able to produce the coverage
+      # report.
       CC: clang-18
       CXX: clang++-18
       # Enable test coverage.
       PROFILE: 1
-      # These flags are necessary for grcov to correctly calculate coverage.
-      CARGO_INCREMENTAL: 0
-      # We add `-A warnings` because we aren't interested in warnings from Rust
-      # Nightly -- GitHub turns them into annotations, which is annoying.
-      RUSTFLAGS: '-A warnings -Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort'
-      RUSTDOCFLAGS: '-Cpanic=abort'
+      CXXFLAGS: '-O0 -fprofile-instr-generate -fcoverage-mapping'
+      RUSTFLAGS: '-Clink-dead-code -Cinstrument-coverage'
       # Some of our tests use ncurses, which require a terminal of some sort.
       # We pretend ours is a simple one.
       TERM: 'dumb'
       # This prevents our tests from hogging too much of the CPU and failing
       # due to races.
       RUST_TEST_THREADS: 2
+      LLVM_PROFILE_FILE: '/tmp/newsboat-profiles/%h_%m.profraw'
 
     steps:
       - name: Install dependencies
@@ -42,7 +40,12 @@ jobs:
           sudo locale-gen
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@nightly
+        # The last version based on LLVM 18. We have to match LLVM versions
+        # used to compile Rust and C++, otherwise they might generate
+        # incompatible coverage reports
+        uses: dtolnay/rust-toolchain@1.81
+        with:
+          components: llvm-tools-preview
 
       - uses: actions/checkout@v4
 
@@ -63,16 +66,8 @@ jobs:
       - name: Run tests
         run: make --jobs=3 NEWSBOAT_RUN_IGNORED_TESTS=1 ci-check
 
-        # gcov tool from gcc doesn't understand profiling info that LLVM
-        # produces, so we trick grcov into using llvm-cov instead. We can't
-        # simply point grcov at llvm-cov, because the latter only behaves like
-        # gcc's gcov when invoked by that name.
-      - name: Prepare to use llvm-cov-18 as gcov
-        run: ln -s $(which llvm-cov-18) gcov
-
       - name: Calculate test coverage
-        # Note that we override the path to gcov tool.
-        run: GCOV=$(pwd)/gcov grcov . --ignore-not-existing --ignore='/*' --ignore='3rd-party/*' --ignore='doc/*' --ignore='test/*' --ignore='target/*' --ignore='newsboat.cpp' --ignore='podboat.cpp' -t lcov -o coverage.lcov
+        run: grcov /tmp/newsboat-profiles/ --source-dir . --binary-path . --ignore-not-existing --ignore='/*' --ignore='3rd-party/*' --ignore='doc/*' --ignore='test/*' --ignore='target/*' --ignore='newsboat.cpp' --ignore='podboat.cpp' -t lcov -o coverage.lcov
 
       - name: Submit coverage to Coveralls
         uses: coverallsapp/github-action@v2
diff --git a/.gitignore b/.gitignore
index d15bc5ba..86a6f067 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,4 @@ doc/podboat-cmds-linked.asciidoc
 *.dSYM
 /.deps/
 /compile_flags.txt
+*.profraw
diff --git a/Makefile b/Makefile
index 70b21077..a05fab2d 100644
--- a/Makefile
+++ b/Makefile
@@ -234,6 +234,7 @@ clean: clean-newsboat clean-podboat clean-libboat clean-libfilter clean-doc clea
 
 profclean:
 	find . -name '*.gc*' -type f -print0 | xargs -0 $(RM) --
+	find . -name '*.profraw' -type f -print0 | xargs -0 $(RM) --
 	$(RM) app*.info
 
 distclean: clean profclean
diff --git a/test/generate_coverage_report.sh b/test/generate_coverage_report.sh
index 66cfe83c..1d73a96e 100755
--- a/test/generate_coverage_report.sh
+++ b/test/generate_coverage_report.sh
@@ -1,37 +1,46 @@
 #!/bin/sh
 
+# Run this script on a machine with Clang 18 and Rust 1.81. These versions both
+# use LLVM 18; matching LLVM versions are required for grcov to produce
+# a report across the entire codebase.
+#
+# One can prepare such environment with Docker:
+#
+# docker build \
+#   --build-arg rust_version=1.81 \
+#   --build-arg cxx_package='clang-18 libclang-rt-18-dev' \
+#   --build-arg cxx=clang++-18 \
+#   --build-arg cc=clang-18 \
+#   --tag=newsboat-clang-18-rust-1.82:24.04 \
+#   --file=docker/ubuntu_24.04-build-tools.dockerfile \
+#   docker
+#
+# docker run \
+#   -it --rm \
+#   --mount type=bind,source=$(pwd),target=/workspace -w /workspace \
+#   --user $(id -u):$(id -g) \
+#   newsboat-clang-18-rust-1.81:24.04 \
+#   bash -c 'rustup component add llvm-tools-preview && cargo install grcov && test/generate_coverage_report.sh'
+
 set -e
 
-APPBASE_INFO=appbase.info
-APPTEST_INFO=apptest.info
-APPTOTAL_INFO=apptotal.info
+export CC=clang-18
+export CXX=clang++-18
+export CXXFLAGS='-O0 -fprofile-instr-generate -fcoverage-mapping'
+export RUSTFLAGS='-Clink-dead-code -Cinstrument-coverage'
+export LLVM_PROFILE_FILE='%h_%m.profraw'
+export PROFILE=1
+
+OUTDIR=html/coverage
 
-rm -rf $APPBASE_INFO $APPTEST_INFO html
-find -name '*.gcda' -print0 | xargs -0 rm --force
-make -j 5 PROFILE=1 all test
-lcov --capture --initial --base-directory . --directory . --output-file $APPBASE_INFO
-( cd test && ./test "$@" )
-lcov --capture --base-directory . --directory . --output-file $APPTEST_INFO
-lcov --base-directory . --directory . --output-file $APPTOTAL_INFO \
-    --add-tracefile $APPBASE_INFO --add-tracefile $APPTEST_INFO
+make --jobs=9 NEWSBOAT_RUN_IGNORED_TESTS=1 ci-check
 
-# Removing info about shared libraries
-lcov --remove $APPTOTAL_INFO '/usr/*' --output-file $APPTOTAL_INFO
-# Removing info about shared libraries (on NixOS)
-lcov --remove $APPTOTAL_INFO '/nix/store/*' --output-file $APPTOTAL_INFO
-# Removing info about compiler internals
-lcov --remove $APPTOTAL_INFO '?.?.?/*' --output-file $APPTOTAL_INFO
-lcov --remove $APPTOTAL_INFO '?.?.??/*' --output-file $APPTOTAL_INFO
-lcov --remove $APPTOTAL_INFO '?.??.?/*' --output-file $APPTOTAL_INFO
-lcov --remove $APPTOTAL_INFO '?.??.??/*' --output-file $APPTOTAL_INFO
-# Removing info about Newsboat's tests
-lcov --remove $APPTOTAL_INFO 'newsboat/test/*' --output-file $APPTOTAL_INFO
-lcov --remove $APPTOTAL_INFO '*/newsboat/test/*' --output-file $APPTOTAL_INFO
-# Removing info about Newsboat's docs
-lcov --remove $APPTOTAL_INFO 'newsboat/doc/*' --output-file $APPTOTAL_INFO
-lcov --remove $APPTOTAL_INFO '*/newsboat/doc/*' --output-file $APPTOTAL_INFO
-# Removing info about third-party libraries
-lcov --remove $APPTOTAL_INFO '*/newsboat/3rd-party/*' --output-file $APPTOTAL_INFO
+rm -rf html
+grcov . --source-dir . --binary-path . \
+    --ignore-not-existing \
+    --ignore='/*' --ignore='3rd-party/*' --ignore='doc/*' --ignore='test/*' \
+    --ignore='target/*' --ignore='newsboat.cpp' --ignore='podboat.cpp' \
+    -t html -o ./$OUTDIR
 
-genhtml -o html $APPTOTAL_INFO
-echo "The coverage report can be found at file://`pwd`/html/index.html"
+echo "Coverage reports:"
+find $OUTDIR -name 'index.html'