diff options
| -rw-r--r-- | Documentation/technotes/2021-05-code-coverage.md | 56 | ||||
| -rw-r--r-- | Documentation/tutorial/part3.md | 2 | ||||
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | tests/Makefile.inc | 56 |
4 files changed, 104 insertions, 14 deletions
diff --git a/Documentation/technotes/2021-05-code-coverage.md b/Documentation/technotes/2021-05-code-coverage.md new file mode 100644 index 0000000000..ec7f4cb5f9 --- /dev/null +++ b/Documentation/technotes/2021-05-code-coverage.md @@ -0,0 +1,56 @@ +# Unit Test Code Coverage + +Code coverage for the coreboot unit tests allows us to see what lines of +code in the coreboot library are covered by unit tests, and allows a test +author to see where they need to add test cases for additional coverage. + +Enable code coverage in your unit test build by setting the environment +variable `COV` to 1; either `export COV=1` in your shell, or add it to your +`make` command, e.g. `COV=1 make unit-tests`. + +The build output directory is either `build/tests` or `build/coverage`, +depending on whether `COV=1` is set in the environment. + +All of the unit test targets are available with and without `COV=1` +* `clean-unit-tests` +* `build-unit-tests` +* `run-unit-tests` +* `unit-tests` (which is just `build-unit-tests` followed by `run-unit-tests`) + +There are two new `make` targets: +* `coverage-report` generates a code coverage report from all of the +GCOV data (`*.gcda` and `*.gcno` files) in the build directory. To view the +coverage report, open `build/coverage/coverage_reports/index.html` in your web +browser. +* `clean-coverage-report` deletes just the coverage report. + +The `coverage-report` and `clean-coverage-report` targets automatically set +`COV=1` if it is not already set in the environment. + + +## Examples + +`COV=1 make unit-tests coverage-report` builds all of the unit tests with code +coverage, runs the unit tests, and generates the code coverage report. + +`COV=1 make build-unit-tests` builds all of the unit tests with code coverage. + +`COV=1 make run-unit-tests` runs the unit tests, building them with code +coverage if they are out-of-date. + +`COV=1 make coverage-report` creates the code coverage report. This +target does not explicitly depend on the tests being built and run; it gathers +the code coverage data from the output directory, which it assumes already +exists. + +`COV=1 make tests/lib/uuid-test coverage-report` builds the uuid test +with code coverage, runs it, and generates a code coverage report just for +that test. + +As a demonstration that building with and without coverage uses different +output directories: +1. `make build-unit-tests` builds unit tests without code coverage into +`build/tests`. +2. `COV=1 make clean-unit-tests` cleans `build/coverage` +3. `make build-unit-tests` doesn't need to build anything in `build/tests`, +because those files weren't affected by the previous `clean-unit-tests`. diff --git a/Documentation/tutorial/part3.md b/Documentation/tutorial/part3.md index 7ccee87754..6d147ff9a9 100644 --- a/Documentation/tutorial/part3.md +++ b/Documentation/tutorial/part3.md @@ -3,6 +3,8 @@ ## Introduction General thoughts about unit testing coreboot can be found in [Unit testing coreboot](../technotes/2020-03-unit-testing-coreboot.md). +Additionally, [code coverage](../technotes/2021-05-code-coverage.md) support +is available for unit tests. This document aims to guide developers through the process of adding and writing unit tests for coreboot modules. @@ -124,8 +124,8 @@ ifneq ($(filter help%, $(MAKECMDGOALS)), ) NOCOMPILE:=1 UNIT_TEST:=1 else -ifneq ($(filter %-test %-tests, $(MAKECMDGOALS)),) -ifneq ($(filter-out %-test %-tests, $(MAKECMDGOALS)),) +ifneq ($(filter %-test %-tests %coverage-report, $(MAKECMDGOALS)),) +ifneq ($(filter-out %-test %-tests %coverage-report, $(MAKECMDGOALS)),) $(error Cannot mix unit-tests targets with other targets) endif UNIT_TEST:=1 diff --git a/tests/Makefile.inc b/tests/Makefile.inc index cd25e0f809..6cf92fa999 100644 --- a/tests/Makefile.inc +++ b/tests/Makefile.inc @@ -1,9 +1,18 @@ # SPDX-License-Identifier: GPL-2.0-only testsrc = $(top)/tests + +# Place the build output in one of two places depending on COV, so that code +# built with code coverage never mixes with code built without code coverage. +ifeq ($(COV),1) +testobj = $(obj)/coverage +else testobj = $(obj)/tests +endif + cmockasrc = 3rdparty/cmocka cmockaobj = $(objutil)/cmocka +coverage_dir = coverage_reports CMOCKA_LIB := $(cmockaobj)/src/libcmocka.so @@ -51,6 +60,12 @@ TEST_LDFLAGS += -Wl,--gc-sections TEST_CFLAGS += -fno-pie -fno-pic TEST_LDFLAGS += -no-pie +# Enable code coverage if COV=1 +ifeq ($(COV),1) +TEST_CFLAGS += --coverage +TEST_LDFLAGS += --coverage +endif + # Extra attributes for unit tests, declared per test attributes:= srcs cflags config mocks stage @@ -99,7 +114,7 @@ $$($(1)-config-file): $(TEST_KCONFIG_AUTOHEADER) $($(1)-objs): TEST_CFLAGS += -I$$(dir $$($(1)-config-file)) \ -D__$$(shell echo $$($(1)-stage) | tr '[:lower:]' '[:upper:]')__ -$($(1)-objs): $(obj)/$(1)/%.o: $$$$*.c $$($(1)-config-file) +$($(1)-objs): $(testobj)/$(1)/%.o: $$$$*.c $$($(1)-config-file) mkdir -p $$(dir $$@) $(HOSTCC) $(HOSTCFLAGS) $$(TEST_CFLAGS) $($(1)-cflags) -MMD \ -MT $$@ -c $$< -o $$@ @@ -111,10 +126,10 @@ $($(1)-bin): $($(1)-objs) $(CMOCKA_LIB) endef $(foreach test, $(alltests), \ - $(eval $(test)-objs:=$(addprefix $(obj)/$(test)/, \ + $(eval $(test)-objs:=$(addprefix $(testobj)/$(test)/, \ $(patsubst %.c,%.o,$($(test)-srcs))))) $(foreach test, $(alltests), \ - $(eval $(test)-bin:=$(obj)/$(test)/run)) + $(eval $(test)-bin:=$(testobj)/$(test)/run)) $(foreach test, $(alltests), \ $(eval $(call TEST_CC_template,$(test)))) @@ -168,15 +183,31 @@ $(alltests): $$($$(@)-bin) rm -f $(testobj)/junit-$(subst /,_,$^).xml $(testobj)/$(subst /,_,$^).failed -./$^ || echo failed > $(testobj)/$(subst /,_,$^).failed -.PHONY: coverage-unit-tests +# Build a code coverage report by collecting all the gcov files into a single +# report. If COV is not set, this might be a user error, and they're trying +# to generate a coverage report without first having built and run the code +# with code coverage. So instead of silently correcting it by adding COV=1, +# let's flag it to the user so they can be sure they're doing the thing they +# want to do. -coverage-unit-tests: TEST_CFLAGS += --coverage -coverage-unit-tests: TEST_LDFLAGS += --coverage -coverage-unit-tests: clean-unit-tests unit-tests - lcov -o $(testobj)/tests.info -c -d $(testobj) --exclude '*/$(testobj)/*' - genhtml -q -o build/tests/coverage_rpt -t "coreboot unit tests" \ +.PHONY: coverage-report clean-coverage-report + +ifeq ($(COV),1) +coverage-report: + lcov -o $(testobj)/tests.info -c -d $(testobj) --exclude '$(testsrc)/*' + genhtml -q -o $(testobj)/$(coverage_dir) -t "coreboot unit tests" \ -s $(testobj)/tests.info +clean-coverage-report: + rm -Rf $(testobj)/$(coverage_dir) +else +coverage-report: + COV=1 V=$(V) $(MAKE) coverage-report + +clean-coverage-report: + COV=1 V=$(V) $(MAKE) clean-coverage-report +endif + unit-tests: build-unit-tests run-unit-tests build-unit-tests: $(test-bins) @@ -195,7 +226,7 @@ run-unit-tests: $(alltests) fi $(addprefix clean-,$(alltests)): clean-%: - rm -rf $(obj)/$* + rm -rf $(testobj)/$* clean-unit-tests: rm -rf $(testobj) @@ -208,11 +239,12 @@ list-unit-tests: help-unit-tests help:: @echo '*** coreboot unit-tests targets ***' + @echo ' Use "COV=1 make [target]" to enable code coverage for unit tests' @echo ' unit-tests - Run all unit-tests from tests/' @echo ' clean-unit-tests - Remove unit-tests build artifacts' @echo ' list-unit-tests - List all unit-tests' @echo ' <unit-test> - Build and run single unit-test' @echo ' clean-<unit-test> - Remove single unit-test build artifacts' - @echo ' coverage-unit-tests - Build unit tests for code coverage and' - @echo ' generate a code coverage report' + @echo ' coverage-report - Generate a code coverage report' + @echo ' clean-coverage-report - Remove the code coverage report' @echo |