Introduce notion of a test group, use for CA tests
Grouping our tests should make it easier to understand the intent than one long poorly-arranged list. It also is convenient for running just the tests for a specific component when working on that component. We need at least one test group so this isn't dead code; I decided to collect the tests for the `ca-derivations` and `dynamic-derivations` experimental features in groups. Do ```bash make ca.test-group -jN ``` and ```bash make dyn-drv.test-group -jN ``` to try running just them. I originally did this as part of #8397 for being able to just the local overlay store alone. I am PRing it separately now so we can separate general infra from new features. Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
This commit is contained in:
John Ericson
parent
a8d5bb5e7e
commit
259e328de8
7 changed files with 97 additions and 29 deletions
2
Makefile
2
Makefile
|
|
@ -27,6 +27,8 @@ makefiles += \
|
|||
src/libstore/tests/local.mk \
|
||||
src/libexpr/tests/local.mk \
|
||||
tests/local.mk \
|
||||
tests/ca/local.mk \
|
||||
tests/dyn-drv/local.mk \
|
||||
tests/test-libstoreconsumer/local.mk \
|
||||
tests/plugins/local.mk
|
||||
else
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c
|
|||
The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
|
||||
Each test is a bash script.
|
||||
|
||||
### Running the whole test suite
|
||||
|
||||
The whole test suite can be run with:
|
||||
|
||||
```shell-session
|
||||
|
|
@ -23,6 +25,33 @@ ran test tests/bar.sh... [PASS]
|
|||
...
|
||||
```
|
||||
|
||||
### Grouping tests
|
||||
|
||||
Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite.
|
||||
Each test group is in a subdirectory of `tests`.
|
||||
For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
|
||||
|
||||
That test group can be run like this:
|
||||
|
||||
```shell-session
|
||||
$ make ca.test-group -j50
|
||||
ran test tests/ca/nix-run.sh... [PASS]
|
||||
ran test tests/ca/import-derivation.sh... [PASS]
|
||||
...
|
||||
```
|
||||
|
||||
The test group is defined in Make like this:
|
||||
```makefile
|
||||
$(test-group-name)-tests := \
|
||||
$(d)/test0.sh \
|
||||
$(d)/test1.sh \
|
||||
...
|
||||
|
||||
install-tests-groups += $(test-group-name)
|
||||
```
|
||||
|
||||
### Running individual tests
|
||||
|
||||
Individual tests can be run with `make`:
|
||||
|
||||
```shell-session
|
||||
|
|
|
|||
20
mk/lib.mk
20
mk/lib.mk
|
|
@ -10,6 +10,7 @@ bin-scripts :=
|
|||
noinst-scripts :=
|
||||
man-pages :=
|
||||
install-tests :=
|
||||
install-tests-groups :=
|
||||
|
||||
ifdef HOST_OS
|
||||
HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
|
||||
|
|
@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b
|
|||
$(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
|
||||
$(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
|
||||
$(foreach template, $(template-files), $(eval $(call instantiate-template,$(template))))
|
||||
$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test))))
|
||||
$(foreach test, $(install-tests), \
|
||||
$(eval $(call run-install-test,$(test))) \
|
||||
$(eval installcheck: $(test).test))
|
||||
$(foreach test-group, $(install-tests-groups), \
|
||||
$(eval $(call run-install-test-group,$(test-group))) \
|
||||
$(eval installcheck: $(test-group).test-group) \
|
||||
$(foreach test, $($(test-group)-tests), \
|
||||
$(eval $(call run-install-test,$(test))) \
|
||||
$(eval $(test-group).test-group: $(test).test)))
|
||||
|
||||
$(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
|
||||
|
||||
|
||||
|
|
@ -151,6 +161,14 @@ ifdef libs-list
|
|||
@echo "The following libraries can be built:"
|
||||
@echo ""
|
||||
@for i in $(libs-list); do echo " $$i"; done
|
||||
endif
|
||||
ifdef install-tests-groups
|
||||
@echo ""
|
||||
@echo "The following groups of functional tests can be run:"
|
||||
@echo ""
|
||||
@for i in $(install-tests-groups); do echo " $$i.test-group"; done
|
||||
@echo ""
|
||||
@echo "(installcheck includes tests in test groups too.)"
|
||||
endif
|
||||
@echo ""
|
||||
@echo "The following variables control the build:"
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ test-deps =
|
|||
|
||||
define run-install-test
|
||||
|
||||
installcheck: $1.test
|
||||
|
||||
.PHONY: $1.test
|
||||
$1.test: $1 $(test-deps)
|
||||
@env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
|
||||
|
|
@ -16,6 +14,12 @@ define run-install-test
|
|||
|
||||
endef
|
||||
|
||||
define run-install-test-group
|
||||
|
||||
.PHONY: $1.test-group
|
||||
|
||||
endef
|
||||
|
||||
.PHONY: check installcheck
|
||||
|
||||
print-top-help += \
|
||||
|
|
|
|||
27
tests/ca/local.mk
Normal file
27
tests/ca/local.mk
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
ca-tests := \
|
||||
$(d)/build-with-garbage-path.sh \
|
||||
$(d)/build.sh \
|
||||
$(d)/concurrent-builds.sh \
|
||||
$(d)/derivation-json.sh \
|
||||
$(d)/duplicate-realisation-in-closure.sh \
|
||||
$(d)/gc.sh \
|
||||
$(d)/import-derivation.sh \
|
||||
$(d)/new-build-cmd.sh \
|
||||
$(d)/nix-copy.sh \
|
||||
$(d)/nix-run.sh \
|
||||
$(d)/nix-shell.sh \
|
||||
$(d)/post-hook.sh \
|
||||
$(d)/recursive.sh \
|
||||
$(d)/repl.sh \
|
||||
$(d)/selfref-gc.sh \
|
||||
$(d)/signatures.sh \
|
||||
$(d)/substitute.sh \
|
||||
$(d)/why-depends.sh
|
||||
|
||||
install-tests-groups += ca
|
||||
|
||||
clean-files += \
|
||||
$(d)/config.nix
|
||||
|
||||
test-deps += \
|
||||
tests/ca/config.nix
|
||||
11
tests/dyn-drv/local.mk
Normal file
11
tests/dyn-drv/local.mk
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
dyn-drv-tests := \
|
||||
$(d)/text-hashed-output.sh \
|
||||
$(d)/recursive-mod-json.sh
|
||||
|
||||
install-tests-groups += dyn-drv
|
||||
|
||||
clean-files += \
|
||||
$(d)/config.nix
|
||||
|
||||
test-deps += \
|
||||
tests/dyn-drv/config.nix
|
||||
|
|
@ -14,7 +14,6 @@ nix_tests = \
|
|||
flakes/absolute-paths.sh \
|
||||
flakes/build-paths.sh \
|
||||
flakes/flake-in-submodule.sh \
|
||||
ca/gc.sh \
|
||||
gc.sh \
|
||||
nix-collect-garbage-d.sh \
|
||||
remote-store.sh \
|
||||
|
|
@ -28,8 +27,6 @@ nix_tests = \
|
|||
user-envs-migration.sh \
|
||||
binary-cache.sh \
|
||||
multiple-outputs.sh \
|
||||
ca/build.sh \
|
||||
ca/new-build-cmd.sh \
|
||||
nix-build.sh \
|
||||
gc-concurrent.sh \
|
||||
repair.sh \
|
||||
|
|
@ -47,24 +44,17 @@ nix_tests = \
|
|||
referrers.sh \
|
||||
optimise-store.sh \
|
||||
substitute-with-invalid-ca.sh \
|
||||
ca/concurrent-builds.sh \
|
||||
signing.sh \
|
||||
ca/build-with-garbage-path.sh \
|
||||
hash.sh \
|
||||
gc-non-blocking.sh \
|
||||
check.sh \
|
||||
ca/substitute.sh \
|
||||
nix-shell.sh \
|
||||
ca/signatures.sh \
|
||||
ca/nix-shell.sh \
|
||||
ca/nix-copy.sh \
|
||||
check-refs.sh \
|
||||
build-remote-input-addressed.sh \
|
||||
secure-drv-outputs.sh \
|
||||
restricted.sh \
|
||||
fetchGitSubmodules.sh \
|
||||
flakes/search-root.sh \
|
||||
ca/duplicate-realisation-in-closure.sh \
|
||||
readfile-context.sh \
|
||||
nix-channel.sh \
|
||||
recursive.sh \
|
||||
|
|
@ -80,10 +70,7 @@ nix_tests = \
|
|||
nar-access.sh \
|
||||
pure-eval.sh \
|
||||
eval.sh \
|
||||
ca/post-hook.sh \
|
||||
repl.sh \
|
||||
ca/repl.sh \
|
||||
ca/recursive.sh \
|
||||
binary-cache-build-remote.sh \
|
||||
search.sh \
|
||||
logging.sh \
|
||||
|
|
@ -109,13 +96,8 @@ nix_tests = \
|
|||
fmt.sh \
|
||||
eval-store.sh \
|
||||
why-depends.sh \
|
||||
ca/why-depends.sh \
|
||||
derivation-json.sh \
|
||||
ca/derivation-json.sh \
|
||||
import-derivation.sh \
|
||||
ca/import-derivation.sh \
|
||||
dyn-drv/text-hashed-output.sh \
|
||||
dyn-drv/recursive-mod-json.sh \
|
||||
nix_path.sh \
|
||||
case-hack.sh \
|
||||
placeholders.sh \
|
||||
|
|
@ -124,8 +106,7 @@ nix_tests = \
|
|||
build.sh \
|
||||
build-delete.sh \
|
||||
output-normalization.sh \
|
||||
ca/nix-run.sh \
|
||||
selfref-gc.sh ca/selfref-gc.sh \
|
||||
selfref-gc.sh \
|
||||
db-migration.sh \
|
||||
bash-profile.sh \
|
||||
pass-as-file.sh \
|
||||
|
|
@ -150,16 +131,12 @@ install-tests += $(foreach x, $(nix_tests), $(d)/$(x))
|
|||
|
||||
clean-files += \
|
||||
$(d)/common/vars-and-functions.sh \
|
||||
$(d)/config.nix \
|
||||
$(d)/ca/config.nix \
|
||||
$(d)/dyn-drv/config.nix
|
||||
$(d)/config.nix
|
||||
|
||||
test-deps += \
|
||||
tests/common/vars-and-functions.sh \
|
||||
tests/config.nix \
|
||||
tests/ca/config.nix \
|
||||
tests/test-libstoreconsumer/test-libstoreconsumer \
|
||||
tests/dyn-drv/config.nix
|
||||
tests/test-libstoreconsumer/test-libstoreconsumer
|
||||
|
||||
ifeq ($(BUILD_SHARED_LIBS), 1)
|
||||
test-deps += tests/plugins/libplugintest.$(SO_EXT)
|
||||
|
|
|
|||
Loading…
Reference in a new issue