1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Run test executables that obey the BYO protocol.
|
4 | #
|
5 | # TODO: doc/byo.md
|
6 | #
|
7 | # Usage:
|
8 | # test/byo-client.sh run-tests ARGS...
|
9 | #
|
10 | # The ARGS... are run with an environment variable, e.g.
|
11 | #
|
12 | # ./myscript.py
|
13 | # python3 ./myscript.py
|
14 | # ./myscript.sh
|
15 | # dash ./myscript.sh
|
16 | # osh ./myscript.sh
|
17 | #
|
18 | # The client creates a clean process state and directory state for each tests.
|
19 |
|
20 | : ${LIB_OSH=stdlib/osh}
|
21 | source $LIB_OSH/bash-strict.sh
|
22 | source $LIB_OSH/two.sh
|
23 |
|
24 | readonly TAB=$'\t'
|
25 |
|
26 | detect() {
|
27 | if test $# -eq 0; then
|
28 | die "Expected argv to run"
|
29 | fi
|
30 |
|
31 | local out
|
32 |
|
33 | local status=0
|
34 | set +o errexit
|
35 | out=$(BYO_COMMAND=detect "$@" < /dev/null)
|
36 | status=$?
|
37 | set -o errexit
|
38 |
|
39 | if test $status -ne 66; then
|
40 | die "$(printf '%q ' "$@") doesn't implement BYO: expected status 66, got $status"
|
41 | fi
|
42 |
|
43 | # Verbose
|
44 | if false; then
|
45 | echo
|
46 | echo "BYO commands detected in $(printf '%q ' "$@"):"
|
47 | echo "$out"
|
48 | fi
|
49 | }
|
50 |
|
51 | run-tests() {
|
52 | # argv is the command to run, like bash foo.sh
|
53 | #
|
54 | # It's an array rather than a single command, so you can run the same scripts
|
55 | # under multiple shells:
|
56 | #
|
57 | # bash myscript.sh
|
58 | # osh myscript.sh
|
59 |
|
60 | if test $# -eq 0; then
|
61 | die "Expected argv to run"
|
62 | fi
|
63 |
|
64 | detect "$@"
|
65 |
|
66 | log '---'
|
67 | log "byo run-tests: $@"
|
68 | log
|
69 |
|
70 |
|
71 | # TODO:
|
72 | # --no-chdir Change directory by default, but this option disables it
|
73 | # --no-stdout-log stdout is not redirected to its own, named file
|
74 | # --max-jobs Parallelism
|
75 | #
|
76 | # And how do we run test binaries that are just one big process?
|
77 | # - Python - inside a file, we probably don't have any tests to run in parallel
|
78 | # - C++ - ditto
|
79 | # - R - we have a few unit tests
|
80 | #
|
81 | # So maybe we need BYO_LIST_TEST_BIN - list the test binaries?
|
82 | # Or mycpp/TEST.sh etc. should list them. Yes that is true!
|
83 | # We can also list things to build.
|
84 | #
|
85 | # BYO_LIST_NINJA=1
|
86 |
|
87 | mkdir -p _tmp
|
88 | local tmp=_tmp/byo-list.txt
|
89 |
|
90 | # First list the tests
|
91 | BYO_COMMAND=list-tests "$@" > $tmp
|
92 |
|
93 | local i=0
|
94 | local status
|
95 |
|
96 | while read -r test_name; do
|
97 |
|
98 | echo "${TAB}${test_name}"
|
99 |
|
100 | set +o errexit
|
101 | BYO_COMMAND=run-test BYO_ARG="$test_name" "$@"
|
102 | status=$?
|
103 | set -o errexit
|
104 |
|
105 | if test $status -eq 0; then
|
106 | echo "${TAB}OK"
|
107 | else
|
108 | echo "${TAB}FAIL with status $status"
|
109 | exit 1
|
110 | fi
|
111 |
|
112 | i=$(( i + 1 ))
|
113 | done < $tmp
|
114 |
|
115 | echo
|
116 | echo "$0: $i tests passed."
|
117 | }
|
118 |
|
119 | # TODO:
|
120 | # BYO_PROBE=1
|
121 | #
|
122 | # must print capabilities
|
123 | #
|
124 | # - testing - success/fail
|
125 | # - benchmarks - output TSV file I think? Or a directory of TSV files?
|
126 | # - BYO_RESULTS
|
127 | # - shell auto-completion
|
128 |
|
129 | "$@"
|