1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Run test executables that obey the BYO protocol.
|
4 | #
|
5 | # TODO: doc/byo.md
|
6 | #
|
7 | # Usage:
|
8 | # test/byo-client.sh run-tests FLAGS* ARGS*
|
9 | #
|
10 | # TODO:
|
11 | # - client creates a clean process state for each test
|
12 | # - clean directory state for each test.
|
13 |
|
14 | : ${LIB_OSH=stdlib/osh}
|
15 | source $LIB_OSH/bash-strict.sh
|
16 | source $LIB_OSH/two.sh
|
17 | source $LIB_OSH/task-five.sh
|
18 |
|
19 | readonly TAB=$'\t'
|
20 |
|
21 | detect() {
|
22 | if test $# -eq 0; then
|
23 | die "Expected argv to run"
|
24 | fi
|
25 |
|
26 | local out
|
27 |
|
28 | local status=0
|
29 | set +o errexit
|
30 | out=$(BYO_COMMAND=detect "$@" < /dev/null)
|
31 | status=$?
|
32 | set -o errexit
|
33 |
|
34 | if test $status -ne 66; then
|
35 | die "$(printf '%q ' "$@") doesn't implement BYO: expected status 66, got $status"
|
36 | fi
|
37 |
|
38 | # Verbose
|
39 | if false; then
|
40 | echo
|
41 | echo "BYO commands detected in $(printf '%q ' "$@"):"
|
42 | echo "$out"
|
43 | fi
|
44 | }
|
45 |
|
46 | print-help() {
|
47 | # Other flags:
|
48 | #
|
49 | # --host host to upload to?
|
50 | # --auth allow writing to host
|
51 | # --no-taskset disable taskset?
|
52 |
|
53 | cat <<EOF
|
54 | Usage: byo run-tests FLAGS*
|
55 | EOF
|
56 | }
|
57 |
|
58 | # Test params:
|
59 | #
|
60 | # SH=dash SH=bash SH=bin/osh # test/spec.sh
|
61 | # OSH=bin/osh OSH=_bin/cxx-asan/osh # e.g. test/*{parse,runtime}-errors.sh
|
62 | # YSH=bin/osh YSH=_bin/cxx-asan/osh # e.g. test/*{parse,runtime}-errors.sh
|
63 | #
|
64 | # benchmarks/compute.sh has 3 dimensions:
|
65 | # ( workload name, param1, param2 )
|
66 | #
|
67 | # Pretty much all tests are parameterized by shell
|
68 | #
|
69 | # There's also python2 vs. python3 vs. awk etc.
|
70 | # benchmarks/compute
|
71 | #
|
72 | # Should it be
|
73 | # BYO_PARAM_OSH
|
74 | #
|
75 | # Usage:
|
76 | #
|
77 | # $ byo run-tests test/osh-usage
|
78 | # $ byo run-tests --param OSH=bin/osh test/osh-usage
|
79 | # $ byo run-tests --param OSH=bin/osh --param OSH=_bin/cxx-asan/osh test/osh-usage
|
80 | # $ byo run-tests --param OSH='bin/osh;_bin/cxx-asan/osh' test/osh-usage
|
81 | #
|
82 | # Run with each value of param in sequence, and then make a big table later?
|
83 | # I think you just loop over the param flags
|
84 |
|
85 | # If no params, we run once. Otherwise we run once per param value
|
86 | FLAG_params=()
|
87 | FLAG_fresh_dir=''
|
88 | FLAG_capture=''
|
89 | FLAG_test_filter=''
|
90 |
|
91 | parse-flags-for-test() {
|
92 | ### Sets global vars FLAG_*
|
93 |
|
94 | while test $# -ne 0; do
|
95 | case "$1" in
|
96 | -h|--help)
|
97 | print-help
|
98 | exit
|
99 | ;;
|
100 |
|
101 | # Capture stdout and stderr? Or let it go to the terminal
|
102 | --capture)
|
103 | FLAG_capture=T
|
104 | ;;
|
105 |
|
106 | # Is each test case run in its own dir? Or set TEST_TEMP_DIR
|
107 | --fresh-dir)
|
108 | FLAG_fresh_dir=T
|
109 | ;;
|
110 |
|
111 | --test-filter)
|
112 | if test $# -eq 1; then
|
113 | die "--test-filter requires an argument"
|
114 | fi
|
115 | shift
|
116 |
|
117 | # Regex in ERE syntax
|
118 | FLAG_test_filter=$1
|
119 | ;;
|
120 |
|
121 | --param)
|
122 | if test $# -eq 1; then
|
123 | die "--param requires an argument"
|
124 | fi
|
125 | shift
|
126 |
|
127 | pat='[A-Z_]+=.*'
|
128 | if ! [[ $1 =~ $pat ]]; then
|
129 | die "Expected string like PARAM_NAME=value, got $1"
|
130 | fi
|
131 | FLAG_params+=( $1 )
|
132 | ;;
|
133 |
|
134 | -*)
|
135 | die "Invalid flag '$1'"
|
136 | break
|
137 | ;;
|
138 |
|
139 | --)
|
140 | shift
|
141 | break
|
142 | ;;
|
143 |
|
144 | *)
|
145 | # Move on to args
|
146 | break
|
147 | ;;
|
148 |
|
149 | esac
|
150 | shift
|
151 | done
|
152 |
|
153 | ARGV=( "$@" )
|
154 | }
|
155 |
|
156 | run-tests() {
|
157 | if test $# -eq 0; then
|
158 | die "Expected argv to run"
|
159 | fi
|
160 |
|
161 | # Set FLAG_* and ARGV
|
162 | parse-flags-for-test "$@"
|
163 |
|
164 | # ARGV is the command to run, like bash foo.sh
|
165 | #
|
166 | # It's an array rather than a single command, so you can run the same scripts
|
167 | # under multiple shells:
|
168 | #
|
169 | # bash myscript.sh
|
170 | # osh myscript.sh
|
171 | #
|
172 | # This matters so two-test.sh can SOURCE two.sh, and run under both bash and
|
173 | # OSH.
|
174 | # That could be or --use-interp bin/osh
|
175 | #
|
176 | # could you have --use-interp python3 too? e.g. I want benchmarks/time_.py
|
177 | # to work under both? See time-tsv3 in tsv-lib.sh
|
178 | #
|
179 | # No that's a test param PYTHON=python3 PYTHON=python2
|
180 |
|
181 | detect "${ARGV[@]}"
|
182 |
|
183 | log '---'
|
184 | log "byo run-tests: ${ARGV[@]}"
|
185 | log
|
186 |
|
187 | mkdir -p _tmp
|
188 | local tmp=_tmp/byo-list.txt
|
189 |
|
190 | # First list the tests
|
191 | BYO_COMMAND=list-tests "${ARGV[@]}" > $tmp
|
192 |
|
193 | local i=0
|
194 | local status
|
195 |
|
196 | while read -r test_name; do
|
197 |
|
198 | echo "${TAB}${test_name}"
|
199 |
|
200 | set +o errexit
|
201 | if test -n "$FLAG_capture"; then
|
202 | # TODO: Capture it to a string
|
203 | # - Write to nice HTML file?
|
204 | BYO_COMMAND=run-test BYO_ARG="$test_name" "${ARGV[@]}" >/dev/null 2>&1
|
205 | else
|
206 | BYO_COMMAND=run-test BYO_ARG="$test_name" "${ARGV[@]}"
|
207 | fi
|
208 | status=$?
|
209 | set -o errexit
|
210 |
|
211 | if test $status -eq 0; then
|
212 | echo "${TAB}OK"
|
213 | else
|
214 | echo "${TAB}FAIL with status $status"
|
215 | exit 1
|
216 | fi
|
217 |
|
218 | i=$(( i + 1 ))
|
219 | done < $tmp
|
220 |
|
221 | echo
|
222 | echo "$0: $i tests passed."
|
223 | }
|
224 |
|
225 | task-five "$@"
|