OILS / devtools / byo.sh View on Github | oilshell.org

225 lines, 114 significant
1#!/usr/bin/env bash
2#
3# Run test executables that obey the BYO protocol.
4#
5# TODO: doc/byo.md
6#
7# Usage:
8# test/byo-client.sh run-tests FLAGS* ARGS*
9#
10# TODO:
11# - client creates a clean process state for each test
12# - clean directory state for each test.
13
14: ${LIB_OSH=stdlib/osh}
15source $LIB_OSH/bash-strict.sh
16source $LIB_OSH/two.sh
17source $LIB_OSH/task-five.sh
18
19readonly TAB=$'\t'
20
21detect() {
22 if test $# -eq 0; then
23 die "Expected argv to run"
24 fi
25
26 local out
27
28 local status=0
29 set +o errexit
30 out=$(BYO_COMMAND=detect "$@" < /dev/null)
31 status=$?
32 set -o errexit
33
34 if test $status -ne 66; then
35 die "$(printf '%q ' "$@") doesn't implement BYO: expected status 66, got $status"
36 fi
37
38 # Verbose
39 if false; then
40 echo
41 echo "BYO commands detected in $(printf '%q ' "$@"):"
42 echo "$out"
43 fi
44}
45
46print-help() {
47 # Other flags:
48 #
49 # --host host to upload to?
50 # --auth allow writing to host
51 # --no-taskset disable taskset?
52
53 cat <<EOF
54Usage: byo run-tests FLAGS*
55EOF
56}
57
58# Test params:
59#
60# SH=dash SH=bash SH=bin/osh # test/spec.sh
61# OSH=bin/osh OSH=_bin/cxx-asan/osh # e.g. test/*{parse,runtime}-errors.sh
62# YSH=bin/osh YSH=_bin/cxx-asan/osh # e.g. test/*{parse,runtime}-errors.sh
63#
64# benchmarks/compute.sh has 3 dimensions:
65# ( workload name, param1, param2 )
66#
67# Pretty much all tests are parameterized by shell
68#
69# There's also python2 vs. python3 vs. awk etc.
70# benchmarks/compute
71#
72# Should it be
73# BYO_PARAM_OSH
74#
75# Usage:
76#
77# $ byo run-tests test/osh-usage
78# $ byo run-tests --param OSH=bin/osh test/osh-usage
79# $ byo run-tests --param OSH=bin/osh --param OSH=_bin/cxx-asan/osh test/osh-usage
80# $ byo run-tests --param OSH='bin/osh;_bin/cxx-asan/osh' test/osh-usage
81#
82# Run with each value of param in sequence, and then make a big table later?
83# I think you just loop over the param flags
84
85# If no params, we run once. Otherwise we run once per param value
86FLAG_params=()
87FLAG_fresh_dir=''
88FLAG_capture=''
89FLAG_test_filter=''
90
91parse-flags-for-test() {
92 ### Sets global vars FLAG_*
93
94 while test $# -ne 0; do
95 case "$1" in
96 -h|--help)
97 print-help
98 exit
99 ;;
100
101 # Capture stdout and stderr? Or let it go to the terminal
102 --capture)
103 FLAG_capture=T
104 ;;
105
106 # Is each test case run in its own dir? Or set TEST_TEMP_DIR
107 --fresh-dir)
108 FLAG_fresh_dir=T
109 ;;
110
111 --test-filter)
112 if test $# -eq 1; then
113 die "--test-filter requires an argument"
114 fi
115 shift
116
117 # Regex in ERE syntax
118 FLAG_test_filter=$1
119 ;;
120
121 --param)
122 if test $# -eq 1; then
123 die "--param requires an argument"
124 fi
125 shift
126
127 pat='[A-Z_]+=.*'
128 if ! [[ $1 =~ $pat ]]; then
129 die "Expected string like PARAM_NAME=value, got $1"
130 fi
131 FLAG_params+=( $1 )
132 ;;
133
134 -*)
135 die "Invalid flag '$1'"
136 break
137 ;;
138
139 --)
140 shift
141 break
142 ;;
143
144 *)
145 # Move on to args
146 break
147 ;;
148
149 esac
150 shift
151 done
152
153 ARGV=( "$@" )
154}
155
156run-tests() {
157 if test $# -eq 0; then
158 die "Expected argv to run"
159 fi
160
161 # Set FLAG_* and ARGV
162 parse-flags-for-test "$@"
163
164 # ARGV is the command to run, like bash foo.sh
165 #
166 # It's an array rather than a single command, so you can run the same scripts
167 # under multiple shells:
168 #
169 # bash myscript.sh
170 # osh myscript.sh
171 #
172 # This matters so two-test.sh can SOURCE two.sh, and run under both bash and
173 # OSH.
174 # That could be or --use-interp bin/osh
175 #
176 # could you have --use-interp python3 too? e.g. I want benchmarks/time_.py
177 # to work under both? See time-tsv3 in tsv-lib.sh
178 #
179 # No that's a test param PYTHON=python3 PYTHON=python2
180
181 detect "${ARGV[@]}"
182
183 log '---'
184 log "byo run-tests: ${ARGV[@]}"
185 log
186
187 mkdir -p _tmp
188 local tmp=_tmp/byo-list.txt
189
190 # First list the tests
191 BYO_COMMAND=list-tests "${ARGV[@]}" > $tmp
192
193 local i=0
194 local status
195
196 while read -r test_name; do
197
198 echo "${TAB}${test_name}"
199
200 set +o errexit
201 if test -n "$FLAG_capture"; then
202 # TODO: Capture it to a string
203 # - Write to nice HTML file?
204 BYO_COMMAND=run-test BYO_ARG="$test_name" "${ARGV[@]}" >/dev/null 2>&1
205 else
206 BYO_COMMAND=run-test BYO_ARG="$test_name" "${ARGV[@]}"
207 fi
208 status=$?
209 set -o errexit
210
211 if test $status -eq 0; then
212 echo "${TAB}OK"
213 else
214 echo "${TAB}FAIL with status $status"
215 exit 1
216 fi
217
218 i=$(( i + 1 ))
219 done < $tmp
220
221 echo
222 echo "$0: $i tests passed."
223}
224
225task-five "$@"