| 1 | #!/usr/bin/env bash
|
| 2 | #
|
| 3 | # Run test executables that obey the BYO protocol.
|
| 4 | #
|
| 5 | # TODO: doc/byo.md
|
| 6 | #
|
| 7 | # Usage:
|
| 8 | # test/byo-client.sh run-tests FLAGS* ARGS*
|
| 9 | #
|
| 10 | # TODO:
|
| 11 | # - client creates a clean process state for each test
|
| 12 | # - clean directory state for each test.
|
| 13 |
|
| 14 | : ${LIB_OSH=stdlib/osh}
|
| 15 | source $LIB_OSH/bash-strict.sh
|
| 16 | source $LIB_OSH/two.sh
|
| 17 | source $LIB_OSH/task-five.sh
|
| 18 |
|
| 19 | readonly TAB=$'\t'
|
| 20 |
|
| 21 | detect() {
|
| 22 | if test $# -eq 0; then
|
| 23 | die "Expected argv to run"
|
| 24 | fi
|
| 25 |
|
| 26 | local out
|
| 27 |
|
| 28 | local status=0
|
| 29 | set +o errexit
|
| 30 | out=$(BYO_COMMAND=detect "$@" < /dev/null)
|
| 31 | status=$?
|
| 32 | set -o errexit
|
| 33 |
|
| 34 | if test $status -ne 66; then
|
| 35 | die "$(printf '%q ' "$@") doesn't implement BYO: expected status 66, got $status"
|
| 36 | fi
|
| 37 |
|
| 38 | # Verbose
|
| 39 | if false; then
|
| 40 | echo
|
| 41 | echo "BYO commands detected in $(printf '%q ' "$@"):"
|
| 42 | echo "$out"
|
| 43 | fi
|
| 44 | }
|
| 45 |
|
| 46 | print-help() {
|
| 47 | # Other flags:
|
| 48 | #
|
| 49 | # --host host to upload to?
|
| 50 | # --auth allow writing to host
|
| 51 | # --no-taskset disable taskset?
|
| 52 |
|
| 53 | cat <<EOF
|
| 54 | Usage: byo run-tests FLAGS*
|
| 55 | EOF
|
| 56 | }
|
| 57 |
|
| 58 | # Test params:
|
| 59 | #
|
| 60 | # SH=dash SH=bash SH=bin/osh # test/spec.sh
|
| 61 | # OSH=bin/osh OSH=_bin/cxx-asan/osh # e.g. test/*{parse,runtime}-errors.sh
|
| 62 | # YSH=bin/osh YSH=_bin/cxx-asan/osh # e.g. test/*{parse,runtime}-errors.sh
|
| 63 | #
|
| 64 | # benchmarks/compute.sh has 3 dimensions:
|
| 65 | # ( workload name, param1, param2 )
|
| 66 | #
|
| 67 | # Pretty much all tests are parameterized by shell
|
| 68 | #
|
| 69 | # There's also python2 vs. python3 vs. awk etc.
|
| 70 | # benchmarks/compute
|
| 71 | #
|
| 72 | # Should it be
|
| 73 | # BYO_PARAM_OSH
|
| 74 | #
|
| 75 | # Usage:
|
| 76 | #
|
| 77 | # $ byo run-tests test/osh-usage
|
| 78 | # $ byo run-tests --param OSH=bin/osh test/osh-usage
|
| 79 | # $ byo run-tests --param OSH=bin/osh --param OSH=_bin/cxx-asan/osh test/osh-usage
|
| 80 | # $ byo run-tests --param OSH='bin/osh;_bin/cxx-asan/osh' test/osh-usage
|
| 81 | #
|
| 82 | # Run with each value of param in sequence, and then make a big table later?
|
| 83 | # I think you just loop over the param flags
|
| 84 |
|
| 85 | # If no params, we run once. Otherwise we run once per param value
|
| 86 | FLAG_params=()
|
| 87 | FLAG_fresh_dir=''
|
| 88 | FLAG_capture=''
|
| 89 | FLAG_test_filter=''
|
| 90 |
|
| 91 | parse-flags-for-test() {
|
| 92 | ### Sets global vars FLAG_*
|
| 93 |
|
| 94 | while test $# -ne 0; do
|
| 95 | case "$1" in
|
| 96 | -h|--help)
|
| 97 | print-help
|
| 98 | exit
|
| 99 | ;;
|
| 100 |
|
| 101 | # Capture stdout and stderr? Or let it go to the terminal
|
| 102 | --capture)
|
| 103 | FLAG_capture=T
|
| 104 | ;;
|
| 105 |
|
| 106 | # Is each test case run in its own dir? Or set TEST_TEMP_DIR
|
| 107 | --fresh-dir)
|
| 108 | FLAG_fresh_dir=T
|
| 109 | ;;
|
| 110 |
|
| 111 | --test-filter)
|
| 112 | if test $# -eq 1; then
|
| 113 | die "--test-filter requires an argument"
|
| 114 | fi
|
| 115 | shift
|
| 116 |
|
| 117 | # Regex in ERE syntax
|
| 118 | FLAG_test_filter=$1
|
| 119 | ;;
|
| 120 |
|
| 121 | --param)
|
| 122 | if test $# -eq 1; then
|
| 123 | die "--param requires an argument"
|
| 124 | fi
|
| 125 | shift
|
| 126 |
|
| 127 | pat='[A-Z_]+=.*'
|
| 128 | if ! [[ $1 =~ $pat ]]; then
|
| 129 | die "Expected string like PARAM_NAME=value, got $1"
|
| 130 | fi
|
| 131 | FLAG_params+=( $1 )
|
| 132 | ;;
|
| 133 |
|
| 134 | -*)
|
| 135 | die "Invalid flag '$1'"
|
| 136 | break
|
| 137 | ;;
|
| 138 |
|
| 139 | --)
|
| 140 | shift
|
| 141 | break
|
| 142 | ;;
|
| 143 |
|
| 144 | *)
|
| 145 | # Move on to args
|
| 146 | break
|
| 147 | ;;
|
| 148 |
|
| 149 | esac
|
| 150 | shift
|
| 151 | done
|
| 152 |
|
| 153 | ARGV=( "$@" )
|
| 154 | }
|
| 155 |
|
| 156 | run-tests() {
|
| 157 | if test $# -eq 0; then
|
| 158 | die "Expected argv to run"
|
| 159 | fi
|
| 160 |
|
| 161 | # Set FLAG_* and ARGV
|
| 162 | parse-flags-for-test "$@"
|
| 163 |
|
| 164 | # ARGV is the command to run, like bash foo.sh
|
| 165 | #
|
| 166 | # It's an array rather than a single command, so you can run the same scripts
|
| 167 | # under multiple shells:
|
| 168 | #
|
| 169 | # bash myscript.sh
|
| 170 | # osh myscript.sh
|
| 171 | #
|
| 172 | # This matters so two-test.sh can SOURCE two.sh, and run under both bash and
|
| 173 | # OSH.
|
| 174 | # That could be or --use-interp bin/osh
|
| 175 | #
|
| 176 | # could you have --use-interp python3 too? e.g. I want benchmarks/time_.py
|
| 177 | # to work under both? See time-tsv3 in tsv-lib.sh
|
| 178 | #
|
| 179 | # No that's a test param PYTHON=python3 PYTHON=python2
|
| 180 |
|
| 181 | detect "${ARGV[@]}"
|
| 182 |
|
| 183 | log '---'
|
| 184 | log "byo run-tests: ${ARGV[@]}"
|
| 185 | log
|
| 186 |
|
| 187 | mkdir -p _tmp
|
| 188 | local tmp=_tmp/byo-list.txt
|
| 189 |
|
| 190 | # First list the tests
|
| 191 | BYO_COMMAND=list-tests "${ARGV[@]}" > $tmp
|
| 192 |
|
| 193 | local i=0
|
| 194 | local status
|
| 195 |
|
| 196 | while read -r test_name; do
|
| 197 |
|
| 198 | echo "${TAB}${test_name}"
|
| 199 |
|
| 200 | set +o errexit
|
| 201 | if test -n "$FLAG_capture"; then
|
| 202 | # TODO: Capture it to a string
|
| 203 | # - Write to nice HTML file?
|
| 204 | BYO_COMMAND=run-test BYO_ARG="$test_name" "${ARGV[@]}" >/dev/null 2>&1
|
| 205 | else
|
| 206 | BYO_COMMAND=run-test BYO_ARG="$test_name" "${ARGV[@]}"
|
| 207 | fi
|
| 208 | status=$?
|
| 209 | set -o errexit
|
| 210 |
|
| 211 | if test $status -eq 0; then
|
| 212 | echo "${TAB}OK"
|
| 213 | else
|
| 214 | echo "${TAB}FAIL with status $status"
|
| 215 | exit 1
|
| 216 | fi
|
| 217 |
|
| 218 | i=$(( i + 1 ))
|
| 219 | done < $tmp
|
| 220 |
|
| 221 | echo
|
| 222 | echo "$0: $i tests passed."
|
| 223 | }
|
| 224 |
|
| 225 | task-five "$@"
|