| 1 | #!/usr/bin/env bash
|
| 2 | #
|
| 3 | # Run test executables that obey the BYO protocol.
|
| 4 | #
|
| 5 | # TODO: doc/byo.md
|
| 6 | #
|
| 7 | # Usage:
|
| 8 | # devtools/byo.sh test FLAGS* ARGS*
|
| 9 | #
|
| 10 | # TODO:
|
| 11 | # - client creates a clean process state for each test
|
| 12 | # - clean directory state for each test.
|
| 13 |
|
| 14 | : ${LIB_OSH=stdlib/osh}
|
| 15 | source $LIB_OSH/bash-strict.sh
|
| 16 | source $LIB_OSH/task-five.sh
|
| 17 |
|
| 18 | readonly TAB=$'\t'
|
| 19 |
|
| 20 | detect() {
|
| 21 | if test $# -eq 0; then
|
| 22 | die "Expected argv to run"
|
| 23 | fi
|
| 24 |
|
| 25 | local out
|
| 26 |
|
| 27 | local status=0
|
| 28 | set +o errexit
|
| 29 | out=$(BYO_COMMAND=detect "$@" < /dev/null)
|
| 30 | status=$?
|
| 31 | set -o errexit
|
| 32 |
|
| 33 | if test $status -ne 66; then
|
| 34 | die "$(printf '%q ' "$@") doesn't implement BYO: expected status 66, got $status"
|
| 35 | fi
|
| 36 |
|
| 37 | # Verbose
|
| 38 | if false; then
|
| 39 | echo
|
| 40 | echo "BYO commands detected in $(printf '%q ' "$@"):"
|
| 41 | echo "$out"
|
| 42 | fi
|
| 43 | }
|
| 44 |
|
| 45 | print-help() {
|
| 46 | # Other flags:
|
| 47 | #
|
| 48 | # --host host to upload to?
|
| 49 | # --auth allow writing to host
|
| 50 | # --no-taskset disable taskset?
|
| 51 |
|
| 52 | cat <<EOF
|
| 53 | Usage: byo run-tests FLAGS*
|
| 54 | EOF
|
| 55 | }
|
| 56 |
|
| 57 | # Test params:
|
| 58 | #
|
| 59 | # SH=dash SH=bash SH=bin/osh # test/spec.sh
|
| 60 | # OSH=bin/osh OSH=_bin/cxx-asan/osh # e.g. test/*{parse,runtime}-errors.sh
|
| 61 | # YSH=bin/osh YSH=_bin/cxx-asan/osh # e.g. test/*{parse,runtime}-errors.sh
|
| 62 | #
|
| 63 | # benchmarks/compute.sh has 3 dimensions:
|
| 64 | # ( workload name, param1, param2 )
|
| 65 | #
|
| 66 | # Pretty much all tests are parameterized by shell
|
| 67 | #
|
| 68 | # There's also python2 vs. python3 vs. awk etc.
|
| 69 | # benchmarks/compute
|
| 70 | #
|
| 71 | # Should it be
|
| 72 | # BYO_PARAM_OSH
|
| 73 | #
|
| 74 | # Usage:
|
| 75 | #
|
| 76 | # $ byo run-tests test/osh-usage
|
| 77 | # $ byo run-tests --param OSH=bin/osh test/osh-usage
|
| 78 | # $ byo run-tests --param OSH=bin/osh --param OSH=_bin/cxx-asan/osh test/osh-usage
|
| 79 | # $ byo run-tests --param OSH='bin/osh;_bin/cxx-asan/osh' test/osh-usage
|
| 80 | #
|
| 81 | # Run with each value of param in sequence, and then make a big table later?
|
| 82 | # I think you just loop over the param flags
|
| 83 |
|
| 84 | # If no params, we run once. Otherwise we run once per param value
|
| 85 | FLAG_params=()
|
| 86 | FLAG_fresh_dir=''
|
| 87 | FLAG_capture=''
|
| 88 | FLAG_test_filter=''
|
| 89 |
|
| 90 | parse-flags-for-test() {
|
| 91 | ### Sets global vars FLAG_*
|
| 92 |
|
| 93 | while test $# -ne 0; do
|
| 94 | case "$1" in
|
| 95 | -h|--help)
|
| 96 | print-help
|
| 97 | exit
|
| 98 | ;;
|
| 99 |
|
| 100 | # Capture stdout and stderr? Or let it go to the terminal
|
| 101 | --capture)
|
| 102 | FLAG_capture=T
|
| 103 | ;;
|
| 104 |
|
| 105 | # Is each test case run in its own dir? Or set TEST_TEMP_DIR
|
| 106 | --fresh-dir)
|
| 107 | FLAG_fresh_dir=T
|
| 108 | ;;
|
| 109 |
|
| 110 | --test-filter)
|
| 111 | if test $# -eq 1; then
|
| 112 | die "--test-filter requires an argument"
|
| 113 | fi
|
| 114 | shift
|
| 115 |
|
| 116 | # Regex in ERE syntax
|
| 117 | FLAG_test_filter=$1
|
| 118 | ;;
|
| 119 |
|
| 120 | --param)
|
| 121 | if test $# -eq 1; then
|
| 122 | die "--param requires an argument"
|
| 123 | fi
|
| 124 | shift
|
| 125 |
|
| 126 | pat='[A-Z_]+=.*'
|
| 127 | if ! [[ $1 =~ $pat ]]; then
|
| 128 | die "Expected string like PARAM_NAME=value, got $1"
|
| 129 | fi
|
| 130 | FLAG_params+=( $1 )
|
| 131 | ;;
|
| 132 |
|
| 133 | -*)
|
| 134 | die "Invalid flag '$1'"
|
| 135 | break
|
| 136 | ;;
|
| 137 |
|
| 138 | --)
|
| 139 | shift
|
| 140 | break
|
| 141 | ;;
|
| 142 |
|
| 143 | *)
|
| 144 | # Move on to args
|
| 145 | break
|
| 146 | ;;
|
| 147 |
|
| 148 | esac
|
| 149 | shift
|
| 150 | done
|
| 151 |
|
| 152 | ARGV=( "$@" )
|
| 153 | }
|
| 154 |
|
| 155 | run-tests() {
|
| 156 | if test $# -eq 0; then
|
| 157 | die "Expected argv to run"
|
| 158 | fi
|
| 159 |
|
| 160 | # Set FLAG_* and ARGV
|
| 161 | parse-flags-for-test "$@"
|
| 162 |
|
| 163 | # ARGV is the command to run, like bash foo.sh
|
| 164 | #
|
| 165 | # It's an array rather than a single command, so you can run the same scripts
|
| 166 | # under multiple shells:
|
| 167 | #
|
| 168 | # bash myscript.sh
|
| 169 | # osh myscript.sh
|
| 170 | #
|
| 171 | # This matters so two-test.sh can SOURCE two.sh, and run under both bash and
|
| 172 | # OSH.
|
| 173 | # That could be or --use-interp bin/osh
|
| 174 | #
|
| 175 | # could you have --use-interp python3 too? e.g. I want benchmarks/time_.py
|
| 176 | # to work under both? See time-tsv3 in tsv-lib.sh
|
| 177 | #
|
| 178 | # No that's a test param PYTHON=python3 PYTHON=python2
|
| 179 |
|
| 180 | detect "${ARGV[@]}"
|
| 181 |
|
| 182 | log '---'
|
| 183 | log "byo run-tests: ${ARGV[@]}"
|
| 184 | log
|
| 185 |
|
| 186 | mkdir -p _tmp
|
| 187 | local tmp=_tmp/byo-list.txt
|
| 188 |
|
| 189 | # First list the tests
|
| 190 | BYO_COMMAND=list-tests "${ARGV[@]}" > $tmp
|
| 191 |
|
| 192 | local i=0
|
| 193 | local status
|
| 194 |
|
| 195 | while read -r test_name; do
|
| 196 |
|
| 197 | echo "${TAB}${test_name}"
|
| 198 |
|
| 199 | set +o errexit
|
| 200 | if test -n "$FLAG_capture"; then
|
| 201 | # TODO: Capture it to a string
|
| 202 | # - Write to nice HTML file?
|
| 203 | BYO_COMMAND=run-test BYO_ARG="$test_name" "${ARGV[@]}" >/dev/null 2>&1
|
| 204 | else
|
| 205 | BYO_COMMAND=run-test BYO_ARG="$test_name" "${ARGV[@]}"
|
| 206 | fi
|
| 207 | status=$?
|
| 208 | set -o errexit
|
| 209 |
|
| 210 | if test $status -eq 0; then
|
| 211 | echo "${TAB}OK"
|
| 212 | else
|
| 213 | echo "${TAB}FAIL with status $status"
|
| 214 | exit 1
|
| 215 | fi
|
| 216 |
|
| 217 | i=$(( i + 1 ))
|
| 218 | done < $tmp
|
| 219 |
|
| 220 | echo
|
| 221 | echo "$0: $i tests passed."
|
| 222 | }
|
| 223 |
|
| 224 | case $1 in
|
| 225 | test) # don't clobber this name
|
| 226 | shift
|
| 227 | run-tests "$@"
|
| 228 | ;;
|
| 229 | *)
|
| 230 | task-five "$@"
|
| 231 | ;;
|
| 232 | esac
|