OILS / test / byo-client.sh View on Github | oilshell.org

129 lines, 53 significant
1#!/usr/bin/env bash
2#
3# Run test executables that obey the BYO protocol.
4#
5# TODO: doc/byo.md
6#
7# Usage:
8# test/byo-client.sh run-tests ARGS...
9#
10# The ARGS... are run with an environment variable, e.g.
11#
12# ./myscript.py
13# python3 ./myscript.py
14# ./myscript.sh
15# dash ./myscript.sh
16# osh ./myscript.sh
17#
18# The client creates a clean process state and directory state for each tests.
19
20: ${LIB_OSH=stdlib/osh}
21source $LIB_OSH/bash-strict.sh
22source $LIB_OSH/two.sh
23
24readonly TAB=$'\t'
25
26detect() {
27 if test $# -eq 0; then
28 die "Expected argv to run"
29 fi
30
31 local out
32
33 local status=0
34 set +o errexit
35 out=$(BYO_COMMAND=detect "$@" < /dev/null)
36 status=$?
37 set -o errexit
38
39 if test $status -ne 66; then
40 die "$(printf '%q ' "$@") doesn't implement BYO: expected status 66, got $status"
41 fi
42
43 # Verbose
44 if false; then
45 echo
46 echo "BYO commands detected in $(printf '%q ' "$@"):"
47 echo "$out"
48 fi
49}
50
51run-tests() {
52 # argv is the command to run, like bash foo.sh
53 #
54 # It's an array rather than a single command, so you can run the same scripts
55 # under multiple shells:
56 #
57 # bash myscript.sh
58 # osh myscript.sh
59
60 if test $# -eq 0; then
61 die "Expected argv to run"
62 fi
63
64 detect "$@"
65
66 log '---'
67 log "byo run-tests: $@"
68 log
69
70
71 # TODO:
72 # --no-chdir Change directory by default, but this option disables it
73 # --no-stdout-log stdout is not redirected to its own, named file
74 # --max-jobs Parallelism
75 #
76 # And how do we run test binaries that are just one big process?
77 # - Python - inside a file, we probably don't have any tests to run in parallel
78 # - C++ - ditto
79 # - R - we have a few unit tests
80 #
81 # So maybe we need BYO_LIST_TEST_BIN - list the test binaries?
82 # Or mycpp/TEST.sh etc. should list them. Yes that is true!
83 # We can also list things to build.
84 #
85 # BYO_LIST_NINJA=1
86
87 mkdir -p _tmp
88 local tmp=_tmp/byo-list.txt
89
90 # First list the tests
91 BYO_COMMAND=list-tests "$@" > $tmp
92
93 local i=0
94 local status
95
96 while read -r test_name; do
97
98 echo "${TAB}${test_name}"
99
100 set +o errexit
101 BYO_COMMAND=run-test BYO_ARG="$test_name" "$@"
102 status=$?
103 set -o errexit
104
105 if test $status -eq 0; then
106 echo "${TAB}OK"
107 else
108 echo "${TAB}FAIL with status $status"
109 exit 1
110 fi
111
112 i=$(( i + 1 ))
113 done < $tmp
114
115 echo
116 echo "$0: $i tests passed."
117}
118
119# TODO:
120# BYO_PROBE=1
121#
122# must print capabilities
123#
124# - testing - success/fail
125# - benchmarks - output TSV file I think? Or a directory of TSV files?
126# - BYO_RESULTS
127# - shell auto-completion
128
129"$@"