OILS / test / stateful.sh View on Github | oilshell.org

229 lines, 98 significant
1#!/usr/bin/env bash
2#
3# Wrapper for test cases in spec/stateful
4#
5# Usage:
6# test/stateful.sh <function name>
7#
8# Examples:
9# test/stateful.sh signals -r 0-1 # run a range of tests
10# test/stateful.sh signals --list # list tests
11# test/stateful.sh job-control --num-retries 0
12#
13# test/stateful.sh signals-quick # not all shells
14#
15# test/stateful.sh soil-run
16#
17# TODO: Should have QUICKLY=1 variants
18
19set -o nounset
20set -o pipefail
21set -o errexit
22
23REPO_ROOT=$(cd $(dirname $0)/.. && pwd) # tsv-lib.sh uses this
24readonly REPO_ROOT
25
26source test/common.sh # log, $OSH
27source test/tsv-lib.sh
28
29source build/dev-shell.sh
30
31readonly BASE_DIR=_tmp/spec/stateful
32
33# Hack for testing the harness
34#readonly FIRST='-r 0'
35readonly FIRST=''
36readonly OSH_CPP=_bin/cxx-asan/osh
37
38#readonly -a QUICK_SHELLS=( $OSH bash )
39readonly -a QUICK_SHELLS=( $OSH $OSH_CPP bash )
40
41#
42# Suites in spec/stateful
43#
44
45signals() {
46 spec/stateful/signals.py $FIRST "$@"
47}
48
49interactive() {
50 spec/stateful/interactive.py $FIRST "$@"
51}
52
53job-control() {
54 spec/stateful/job_control.py $FIRST --oils-failures-allowed 0 "$@"
55}
56
57# Run on just 2 shells
58
59signals-quick() { signals "${QUICK_SHELLS[@]}" "$@"; }
60interactive-quick() { interactive "${QUICK_SHELLS[@]}" "$@"; }
61job-control-quick() { job-control "${QUICK_SHELLS[@]}" "$@"; }
62
63# Run on all shells we can
64
65# They now pass for dash and mksh, with wait -n and PIPESTATUS skipped. zsh
66# doesn't work now, but could if the prompt was changed to $ ?
67signals-all() { signals "${QUICK_SHELLS[@]}" dash mksh "$@"; }
68
69interactive-all() { interactive "${QUICK_SHELLS[@]}" dash mksh "$@"; }
70
71job-control-all() { job-control "${QUICK_SHELLS[@]}" dash "$@"; }
72
73#
74# More automation
75#
76
77print-tasks() {
78 ### List all tests
79
80 # TODO:
81 # - Print a table with --osh-allowed-failures and shells. It can be filtered
82
83 if test -n "${QUICKLY:-}"; then
84 echo 'interactive'
85 else
86 echo 'interactive'
87 echo 'job-control'
88 echo 'signals'
89 fi
90}
91
92run-file() {
93 ### Run a spec/stateful file, logging output
94
95 local spec_name=$1
96
97 log "__ $spec_name"
98
99 local base_dir=$BASE_DIR
100
101 local log_filename=$spec_name.log.txt
102 local results_filename=$spec_name.results.txt
103
104 time-tsv -o $base_dir/${spec_name}.task.txt \
105 --field $spec_name --field $log_filename --field $results_filename -- \
106 $0 "$spec_name-all" --results-file $base_dir/$results_filename \
107 >$base_dir/$log_filename 2>&1 || true
108}
109
110html-summary() {
111 ### Summarize all files
112
113 # Note: In retrospect, it would be better if every process writes a "long"
114 # TSV file of results.
115 # And then we concatenate them and write the "wide" summary here.
116
117 html-head --title 'Stateful Tests' \
118 ../../../web/base.css ../../../web/spec-tests.css
119
120 # Similar to test/spec-runner.sh and soil format-wwz-index
121
122 cat <<EOF
123 <body class="width50">
124
125<p id="home-link">
126 <!-- up to .wwz index -->
127 <a href="../..">Up</a> |
128 <a href="/">Home</a>
129</p>
130
131 <h1>Stateful Tests with <a href="//www.oilshell.org/cross-ref.html#pexpect">pexpect</a> </h1>
132
133 <table>
134 <thead>
135 <tr>
136 <td>Test File</td>
137 <td>Elapsed seconds</td>
138 <td>Status</td>
139 </tr>
140 </thead>
141EOF
142
143 local all_passed=0
144
145 shopt -s lastpipe # to mutate all_passed in while
146
147 local results_tmp=$BASE_DIR/results.html
148 echo '' > $results_tmp # Accumulate more here
149
150 print-tasks | while read spec_name; do
151
152 # Note: in test/spec-runner.sh, an awk script creates this table. It reads
153 # *.task.txt and *.stats.txt. I could add --stats-file to harness.py
154 # with pass/fail stats
155 read status elapsed _ log_filename results_filename < $BASE_DIR/${spec_name}.task.txt
156
157 echo '<tr>'
158 echo "<td> <a href="$log_filename">$spec_name</a> </td>"
159
160 printf -v elapsed_str '%.1f' $elapsed
161 echo "<td>$elapsed_str</td>"
162
163 case $status in
164 (0) # exit code 0 is success
165 echo " <td>$status</td>"
166 ;;
167 (*) # everything else is a failure
168 # Add extra text to make red stand out.
169 echo " <td class=\"fail\">status: $status</td>"
170
171 # Mark failure
172 all_passed=1
173 ;;
174 esac
175 echo '</tr>'
176
177 # Append to temp file
178 {
179 echo "<h2>$spec_name</h2>"
180 echo '<pre>'
181 escape-html $BASE_DIR/$results_filename
182 echo '</pre>'
183 } >> $results_tmp
184
185 done
186 echo '</table>'
187
188 cat $results_tmp
189
190 cat <<EOF
191 </table>
192 </body>
193</html>
194EOF
195
196 log "all_passed = $all_passed"
197
198 return $all_passed
199}
200
201soil-run() {
202 ninja $OSH_CPP
203
204 mkdir -p $BASE_DIR
205
206 print-tasks | xargs -n 1 -- $0 run-file
207
208 # Returns whether all passed
209 html-summary > $BASE_DIR/index.html
210}
211
212#
213# Debugging
214#
215
216test-stop() {
217 python3 spec/stateful/harness.py test-stop demo/cpython/fork_signal_state.py
218}
219
220strace-py-fork() {
221 rm -f -v _tmp/py-fork.*
222 strace -ff -o _tmp/py-fork demo/cpython/fork_signal_state.py
223 ls -l _tmp/py-fork.*
224
225 # I see rt_sigaction(SIGSTP, ...) which is good
226 # so yeah this seems perfectly fine -- why is it ignoring SIGTSTP? :-(
227}
228
229"$@"