1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Wrapper for test cases in spec/stateful
|
4 | #
|
5 | # Usage:
|
6 | # test/stateful.sh <function name>
|
7 | #
|
8 | # Examples:
|
9 | # test/stateful.sh signals -r 0-1 # run a range of tests
|
10 | # test/stateful.sh signals --list # list tests
|
11 | # test/stateful.sh job-control --num-retries 0
|
12 | #
|
13 | # test/stateful.sh signals-quick # not all shells
|
14 | #
|
15 | # test/stateful.sh soil-run
|
16 | #
|
17 | # TODO: Should have QUICKLY=1 variants
|
18 |
|
19 | set -o nounset
|
20 | set -o pipefail
|
21 | set -o errexit
|
22 |
|
23 | REPO_ROOT=$(cd $(dirname $0)/.. && pwd) # tsv-lib.sh uses this
|
24 | readonly REPO_ROOT
|
25 |
|
26 | source test/common.sh # log, $OSH
|
27 | source test/tsv-lib.sh
|
28 |
|
29 | source build/dev-shell.sh
|
30 |
|
31 | readonly BASE_DIR=_tmp/spec/stateful
|
32 |
|
33 | # Hack for testing the harness
|
34 | #readonly FIRST='-r 0'
|
35 | readonly FIRST=''
|
36 | readonly OSH_CPP=_bin/cxx-asan/osh
|
37 |
|
38 | #readonly -a QUICK_SHELLS=( $OSH bash )
|
39 | readonly -a QUICK_SHELLS=( $OSH $OSH_CPP bash )
|
40 |
|
41 | #
|
42 | # Suites in spec/stateful
|
43 | #
|
44 |
|
45 | signals() {
|
46 | spec/stateful/signals.py $FIRST "$@"
|
47 | }
|
48 |
|
49 | interactive() {
|
50 | spec/stateful/interactive.py $FIRST "$@"
|
51 | }
|
52 |
|
53 | job-control() {
|
54 | spec/stateful/job_control.py $FIRST --oils-failures-allowed 0 "$@"
|
55 | }
|
56 |
|
57 | # Run on just 2 shells
|
58 |
|
59 | signals-quick() { signals "${QUICK_SHELLS[@]}" "$@"; }
|
60 | interactive-quick() { interactive "${QUICK_SHELLS[@]}" "$@"; }
|
61 | job-control-quick() { job-control "${QUICK_SHELLS[@]}" "$@"; }
|
62 |
|
63 | # Run on all shells we can
|
64 |
|
65 | # They now pass for dash and mksh, with wait -n and PIPESTATUS skipped. zsh
|
66 | # doesn't work now, but could if the prompt was changed to $ ?
|
67 | signals-all() { signals "${QUICK_SHELLS[@]}" dash mksh "$@"; }
|
68 |
|
69 | interactive-all() { interactive "${QUICK_SHELLS[@]}" dash mksh "$@"; }
|
70 |
|
71 | job-control-all() { job-control "${QUICK_SHELLS[@]}" dash "$@"; }
|
72 |
|
73 | #
|
74 | # More automation
|
75 | #
|
76 |
|
77 | print-tasks() {
|
78 | ### List all tests
|
79 |
|
80 | # TODO:
|
81 | # - Print a table with --osh-allowed-failures and shells. It can be filtered
|
82 |
|
83 | if test -n "${QUICKLY:-}"; then
|
84 | echo 'interactive'
|
85 | else
|
86 | echo 'interactive'
|
87 | echo 'job-control'
|
88 | echo 'signals'
|
89 | fi
|
90 | }
|
91 |
|
92 | run-file() {
|
93 | ### Run a spec/stateful file, logging output
|
94 |
|
95 | local spec_name=$1
|
96 |
|
97 | log "__ $spec_name"
|
98 |
|
99 | local base_dir=$BASE_DIR
|
100 |
|
101 | local log_filename=$spec_name.log.txt
|
102 | local results_filename=$spec_name.results.txt
|
103 |
|
104 | time-tsv -o $base_dir/${spec_name}.task.txt \
|
105 | --field $spec_name --field $log_filename --field $results_filename -- \
|
106 | $0 "$spec_name-all" --results-file $base_dir/$results_filename \
|
107 | >$base_dir/$log_filename 2>&1 || true
|
108 | }
|
109 |
|
110 | html-summary() {
|
111 | ### Summarize all files
|
112 |
|
113 | # Note: In retrospect, it would be better if every process writes a "long"
|
114 | # TSV file of results.
|
115 | # And then we concatenate them and write the "wide" summary here.
|
116 |
|
117 | html-head --title 'Stateful Tests' \
|
118 | ../../../web/base.css ../../../web/spec-tests.css
|
119 |
|
120 | # Similar to test/spec-runner.sh and soil format-wwz-index
|
121 |
|
122 | cat <<EOF
|
123 | <body class="width50">
|
124 |
|
125 | <p id="home-link">
|
126 | <!-- up to .wwz index -->
|
127 | <a href="../..">Up</a> |
|
128 | <a href="/">Home</a>
|
129 | </p>
|
130 |
|
131 | <h1>Stateful Tests with <a href="//www.oilshell.org/cross-ref.html#pexpect">pexpect</a> </h1>
|
132 |
|
133 | <table>
|
134 | <thead>
|
135 | <tr>
|
136 | <td>Test File</td>
|
137 | <td>Elapsed seconds</td>
|
138 | <td>Status</td>
|
139 | </tr>
|
140 | </thead>
|
141 | EOF
|
142 |
|
143 | local all_passed=0
|
144 |
|
145 | shopt -s lastpipe # to mutate all_passed in while
|
146 |
|
147 | local results_tmp=$BASE_DIR/results.html
|
148 | echo '' > $results_tmp # Accumulate more here
|
149 |
|
150 | print-tasks | while read spec_name; do
|
151 |
|
152 | # Note: in test/spec-runner.sh, an awk script creates this table. It reads
|
153 | # *.task.txt and *.stats.txt. I could add --stats-file to harness.py
|
154 | # with pass/fail stats
|
155 | read status elapsed _ log_filename results_filename < $BASE_DIR/${spec_name}.task.txt
|
156 |
|
157 | echo '<tr>'
|
158 | echo "<td> <a href="$log_filename">$spec_name</a> </td>"
|
159 |
|
160 | printf -v elapsed_str '%.1f' $elapsed
|
161 | echo "<td>$elapsed_str</td>"
|
162 |
|
163 | case $status in
|
164 | (0) # exit code 0 is success
|
165 | echo " <td>$status</td>"
|
166 | ;;
|
167 | (*) # everything else is a failure
|
168 | # Add extra text to make red stand out.
|
169 | echo " <td class=\"fail\">status: $status</td>"
|
170 |
|
171 | # Mark failure
|
172 | all_passed=1
|
173 | ;;
|
174 | esac
|
175 | echo '</tr>'
|
176 |
|
177 | # Append to temp file
|
178 | {
|
179 | echo "<h2>$spec_name</h2>"
|
180 | echo '<pre>'
|
181 | escape-html $BASE_DIR/$results_filename
|
182 | echo '</pre>'
|
183 | } >> $results_tmp
|
184 |
|
185 | done
|
186 | echo '</table>'
|
187 |
|
188 | cat $results_tmp
|
189 |
|
190 | cat <<EOF
|
191 | </table>
|
192 | </body>
|
193 | </html>
|
194 | EOF
|
195 |
|
196 | log "all_passed = $all_passed"
|
197 |
|
198 | return $all_passed
|
199 | }
|
200 |
|
201 | soil-run() {
|
202 | ninja $OSH_CPP
|
203 |
|
204 | mkdir -p $BASE_DIR
|
205 |
|
206 | print-tasks | xargs -n 1 -- $0 run-file
|
207 |
|
208 | # Returns whether all passed
|
209 | html-summary > $BASE_DIR/index.html
|
210 | }
|
211 |
|
212 | #
|
213 | # Debugging
|
214 | #
|
215 |
|
216 | test-stop() {
|
217 | python3 spec/stateful/harness.py test-stop demo/cpython/fork_signal_state.py
|
218 | }
|
219 |
|
220 | strace-py-fork() {
|
221 | rm -f -v _tmp/py-fork.*
|
222 | strace -ff -o _tmp/py-fork demo/cpython/fork_signal_state.py
|
223 | ls -l _tmp/py-fork.*
|
224 |
|
225 | # I see rt_sigaction(SIGSTP, ...) which is good
|
226 | # so yeah this seems perfectly fine -- why is it ignoring SIGTSTP? :-(
|
227 | }
|
228 |
|
229 | "$@"
|