OILS / mycpp / TEST.sh View on Github | oilshell.org

360 lines, 232 significant
1#!/usr/bin/env bash
2#
3# Run tests in this directory.
4#
5# Usage:
6# mycpp/TEST.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13source build/common.sh
14source build/ninja-rules-cpp.sh
15source devtools/common.sh
16source test/common.sh # run-test-bin, can-compile-32-bit
17
18# in case binaries weren't built
19shopt -s failglob
20
21# Will be needed to pass ASAN leak detector? Or only do this for the main binary?
22# export OILS_GC_ON_EXIT=1
23
24examples-variant() {
25 ### Run all examples using a variant -- STATS only
26
27 local compiler=${1:-cxx}
28 local variant=${2:-asan+gcalways}
29 local do_benchmark=${3:-}
30
31 banner "$0 examples-variant $compiler $variant"
32
33 ninja mycpp-examples-$compiler-$variant
34
35 local num_tests=0
36 local num_failed=0
37 local status=0
38
39 local log_dir=_test/$compiler-$variant/mycpp/examples
40 mkdir -p $log_dir
41
42 for b in _bin/$compiler-$variant/mycpp/examples/*; do
43 case $b in
44 (*.stripped) # just run the unstripped binary
45 continue
46 ;;
47 esac
48
49 local prefix="$log_dir/$(basename $b)"
50
51 case $variant in
52 (coverage)
53 export LLVM_PROFILE_FILE=$prefix.profraw
54 ;;
55 esac
56
57 local log="${prefix}${do_benchmark}.log"
58
59 log "RUN $b > $log"
60
61 local test_name=$(basename $b)
62 if test -n "$do_benchmark" && [[ $test_name == test_* ]]; then
63 log "Skipping $test_name in benchmark mode"
64 continue
65 fi
66
67 set +o errexit
68 BENCHMARK="$do_benchmark" $b >$log 2>&1
69 status=$?
70 set -o errexit
71
72 if test "$status" -eq 0; then
73 log 'OK'
74 else
75 log "FAIL with status $?"
76 log ''
77 #return $status
78 num_failed=$((num_failed + 1))
79 fi
80
81 num_tests=$((num_tests + 1))
82 done
83
84 log ''
85 log "$num_failed of $num_tests tests failed"
86 log ''
87
88 if test $num_failed -ne 0; then
89 echo "FAIL: Expected no failures, got $num_failed"
90 return 1
91 fi
92
93 return 0
94}
95
96#
97# 3 Variants x {test, benchmark}
98#
99
100ex-gcalways() {
101 local compiler=${1:-}
102 examples-variant "$compiler" asan+gcalways
103}
104
105# TOO SLOW to run. It's garbage collecting all the time.
106ex-gcalways-bench() {
107 local compiler=${1:-}
108 examples-variant "$compiler" asan+gcalways '.BENCHMARK'
109}
110
111ex-asan() {
112 local compiler=${1:-}
113 examples-variant "$compiler" asan
114}
115
116# 2 of 18 tests failed: cartesian, parse
117# So it does not catch the 10 segfaults that 'asan+gcalways' catches with a few
118# iterations!
119ex-asan-bench() {
120 local compiler=${1:-}
121 examples-variant "$compiler" asan '.BENCHMARK'
122}
123
124# PASS! Under both clang and GCC.
125ex-ubsan() {
126 local compiler=${1:-}
127 examples-variant "$compiler" ubsan
128}
129
130# same as ASAN: 2 of 18
131ex-ubsan-bench() {
132 local compiler=${1:-}
133 examples-variant "$compiler" ubsan '.BENCHMARK'
134}
135
136# PASS!
137ex-opt() {
138 local compiler=${1:-}
139 examples-variant "$compiler" opt
140}
141
142# 2 of 18 tests failed
143ex-opt-bench() {
144 local compiler=${1:-}
145 examples-variant "$compiler" opt '.BENCHMARK'
146}
147
148#
149# Unit Tests
150#
151
152unit() {
153 ### Run by test/cpp-unit.sh
154
155 local compiler=${1:-cxx}
156 local variant=${2:-asan+gcalways}
157
158 log ''
159 log "$0 unit $compiler $variant"
160 log ''
161
162 ninja mycpp-unit-$compiler-$variant
163
164 local -a binaries=(_bin/$compiler-$variant/mycpp/*)
165
166 # Add these files if they exist in the variant
167 if test -d _bin/$compiler-$variant/mycpp/demo; then
168 binaries+=(_bin/$compiler-$variant/mycpp/demo/*)
169 fi
170
171 for b in "${binaries[@]}"; do
172 if ! test -f $b; then
173 continue
174 fi
175
176 local prefix=${b//_bin/_test/}
177 local log=$prefix.log
178 mkdir -p $(dirname $log)
179
180 local asan_options=''
181 case $b in
182 # leaks with malloc
183 (*/demo/hash_table|*/demo/target_lang|*/demo/gc_header|*/small_str_test)
184 asan_options='detect_leaks=0'
185 ;;
186 esac
187
188 ASAN_OPTIONS="$asan_options" run-test-bin $b
189
190 done
191}
192
193#
194# Test failures
195#
196
197translate-example() {
198 local ex=$1
199
200 local mycpp=_bin/shwrap/mycpp_main
201 $mycpp '.:pyext' _tmp/mycpp-invalid $ex
202}
203
204test-invalid-examples() {
205 local mycpp=_bin/shwrap/mycpp_main
206 ninja $mycpp
207 for ex in mycpp/examples/invalid_*; do
208
209 banner "$ex"
210
211 set +o errexit
212 translate-example $ex
213 local status=$?
214 set -o errexit
215
216 local expected_status=1
217
218 case $ex in
219 */invalid_condition.py)
220 expected_status=8
221 ;;
222 */invalid_default_args.py)
223 expected_status=4
224 ;;
225 */invalid_try_else.py)
226 expected_status=3
227 ;;
228 */invalid_except.py)
229 expected_status=3
230 ;;
231 */invalid_global.py)
232 expected_status=2
233 ;;
234 */invalid_python.py)
235 expected_status=5
236 ;;
237 */invalid_switch.py)
238 expected_status=5
239 ;;
240 esac
241
242 if test $status -ne $expected_status; then
243 die "mycpp $ex: expected status $expected_status, got $status"
244 fi
245
246 done
247}
248
249test-control-flow-graph() {
250 local mycpp=_bin/shwrap/mycpp_main
251 ninja $mycpp
252 for ex in mycpp/examples/*.py; do
253 local data_dir=testdata/control-flow-graph/$(basename -s .py $ex)
254 if ! test -d $data_dir; then
255 continue
256 fi
257 banner "$ex"
258
259 translate-example $ex
260 for fact_path in $data_dir/*.facts; do
261 local fact_file=$(basename $fact_path)
262 diff -u $data_dir/$fact_file _tmp/mycpp-facts/$fact_file
263 done
264 done
265}
266
267golden-control-flow-graph() {
268 run-test-func test-control-flow-graph _test/mycpp/test-cfg-examples.log
269}
270
271test-runtime() {
272 # Run other unit tests, e.g. the GC tests
273
274 if can-compile-32-bit; then
275 unit '' asan32+gcalways # ASAN on 32-bit
276 else
277 log ''
278 log "*** Can't compile 32-bit binaries (gcc-multilib g++-multilib needed on Debian)"
279 log ''
280 fi
281
282 # TODO: Run with Clang UBSAN in CI as well
283 local ubsan_compiler=cxx
284 #local ubsan_compiler=clang
285
286 for config in cxx-asan+bumpleak $ubsan_compiler-ubsan+bumpleak; do
287 local bin=_bin/$config/mycpp/bump_leak_heap_test
288 ninja $bin
289 run-test-bin $bin
290 done
291
292 # Run other tests with all variants
293
294 unit $ubsan_compiler ubsan
295
296 unit '' asan
297 unit '' asan+gcalways
298 unit '' opt
299}
300
301#
302# Translator
303#
304
305test-translator() {
306 ### Invoked by soil/worker.sh
307
308 examples-variant '' asan
309
310 # Test with more collections
311 examples-variant '' asan+gcalways
312
313 run-test-func test-invalid-examples _test/mycpp/test-invalid-examples.log
314
315 golden-control-flow-graph
316
317 # Runs tests in cxx-asan variant, and benchmarks in cxx-opt variant
318 if ! ninja mycpp-logs-equal; then
319 log 'FAIL mycpp-logs-equal'
320 return 1
321 fi
322}
323
324soil-run() {
325 set +o errexit
326 $0 test-translator
327 local status=$?
328 set -o errexit
329
330 return $status
331}
332
333unit-test-coverage() {
334 ### Invoked by Soil
335
336 local bin=_bin/clang-coverage+bumpleak/mycpp/bump_leak_heap_test
337 ninja $bin
338 run-test-bin $bin
339
340 unit clang coverage
341
342 local out_dir=_test/clang-coverage/mycpp
343 test/coverage.sh html-report $out_dir \
344 clang-coverage/mycpp clang-coverage+bumpleak/mycpp
345}
346
347examples-coverage() {
348 ### Invoked by Soil
349
350 examples-variant clang coverage
351
352 local out_dir=_test/clang-coverage/mycpp/examples
353 test/coverage.sh html-report $out_dir clang-coverage/mycpp/examples
354}
355
356# Call function $1 with arguments $2 $3 $4
357#
358# mycpp/TEST.sh examples-variant '' asan
359
360"$@"