OILS / mycpp / TEST.sh View on Github | oilshell.org

357 lines, 228 significant
1#!/usr/bin/env bash
2#
3# Run tests in this directory.
4#
5# Usage:
6# mycpp/TEST.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13source build/common.sh
14source build/ninja-rules-cpp.sh
15source devtools/common.sh
16source test/common.sh # run-test-bin, can-compile-32-bit
17
18# in case binaries weren't built
19shopt -s failglob
20
21# Will be needed to pass ASAN leak detector? Or only do this for the main binary?
22# export OILS_GC_ON_EXIT=1
23
24examples-variant() {
25 ### Run all examples using a variant -- STATS only
26
27 local compiler=${1:-cxx}
28 local variant=${2:-asan+gcalways}
29 local do_benchmark=${3:-}
30
31 banner "$0 examples-variant $compiler $variant"
32
33 ninja mycpp-examples-$compiler-$variant
34
35 local num_tests=0
36 local num_failed=0
37 local status=0
38
39 local log_dir=_test/$compiler-$variant/mycpp/examples
40 mkdir -p $log_dir
41
42 for b in _bin/$compiler-$variant/mycpp/examples/*; do
43 case $b in
44 (*.stripped) # just run the unstripped binary
45 continue
46 ;;
47 esac
48
49 local prefix="$log_dir/$(basename $b)"
50
51 case $variant in
52 (coverage)
53 export LLVM_PROFILE_FILE=$prefix.profraw
54 ;;
55 esac
56
57 local log="${prefix}${do_benchmark}.log"
58
59 log "RUN $b > $log"
60
61 local test_name=$(basename $b)
62 if test -n "$do_benchmark" && [[ $test_name == test_* ]]; then
63 log "Skipping $test_name in benchmark mode"
64 continue
65 fi
66
67 set +o errexit
68 BENCHMARK="$do_benchmark" $b >$log 2>&1
69 status=$?
70 set -o errexit
71
72 if test "$status" -eq 0; then
73 log 'OK'
74 else
75 log "FAIL with status $?"
76 log ''
77 #return $status
78 num_failed=$((num_failed + 1))
79 fi
80
81 num_tests=$((num_tests + 1))
82 done
83
84 log ''
85 log "$num_failed of $num_tests tests failed"
86 log ''
87
88 if test $num_failed -ne 0; then
89 echo "FAIL: Expected no failures, got $num_failed"
90 return 1
91 fi
92
93 return 0
94}
95
96#
97# 3 Variants x {test, benchmark}
98#
99
100ex-gcalways() {
101 local compiler=${1:-}
102 examples-variant "$compiler" asan+gcalways
103}
104
105# TOO SLOW to run. It's garbage collecting all the time.
106ex-gcalways-bench() {
107 local compiler=${1:-}
108 examples-variant "$compiler" asan+gcalways '.BENCHMARK'
109}
110
111ex-asan() {
112 local compiler=${1:-}
113 examples-variant "$compiler" asan
114}
115
116# 2 of 18 tests failed: cartesian, parse
117# So it does not catch the 10 segfaults that 'asan+gcalways' catches with a few
118# iterations!
119ex-asan-bench() {
120 local compiler=${1:-}
121 examples-variant "$compiler" asan '.BENCHMARK'
122}
123
124# PASS! Under both clang and GCC.
125ex-ubsan() {
126 local compiler=${1:-}
127 examples-variant "$compiler" ubsan
128}
129
130# same as ASAN: 2 of 18
131ex-ubsan-bench() {
132 local compiler=${1:-}
133 examples-variant "$compiler" ubsan '.BENCHMARK'
134}
135
136# PASS!
137ex-opt() {
138 local compiler=${1:-}
139 examples-variant "$compiler" opt
140}
141
142# 2 of 18 tests failed
143ex-opt-bench() {
144 local compiler=${1:-}
145 examples-variant "$compiler" opt '.BENCHMARK'
146}
147
148#
149# Unit Tests
150#
151
152unit() {
153 ### Run by test/cpp-unit.sh
154
155 local compiler=${1:-cxx}
156 local variant=${2:-asan+gcalways}
157
158 log ''
159 log "$0 unit $compiler $variant"
160 log ''
161
162 ninja mycpp-unit-$compiler-$variant
163
164 local -a binaries=(_bin/$compiler-$variant/mycpp/*)
165
166 # Add these files if they exist in the variant
167 if test -d _bin/$compiler-$variant/mycpp/demo; then
168 binaries+=(_bin/$compiler-$variant/mycpp/demo/*)
169 fi
170
171 for b in "${binaries[@]}"; do
172 if ! test -f $b; then
173 continue
174 fi
175
176 local prefix=${b//_bin/_test/}
177 local log=$prefix.log
178 mkdir -p $(dirname $log)
179
180 local asan_options=''
181 case $b in
182 # leaks with malloc
183 (*/demo/hash_table|*/demo/target_lang|*/demo/gc_header|*/small_str_test)
184 asan_options='detect_leaks=0'
185 ;;
186 esac
187
188 ASAN_OPTIONS="$asan_options" run-test-bin $b
189
190 done
191}
192
193#
194# Test failures
195#
196
197translate-example() {
198 local ex=$1
199
200 local mycpp=_bin/shwrap/mycpp_main
201 $mycpp '.:pyext' _tmp/mycpp-invalid $ex
202}
203
204test-invalid-examples() {
205 local mycpp=_bin/shwrap/mycpp_main
206 ninja $mycpp
207 for ex in mycpp/examples/invalid_*; do
208
209 banner "$ex"
210
211 set +o errexit
212 translate-example $ex
213 local status=$?
214 set -o errexit
215
216 local expected_status=1
217
218 case $ex in
219 */invalid_condition.py)
220 expected_status=8
221 ;;
222 */invalid_default_args.py)
223 expected_status=4
224 ;;
225 */invalid_try_else.py)
226 expected_status=3
227 ;;
228 */invalid_except.py)
229 expected_status=3
230 ;;
231 */invalid_global.py)
232 expected_status=2
233 ;;
234 */invalid_python.py)
235 expected_status=5
236 ;;
237 */invalid_switch.py)
238 expected_status=5
239 ;;
240 esac
241
242 if test $status -ne $expected_status; then
243 die "mycpp $ex: expected status $expected_status, got $status"
244 fi
245
246 done
247}
248
249test-control-flow-graph() {
250 local mycpp=_bin/shwrap/mycpp_main
251 ninja $mycpp
252 for ex in mycpp/examples/*.py; do
253 local data_dir=testdata/control-flow-graph/$(basename -s .py $ex)
254 if ! test -d $data_dir; then
255 continue
256 fi
257 banner "$ex"
258
259 translate-example $ex
260 diff -u $data_dir/cf_edge.facts _tmp/mycpp-facts/cf_edge.facts
261 done
262}
263
264golden-control-flow-graph() {
265 echo 'TODO: regenerate data in testdata/control-flow-graph'
266}
267
268test-runtime() {
269 # Run other unit tests, e.g. the GC tests
270
271 if can-compile-32-bit; then
272 unit '' asan32+gcalways # ASAN on 32-bit
273 else
274 log ''
275 log "*** Can't compile 32-bit binaries (gcc-multilib g++-multilib needed on Debian)"
276 log ''
277 fi
278
279 # TODO: Run with Clang UBSAN in CI as well
280 local ubsan_compiler=cxx
281 #local ubsan_compiler=clang
282
283 for config in cxx-asan+bumpleak $ubsan_compiler-ubsan+bumpleak; do
284 local bin=_bin/$config/mycpp/bump_leak_heap_test
285 ninja $bin
286 run-test-bin $bin
287 done
288
289 # Run other tests with all variants
290
291 unit $ubsan_compiler ubsan
292
293 unit '' asan
294 unit '' asan+gcalways
295 unit '' opt
296}
297
298#
299# Translator
300#
301
302test-translator() {
303 ### Invoked by soil/worker.sh
304
305 examples-variant '' asan
306
307 # Test with more collections
308 examples-variant '' asan+gcalways
309
310 run-test-func test-invalid-examples _test/mycpp/test-invalid-examples.log
311
312 #run-test-func test-control-flow-graph _test/mycpp/test-cfg-examples.log
313
314 # Runs tests in cxx-asan variant, and benchmarks in cxx-opt variant
315 if ! ninja mycpp-logs-equal; then
316 log 'FAIL mycpp-logs-equal'
317 return 1
318 fi
319}
320
321soil-run() {
322 set +o errexit
323 $0 test-translator
324 local status=$?
325 set -o errexit
326
327 return $status
328}
329
330unit-test-coverage() {
331 ### Invoked by Soil
332
333 local bin=_bin/clang-coverage+bumpleak/mycpp/bump_leak_heap_test
334 ninja $bin
335 run-test-bin $bin
336
337 unit clang coverage
338
339 local out_dir=_test/clang-coverage/mycpp
340 test/coverage.sh html-report $out_dir \
341 clang-coverage/mycpp clang-coverage+bumpleak/mycpp
342}
343
344examples-coverage() {
345 ### Invoked by Soil
346
347 examples-variant clang coverage
348
349 local out_dir=_test/clang-coverage/mycpp/examples
350 test/coverage.sh html-report $out_dir clang-coverage/mycpp/examples
351}
352
353# Call function $1 with arguments $2 $3 $4
354#
355# mycpp/TEST.sh examples-variant '' asan
356
357"$@"