1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Run tests in this directory.
|
4 | #
|
5 | # Usage:
|
6 | # mycpp/TEST.sh <function name>
|
7 |
|
8 | set -o nounset
|
9 | set -o pipefail
|
10 | set -o errexit
|
11 |
|
12 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
13 | source build/common.sh
|
14 | source build/ninja-rules-cpp.sh
|
15 | source devtools/common.sh
|
16 | source test/common.sh # run-test-bin, can-compile-32-bit
|
17 |
|
18 | # in case binaries weren't built
|
19 | shopt -s failglob
|
20 |
|
21 | # Will be needed to pass ASAN leak detector? Or only do this for the main binary?
|
22 | # export OILS_GC_ON_EXIT=1
|
23 |
|
24 | examples-variant() {
|
25 | ### Run all examples using a variant -- STATS only
|
26 |
|
27 | local compiler=${1:-cxx}
|
28 | local variant=${2:-asan+gcalways}
|
29 | local do_benchmark=${3:-}
|
30 |
|
31 | banner "$0 examples-variant $compiler $variant"
|
32 |
|
33 | ninja mycpp-examples-$compiler-$variant
|
34 |
|
35 | local num_tests=0
|
36 | local num_failed=0
|
37 | local status=0
|
38 |
|
39 | local log_dir=_test/$compiler-$variant/mycpp/examples
|
40 | mkdir -p $log_dir
|
41 |
|
42 | for b in _bin/$compiler-$variant/mycpp/examples/*; do
|
43 | case $b in
|
44 | (*.stripped) # just run the unstripped binary
|
45 | continue
|
46 | ;;
|
47 | esac
|
48 |
|
49 | local prefix="$log_dir/$(basename $b)"
|
50 |
|
51 | case $variant in
|
52 | (coverage)
|
53 | export LLVM_PROFILE_FILE=$prefix.profraw
|
54 | ;;
|
55 | esac
|
56 |
|
57 | local log="${prefix}${do_benchmark}.log"
|
58 |
|
59 | log "RUN $b > $log"
|
60 |
|
61 | local test_name=$(basename $b)
|
62 | if test -n "$do_benchmark" && [[ $test_name == test_* ]]; then
|
63 | log "Skipping $test_name in benchmark mode"
|
64 | continue
|
65 | fi
|
66 |
|
67 | set +o errexit
|
68 | BENCHMARK="$do_benchmark" $b >$log 2>&1
|
69 | status=$?
|
70 | set -o errexit
|
71 |
|
72 | if test "$status" -eq 0; then
|
73 | log 'OK'
|
74 | else
|
75 | log "FAIL with status $?"
|
76 | log ''
|
77 | #return $status
|
78 | num_failed=$((num_failed + 1))
|
79 | fi
|
80 |
|
81 | num_tests=$((num_tests + 1))
|
82 | done
|
83 |
|
84 | log ''
|
85 | log "$num_failed of $num_tests tests failed"
|
86 | log ''
|
87 |
|
88 | if test $num_failed -ne 0; then
|
89 | echo "FAIL: Expected no failures, got $num_failed"
|
90 | return 1
|
91 | fi
|
92 |
|
93 | return 0
|
94 | }
|
95 |
|
96 | #
|
97 | # 3 Variants x {test, benchmark}
|
98 | #
|
99 |
|
100 | ex-gcalways() {
|
101 | local compiler=${1:-}
|
102 | examples-variant "$compiler" asan+gcalways
|
103 | }
|
104 |
|
105 | # TOO SLOW to run. It's garbage collecting all the time.
|
106 | ex-gcalways-bench() {
|
107 | local compiler=${1:-}
|
108 | examples-variant "$compiler" asan+gcalways '.BENCHMARK'
|
109 | }
|
110 |
|
111 | ex-asan() {
|
112 | local compiler=${1:-}
|
113 | examples-variant "$compiler" asan
|
114 | }
|
115 |
|
116 | # 2 of 18 tests failed: cartesian, parse
|
117 | # So it does not catch the 10 segfaults that 'asan+gcalways' catches with a few
|
118 | # iterations!
|
119 | ex-asan-bench() {
|
120 | local compiler=${1:-}
|
121 | examples-variant "$compiler" asan '.BENCHMARK'
|
122 | }
|
123 |
|
124 | # PASS! Under both clang and GCC.
|
125 | ex-ubsan() {
|
126 | local compiler=${1:-}
|
127 | examples-variant "$compiler" ubsan
|
128 | }
|
129 |
|
130 | # same as ASAN: 2 of 18
|
131 | ex-ubsan-bench() {
|
132 | local compiler=${1:-}
|
133 | examples-variant "$compiler" ubsan '.BENCHMARK'
|
134 | }
|
135 |
|
136 | # PASS!
|
137 | ex-opt() {
|
138 | local compiler=${1:-}
|
139 | examples-variant "$compiler" opt
|
140 | }
|
141 |
|
142 | # 2 of 18 tests failed
|
143 | ex-opt-bench() {
|
144 | local compiler=${1:-}
|
145 | examples-variant "$compiler" opt '.BENCHMARK'
|
146 | }
|
147 |
|
148 | #
|
149 | # Unit Tests
|
150 | #
|
151 |
|
152 | unit() {
|
153 | ### Run by test/cpp-unit.sh
|
154 |
|
155 | local compiler=${1:-cxx}
|
156 | local variant=${2:-asan+gcalways}
|
157 |
|
158 | log ''
|
159 | log "$0 unit $compiler $variant"
|
160 | log ''
|
161 |
|
162 | ninja mycpp-unit-$compiler-$variant
|
163 |
|
164 | local -a binaries=(_bin/$compiler-$variant/mycpp/*)
|
165 |
|
166 | # Add these files if they exist in the variant
|
167 | if test -d _bin/$compiler-$variant/mycpp/demo; then
|
168 | binaries+=(_bin/$compiler-$variant/mycpp/demo/*)
|
169 | fi
|
170 |
|
171 | for b in "${binaries[@]}"; do
|
172 | if ! test -f $b; then
|
173 | continue
|
174 | fi
|
175 |
|
176 | local prefix=${b//_bin/_test/}
|
177 | local log=$prefix.log
|
178 | mkdir -p $(dirname $log)
|
179 |
|
180 | local asan_options=''
|
181 | case $b in
|
182 | # leaks with malloc
|
183 | (*/demo/hash_table|*/demo/target_lang|*/demo/gc_header|*/small_str_test)
|
184 | asan_options='detect_leaks=0'
|
185 | ;;
|
186 | esac
|
187 |
|
188 | ASAN_OPTIONS="$asan_options" run-test-bin $b
|
189 |
|
190 | done
|
191 | }
|
192 |
|
193 | #
|
194 | # Test failures
|
195 | #
|
196 |
|
197 | translate-example() {
|
198 | local ex=$1
|
199 |
|
200 | local mycpp=_bin/shwrap/mycpp_main
|
201 | $mycpp '.:pyext' _tmp/mycpp-invalid $ex
|
202 | }
|
203 |
|
204 | test-invalid-examples() {
|
205 | local mycpp=_bin/shwrap/mycpp_main
|
206 | ninja $mycpp
|
207 | for ex in mycpp/examples/invalid_*; do
|
208 |
|
209 | banner "$ex"
|
210 |
|
211 | set +o errexit
|
212 | translate-example $ex
|
213 | local status=$?
|
214 | set -o errexit
|
215 |
|
216 | local expected_status=1
|
217 |
|
218 | case $ex in
|
219 | */invalid_condition.py)
|
220 | expected_status=8
|
221 | ;;
|
222 | */invalid_default_args.py)
|
223 | expected_status=4
|
224 | ;;
|
225 | */invalid_try_else.py)
|
226 | expected_status=3
|
227 | ;;
|
228 | */invalid_except.py)
|
229 | expected_status=3
|
230 | ;;
|
231 | */invalid_global.py)
|
232 | expected_status=2
|
233 | ;;
|
234 | */invalid_python.py)
|
235 | expected_status=5
|
236 | ;;
|
237 | */invalid_switch.py)
|
238 | expected_status=5
|
239 | ;;
|
240 | esac
|
241 |
|
242 | if test $status -ne $expected_status; then
|
243 | die "mycpp $ex: expected status $expected_status, got $status"
|
244 | fi
|
245 |
|
246 | done
|
247 | }
|
248 |
|
249 | test-control-flow-graph() {
|
250 | local mycpp=_bin/shwrap/mycpp_main
|
251 | ninja $mycpp
|
252 | for ex in mycpp/examples/*.py; do
|
253 | local data_dir=testdata/control-flow-graph/$(basename -s .py $ex)
|
254 | if ! test -d $data_dir; then
|
255 | continue
|
256 | fi
|
257 | banner "$ex"
|
258 |
|
259 | translate-example $ex
|
260 | diff -u $data_dir/cf_edge.facts _tmp/mycpp-facts/cf_edge.facts
|
261 | done
|
262 | }
|
263 |
|
264 | golden-control-flow-graph() {
|
265 | echo 'TODO: regenerate data in testdata/control-flow-graph'
|
266 | }
|
267 |
|
268 | test-runtime() {
|
269 | # Run other unit tests, e.g. the GC tests
|
270 |
|
271 | if can-compile-32-bit; then
|
272 | unit '' asan32+gcalways # ASAN on 32-bit
|
273 | else
|
274 | log ''
|
275 | log "*** Can't compile 32-bit binaries (gcc-multilib g++-multilib needed on Debian)"
|
276 | log ''
|
277 | fi
|
278 |
|
279 | # TODO: Run with Clang UBSAN in CI as well
|
280 | local ubsan_compiler=cxx
|
281 | #local ubsan_compiler=clang
|
282 |
|
283 | for config in cxx-asan+bumpleak $ubsan_compiler-ubsan+bumpleak; do
|
284 | local bin=_bin/$config/mycpp/bump_leak_heap_test
|
285 | ninja $bin
|
286 | run-test-bin $bin
|
287 | done
|
288 |
|
289 | # Run other tests with all variants
|
290 |
|
291 | unit $ubsan_compiler ubsan
|
292 |
|
293 | unit '' asan
|
294 | unit '' asan+gcalways
|
295 | unit '' opt
|
296 | }
|
297 |
|
298 | #
|
299 | # Translator
|
300 | #
|
301 |
|
302 | test-translator() {
|
303 | ### Invoked by soil/worker.sh
|
304 |
|
305 | examples-variant '' asan
|
306 |
|
307 | # Test with more collections
|
308 | examples-variant '' asan+gcalways
|
309 |
|
310 | run-test-func test-invalid-examples _test/mycpp/test-invalid-examples.log
|
311 |
|
312 | #run-test-func test-control-flow-graph _test/mycpp/test-cfg-examples.log
|
313 |
|
314 | # Runs tests in cxx-asan variant, and benchmarks in cxx-opt variant
|
315 | if ! ninja mycpp-logs-equal; then
|
316 | log 'FAIL mycpp-logs-equal'
|
317 | return 1
|
318 | fi
|
319 | }
|
320 |
|
321 | soil-run() {
|
322 | set +o errexit
|
323 | $0 test-translator
|
324 | local status=$?
|
325 | set -o errexit
|
326 |
|
327 | return $status
|
328 | }
|
329 |
|
330 | unit-test-coverage() {
|
331 | ### Invoked by Soil
|
332 |
|
333 | local bin=_bin/clang-coverage+bumpleak/mycpp/bump_leak_heap_test
|
334 | ninja $bin
|
335 | run-test-bin $bin
|
336 |
|
337 | unit clang coverage
|
338 |
|
339 | local out_dir=_test/clang-coverage/mycpp
|
340 | test/coverage.sh html-report $out_dir \
|
341 | clang-coverage/mycpp clang-coverage+bumpleak/mycpp
|
342 | }
|
343 |
|
344 | examples-coverage() {
|
345 | ### Invoked by Soil
|
346 |
|
347 | examples-variant clang coverage
|
348 |
|
349 | local out_dir=_test/clang-coverage/mycpp/examples
|
350 | test/coverage.sh html-report $out_dir clang-coverage/mycpp/examples
|
351 | }
|
352 |
|
353 | # Call function $1 with arguments $2 $3 $4
|
354 | #
|
355 | # mycpp/TEST.sh examples-variant '' asan
|
356 |
|
357 | "$@"
|