OILS / test / spec-cpp.sh View on Github | oilshell.org

361 lines, 159 significant
1#!/usr/bin/env bash
2#
3# Test the C++ translation of Oils.
4#
5# Usage:
6# test/spec-cpp.sh <function name>
7#
8# Examples:
9# test/spec-cpp.sh run-file smoke -r 0 -v
10# NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
11
12set -o nounset
13set -o pipefail
14set -o errexit
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source build/dev-shell.sh # PYTHONPATH
19source devtools/run-task.sh
20source test/common.sh # html-head
21source test/spec-common.sh
22source web/table/html.sh
23
24shopt -s failglob # to debug TSV expansion failure below
25
26OSH_PY=$REPO_ROOT/bin/osh
27YSH_PY=$REPO_ROOT/bin/ysh
28
29# Run with ASAN binary by default. Release overrides this
30OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
31YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
32
33# Same variable in test/spec-runner.sh
34NUM_SPEC_TASKS=${NUM_SPEC_TASKS:-400}
35
36# So we can pass ASAN. Note that test/spec-common.sh has to pass this to
37# sh_spec.py.
38export OILS_GC_ON_EXIT=1
39
40#
41# For translation
42#
43
44run-file() {
45 local spec_name=$1
46 shift
47
48 local spec_file=spec/$spec_name.test.sh
49
50 local suite
51 suite=$(test/sh_spec.py --print-spec-suite $spec_file)
52
53 local spec_subdir
54 case $suite in
55 osh) spec_subdir='osh-cpp' ;;
56 ysh) spec_subdir='ysh-cpp' ;;
57 *) die "Invalid suite $suite" ;;
58 esac
59
60 local base_dir=_tmp/spec/$spec_subdir
61 mkdir -v -p $base_dir
62
63 # Compare Python and C++ shells by passing --oils-cpp-bin-dir
64 sh-spec $spec_file \
65 --timeout 10 \
66 --oils-bin-dir $PWD/bin \
67 --oils-cpp-bin-dir $REPO_ROOT/_bin/cxx-asan \
68 --tsv-output $base_dir/${spec_name}.tsv \
69 "$@"
70}
71
72osh-all() {
73 # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
74
75 # For debugging hangs
76 #export MAX_PROCS=1
77
78 ninja _bin/cxx-asan/{osh,ysh}
79
80 test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC
81
82 local spec_subdir=osh-cpp
83
84 # $suite $compare_mode
85 test/spec-runner.sh all-parallel \
86 osh compare-cpp $spec_subdir "$@" || true # OK if it fails
87
88 write-compare-html $spec_subdir
89}
90
91ysh-all() {
92 ninja _bin/cxx-asan/{osh,ysh}
93
94 local spec_subdir=ysh-cpp
95
96 # $suite $compare_mode
97 test/spec-runner.sh all-parallel \
98 ysh compare-cpp $spec_subdir "$@" || true # OK if it fails
99
100 write-compare-html $spec_subdir
101}
102
103console-row() {
104 ### Print out a histogram of results
105
106 awk '
107FNR == 1 {
108 #print FILENAME > "/dev/stderr"
109}
110FNR != 1 {
111 case_num = $1
112 sh = $2
113 result = $3
114
115 if (sh == "osh") {
116 osh[result] += 1
117 } else if (sh == "osh_cpp") { # bin/osh_cpp
118 oe_py[result] += 1
119 } else if (sh == "osh_ALT") { # _bin/*/osh
120 oe_cpp[result] += 1
121 }
122}
123
124function print_hist(sh, hist) {
125 printf("%s\t", sh)
126
127 k = "pass"
128 printf("%s %4d\t", k, hist[k])
129 k = "FAIL"
130 printf("%s %4d\t", k, hist[k])
131
132 print ""
133
134 # This prints N-I, ok, bug, etc.
135 #for (k in hist) {
136 # printf("%s %s\t", k, hist[k])
137 #}
138
139}
140
141END {
142 print_hist("osh", osh)
143 print_hist("osh_cpp", oe_py)
144 print_hist("osh_ALT", oe_cpp)
145}
146 ' "$@"
147}
148
149console-summary() {
150 ### Report on our progress translating
151
152 local spec_subdir=$1
153
154 # Can't go at the top level because files won't exist!
155 readonly TSV=(_tmp/spec/$spec_subdir/*.tsv)
156
157 wc -l "${TSV[@]}"
158
159 for file in "${TSV[@]}"; do
160 echo
161 echo "$file"
162 console-row $file
163 done
164
165 echo
166 echo "TOTAL"
167 console-row "${TSV[@]}"
168}
169
170#
171# HTML
172#
173
174summary-csv-row() {
175 ### Print one row or the last total row
176
177 local spec_subdir=$1
178 shift
179
180 if test $# -eq 1; then
181 local spec_name=$1
182 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.tsv )
183 else
184 local spec_name='TOTAL'
185 local -a tsv_files=( "$@" )
186 fi
187
188 awk -v spec_name=$spec_name '
189# skip the first row
190FNR != 1 {
191 case_num = $1
192 sh = $2
193 result = $3
194
195 if (sh == "osh" || sh == "ysh") {
196 osh[result] += 1
197 } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
198 osh_native[result] += 1
199 }
200}
201
202END {
203 num_py = osh["pass"]
204 num_cpp = osh_native["pass"]
205 if (spec_name == "TOTAL") {
206 href = ""
207 } else {
208 href = sprintf("%s.html", spec_name)
209 }
210
211 if (num_py == num_cpp) {
212 row_css_class = "cpp-good" # green
213 }
214
215 printf("%s,%s,%s,%d,%d,%d\n",
216 row_css_class,
217 spec_name, href,
218 num_py,
219 num_cpp,
220 num_py - num_cpp)
221}
222' "${tsv_files[@]}"
223}
224
225summary-csv() {
226 local spec_subdir=$1
227
228 local sh_label
229 local manifest
230
231 case $spec_subdir in
232 osh-cpp)
233 sh_label=osh
234 manifest=_tmp/spec/SUITE-osh.txt
235 ;;
236 ysh-cpp)
237 sh_label=ysh
238 manifest=_tmp/spec/SUITE-ysh.txt
239 ;;
240 *)
241 die "Invalid dir $spec_subdir"
242 ;;
243 esac
244
245 # Can't go at the top level because files might not exist!
246 cat <<EOF
247ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta
248EOF
249
250 # total row rows goes at the TOP, so it's in <thead> and not sorted.
251 summary-csv-row $spec_subdir _tmp/spec/$spec_subdir/*.tsv
252
253 head -n $NUM_SPEC_TASKS $manifest | sort |
254 while read spec_name; do
255 summary-csv-row $spec_subdir $spec_name
256 done
257}
258
259html-summary-header() {
260 local prefix=../../..
261 html-head --title 'Passing Spec Tests in C++' \
262 $prefix/web/ajax.js \
263 $prefix/web/table/table-sort.js $prefix/web/table/table-sort.css \
264 $prefix/web/base.css \
265 $prefix/web/spec-cpp.css
266
267 table-sort-begin "width50"
268
269 cat <<EOF
270<p id="home-link">
271 <!-- The release index is two dirs up -->
272 <a href="../..">Up</a> |
273 <a href="/">oilshell.org</a>
274</p>
275
276<h1>Python vs C++</h1>
277
278<p>These numbers measure the progress of the C++ translation.
279Compare with <a href=".">index.html</a>.
280</p>
281
282EOF
283}
284
285html-summary-footer() {
286 cat <<EOF
287<p>Generated by <code>test/spec-cpp.sh</code>.</p>
288EOF
289 table-sort-end 'summary' # The table name
290}
291
292# TODO: Use here-schema-tsv in test/tsv-lib.sh
293here-schema() {
294 ### Read a legible text format on stdin, and write CSV on stdout
295
296 # This is a little like: https://wiki.xxiivv.com/site/tablatal.html
297 # TODO: generalize this in stdlib/here.sh
298 while read one two; do
299 echo "$one,$two"
300 done
301}
302
303write-compare-html() {
304 local spec_subdir=$1
305
306 local sh_label
307 case $spec_subdir in
308 osh-cpp)
309 sh_label=osh
310 ;;
311 ysh-cpp)
312 sh_label=ysh
313 ;;
314 *)
315 die "Invalid dir $spec_subdir"
316 ;;
317 esac
318
319 local dir=_tmp/spec/$spec_subdir
320 local out=$dir/compare.html
321
322 summary-csv $spec_subdir >$dir/summary.csv
323
324 # The underscores are stripped when we don't want them to be!
325 # Note: we could also put "pretty_heading" in the schema
326
327 here-schema >$dir/summary.schema.csv <<EOF
328column_name type
329ROW_CSS_CLASS string
330name string
331name_HREF string
332${sh_label}_py integer
333${sh_label}_cpp integer
334delta integer
335EOF
336
337 { html-summary-header
338 # total row isn't sorted
339 web/table/csv2html.py --thead-offset 1 $dir/summary.csv
340 html-summary-footer
341 } > $out
342
343 log "Comparison: file://$REPO_ROOT/$out"
344}
345
346tsv-demo() {
347 sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
348 cat _tmp/arith.tsv
349}
350
351repro() {
352 test/spec.sh alias -r 0 -p > _tmp/a
353 ninja _bin/clang-dbg/osh
354 _bin/clang-dbg/osh _tmp/a
355}
356
357repro-all() {
358 OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
359}
360
361run-task "$@"