1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Test the C++ translation of Oils.
|
4 | #
|
5 | # Usage:
|
6 | # test/spec-cpp.sh <function name>
|
7 | #
|
8 | # Examples:
|
9 | # test/spec-cpp.sh run-file smoke -r 0 -v
|
10 | # NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
|
11 |
|
12 | : ${LIB_OSH=stdlib/osh}
|
13 | source $LIB_OSH/bash-strict.sh
|
14 | source $LIB_OSH/task-five.sh
|
15 |
|
16 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
17 |
|
18 | source build/dev-shell.sh # PYTHONPATH
|
19 | source test/common.sh # html-head
|
20 | source test/spec-common.sh
|
21 | source web/table/html.sh
|
22 |
|
23 | shopt -s failglob # to debug TSV expansion failure below
|
24 |
|
25 | OSH_PY=$REPO_ROOT/bin/osh
|
26 | YSH_PY=$REPO_ROOT/bin/ysh
|
27 |
|
28 | # Run with ASAN binary by default. Release overrides this
|
29 | OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
|
30 | YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
|
31 |
|
32 | # Same variable in test/spec-runner.sh
|
33 | NUM_SPEC_TASKS=${NUM_SPEC_TASKS:-400}
|
34 |
|
35 | # So we can pass ASAN. Note that test/spec-common.sh has to pass this to
|
36 | # sh_spec.py.
|
37 | export OILS_GC_ON_EXIT=1
|
38 |
|
39 | #
|
40 | # For translation
|
41 | #
|
42 |
|
43 | run-file() {
|
44 | local spec_name=$1
|
45 | shift
|
46 |
|
47 | local spec_file=spec/$spec_name.test.sh
|
48 |
|
49 | local suite
|
50 | suite=$(test/sh_spec.py --print-spec-suite $spec_file)
|
51 |
|
52 | local spec_subdir
|
53 | case $suite in
|
54 | osh) spec_subdir='osh-cpp' ;;
|
55 | ysh) spec_subdir='ysh-cpp' ;;
|
56 | *) die "Invalid suite $suite" ;;
|
57 | esac
|
58 |
|
59 | local base_dir=_tmp/spec/$spec_subdir
|
60 | mkdir -v -p $base_dir
|
61 |
|
62 | # Compare Python and C++ shells by passing --oils-cpp-bin-dir
|
63 | sh-spec $spec_file \
|
64 | --timeout 10 \
|
65 | --oils-bin-dir $PWD/bin \
|
66 | --oils-cpp-bin-dir $REPO_ROOT/_bin/cxx-asan \
|
67 | --tsv-output $base_dir/${spec_name}.tsv \
|
68 | "$@"
|
69 | }
|
70 |
|
71 | osh-all() {
|
72 | # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
|
73 |
|
74 | # For debugging hangs
|
75 | #export MAX_PROCS=1
|
76 |
|
77 | ninja _bin/cxx-asan/{osh,ysh}
|
78 |
|
79 | test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC
|
80 |
|
81 | local spec_subdir=osh-cpp
|
82 |
|
83 | # $suite $compare_mode
|
84 | test/spec-runner.sh all-parallel \
|
85 | osh compare-cpp $spec_subdir "$@" || true # OK if it fails
|
86 |
|
87 | write-compare-html $spec_subdir
|
88 | }
|
89 |
|
90 | ysh-all() {
|
91 | ninja _bin/cxx-asan/{osh,ysh}
|
92 |
|
93 | local spec_subdir=ysh-cpp
|
94 |
|
95 | # $suite $compare_mode
|
96 | test/spec-runner.sh all-parallel \
|
97 | ysh compare-cpp $spec_subdir "$@" || true # OK if it fails
|
98 |
|
99 | write-compare-html $spec_subdir
|
100 | }
|
101 |
|
102 | console-row() {
|
103 | ### Print out a histogram of results
|
104 |
|
105 | awk '
|
106 | FNR == 1 {
|
107 | #print FILENAME > "/dev/stderr"
|
108 | }
|
109 | FNR != 1 {
|
110 | case_num = $1
|
111 | sh = $2
|
112 | result = $3
|
113 |
|
114 | if (sh == "osh") {
|
115 | osh[result] += 1
|
116 | } else if (sh == "osh_cpp") { # bin/osh_cpp
|
117 | oe_py[result] += 1
|
118 | } else if (sh == "osh_ALT") { # _bin/*/osh
|
119 | oe_cpp[result] += 1
|
120 | }
|
121 | }
|
122 |
|
123 | function print_hist(sh, hist) {
|
124 | printf("%s\t", sh)
|
125 |
|
126 | k = "pass"
|
127 | printf("%s %4d\t", k, hist[k])
|
128 | k = "FAIL"
|
129 | printf("%s %4d\t", k, hist[k])
|
130 |
|
131 | print ""
|
132 |
|
133 | # This prints N-I, ok, bug, etc.
|
134 | #for (k in hist) {
|
135 | # printf("%s %s\t", k, hist[k])
|
136 | #}
|
137 |
|
138 | }
|
139 |
|
140 | END {
|
141 | print_hist("osh", osh)
|
142 | print_hist("osh_cpp", oe_py)
|
143 | print_hist("osh_ALT", oe_cpp)
|
144 | }
|
145 | ' "$@"
|
146 | }
|
147 |
|
148 | console-summary() {
|
149 | ### Report on our progress translating
|
150 |
|
151 | local spec_subdir=$1
|
152 |
|
153 | # Can't go at the top level because files won't exist!
|
154 | readonly TSV=(_tmp/spec/$spec_subdir/*.tsv)
|
155 |
|
156 | wc -l "${TSV[@]}"
|
157 |
|
158 | for file in "${TSV[@]}"; do
|
159 | echo
|
160 | echo "$file"
|
161 | console-row $file
|
162 | done
|
163 |
|
164 | echo
|
165 | echo "TOTAL"
|
166 | console-row "${TSV[@]}"
|
167 | }
|
168 |
|
169 | #
|
170 | # HTML
|
171 | #
|
172 |
|
173 | summary-csv-row() {
|
174 | ### Print one row or the last total row
|
175 |
|
176 | local spec_subdir=$1
|
177 | shift
|
178 |
|
179 | if test $# -eq 1; then
|
180 | local spec_name=$1
|
181 | local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.tsv )
|
182 | else
|
183 | local spec_name='TOTAL'
|
184 | local -a tsv_files=( "$@" )
|
185 | fi
|
186 |
|
187 | awk -v spec_name=$spec_name '
|
188 | # skip the first row
|
189 | FNR != 1 {
|
190 | case_num = $1
|
191 | sh = $2
|
192 | result = $3
|
193 |
|
194 | if (sh == "osh" || sh == "ysh") {
|
195 | osh[result] += 1
|
196 | } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
|
197 | osh_native[result] += 1
|
198 | }
|
199 | }
|
200 |
|
201 | END {
|
202 | num_py = osh["pass"]
|
203 | num_cpp = osh_native["pass"]
|
204 | if (spec_name == "TOTAL") {
|
205 | href = ""
|
206 | } else {
|
207 | href = sprintf("%s.html", spec_name)
|
208 | }
|
209 |
|
210 | if (num_py == num_cpp) {
|
211 | row_css_class = "cpp-good" # green
|
212 | }
|
213 |
|
214 | printf("%s,%s,%s,%d,%d,%d\n",
|
215 | row_css_class,
|
216 | spec_name, href,
|
217 | num_py,
|
218 | num_cpp,
|
219 | num_py - num_cpp)
|
220 | }
|
221 | ' "${tsv_files[@]}"
|
222 | }
|
223 |
|
224 | summary-csv() {
|
225 | local spec_subdir=$1
|
226 |
|
227 | local sh_label
|
228 | local manifest
|
229 |
|
230 | case $spec_subdir in
|
231 | osh-cpp)
|
232 | sh_label=osh
|
233 | manifest=_tmp/spec/SUITE-osh.txt
|
234 | ;;
|
235 | ysh-cpp)
|
236 | sh_label=ysh
|
237 | manifest=_tmp/spec/SUITE-ysh.txt
|
238 | ;;
|
239 | *)
|
240 | die "Invalid dir $spec_subdir"
|
241 | ;;
|
242 | esac
|
243 |
|
244 | # Can't go at the top level because files might not exist!
|
245 | cat <<EOF
|
246 | ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta
|
247 | EOF
|
248 |
|
249 | # total row rows goes at the TOP, so it's in <thead> and not sorted.
|
250 | summary-csv-row $spec_subdir _tmp/spec/$spec_subdir/*.tsv
|
251 |
|
252 | head -n $NUM_SPEC_TASKS $manifest | sort |
|
253 | while read spec_name; do
|
254 | summary-csv-row $spec_subdir $spec_name
|
255 | done
|
256 | }
|
257 |
|
258 | html-summary-header() {
|
259 | local prefix=../../..
|
260 | html-head --title 'Passing Spec Tests in C++' \
|
261 | $prefix/web/ajax.js \
|
262 | $prefix/web/table/table-sort.js $prefix/web/table/table-sort.css \
|
263 | $prefix/web/base.css \
|
264 | $prefix/web/spec-cpp.css
|
265 |
|
266 | table-sort-begin "width50"
|
267 |
|
268 | cat <<EOF
|
269 | <p id="home-link">
|
270 | <!-- The release index is two dirs up -->
|
271 | <a href="../..">Up</a> |
|
272 | <a href="/">oilshell.org</a>
|
273 | </p>
|
274 |
|
275 | <h1>Python vs C++</h1>
|
276 |
|
277 | <p>These numbers measure the progress of the C++ translation.
|
278 | Compare with <a href=".">index.html</a>.
|
279 | </p>
|
280 |
|
281 | EOF
|
282 | }
|
283 |
|
284 | html-summary-footer() {
|
285 | cat <<EOF
|
286 | <p>Generated by <code>test/spec-cpp.sh</code>.</p>
|
287 | EOF
|
288 | table-sort-end 'summary' # The table name
|
289 | }
|
290 |
|
291 | # TODO: Use here-schema-tsv in test/tsv-lib.sh
|
292 | here-schema() {
|
293 | ### Read a legible text format on stdin, and write CSV on stdout
|
294 |
|
295 | # This is a little like: https://wiki.xxiivv.com/site/tablatal.html
|
296 | # TODO: generalize this in stdlib/here.sh
|
297 | while read one two; do
|
298 | echo "$one,$two"
|
299 | done
|
300 | }
|
301 |
|
302 | write-compare-html() {
|
303 | local spec_subdir=$1
|
304 |
|
305 | local sh_label
|
306 | case $spec_subdir in
|
307 | osh-cpp)
|
308 | sh_label=osh
|
309 | ;;
|
310 | ysh-cpp)
|
311 | sh_label=ysh
|
312 | ;;
|
313 | *)
|
314 | die "Invalid dir $spec_subdir"
|
315 | ;;
|
316 | esac
|
317 |
|
318 | local dir=_tmp/spec/$spec_subdir
|
319 | local out=$dir/compare.html
|
320 |
|
321 | summary-csv $spec_subdir >$dir/summary.csv
|
322 |
|
323 | # The underscores are stripped when we don't want them to be!
|
324 | # Note: we could also put "pretty_heading" in the schema
|
325 |
|
326 | here-schema >$dir/summary.schema.csv <<EOF
|
327 | column_name type
|
328 | ROW_CSS_CLASS string
|
329 | name string
|
330 | name_HREF string
|
331 | ${sh_label}_py integer
|
332 | ${sh_label}_cpp integer
|
333 | delta integer
|
334 | EOF
|
335 |
|
336 | { html-summary-header
|
337 | # total row isn't sorted
|
338 | web/table/csv2html.py --thead-offset 1 $dir/summary.csv
|
339 | html-summary-footer
|
340 | } > $out
|
341 |
|
342 | log "Comparison: file://$REPO_ROOT/$out"
|
343 | }
|
344 |
|
345 | tsv-demo() {
|
346 | sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
|
347 | cat _tmp/arith.tsv
|
348 | }
|
349 |
|
350 | repro() {
|
351 | test/spec.sh alias -r 0 -p > _tmp/a
|
352 | ninja _bin/clang-dbg/osh
|
353 | _bin/clang-dbg/osh _tmp/a
|
354 | }
|
355 |
|
356 | repro-all() {
|
357 | OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
|
358 | }
|
359 |
|
360 | task-five "$@"
|