OILS / test / wild-runner.sh View on Github | oilshell.org

204 lines, 127 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# test/wild-runner.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9shopt -s strict:all 2>/dev/null || true # dogfood for OSH
10
11source test/common.sh # $OSH, log
12
13REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
14
15dump-html-and-translate-file() {
16 local rel_path=$1
17 local abs_path=$2
18
19 local raw_base=_tmp/wild/raw/$rel_path
20 local www_base=_tmp/wild-www/$rel_path
21 mkdir -p $(dirname $raw_base) $(dirname $www_base)
22
23 log "--- Processing $rel_path"
24
25 # Count the number of lines. This creates a tiny file, but we're doing
26 # everything involving $abs_path at once so it's in the FS cache.
27 #
28 # TODO: Could replace with a single invocation of micro-syntax, then join it
29 wc $abs_path > ${raw_base}__wc.txt
30
31 # Make a literal copy with .txt extension, so we can browse it
32 cp $abs_path ${www_base}.txt
33
34 # Parse the file.
35 local task_file=${raw_base}__parse.task.txt
36 local stderr_file=${raw_base}__parse.stderr.txt
37
38 # Note: abbrev-html is SLOW, much slower than 'none'
39 # e.g. 175 ms vs. 7 ms on 'configure'
40 run-task-with-status $task_file \
41 $OSH --ast-format none -n $abs_path \
42 2> $stderr_file
43
44 # Convert the file.
45 task_file=${raw_base}__ysh-ify.task.txt
46 stderr_file=${raw_base}__ysh-ify.stderr.txt
47 out_file=${www_base}__ysh.txt
48
49 # ysh-ify is fast
50 run-task-with-status $task_file \
51 $OSH --tool ysh-ify $abs_path \
52 > $out_file 2> $stderr_file
53}
54
55# In case we built with ASAN
56export ASAN_OPTIONS='detect_leaks=0'
57
58dump-text-for-file() {
59 local rel_path=$1
60 local abs_path=$2
61
62 local py_base=_tmp/wild/py/$rel_path
63 local cpp_base=_tmp/wild/cpp/$rel_path
64
65 mkdir -p $(dirname $py_base) $(dirname $cpp_base)
66
67 log "--- Processing $rel_path"
68
69 # Parse the file with Python
70 local task_file=${py_base}.task.txt
71 local stderr_file=${py_base}.stderr.txt
72 local out_file=${py_base}.ast.txt
73
74 run-task-with-status $task_file \
75 $OSH --ast-format text -n $abs_path \
76 > $out_file #2> $stderr_file
77
78 # Parse the file with C++
79 local task_file=${cpp_base}.task.txt
80 local stderr_file=${cpp_base}.stderr.txt
81 local out_file=${cpp_base}.ast.txt
82
83 run-task-with-status $task_file \
84 $OSH -n $abs_path \
85 > $out_file #2> $stderr_file
86}
87
88
89readonly NUM_TASKS=200
90readonly MANIFEST=_tmp/wild/MANIFEST.txt
91
92parse-in-parallel() {
93 local func=${1:-dump-html-and-translate-file}
94
95 log ''
96 log "$0: Making wild report with $MAX_PROCS parallel processes"
97 log ''
98
99 local failed=''
100 xargs -n 2 -P $MAX_PROCS -- $0 $func || failed=1
101
102 # Limit the output depth
103 if command -v tree > /dev/null; then
104 tree -L 3 _tmp/wild
105 fi
106}
107
108filter-manifest() {
109 local manifest_regex=${1:-} # egrep regex for manifest line
110 if test -n "$manifest_regex"; then
111 egrep -- "$manifest_regex" $MANIFEST
112 else
113 cat $MANIFEST
114 fi
115}
116
117dump-text-asts() {
118 local manifest_regex=${1:-} # egrep regex for manifest line
119
120 local func=dump-text-for-file
121
122 if test -n "$manifest_regex"; then
123 egrep -- "$manifest_regex" $MANIFEST | parse-in-parallel $func
124 else
125 cat $MANIFEST | parse-in-parallel $func
126 fi
127}
128
129compare-one-ast() {
130 local left=$1
131
132 local old='/py/'
133 local new='/cpp/'
134 local right=${left/$old/$new}
135
136 #echo $left $right
137 diff -q -u $left $right
138 #md5sum $left $right
139}
140
141compare-asts() {
142 local manifest=_tmp/wild/compare.txt
143 find _tmp/wild/py -name '*.ast.txt' > $manifest
144
145 log "Comparing ..."
146 wc -l $manifest
147 echo
148
149 cat $manifest | xargs -n 1 -- $0 compare-one-ast
150}
151
152wild-report() {
153 PYTHONPATH=$REPO_ROOT $REPO_ROOT/test/wild_report.py "$@"
154}
155
156_link() {
157 ln -s -f -v "$@"
158}
159
160version-text() {
161 date-and-git-info
162 echo "\$ $OSH --version"
163 $OSH --version
164}
165
166make-report() {
167 local manifest_regex=${1:-}
168 local in_dir=_tmp/wild/raw
169 local out_dir=_tmp/wild-www
170
171 # TODO: This could also go in 'raw', and then be processed by Python?
172 version-text > $out_dir/version-info.txt
173
174 filter-manifest "$manifest_regex" | wild-report summarize-dirs \
175 --not-shell test/wild-not-shell.txt \
176 --not-osh test/wild-not-osh.txt \
177 $in_dir $out_dir
178
179 # This has to go inside the www dir because of the way that relative links
180 # are calculated.
181 # TODO: Isn't this redundant?
182 _link $PWD/web/osh-to-oil.{html,js} $out_dir
183 _link $PWD/web _tmp
184}
185
186# Takes 3m 47s on 7 cores for 513K lines.
187# So that's like 230 seconds or so. It should really take 1 second!
188
189parse-and-report() {
190 local manifest_regex=${1:-} # egrep regex for manifest line
191 local func=${2:-dump-html-and-translate-file}
192
193 time {
194 #test/wild.sh write-manifest
195 test/wild.sh manifest-from-archive
196
197 filter-manifest "$manifest_regex" | parse-in-parallel $func
198 make-report "$manifest_regex"
199 }
200}
201
202if test "$(basename $0)" = 'wild-runner.sh'; then
203 "$@"
204fi