1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Usage:
|
4 | # test/wild-runner.sh <function name>
|
5 |
|
6 | set -o nounset
|
7 | set -o pipefail
|
8 | set -o errexit
|
9 | shopt -s strict:all 2>/dev/null || true # dogfood for OSH
|
10 |
|
11 | source test/common.sh # $OSH, log
|
12 |
|
13 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
14 |
|
15 | dump-html-and-translate-file() {
|
16 | local rel_path=$1
|
17 | local abs_path=$2
|
18 |
|
19 | local raw_base=_tmp/wild/raw/$rel_path
|
20 | local www_base=_tmp/wild-www/$rel_path
|
21 | mkdir -p $(dirname $raw_base) $(dirname $www_base)
|
22 |
|
23 | log "--- Processing $rel_path"
|
24 |
|
25 | # Count the number of lines. This creates a tiny file, but we're doing
|
26 | # everything involving $abs_path at once so it's in the FS cache.
|
27 | #
|
28 | # TODO: Could replace with a single invocation of micro-syntax, then join it
|
29 | wc $abs_path > ${raw_base}__wc.txt
|
30 |
|
31 | # Make a literal copy with .txt extension, so we can browse it
|
32 | cp $abs_path ${www_base}.txt
|
33 |
|
34 | # Parse the file.
|
35 | local task_file=${raw_base}__parse.task.txt
|
36 | local stderr_file=${raw_base}__parse.stderr.txt
|
37 |
|
38 | # Note: abbrev-html is SLOW, much slower than 'none'
|
39 | # e.g. 175 ms vs. 7 ms on 'configure'
|
40 | run-task-with-status $task_file \
|
41 | $OSH --ast-format none -n $abs_path \
|
42 | 2> $stderr_file
|
43 |
|
44 | # Convert the file.
|
45 | task_file=${raw_base}__ysh-ify.task.txt
|
46 | stderr_file=${raw_base}__ysh-ify.stderr.txt
|
47 | out_file=${www_base}__ysh.txt
|
48 |
|
49 | # ysh-ify is fast
|
50 | run-task-with-status $task_file \
|
51 | $OSH --tool ysh-ify $abs_path \
|
52 | > $out_file 2> $stderr_file
|
53 | }
|
54 |
|
55 | # In case we built with ASAN
|
56 | export ASAN_OPTIONS='detect_leaks=0'
|
57 |
|
58 | dump-text-for-file() {
|
59 | local rel_path=$1
|
60 | local abs_path=$2
|
61 |
|
62 | local py_base=_tmp/wild/py/$rel_path
|
63 | local cpp_base=_tmp/wild/cpp/$rel_path
|
64 |
|
65 | mkdir -p $(dirname $py_base) $(dirname $cpp_base)
|
66 |
|
67 | log "--- Processing $rel_path"
|
68 |
|
69 | # Parse the file with Python
|
70 | local task_file=${py_base}.task.txt
|
71 | local stderr_file=${py_base}.stderr.txt
|
72 | local out_file=${py_base}.ast.txt
|
73 |
|
74 | run-task-with-status $task_file \
|
75 | $OSH --ast-format text -n $abs_path \
|
76 | > $out_file #2> $stderr_file
|
77 |
|
78 | # Parse the file with C++
|
79 | local task_file=${cpp_base}.task.txt
|
80 | local stderr_file=${cpp_base}.stderr.txt
|
81 | local out_file=${cpp_base}.ast.txt
|
82 |
|
83 | run-task-with-status $task_file \
|
84 | $OSH -n $abs_path \
|
85 | > $out_file #2> $stderr_file
|
86 | }
|
87 |
|
88 |
|
89 | readonly NUM_TASKS=200
|
90 | readonly MANIFEST=_tmp/wild/MANIFEST.txt
|
91 |
|
92 | parse-in-parallel() {
|
93 | local func=${1:-dump-html-and-translate-file}
|
94 |
|
95 | log ''
|
96 | log "$0: Making wild report with $MAX_PROCS parallel processes"
|
97 | log ''
|
98 |
|
99 | local failed=''
|
100 | xargs -n 2 -P $MAX_PROCS -- $0 $func || failed=1
|
101 |
|
102 | # Limit the output depth
|
103 | if command -v tree > /dev/null; then
|
104 | tree -L 3 _tmp/wild
|
105 | fi
|
106 | }
|
107 |
|
108 | filter-manifest() {
|
109 | local manifest_regex=${1:-} # egrep regex for manifest line
|
110 | if test -n "$manifest_regex"; then
|
111 | egrep -- "$manifest_regex" $MANIFEST
|
112 | else
|
113 | cat $MANIFEST
|
114 | fi
|
115 | }
|
116 |
|
117 | dump-text-asts() {
|
118 | local manifest_regex=${1:-} # egrep regex for manifest line
|
119 |
|
120 | local func=dump-text-for-file
|
121 |
|
122 | if test -n "$manifest_regex"; then
|
123 | egrep -- "$manifest_regex" $MANIFEST | parse-in-parallel $func
|
124 | else
|
125 | cat $MANIFEST | parse-in-parallel $func
|
126 | fi
|
127 | }
|
128 |
|
129 | compare-one-ast() {
|
130 | local left=$1
|
131 |
|
132 | local old='/py/'
|
133 | local new='/cpp/'
|
134 | local right=${left/$old/$new}
|
135 |
|
136 | #echo $left $right
|
137 | diff -q -u $left $right
|
138 | #md5sum $left $right
|
139 | }
|
140 |
|
141 | compare-asts() {
|
142 | local manifest=_tmp/wild/compare.txt
|
143 | find _tmp/wild/py -name '*.ast.txt' > $manifest
|
144 |
|
145 | log "Comparing ..."
|
146 | wc -l $manifest
|
147 | echo
|
148 |
|
149 | cat $manifest | xargs -n 1 -- $0 compare-one-ast
|
150 | }
|
151 |
|
152 | wild-report() {
|
153 | PYTHONPATH=$REPO_ROOT $REPO_ROOT/test/wild_report.py "$@"
|
154 | }
|
155 |
|
156 | _link() {
|
157 | ln -s -f -v "$@"
|
158 | }
|
159 |
|
160 | version-text() {
|
161 | date-and-git-info
|
162 | echo "\$ $OSH --version"
|
163 | $OSH --version
|
164 | }
|
165 |
|
166 | make-report() {
|
167 | local manifest_regex=${1:-}
|
168 | local in_dir=_tmp/wild/raw
|
169 | local out_dir=_tmp/wild-www
|
170 |
|
171 | # TODO: This could also go in 'raw', and then be processed by Python?
|
172 | version-text > $out_dir/version-info.txt
|
173 |
|
174 | filter-manifest "$manifest_regex" | wild-report summarize-dirs \
|
175 | --not-shell test/wild-not-shell.txt \
|
176 | --not-osh test/wild-not-osh.txt \
|
177 | $in_dir $out_dir
|
178 |
|
179 | # This has to go inside the www dir because of the way that relative links
|
180 | # are calculated.
|
181 | # TODO: Isn't this redundant?
|
182 | _link $PWD/web/osh-to-oil.{html,js} $out_dir
|
183 | _link $PWD/web _tmp
|
184 | }
|
185 |
|
186 | # Takes 3m 47s on 7 cores for 513K lines.
|
187 | # So that's like 230 seconds or so. It should really take 1 second!
|
188 |
|
189 | parse-and-report() {
|
190 | local manifest_regex=${1:-} # egrep regex for manifest line
|
191 | local func=${2:-dump-html-and-translate-file}
|
192 |
|
193 | time {
|
194 | #test/wild.sh write-manifest
|
195 | test/wild.sh manifest-from-archive
|
196 |
|
197 | filter-manifest "$manifest_regex" | parse-in-parallel $func
|
198 | make-report "$manifest_regex"
|
199 | }
|
200 | }
|
201 |
|
202 | if test "$(basename $0)" = 'wild-runner.sh'; then
|
203 | "$@"
|
204 | fi
|