1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Usage:
|
4 | # ./run.sh <function name>
|
5 |
|
6 | set -o nounset
|
7 | set -o pipefail
|
8 | set -o errexit
|
9 |
|
10 | source common.sh
|
11 | source compare.sh
|
12 |
|
13 | readonly PY=$PY36
|
14 |
|
15 | _parse-one() {
|
16 | #PYTHONPATH=. ./opy_main.py 2to3.grammar parse "$@"
|
17 | opyg parse "$@"
|
18 | }
|
19 |
|
20 | parse-test() {
|
21 | _parse-one testdata/hello_py3.py # Python 3 print syntax
|
22 | echo ---
|
23 | _parse-one testdata/hello_py2.py
|
24 | }
|
25 |
|
26 | # It has problems without EOL!
|
27 | parser-bug() {
|
28 | local out=_tmp/opy_parser_bug.py
|
29 | echo -n 'foo = {}' > $out
|
30 | _parse-one $out
|
31 | }
|
32 |
|
33 | _compile-and-run() {
|
34 | local path=$1
|
35 | local basename=$(basename $path .py)
|
36 |
|
37 | mkdir -p _tmp
|
38 | local out=_tmp/${basename}.pyc
|
39 |
|
40 | #_parse-one $path
|
41 |
|
42 | # new opy compile
|
43 | _compile-one $path $out
|
44 | # unmodified pgen2
|
45 | #_compile2-one $path $out
|
46 |
|
47 | ls -l $out
|
48 | xxd $out
|
49 |
|
50 | python $out
|
51 | }
|
52 |
|
53 | _stdlib-compile-and-run() {
|
54 | local path=$1
|
55 | local basename=$(basename $path .py)
|
56 |
|
57 | mkdir -p _tmp
|
58 | local out=_tmp/${basename}.pyc_stdlib
|
59 | misc/stdlib_compile.py $path $out
|
60 |
|
61 | ls -l $out
|
62 | xxd $out
|
63 |
|
64 | python $out
|
65 | }
|
66 |
|
67 | stdlib-compile-test() {
|
68 | _stdlib-compile-and-run testdata/hello_py2.py
|
69 | }
|
70 |
|
71 | # Bad code object because it only has 14 fields. Gah!
|
72 | # We have to marshal the old one I guess.
|
73 | compile-hello2() {
|
74 | local out=_tmp/hello_py2.pyc27
|
75 | _compile-and-run testdata/hello_py2.py $out
|
76 | }
|
77 |
|
78 | # This compiles Python 3 to Python 2 bytecode, and runs it.
|
79 | compile-hello3() {
|
80 | _compile-and-run testdata/hello_py3.py
|
81 | }
|
82 |
|
83 | stdlib-determinism() {
|
84 | mkdir -p _tmp/det
|
85 | local file=./opy_main.py
|
86 |
|
87 | # 1 in 10 times we get a diff! And sometimes same diff! wtf!
|
88 | #
|
89 | # Code is definitely reordered. Basic block. But then there are tiny byte
|
90 | # differences too!
|
91 |
|
92 | #local file=./pytree.py
|
93 | _stdlib-compile-one $file _tmp/det/$file.1
|
94 | _stdlib-compile-one $file _tmp/det/$file.2
|
95 |
|
96 | compare-files _tmp/det/$file.{1,2}
|
97 | }
|
98 |
|
99 | stdlib-determinism-loop() {
|
100 | determinism-loop _stdlib-compile-one
|
101 | }
|
102 |
|
103 | # We want to fix the bug here. Hm not able to hit it?
|
104 | compile2-determinism() {
|
105 | mkdir -p _tmp/det
|
106 | local file=./opy_main.py
|
107 |
|
108 | #local file=./pytree.py
|
109 | _compile2-one $file _tmp/det/$file.1
|
110 | _compile2-one $file _tmp/det/$file.2
|
111 |
|
112 | compare-files _tmp/det/$file.{1,2}
|
113 | }
|
114 |
|
115 | # Compare stdlib and compile2. They differ every time! Is it because somehow
|
116 | # the Python interpreter is in a different state? TODO: Could force iteration
|
117 | # order.
|
118 |
|
119 | stdlib-compile2() {
|
120 | mkdir -p _tmp/det
|
121 | local file=./opy_main.py
|
122 |
|
123 | #local file=./pytree.py
|
124 | _stdlib-compile-one $file _tmp/det/$file.stdlib
|
125 | _compile2-one $file _tmp/det/$file.compile2
|
126 |
|
127 | compare-files _tmp/det/$file.{stdlib,compile2}
|
128 | }
|
129 |
|
130 | export PYTHONHASHSEED=0
|
131 | #export PYTHONHASHSEED=random
|
132 |
|
133 | compare-opy-tree() {
|
134 | diff -u _tmp/opy-{stdlib,stdlib2}/SIZES.txt || true
|
135 | #diff -u _tmp/opy-{stdlib,stdlib2}/MD5.txt || true
|
136 |
|
137 | # Hm even two stdlib runs are different!
|
138 | # TODO: find the smallest ones that are different
|
139 |
|
140 | # Same strings output
|
141 | compare-files _tmp/opy-{stdlib,stdlib2}/pytree.pyc
|
142 | return
|
143 | compare-files _tmp/opy-{stdlib,stdlib2}/opy_main.pyc
|
144 | compare-files _tmp/opy-{stdlib,stdlib2}/compiler2/pyassem.pyc
|
145 | compare-files _tmp/opy-{stdlib,stdlib2}/compiler2/pycodegen.pyc
|
146 | compare-files _tmp/opy-{stdlib,stdlib2}/compiler2/symbols.pyc
|
147 | compare-files _tmp/opy-{stdlib,stdlib2}/compiler2/transformer.pyc
|
148 | return
|
149 |
|
150 | #diff -u _tmp/opy-{stdlib,compile2}/MANIFEST.txt
|
151 |
|
152 | compare-files _tmp/opy-{stdlib,compile2}/util.pyc
|
153 | compare-files _tmp/opy-{stdlib,compile2}/pgen2/driver.pyc
|
154 | compare-files _tmp/opy-{stdlib,compile2}/opy_main.pyc
|
155 | }
|
156 |
|
157 | compare-osh-tree() {
|
158 | #diff -u _tmp/opy-{stdlib,stdlib2}/SIZES.txt || true
|
159 | #compare-files _tmp/osh-{ccompile,compile2}/core/id_kind_test.pyc
|
160 | compare-files _tmp/osh-{ccompile,compile2}/core/testdbg.pyc
|
161 | }
|
162 |
|
163 | unit-osh() {
|
164 | local dir=${1:-_tmp/osh-stdlib}
|
165 | local vm=${2:-byterun} # or cpython
|
166 | shift 2
|
167 | pushd $dir
|
168 | if test $vm = byterun; then
|
169 | PYTHONPATH=. byterun -c "$@"
|
170 | else
|
171 | PYTHONPATH=. python "$@"
|
172 | fi
|
173 | popd
|
174 | }
|
175 |
|
176 | # Combinations of {ccompile, compiler2} x {cpython, byterun}
|
177 | compile-run-one() {
|
178 | local compiler=${1:-ccompile} # or compile2
|
179 | local vm=${2:-byterun} # or cpython
|
180 | local py=$3
|
181 | shift 3
|
182 |
|
183 | if ! { test $compiler = ccompile || test $compiler = compile2; } then
|
184 | die "Invalid compiler $compiler"
|
185 | fi
|
186 |
|
187 | local dir="_tmp/osh-$compiler"
|
188 | local pyc="$dir/$(basename $py)c"
|
189 | _$compiler-one $py $pyc
|
190 |
|
191 | export PYTHONPATH=$dir
|
192 | if test $vm = cpython; then
|
193 | python $pyc "$@"
|
194 | elif test $vm = byterun; then
|
195 | #byterun -v -c $pyc "$@"
|
196 | byterun -c $pyc "$@"
|
197 | else
|
198 | die $vm
|
199 | fi
|
200 | }
|
201 |
|
202 | compare-sizes() {
|
203 | local left=$1
|
204 | local right=$2
|
205 | find $left -name '*.pyc' -a -printf '%s %P\n' | sort -n
|
206 | echo ---
|
207 | # Wow, opyc files are bigger! Code is not as optimal or what?
|
208 | # Order is roughly the same.
|
209 | find $right -name '*.opyc' -a -printf '%s %P\n' | sort -n
|
210 | }
|
211 |
|
212 | compare-opy-sizes() {
|
213 | compare-sizes .. _tmp/opy
|
214 | }
|
215 |
|
216 | compare-osh-sizes() {
|
217 | # TODO: filter opy out of the left
|
218 | compare-sizes .. _tmp/osh
|
219 | }
|
220 |
|
221 | # Doesn't work because of compileFile.
|
222 | old-compile-test() {
|
223 | PYTHONPATH=. tools/compile.py testdata/hello_py3.py
|
224 | }
|
225 |
|
226 | #
|
227 | # Parsing tests subsummed by compiling
|
228 | #
|
229 |
|
230 | # 2to3.grammar is from Python-3.6.1/ Lib/lib2to3/Grammar.txt
|
231 | parse-with-pgen2() {
|
232 | set +o errexit
|
233 | for py in "$@"; do
|
234 | _parse-one $py >/dev/null #2>&1
|
235 | echo $? $py
|
236 | done
|
237 | }
|
238 |
|
239 | # This fails due to some files not using __future__ print_function.
|
240 | parse-oil() {
|
241 | parse-with-pgen2 *.py ../*.py ../osh/*.py ../core/*.py ../asdl/*.py
|
242 | }
|
243 |
|
244 | # Parse the old Python2 code
|
245 | parse-pycompiler2() {
|
246 | # parse print statement
|
247 | parse-with-pgen2 ~/src/Python-2.7.6/Lib/compiler/*.py
|
248 | }
|
249 |
|
250 | # After lib2to3
|
251 | parse-pycompiler() {
|
252 | # parse print statement
|
253 | parse-with-pgen2 compiler/*.py tools/*.py
|
254 | }
|
255 |
|
256 | #
|
257 | # File Management
|
258 | #
|
259 |
|
260 | clear-tokens() {
|
261 | rm token.py tokenize.py
|
262 | rm -rf --verbose __pycache ../__pycache__
|
263 | }
|
264 |
|
265 | copy-lib2to3() {
|
266 | #cp -v $PY/Lib/{token,tokenize}.py .
|
267 | #return
|
268 |
|
269 | # For comparison
|
270 | mkdir -p pgen2
|
271 |
|
272 | cp -v $PY/Lib/lib2to3/{pytree,pygram}.py .
|
273 | cp -v $PY/Lib/lib2to3/pgen2/{__init__,driver,grammar,parse,token,tokenize,pgen}.py pgen2
|
274 | # The 2to3 grammar supports both Python 2 and Python 3.
|
275 | # - it has the old print statement. Well I guess you still want that! Gah.
|
276 | cp -v $PY/Lib/lib2to3/Grammar.txt .
|
277 | return
|
278 |
|
279 | cp -v $PY/Parser/Python.asdl .
|
280 |
|
281 | }
|
282 |
|
283 | # For compatibility with the compiler package.
|
284 | # I kind of want the static type syntax though?
|
285 | copy-old-grammar() {
|
286 | cp -v $PY27/Grammar/Grammar py27.grammar
|
287 | }
|
288 |
|
289 | copy-pycompiler() {
|
290 | # The last version of the pure Python compile package.
|
291 | mkdir -p compiler2
|
292 | cp -v ~/src/Python-2.7.6/Lib/compiler/*.py compiler2
|
293 | }
|
294 |
|
295 | copy-pycompiler-tools() {
|
296 | cp -v ~/src/Python-2.7.6/Tools/compiler/{ast.txt,ACKS,README,*.py} tools/
|
297 | }
|
298 |
|
299 | # Features from Python 3 used? Static types? I guess Python 3.6 has locals with
|
300 | # foo: str = 1
|
301 | #
|
302 | # Do I want that?
|
303 | #
|
304 | # Main things I ran into were:
|
305 | # - print statement
|
306 | # - next() is now __next__
|
307 | # - io.StringIO vs. cStringIO.cstringIO()
|
308 | #
|
309 | # And occasional exceptions about encoding. Had to add .encode('utf-8') in a
|
310 | # few places.
|
311 | #
|
312 | # So mostly cosmetic issues.
|
313 |
|
314 | "$@"
|