OILS / opy / old / run.sh View on Github | oilshell.org

314 lines, 177 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# ./run.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10source common.sh
11source compare.sh
12
13readonly PY=$PY36
14
15_parse-one() {
16 #PYTHONPATH=. ./opy_main.py 2to3.grammar parse "$@"
17 opyg parse "$@"
18}
19
20parse-test() {
21 _parse-one testdata/hello_py3.py # Python 3 print syntax
22 echo ---
23 _parse-one testdata/hello_py2.py
24}
25
26# It has problems without EOL!
27parser-bug() {
28 local out=_tmp/opy_parser_bug.py
29 echo -n 'foo = {}' > $out
30 _parse-one $out
31}
32
33_compile-and-run() {
34 local path=$1
35 local basename=$(basename $path .py)
36
37 mkdir -p _tmp
38 local out=_tmp/${basename}.pyc
39
40 #_parse-one $path
41
42 # new opy compile
43 _compile-one $path $out
44 # unmodified pgen2
45 #_compile2-one $path $out
46
47 ls -l $out
48 xxd $out
49
50 python $out
51}
52
53_stdlib-compile-and-run() {
54 local path=$1
55 local basename=$(basename $path .py)
56
57 mkdir -p _tmp
58 local out=_tmp/${basename}.pyc_stdlib
59 misc/stdlib_compile.py $path $out
60
61 ls -l $out
62 xxd $out
63
64 python $out
65}
66
67stdlib-compile-test() {
68 _stdlib-compile-and-run testdata/hello_py2.py
69}
70
71# Bad code object because it only has 14 fields. Gah!
72# We have to marshal the old one I guess.
73compile-hello2() {
74 local out=_tmp/hello_py2.pyc27
75 _compile-and-run testdata/hello_py2.py $out
76}
77
78# This compiles Python 3 to Python 2 bytecode, and runs it.
79compile-hello3() {
80 _compile-and-run testdata/hello_py3.py
81}
82
83stdlib-determinism() {
84 mkdir -p _tmp/det
85 local file=./opy_main.py
86
87 # 1 in 10 times we get a diff! And sometimes same diff! wtf!
88 #
89 # Code is definitely reordered. Basic block. But then there are tiny byte
90 # differences too!
91
92 #local file=./pytree.py
93 _stdlib-compile-one $file _tmp/det/$file.1
94 _stdlib-compile-one $file _tmp/det/$file.2
95
96 compare-files _tmp/det/$file.{1,2}
97}
98
99stdlib-determinism-loop() {
100 determinism-loop _stdlib-compile-one
101}
102
103# We want to fix the bug here. Hm not able to hit it?
104compile2-determinism() {
105 mkdir -p _tmp/det
106 local file=./opy_main.py
107
108 #local file=./pytree.py
109 _compile2-one $file _tmp/det/$file.1
110 _compile2-one $file _tmp/det/$file.2
111
112 compare-files _tmp/det/$file.{1,2}
113}
114
115# Compare stdlib and compile2. They differ every time! Is it because somehow
116# the Python interpreter is in a different state? TODO: Could force iteration
117# order.
118
119stdlib-compile2() {
120 mkdir -p _tmp/det
121 local file=./opy_main.py
122
123 #local file=./pytree.py
124 _stdlib-compile-one $file _tmp/det/$file.stdlib
125 _compile2-one $file _tmp/det/$file.compile2
126
127 compare-files _tmp/det/$file.{stdlib,compile2}
128}
129
130export PYTHONHASHSEED=0
131#export PYTHONHASHSEED=random
132
133compare-opy-tree() {
134 diff -u _tmp/opy-{stdlib,stdlib2}/SIZES.txt || true
135 #diff -u _tmp/opy-{stdlib,stdlib2}/MD5.txt || true
136
137 # Hm even two stdlib runs are different!
138 # TODO: find the smallest ones that are different
139
140 # Same strings output
141 compare-files _tmp/opy-{stdlib,stdlib2}/pytree.pyc
142 return
143 compare-files _tmp/opy-{stdlib,stdlib2}/opy_main.pyc
144 compare-files _tmp/opy-{stdlib,stdlib2}/compiler2/pyassem.pyc
145 compare-files _tmp/opy-{stdlib,stdlib2}/compiler2/pycodegen.pyc
146 compare-files _tmp/opy-{stdlib,stdlib2}/compiler2/symbols.pyc
147 compare-files _tmp/opy-{stdlib,stdlib2}/compiler2/transformer.pyc
148 return
149
150 #diff -u _tmp/opy-{stdlib,compile2}/MANIFEST.txt
151
152 compare-files _tmp/opy-{stdlib,compile2}/util.pyc
153 compare-files _tmp/opy-{stdlib,compile2}/pgen2/driver.pyc
154 compare-files _tmp/opy-{stdlib,compile2}/opy_main.pyc
155}
156
157compare-osh-tree() {
158 #diff -u _tmp/opy-{stdlib,stdlib2}/SIZES.txt || true
159 #compare-files _tmp/osh-{ccompile,compile2}/core/id_kind_test.pyc
160 compare-files _tmp/osh-{ccompile,compile2}/core/testdbg.pyc
161}
162
163unit-osh() {
164 local dir=${1:-_tmp/osh-stdlib}
165 local vm=${2:-byterun} # or cpython
166 shift 2
167 pushd $dir
168 if test $vm = byterun; then
169 PYTHONPATH=. byterun -c "$@"
170 else
171 PYTHONPATH=. python "$@"
172 fi
173 popd
174}
175
176# Combinations of {ccompile, compiler2} x {cpython, byterun}
177compile-run-one() {
178 local compiler=${1:-ccompile} # or compile2
179 local vm=${2:-byterun} # or cpython
180 local py=$3
181 shift 3
182
183 if ! { test $compiler = ccompile || test $compiler = compile2; } then
184 die "Invalid compiler $compiler"
185 fi
186
187 local dir="_tmp/osh-$compiler"
188 local pyc="$dir/$(basename $py)c"
189 _$compiler-one $py $pyc
190
191 export PYTHONPATH=$dir
192 if test $vm = cpython; then
193 python $pyc "$@"
194 elif test $vm = byterun; then
195 #byterun -v -c $pyc "$@"
196 byterun -c $pyc "$@"
197 else
198 die $vm
199 fi
200}
201
202compare-sizes() {
203 local left=$1
204 local right=$2
205 find $left -name '*.pyc' -a -printf '%s %P\n' | sort -n
206 echo ---
207 # Wow, opyc files are bigger! Code is not as optimal or what?
208 # Order is roughly the same.
209 find $right -name '*.opyc' -a -printf '%s %P\n' | sort -n
210}
211
212compare-opy-sizes() {
213 compare-sizes .. _tmp/opy
214}
215
216compare-osh-sizes() {
217 # TODO: filter opy out of the left
218 compare-sizes .. _tmp/osh
219}
220
221# Doesn't work because of compileFile.
222old-compile-test() {
223 PYTHONPATH=. tools/compile.py testdata/hello_py3.py
224}
225
226#
227# Parsing tests subsummed by compiling
228#
229
230# 2to3.grammar is from Python-3.6.1/ Lib/lib2to3/Grammar.txt
231parse-with-pgen2() {
232 set +o errexit
233 for py in "$@"; do
234 _parse-one $py >/dev/null #2>&1
235 echo $? $py
236 done
237}
238
239# This fails due to some files not using __future__ print_function.
240parse-oil() {
241 parse-with-pgen2 *.py ../*.py ../osh/*.py ../core/*.py ../asdl/*.py
242}
243
244# Parse the old Python2 code
245parse-pycompiler2() {
246 # parse print statement
247 parse-with-pgen2 ~/src/Python-2.7.6/Lib/compiler/*.py
248}
249
250# After lib2to3
251parse-pycompiler() {
252 # parse print statement
253 parse-with-pgen2 compiler/*.py tools/*.py
254}
255
256#
257# File Management
258#
259
260clear-tokens() {
261 rm token.py tokenize.py
262 rm -rf --verbose __pycache ../__pycache__
263}
264
265copy-lib2to3() {
266 #cp -v $PY/Lib/{token,tokenize}.py .
267 #return
268
269 # For comparison
270 mkdir -p pgen2
271
272 cp -v $PY/Lib/lib2to3/{pytree,pygram}.py .
273 cp -v $PY/Lib/lib2to3/pgen2/{__init__,driver,grammar,parse,token,tokenize,pgen}.py pgen2
274 # The 2to3 grammar supports both Python 2 and Python 3.
275 # - it has the old print statement. Well I guess you still want that! Gah.
276 cp -v $PY/Lib/lib2to3/Grammar.txt .
277 return
278
279 cp -v $PY/Parser/Python.asdl .
280
281}
282
283# For compatibility with the compiler package.
284# I kind of want the static type syntax though?
285copy-old-grammar() {
286 cp -v $PY27/Grammar/Grammar py27.grammar
287}
288
289copy-pycompiler() {
290 # The last version of the pure Python compile package.
291 mkdir -p compiler2
292 cp -v ~/src/Python-2.7.6/Lib/compiler/*.py compiler2
293}
294
295copy-pycompiler-tools() {
296 cp -v ~/src/Python-2.7.6/Tools/compiler/{ast.txt,ACKS,README,*.py} tools/
297}
298
299# Features from Python 3 used? Static types? I guess Python 3.6 has locals with
300# foo: str = 1
301#
302# Do I want that?
303#
304# Main things I ran into were:
305# - print statement
306# - next() is now __next__
307# - io.StringIO vs. cStringIO.cstringIO()
308#
309# And occasional exceptions about encoding. Had to add .encode('utf-8') in a
310# few places.
311#
312# So mostly cosmetic issues.
313
314"$@"