1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Calculate and filter deps of Python apps.
|
4 | #
|
5 | # Usage:
|
6 | # build/dynamic-deps.sh <function name>
|
7 |
|
8 | set -o nounset
|
9 | set -o pipefail
|
10 | set -o errexit
|
11 |
|
12 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
13 |
|
14 | source mycpp/common.sh # $MYPY_REPO
|
15 |
|
16 | readonly PY_PATH='.:vendor/'
|
17 |
|
18 | # Temporary
|
19 | readonly DIR=_build/NINJA
|
20 |
|
21 | # In git
|
22 | readonly FILTER_DIR='prebuilt/dynamic-deps'
|
23 |
|
24 | make-egrep() {
|
25 | # match chars until # or space, and line must be non-empty
|
26 | gawk '
|
27 | match($0, /([^# ]*)/, m) {
|
28 | contents = m[0]
|
29 | if (contents) { # skip empty lines
|
30 | print(contents)
|
31 | }
|
32 | }
|
33 | '
|
34 | }
|
35 |
|
36 | write-filters() {
|
37 | ### Write filename filters in the egrep -f format
|
38 |
|
39 | # For ./NINJA-config.sh to use.
|
40 | # This style lets us add comments.
|
41 |
|
42 | # For asdl.asdl_main and other tools
|
43 | make-egrep >$FILTER_DIR/filter-py-tool.txt <<'EOF'
|
44 | __init__.py
|
45 | typing.py # vendor/typing.py isn't imported normally
|
46 | EOF
|
47 |
|
48 | # Don't typecheck these files.
|
49 |
|
50 | make-egrep >$FILTER_DIR/filter-typecheck.txt <<'EOF'
|
51 | __init__.py
|
52 | typing.py
|
53 |
|
54 | # OrderedDict is polymorphic
|
55 | pylib/collections_.py
|
56 |
|
57 | # lots of polymorphic stuff etc.
|
58 | mycpp/mylib.py
|
59 |
|
60 | # TODO: move or remove these
|
61 | tools/deps.py
|
62 | tools/readlink.py
|
63 | EOF
|
64 |
|
65 | # On top of the typecheck filter, exclude these from translation. They are
|
66 | # not inputs to mycpp.
|
67 |
|
68 | make-egrep >$FILTER_DIR/filter-translate.txt <<'EOF'
|
69 | # generated code shouldn't be translated
|
70 | _devbuild/
|
71 | _gen/
|
72 |
|
73 | # definitions that are used by */*_gen.py
|
74 | .*_def\.py
|
75 | .*_spec\.py
|
76 |
|
77 | asdl/py.* # pybase.py ported by hand to C++
|
78 |
|
79 | core/py.* # pyos.py, pyutil.py ported by hand to C++
|
80 | core/optview\.py # core/optview_gen.py
|
81 |
|
82 | data_lang/py.* # pyj8.py
|
83 |
|
84 | frontend/py.*\.py # py_readline.py ported by hand to C++
|
85 | frontend/consts.py # frontend/consts_gen.py
|
86 | frontend/match.py # frontend/lexer_gen.py
|
87 |
|
88 | mycpp/mops.py # Implemented in gc_mops.{h,cC}
|
89 |
|
90 | pgen2/grammar.py # These files are re-done in C++
|
91 | pgen2/pnode.py
|
92 | pgen2/token.py
|
93 |
|
94 | # should be py_path_stat.py, because it's ported by hand to C++
|
95 | pylib/path_stat.py
|
96 |
|
97 | # should be py_bool_stat.py, because it's ported by hand to C++
|
98 | osh/bool_stat.py
|
99 | EOF
|
100 |
|
101 | wc -l $FILTER_DIR/filter-*
|
102 | }
|
103 |
|
104 | repo-filter() {
|
105 | ### Select files from the dynamic_deps.py output
|
106 |
|
107 | # select what's in the repo; eliminating stdlib stuff
|
108 | # eliminate _cache for mycpp running under Python-3.10
|
109 | fgrep -v "$REPO_ROOT/_cache" | fgrep "$REPO_ROOT" | awk '{ print $2 }'
|
110 | }
|
111 |
|
112 | exclude-filter() {
|
113 | ### Exclude repo-relative paths
|
114 |
|
115 | local filter_name=$1
|
116 |
|
117 | egrep -v -f $FILTER_DIR/filter-$filter_name.txt
|
118 | }
|
119 |
|
120 | mysort() {
|
121 | LC_ALL=C sort
|
122 | }
|
123 |
|
124 | #
|
125 | # Programs
|
126 | #
|
127 |
|
128 | py-tool() {
|
129 | local py_module=$1
|
130 |
|
131 | local dir=$DIR/$py_module
|
132 | mkdir -p $dir
|
133 |
|
134 | PYTHONPATH=$PY_PATH /usr/bin/env python2 \
|
135 | build/dynamic_deps.py py-manifest $py_module \
|
136 | > $dir/all-pairs.txt
|
137 |
|
138 | cat $dir/all-pairs.txt | repo-filter | exclude-filter py-tool | mysort \
|
139 | > $dir/deps.txt
|
140 |
|
141 | echo "DEPS $dir/deps.txt"
|
142 | }
|
143 |
|
144 | # Code generators
|
145 | list-gen() {
|
146 | ls */*_gen.py
|
147 | }
|
148 |
|
149 | # mycpp and pea deps are committed to git instead of in _build/NINJA/ because
|
150 | # users might not have Python 3.10
|
151 |
|
152 | write-pea() {
|
153 | # PYTHONPATH=$PY_PATH
|
154 | local module='pea.pea_main'
|
155 | local dir=prebuilt/ninja/$module
|
156 | mkdir -p $dir
|
157 |
|
158 | source build/dev-shell.sh # python3
|
159 |
|
160 | # Can't use vendor/typing.py
|
161 | PYTHONPATH=. python3 \
|
162 | build/dynamic_deps.py py-manifest $module \
|
163 | > $dir/all-pairs.txt
|
164 |
|
165 | cat $dir/all-pairs.txt | repo-filter | mysort | tee $dir/deps.txt
|
166 |
|
167 | echo
|
168 | echo $dir/*
|
169 | }
|
170 |
|
171 | write-mycpp() {
|
172 | local module='mycpp.mycpp_main'
|
173 | local dir=prebuilt/ninja/$module
|
174 | mkdir -p $dir
|
175 |
|
176 | ( source $MYCPP_VENV/bin/activate
|
177 | PYTHONPATH=$REPO_ROOT:$REPO_ROOT/mycpp:$MYPY_REPO maybe-our-python3 \
|
178 | build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt
|
179 | )
|
180 |
|
181 | cat $dir/all-pairs.txt \
|
182 | | grep -v oilshell/oil_DEPS \
|
183 | | repo-filter \
|
184 | | exclude-filter py-tool \
|
185 | | mysort \
|
186 | | tee $dir/deps.txt
|
187 |
|
188 | echo
|
189 | echo $dir/*
|
190 | }
|
191 |
|
192 | mycpp-example-parse() {
|
193 | ### Manifests for mycpp/examples/parse are committed to git
|
194 |
|
195 | local dir=$DIR/parse
|
196 | mkdir -p $dir
|
197 |
|
198 | PYTHONPATH=$PY_PATH /usr/bin/env python2 \
|
199 | build/dynamic_deps.py py-manifest mycpp.examples.parse \
|
200 | > $dir/all-pairs.txt
|
201 |
|
202 | local ty=mycpp/examples/parse.typecheck.txt
|
203 | local tr=mycpp/examples/parse.translate.txt
|
204 |
|
205 | cat $dir/all-pairs.txt | repo-filter | exclude-filter typecheck | mysort > $ty
|
206 |
|
207 | cat $ty | exclude-filter translate > $tr
|
208 |
|
209 | wc -l $ty $tr
|
210 |
|
211 | #head $ty $tr
|
212 | }
|
213 |
|
214 | pea-hack() {
|
215 | # Leave out help_.py for Soil
|
216 | grep -v '_devbuild/gen/help_meta.py' $DIR/bin.oils_for_unix/typecheck.txt \
|
217 | > pea/oils-typecheck.txt
|
218 | }
|
219 |
|
220 | # Sourced by NINJA-config.sh
|
221 | if test $(basename $0) = 'dynamic-deps.sh'; then
|
222 | "$@"
|
223 | fi
|