1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Script for contributors to build dev dependencies -- packaged as cross-distro
|
4 | # "wedges". Tested in the Soil CI.
|
5 | #
|
6 | # Usage:
|
7 | # build/deps.sh <function name>
|
8 | #
|
9 | # Examples:
|
10 | # build/deps.sh fetch
|
11 | # build/deps.sh install-wedges-fast # for both Python and C++
|
12 | #
|
13 | # build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
|
14 | #
|
15 | # TODO: Do we need something faster, just python2, re2c, and cmark?
|
16 | #
|
17 | # - build/deps.sh fetch-py
|
18 | # - build/deps.sh install-wedges-py
|
19 | #
|
20 | # TODO: Can we make most of them non-root deps? This requires rebuilding
|
21 | # containers, which requires podman.
|
22 | #
|
23 | # rm -r -f ~/wedge # would be better
|
24 |
|
25 | : ${LIB_OSH=stdlib/osh}
|
26 | source $LIB_OSH/bash-strict.sh
|
27 | source $LIB_OSH/task-five.sh
|
28 |
|
29 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
30 |
|
31 | source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
|
32 | source deps/from-apt.sh # PY3_BUILD_DEPS
|
33 | #source deps/podman.sh
|
34 | source test/tsv-lib.sh # tsv-concat
|
35 | source web/table/html.sh # table-sort-{begin,end}
|
36 |
|
37 | # Also in build/dev-shell.sh
|
38 | USER_WEDGE_DIR=~/wedge/oils-for-unix.org
|
39 | ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
|
40 |
|
41 | readonly DEPS_SOURCE_DIR=_build/deps-source
|
42 |
|
43 | readonly RE2C_VERSION=3.0
|
44 | readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
|
45 |
|
46 | readonly CMARK_VERSION=0.29.0
|
47 | readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
|
48 |
|
49 | readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
|
50 |
|
51 | readonly PY2_VERSION=2.7.18
|
52 | readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
|
53 |
|
54 | readonly PY3_VERSION=3.10.4
|
55 | readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
|
56 |
|
57 | readonly BASH_VER=4.4 # don't clobber BASH_VERSION
|
58 | readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
|
59 |
|
60 | # Another version of bash to test
|
61 | readonly BASH5_VER=5.2.21
|
62 | readonly BASH5_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH5_VER.tar.gz"
|
63 |
|
64 | readonly DASH_VERSION=0.5.10.2
|
65 | readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
|
66 |
|
67 | readonly ZSH_VERSION=5.1.1
|
68 | readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
|
69 |
|
70 | readonly MKSH_VERSION=R52c
|
71 | readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
|
72 |
|
73 | readonly BUSYBOX_VERSION='1.35.0'
|
74 | readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
|
75 |
|
76 | readonly YASH_VERSION=2.49
|
77 | readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
|
78 |
|
79 | readonly MYPY_GIT_URL=https://github.com/python/mypy
|
80 | readonly MYPY_VERSION=0.780
|
81 |
|
82 | readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
|
83 |
|
84 | # Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
|
85 | # https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
|
86 | readonly PYFLAKES_VERSION=2.4.0
|
87 | #readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
|
88 | # 2023-07: Mirrored to avoid network problem on broome during release
|
89 | readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
|
90 |
|
91 | readonly BLOATY_VERSION=1.1
|
92 | readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
|
93 |
|
94 | readonly UFTRACE_VERSION=0.13
|
95 | readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
|
96 |
|
97 | readonly SOUFFLE_VERSION=2.4.1
|
98 | readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
|
99 |
|
100 | log() {
|
101 | echo "$@" >& 2
|
102 | }
|
103 |
|
104 | die() {
|
105 | log "$0: fatal: $@"
|
106 | exit 1
|
107 | }
|
108 |
|
109 | rm-oils-crap() {
|
110 | ### When you want to start over
|
111 |
|
112 | rm -r -f -v ~/wedge
|
113 | sudo rm -r -f -v /wedge
|
114 | }
|
115 |
|
116 | # Note: git is an implicit dependency -- that's how we got the repo in the
|
117 | # first place!
|
118 |
|
119 | # python2-dev is no longer available on Debian 12
|
120 | # python-dev also seems gone
|
121 | #
|
122 | # wget: for fetching wedges (not on Debian by default!)
|
123 | # tree: tiny package that's useful for showing what we installed
|
124 | # g++: essential
|
125 | # libreadline-dev: needed for the build/prepare.sh Python build.
|
126 | # gawk: used by spec-runner.sh for the special match() function.
|
127 | # cmake: for cmark
|
128 | # PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
|
129 | # as well
|
130 | readonly -a WEDGE_DEPS_DEBIAN=(
|
131 | bzip2
|
132 | wget
|
133 | tree
|
134 | gawk
|
135 | g++
|
136 | ninja-build
|
137 | cmake
|
138 | libreadline-dev
|
139 | systemtap-sdt-dev
|
140 |
|
141 | # for Souffle, flex and bison
|
142 | #flex bison
|
143 |
|
144 | "${PY3_BUILD_DEPS[@]}"
|
145 | )
|
146 |
|
147 | readonly -a WEDGE_DEPS_ALPINE=(
|
148 | bzip2
|
149 | xz
|
150 |
|
151 | wget tree gawk
|
152 |
|
153 | gcc g++
|
154 | ninja-build
|
155 | # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
|
156 | ninja-is-really-ninja
|
157 | cmake
|
158 |
|
159 | readline-dev
|
160 | zlib-dev
|
161 | libffi-dev
|
162 | openssl-dev
|
163 |
|
164 | ncurses-dev
|
165 |
|
166 | # for Souffle, flex and bison
|
167 | #flex bison
|
168 | )
|
169 |
|
170 | readonly -a WEDGE_DEPS_FEDORA=(
|
171 |
|
172 | # Weird, Fedora doesn't have these by default!
|
173 | hostname
|
174 | tar
|
175 | bzip2
|
176 |
|
177 | # https://packages.fedoraproject.org/pkgs/wget/wget/
|
178 | wget
|
179 | # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
|
180 | tree
|
181 | gawk
|
182 |
|
183 | # https://packages.fedoraproject.org/pkgs/gcc/gcc/
|
184 | gcc gcc-c++
|
185 |
|
186 | ninja-build
|
187 | cmake
|
188 |
|
189 | readline-devel
|
190 |
|
191 | # Like PY3_BUILD_DEPS
|
192 | # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
|
193 | zlib-devel
|
194 | # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
|
195 | libffi-devel
|
196 | # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
|
197 | openssl-devel
|
198 |
|
199 | # For building zsh from source?
|
200 | # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
|
201 | ncurses-devel
|
202 | #libcap-devel
|
203 |
|
204 | # still have a job control error compiling bash
|
205 | # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
|
206 | # glibc-devel
|
207 | )
|
208 |
|
209 | install-debian-packages() {
|
210 | ### Packages for build/py.sh all, building wedges, etc.
|
211 |
|
212 | set -x # show what needs sudo
|
213 |
|
214 | # pass -y for say gitpod
|
215 | sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
|
216 | set +x
|
217 |
|
218 | # maybe pass -y through
|
219 | test/spec-bin.sh install-shells-with-apt "$@"
|
220 | }
|
221 |
|
222 | install-ubuntu-packages() {
|
223 | ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
|
224 | install-debian-packages "$@"
|
225 | }
|
226 |
|
227 | wedge-deps-debian() {
|
228 | # Install packages without prompt
|
229 |
|
230 | # 2024-02 - there was an Ubuntu update, and we started needing this
|
231 | sudo apt-get -y update
|
232 |
|
233 | install-debian-packages -y
|
234 | }
|
235 |
|
236 | wedge-deps-fedora() {
|
237 | # https://linuxconfig.org/install-development-tools-on-redhat-8
|
238 | # Trying to get past compile errors
|
239 | # sudo dnf group install --assumeyes 'Development Tools'
|
240 |
|
241 | sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
|
242 | }
|
243 |
|
244 | wedge-deps-alpine() {
|
245 | # https://linuxconfig.org/install-development-tools-on-redhat-8
|
246 | # Trying to get past compile errors
|
247 | # sudo dnf group install --assumeyes 'Development Tools'
|
248 |
|
249 | sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
|
250 | }
|
251 |
|
252 | #
|
253 | # Unused patch, was experiment for Fedora
|
254 | #
|
255 |
|
256 | get-typed-ast-patch() {
|
257 | curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
|
258 | }
|
259 |
|
260 | # Work around typed_ast bug:
|
261 | # https://github.com/python/typed_ast/issues/169
|
262 | #
|
263 | # Apply this patch
|
264 | # https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
|
265 | #
|
266 | # typed_ast is tarred up though
|
267 | patch-typed-ast() {
|
268 | local package_dir=_cache/py3-libs
|
269 | local patch=$PWD/deps/typed_ast.patch
|
270 |
|
271 | pushd $package_dir
|
272 | cat $patch
|
273 | echo
|
274 |
|
275 | local dir=typed_ast-1.4.3
|
276 | local tar=typed_ast-1.4.3.tar.gz
|
277 |
|
278 | echo OLD
|
279 | ls -l $tar
|
280 | echo
|
281 |
|
282 | rm -r -f -v $dir
|
283 | tar -x -z < $tar
|
284 |
|
285 | pushd $dir
|
286 | patch -p1 < $patch
|
287 | popd
|
288 | #find $dir
|
289 |
|
290 | # Create a new one
|
291 | tar --create --gzip --file $tar typed_ast-1.4.3
|
292 |
|
293 | echo NEW
|
294 | ls -l $tar
|
295 | echo
|
296 |
|
297 | popd
|
298 | }
|
299 |
|
300 | #
|
301 | # Fetch
|
302 | #
|
303 |
|
304 | download-to() {
|
305 | local dir=$1
|
306 | local url=$2
|
307 | wget --no-clobber --directory-prefix "$dir" "$url"
|
308 | }
|
309 |
|
310 | maybe-extract() {
|
311 | local wedge_dir=$1
|
312 | local tar_name=$2
|
313 | local out_dir=$3
|
314 |
|
315 | if test -d "$wedge_dir/$out_dir"; then
|
316 | log "Not extracting because $wedge_dir/$out_dir exists"
|
317 | return
|
318 | fi
|
319 |
|
320 | local tar=$wedge_dir/$tar_name
|
321 | case $tar_name in
|
322 | *.gz|*.tgz) # mksh ends with .tgz
|
323 | flag='--gzip'
|
324 | ;;
|
325 | *.bz2)
|
326 | flag='--bzip2'
|
327 | ;;
|
328 | *.xz)
|
329 | flag='--xz'
|
330 | ;;
|
331 | *)
|
332 | die "tar with unknown extension: $tar_name"
|
333 | ;;
|
334 | esac
|
335 |
|
336 | tar --extract $flag --file $tar --directory $wedge_dir
|
337 | }
|
338 |
|
339 | clone-mypy() {
|
340 | ### replaces deps/from-git
|
341 | local dest_dir=$1
|
342 | local version=${2:-$MYPY_VERSION}
|
343 |
|
344 | local dest=$dest_dir/mypy-$version
|
345 | if test -d $dest; then
|
346 | log "Not cloning because $dest exists"
|
347 | return
|
348 | fi
|
349 |
|
350 | # v$VERSION is a tag, not a branch
|
351 |
|
352 | # size optimization: --depth=1 --shallow-submodules
|
353 | # https://git-scm.com/docs/git-clone
|
354 |
|
355 | git clone --recursive --branch v$version \
|
356 | --depth=1 --shallow-submodules \
|
357 | $MYPY_GIT_URL $dest
|
358 |
|
359 | # TODO: verify commit checksum
|
360 | }
|
361 |
|
362 | copy-source-medo() {
|
363 | mkdir -p $DEPS_SOURCE_DIR
|
364 |
|
365 | # Copy the whole tree, including the .treeptr files
|
366 | cp --verbose --recursive --no-target-directory \
|
367 | deps/source.medo/ $DEPS_SOURCE_DIR/
|
368 | }
|
369 |
|
370 | fetch-spec-bin() {
|
371 | download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
|
372 | maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" bash-$BASH_VER
|
373 |
|
374 | download-to $DEPS_SOURCE_DIR/bash "$BASH5_URL"
|
375 | maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH5_URL)" bash-$BASH5_VER
|
376 |
|
377 | download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
|
378 | maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
|
379 |
|
380 | download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
|
381 | maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
|
382 |
|
383 | download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
|
384 | maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
|
385 |
|
386 | download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
|
387 | maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
|
388 |
|
389 | download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
|
390 | maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$YASH_VERSION
|
391 |
|
392 | # Patch: this tarball doesn't follow the convention $name-$version
|
393 | if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
|
394 | pushd $DEPS_SOURCE_DIR/mksh
|
395 | mv -v mksh mksh-$MKSH_VERSION
|
396 | popd
|
397 | fi
|
398 | }
|
399 |
|
400 | fetch() {
|
401 | local py_only=${1:-}
|
402 |
|
403 | # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
|
404 | # would do: fetch compressed tarballs designated by .treeptr files, and
|
405 | # expand them.
|
406 |
|
407 | # _build/deps-source/
|
408 | # re2c/
|
409 | # WEDGE
|
410 | # re2c-3.0/ # expanded .tar.xz file
|
411 |
|
412 | copy-source-medo
|
413 |
|
414 | download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
|
415 | download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
|
416 | maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
|
417 | maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
|
418 |
|
419 | if test -n "$py_only"; then
|
420 | log "Fetched dependencies for 'build/py.sh'"
|
421 | return
|
422 | fi
|
423 |
|
424 | download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
|
425 | maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
|
426 | pyflakes-$PYFLAKES_VERSION
|
427 |
|
428 | download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
|
429 | download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
|
430 | maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
|
431 | maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
|
432 |
|
433 | fetch-spec-bin
|
434 |
|
435 | # bloaty and uftrace are for benchmarks, in containers
|
436 | download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
|
437 | download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
|
438 | maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" bloaty-$BLOATY_VERSION
|
439 | maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" uftrace-$UFTRACE_VERSION
|
440 |
|
441 | # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
|
442 |
|
443 | # It's also copied into a wedge in install-wedges.
|
444 | clone-mypy $DEPS_SOURCE_DIR/mypy
|
445 |
|
446 | if false; then
|
447 | download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
|
448 | maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
|
449 | fi
|
450 |
|
451 | if command -v tree > /dev/null; then
|
452 | tree -L 2 $DEPS_SOURCE_DIR
|
453 | fi
|
454 | }
|
455 |
|
456 | fetch-py() {
|
457 | fetch py_only
|
458 | }
|
459 |
|
460 | mirror-pyflakes() {
|
461 | ### Workaround for network error during release
|
462 | scp \
|
463 | $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
|
464 | oilshell.org:oilshell.org/blob/
|
465 | }
|
466 |
|
467 | wedge-exists() {
|
468 | ### Does an installed wedge already exist?
|
469 |
|
470 | local name=$1
|
471 | local version=$2
|
472 | local wedge_dir=${3:-/wedge/oils-for-unix.org}
|
473 |
|
474 | local installed=$wedge_dir/pkg/$name/$version
|
475 |
|
476 | if test -d $installed; then
|
477 | log "$installed already exists"
|
478 | return 0
|
479 | else
|
480 | return 1
|
481 | fi
|
482 | }
|
483 |
|
484 | #
|
485 | # Install
|
486 | #
|
487 |
|
488 | # TODO: py3-libs needs to be a WEDGE, so that that you can run
|
489 | # 'wedge build deps/source.medo/py3-libs/' and then get it in
|
490 | #
|
491 | # _build/wedge/{absolute,relative} # which one?
|
492 | #
|
493 | # It needs a BUILD DEPENDENCY on:
|
494 | # - the python3 wedge, so you can do python3 -m pip install.
|
495 | # - the mypy repo, which has test-requirements.txt
|
496 |
|
497 | download-py3-libs() {
|
498 | ### Download source/binary packages, AFTER python3 is installed
|
499 |
|
500 | # Note that this is NOT source code; there is binary code, e.g. in
|
501 | # lxml-*.whl
|
502 |
|
503 | local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
|
504 | local py_package_dir=_cache/py3-libs
|
505 | mkdir -p $py_package_dir
|
506 |
|
507 | # Avoids a warning, but doesn't fix typed_ast
|
508 | #python3 -m pip download -d $py_package_dir wheel
|
509 |
|
510 | python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
|
511 | python3 -m pip download -d $py_package_dir pexpect
|
512 | }
|
513 |
|
514 | install-py3-libs-in-venv() {
|
515 | local venv_dir=$1
|
516 | local mypy_dir=$2 # This is a param for host build vs. container build
|
517 | local package_dir=_cache/py3-libs
|
518 |
|
519 | source $venv_dir/bin/activate # enter virtualenv
|
520 |
|
521 | # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
|
522 | # conflicts with MyPy deps!
|
523 | # "ERROR: pip's dependency resolver does not currently take into account all
|
524 | # the packages that are installed."
|
525 |
|
526 | # --find-links uses a "cache dir" for packages (weird flag name!)
|
527 |
|
528 | # Avoids a warning, but doesn't fix typed_ast
|
529 | #time python3 -m pip install --find-links $package_dir wheel
|
530 |
|
531 | upgrade-typed-ast $mypy_dir/mypy-requirements.txt
|
532 |
|
533 | # for mycpp/
|
534 | time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
|
535 |
|
536 | # pexpect: for spec/stateful/*.py
|
537 | time python3 -m pip install --find-links $package_dir pexpect
|
538 | }
|
539 |
|
540 | upgrade-typed-ast() {
|
541 | local file=$1
|
542 | sed -i 's/typed_ast.*/typed_ast==1.5.0/' $file
|
543 | }
|
544 |
|
545 | test-typed-ast() {
|
546 | local dir=~/wedge/oils-for-unix.org/pkg/mypy/0.780
|
547 |
|
548 | cp -v $dir/mypy-requirements.txt _tmp
|
549 |
|
550 | local file=_tmp/mypy-requirements.txt
|
551 | cat $file
|
552 | #echo
|
553 |
|
554 | # 1.5.0 fixed this bug
|
555 | # https://github.com/python/typed_ast/issues/169
|
556 |
|
557 | upgrade-typed-ast $file
|
558 | echo
|
559 | cat $file
|
560 | }
|
561 |
|
562 | install-py3-libs-from-cache() {
|
563 |
|
564 | # As well as end users
|
565 |
|
566 | local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
|
567 |
|
568 | local py3
|
569 | py3=$(command -v python3)
|
570 | case $py3 in
|
571 | *wedge/oils-for-unix.org/*)
|
572 | ;;
|
573 | *)
|
574 | die "python3 is '$py3', but expected it to be in a wedge"
|
575 | ;;
|
576 | esac
|
577 |
|
578 | log "Ensuring pip is installed (interpreter $(command -v python3)"
|
579 | python3 -m ensurepip
|
580 |
|
581 | local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
|
582 | log "Creating venv in $venv_dir"
|
583 |
|
584 | # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
|
585 | # the /wedge we just built
|
586 | python3 -m venv $venv_dir
|
587 |
|
588 | log "Installing MyPy deps in venv"
|
589 |
|
590 | # Run in a subshell because it mutates shell state
|
591 | $0 install-py3-libs-in-venv $venv_dir $mypy_dir
|
592 | }
|
593 |
|
594 | install-py3-libs() {
|
595 | ### Invoked by Dockerfile.cpp-small, etc.
|
596 |
|
597 | download-py3-libs
|
598 | install-py3-libs-from-cache
|
599 | }
|
600 |
|
601 |
|
602 | # zsh notes
|
603 | # Fedora compiler error
|
604 | # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
|
605 | # fallback in Src/Modules/termcap.c that causes a compile error! It seems
|
606 | # like ncurses-devel should fix this, but it doesn't
|
607 | #
|
608 | # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
|
609 | #
|
610 | # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
|
611 | # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
|
612 | # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
|
613 | #
|
614 | # I think the ./configure is out of sync with the actual build?
|
615 |
|
616 |
|
617 | # TODO:
|
618 | # - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
|
619 | # already in each WEDGE file
|
620 |
|
621 | py-wedges() {
|
622 | ### for build/py.sh all
|
623 |
|
624 | echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
|
625 | echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
|
626 | echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
|
627 | echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
|
628 | }
|
629 |
|
630 | cpp-wedges() {
|
631 | ### for ninja / mycpp translation
|
632 |
|
633 | echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
|
634 | echo mypy $MYPY_VERSION $USER_WEDGE_DIR
|
635 |
|
636 | # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
|
637 | # separately for now
|
638 | #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
|
639 | }
|
640 |
|
641 | spec-bin-wedges() {
|
642 | ### for test/spec-py.sh osh-all
|
643 |
|
644 | echo dash $DASH_VERSION $USER_WEDGE_DIR
|
645 | echo bash $BASH_VER $USER_WEDGE_DIR
|
646 | echo bash $BASH5_VER $USER_WEDGE_DIR
|
647 | echo mksh $MKSH_VERSION $USER_WEDGE_DIR
|
648 | echo zsh $ZSH_VERSION $USER_WEDGE_DIR
|
649 | echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
|
650 | echo yash $YASH_VERSION $USER_WEDGE_DIR
|
651 | }
|
652 |
|
653 | contributor-wedges() {
|
654 | py-wedges
|
655 | cpp-wedges
|
656 | spec-bin-wedges
|
657 | }
|
658 |
|
659 | extra-wedges() {
|
660 | # Contributors don't need uftrace, bloaty, and probably R-libs
|
661 | # Although R-libs could be useful for benchmarks
|
662 |
|
663 | # Test both outside the contianer, as well as inside?
|
664 | echo uftrace $UFTRACE_VERSION $ROOT_WEDGE_DIR
|
665 |
|
666 | #echo souffle $SOUFFLE_VERSION $USER_WEDGE_DIR
|
667 | }
|
668 |
|
669 | timestamp() {
|
670 | date '+%H:%M:%S'
|
671 | }
|
672 |
|
673 | my-time-tsv() {
|
674 | python3 benchmarks/time_.py \
|
675 | --tsv \
|
676 | --time-span --rusage \
|
677 | "$@"
|
678 | }
|
679 |
|
680 | maybe-install-wedge() {
|
681 | local name=$1
|
682 | local version=$2
|
683 | local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
|
684 |
|
685 | local task_file=$WEDGE_LOG_DIR/$name-$version.task.tsv
|
686 | local log_file=$WEDGE_LOG_DIR/$name-$version.log.txt
|
687 |
|
688 | echo " TASK $(timestamp) $name $version > $log_file"
|
689 |
|
690 | # python3 because it's OUTSIDE the container
|
691 | # Separate columns that could be joined: number of files, total size
|
692 | my-time-tsv --print-header \
|
693 | --field xargs_slot \
|
694 | --field wedge \
|
695 | --field wedge_HREF \
|
696 | --field version \
|
697 | --output $task_file
|
698 |
|
699 | if wedge-exists "$name" "$version" "$wedge_dir"; then
|
700 | echo "CACHED $(timestamp) $name $version"
|
701 | return
|
702 | fi
|
703 |
|
704 | local -a cmd=( deps/wedge.sh unboxed _build/deps-source/$name/ $version)
|
705 |
|
706 | set +o errexit
|
707 | my-time-tsv \
|
708 | --field "$XARGS_SLOT" \
|
709 | --field "$name" \
|
710 | --field "$name-$version.log.txt" \
|
711 | --field "$version" \
|
712 | --append \
|
713 | --output $task_file \
|
714 | "${cmd[@]}" "$@" >$log_file 2>&1
|
715 | local status=$?
|
716 | set -o errexit
|
717 |
|
718 | if test "$status" -eq 0; then
|
719 | echo " OK $(timestamp) $name $version"
|
720 | else
|
721 | echo " FAIL $(timestamp) $name $version"
|
722 | fi
|
723 | }
|
724 |
|
725 | dummy-task() {
|
726 | ### For testing log capture
|
727 | local name=$1
|
728 | local version=$2
|
729 |
|
730 | echo "Building $name $version"
|
731 |
|
732 | # random float between 0 and 3
|
733 | # weirdly we need a seed from bash
|
734 | # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
|
735 | local secs
|
736 | secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
|
737 |
|
738 | echo "sleep $secs"
|
739 | sleep $secs
|
740 |
|
741 | echo 'stdout'
|
742 | log 'stderr'
|
743 |
|
744 | if test $name = 'mksh'; then
|
745 | echo "simulate failure for $name"
|
746 | exit 2
|
747 | fi
|
748 | }
|
749 |
|
750 | readonly WEDGE_LOG_DIR=_build/wedge/logs
|
751 |
|
752 | dummy-task-wrapper() {
|
753 | # Similar to test/common.sh run-task-with-status, used by
|
754 | # test/{spec,wild}-runner.sh
|
755 | local name=$1
|
756 | local version=$2
|
757 |
|
758 | local task_file=$WEDGE_LOG_DIR/$name.task.tsv
|
759 | local log_file=$WEDGE_LOG_DIR/$name.log.txt
|
760 |
|
761 | echo " TASK $(timestamp) $name $version > $log_file"
|
762 |
|
763 | # python3 because it's OUTSIDE the container
|
764 | # Separate columns that could be joined: number of files, total size
|
765 | my-time-tsv --print-header \
|
766 | --field xargs_slot \
|
767 | --field wedge \
|
768 | --field wedge_HREF \
|
769 | --field version \
|
770 | --output $task_file
|
771 |
|
772 | my-time-tsv \
|
773 | --field "$XARGS_SLOT" \
|
774 | --field "$name" \
|
775 | --field "$name.log.txt" \
|
776 | --field "$version" \
|
777 | --append \
|
778 | --output $task_file \
|
779 | $0 dummy-task "$@" >$log_file 2>&1 || true
|
780 |
|
781 | echo " DONE $(timestamp) $name $version"
|
782 | }
|
783 |
|
784 | html-head() {
|
785 | # python3 because we're outside containers
|
786 | PYTHONPATH=. python3 doctools/html_head.py "$@"
|
787 | }
|
788 |
|
789 | index-html() {
|
790 | local tasks_tsv=$1
|
791 |
|
792 | local base_url='../../../web'
|
793 | html-head --title 'Wedge Builds' \
|
794 | "$base_url/ajax.js" \
|
795 | "$base_url/table/table-sort.js" \
|
796 | "$base_url/table/table-sort.css" \
|
797 | "$base_url/base.css"\
|
798 |
|
799 | table-sort-begin 'width60'
|
800 |
|
801 | cat <<EOF
|
802 | <p id="home-link">
|
803 | <a href="/">oilshell.org</a>
|
804 | </p>
|
805 |
|
806 | <h1>Wedge Builds</h1>
|
807 | EOF
|
808 |
|
809 | tsv2html3 $tasks_tsv
|
810 |
|
811 | cat <<EOF
|
812 | <p>
|
813 | <a href="tasks.tsv">tasks.tsv</a>
|
814 | </p>
|
815 | EOF
|
816 |
|
817 | table-sort-end 'tasks' # ID for sorting
|
818 | }
|
819 |
|
820 | NPROC=$(nproc)
|
821 | #NPROC=1
|
822 |
|
823 | install-wedge-list() {
|
824 | ### Reads task rows from stdin
|
825 | local parallel=${1:-}
|
826 |
|
827 | mkdir -p _build/wedge/logs
|
828 |
|
829 | local -a flags
|
830 | if test -n "$parallel"; then
|
831 | log ""
|
832 | log "=== Installing wedges with $NPROC jobs in parallel"
|
833 | log ""
|
834 | flags=( -P $NPROC )
|
835 | else
|
836 | log ""
|
837 | log "=== Installing wedges serially"
|
838 | log ""
|
839 | fi
|
840 |
|
841 | # Reads from stdin
|
842 | # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
|
843 | #
|
844 | # $name $version $wedge_dir
|
845 | xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
|
846 |
|
847 | #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
|
848 | }
|
849 |
|
850 | write-task-report() {
|
851 | local tasks_tsv=_build/wedge/logs/tasks.tsv
|
852 |
|
853 | python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
|
854 | log "Wrote $tasks_tsv"
|
855 |
|
856 | # TODO: version can be right-justified?
|
857 | here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
|
858 | column_name type precision strftime
|
859 | status integer 0 -
|
860 | elapsed_secs float 1 -
|
861 | user_secs float 1 -
|
862 | start_time float 1 %H:%M:%S
|
863 | end_time float 1 %H:%M:%S
|
864 | sys_secs float 1 -
|
865 | max_rss_KiB integer 0 -
|
866 | xargs_slot integer 0 -
|
867 | wedge string 0 -
|
868 | wedge_HREF string 0 -
|
869 | version string 0 -
|
870 | EOF
|
871 |
|
872 | index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
|
873 | log "Wrote $WEDGE_LOG_DIR/index.html"
|
874 | }
|
875 |
|
876 | install-spec-bin-fast() {
|
877 | spec-bin-wedges | install-wedge-list T
|
878 | write-task-report
|
879 | }
|
880 |
|
881 | fake-py3-libs-wedge() {
|
882 | local name=py3-libs
|
883 | local version=$PY3_LIBS_VERSION
|
884 |
|
885 | local task_file=$WEDGE_LOG_DIR/$name.task.tsv
|
886 | local log_file=$WEDGE_LOG_DIR/$name.log.txt
|
887 |
|
888 | my-time-tsv --print-header \
|
889 | --field xargs_slot \
|
890 | --field wedge \
|
891 | --field wedge_HREF \
|
892 | --field version \
|
893 | --output $task_file
|
894 |
|
895 | # There is no xargs slot!
|
896 | my-time-tsv \
|
897 | --field "-1" \
|
898 | --field "$name" \
|
899 | --field "$name.log.txt" \
|
900 | --field "$version" \
|
901 | --append \
|
902 | --output $task_file \
|
903 | $0 install-py3-libs >$log_file 2>&1 || true
|
904 |
|
905 | echo " FAKE $(timestamp) $name $version"
|
906 | }
|
907 |
|
908 | install-wedges-fast() {
|
909 | local extra=${1:-}
|
910 |
|
911 | # For contributor setup: we need to use this BEFORE running build/py.sh all
|
912 | build/py.sh time-helper
|
913 |
|
914 | echo " START $(timestamp)"
|
915 |
|
916 | # Do all of them in parallel
|
917 | if test -n "$extra"; then
|
918 | { contributor-wedges; extra-wedges; } | install-wedge-list T
|
919 | else
|
920 | contributor-wedges | install-wedge-list T
|
921 | fi
|
922 |
|
923 | fake-py3-libs-wedge
|
924 | echo " END $(timestamp)"
|
925 |
|
926 | write-task-report
|
927 | }
|
928 |
|
929 | install-wedges-soil() {
|
930 | install-wedges-fast extra
|
931 | }
|
932 |
|
933 | #
|
934 | # Unboxed wedge builds
|
935 | #
|
936 |
|
937 | uftrace-host() {
|
938 | ### built on demand; run $0 first
|
939 |
|
940 | # BUG: doesn't detect python3
|
941 | # WEDGE tells me that it depends on pkg-config
|
942 | # 'apt-get install pkgconf' gets it
|
943 | # TODO: Should use python3 WEDGE instead of SYSTEM python3?
|
944 | deps/wedge.sh unboxed _build/deps-source/uftrace
|
945 | }
|
946 |
|
947 | bloaty-host() {
|
948 | deps/wedge.sh unboxed _build/deps-source/bloaty
|
949 | }
|
950 |
|
951 | R-libs-host() {
|
952 | deps/wedge.sh unboxed _build/deps-source/R-libs
|
953 | }
|
954 |
|
955 | #
|
956 | # Wedges built inside a container, for copying into a container
|
957 | #
|
958 |
|
959 | boxed-wedges() {
|
960 | #### host _build/wedge/binary -> guest container /wedge or ~/wedge
|
961 |
|
962 | #export-podman
|
963 |
|
964 | # TODO:
|
965 | #
|
966 | # - Add equivalents of spec-bin
|
967 | # - Use the same manifest as install-wedges-fast
|
968 | # - so then you can delete the _build/wedge dir to re-run it
|
969 | # - use xargs -n 1 so it's done serially
|
970 |
|
971 | # - Do these lazily like we do in install-wedges-fast?
|
972 |
|
973 | # We can test if the dir _build/wedge/binary/oils-for-unix.org/pkg/FOO exists
|
974 | # if wedge-exists "$name" "$version" "$wedge_dir"; then
|
975 | # echo "CACHED $(timestamp) $name $version"
|
976 | # return
|
977 | # fi
|
978 |
|
979 | if false; then
|
980 | deps/wedge.sh boxed deps/source.medo/time-helper
|
981 | deps/wedge.sh boxed deps/source.medo/cmark/
|
982 | deps/wedge.sh boxed deps/source.medo/re2c/
|
983 | deps/wedge.sh boxed deps/source.medo/python3/
|
984 | fi
|
985 |
|
986 | if false; then
|
987 | deps/wedge.sh boxed deps/source.medo/bloaty/
|
988 | fi
|
989 |
|
990 | if true; then
|
991 | # build with debian-12, because soil-benchmarks2 is, because it has R
|
992 | deps/wedge.sh boxed deps/source.medo/uftrace/ '' debian-12
|
993 | # python2 needed everywhere
|
994 | #deps/wedge.sh boxed deps/source.medo/python2/ '' debian-12
|
995 |
|
996 | # TODO: build with debian-12
|
997 | # Used in {benchmarks,benchmarks2,other-tests}
|
998 | #deps/wedge.sh boxed deps/source.medo/R-libs/ '' debian-12
|
999 | fi
|
1000 | }
|
1001 |
|
1002 | boxed-spec-bin() {
|
1003 | if true; then
|
1004 | #deps/wedge.sh boxed deps/source.medo/bash '4.4'
|
1005 | deps/wedge.sh boxed deps/source.medo/bash '5.2.21'
|
1006 | fi
|
1007 |
|
1008 | if false; then
|
1009 | deps/wedge.sh boxed deps/source.medo/dash
|
1010 | deps/wedge.sh boxed deps/source.medo/mksh
|
1011 | fi
|
1012 |
|
1013 | if false; then
|
1014 | # Note: zsh requires libncursesw5-dev
|
1015 | #deps/wedge.sh boxed deps/source.medo/zsh
|
1016 |
|
1017 | #deps/wedge.sh boxed deps/source.medo/busybox
|
1018 |
|
1019 | # Problem with out of tree build, as above. Skipping for now
|
1020 | deps/wedge.sh boxed deps/source.medo/yash
|
1021 | echo
|
1022 | fi
|
1023 | }
|
1024 |
|
1025 | #
|
1026 | # Report
|
1027 | #
|
1028 |
|
1029 | commas() {
|
1030 | # Wow I didn't know this :a trick
|
1031 | #
|
1032 | # OK this is a label and a loop, which makes sense. You can't do it with
|
1033 | # pure regex.
|
1034 | #
|
1035 | # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
|
1036 | # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
|
1037 | sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
|
1038 | }
|
1039 |
|
1040 | wedge-sizes() {
|
1041 | local tmp=_tmp/wedge-sizes.txt
|
1042 |
|
1043 | # -b is --bytes, but use short flag for busybox compat
|
1044 | du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
|
1045 | { print $0 # print the line
|
1046 | total_bytes += $1 # accumulate
|
1047 | }
|
1048 | END { print total_bytes " TOTAL" }
|
1049 | ' > $tmp
|
1050 |
|
1051 | # printf justifies du output
|
1052 | cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
|
1053 | echo
|
1054 |
|
1055 | #du -s --si /wedge/*/*/* ~/wedge/*/*/*
|
1056 | #echo
|
1057 | }
|
1058 |
|
1059 | wedge-report() {
|
1060 | # 4 levels deep shows the package
|
1061 | if command -v tree > /dev/null; then
|
1062 | tree -L 4 /wedge ~/wedge
|
1063 | echo
|
1064 | fi
|
1065 |
|
1066 | wedge-sizes
|
1067 |
|
1068 | local tmp=_tmp/wedge-manifest.txt
|
1069 |
|
1070 | echo 'Biggest files'
|
1071 | if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
|
1072 | # busybox find doesn't have -printf
|
1073 | echo 'find -printf failed'
|
1074 | return
|
1075 | fi
|
1076 |
|
1077 | set +o errexit # ignore SIGPIPE
|
1078 | sort -n --reverse $tmp | head -n 20 | commas
|
1079 | set -o errexit
|
1080 |
|
1081 | echo
|
1082 |
|
1083 | # Show the most common file extensions
|
1084 | #
|
1085 | # I feel like we should be able to get rid of .a files? That's 92 MB, second
|
1086 | # most common
|
1087 | #
|
1088 | # There are also duplicate .a files for Python -- should look at how distros
|
1089 | # get rid of those
|
1090 |
|
1091 | cat $tmp | python3 -c '
|
1092 | import os, sys, collections
|
1093 |
|
1094 | bytes = collections.Counter()
|
1095 | files = collections.Counter()
|
1096 |
|
1097 | for line in sys.stdin:
|
1098 | size, path = line.split(None, 1)
|
1099 | path = path.strip() # remove newline
|
1100 | _, ext = os.path.splitext(path)
|
1101 | size = int(size)
|
1102 |
|
1103 | bytes[ext] += size
|
1104 | files[ext] += 1
|
1105 |
|
1106 | #print(bytes)
|
1107 | #print(files)
|
1108 |
|
1109 | n = 20
|
1110 |
|
1111 | print("Most common file types")
|
1112 | for ext, count in files.most_common()[:n]:
|
1113 | print("%10d %s" % (count, ext))
|
1114 |
|
1115 | print()
|
1116 |
|
1117 | print("Total bytes by file type")
|
1118 | for ext, total_bytes in bytes.most_common()[:n]:
|
1119 | print("%10d %s" % (total_bytes, ext))
|
1120 | ' | commas
|
1121 | }
|
1122 |
|
1123 | task-five "$@"
|