OILS / build / deps.sh View on Github | oilshell.org

1124 lines, 572 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges-fast # for both Python and C++
12#
13# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
14#
15# TODO: Do we need something faster, just python2, re2c, and cmark?
16#
17# - build/deps.sh fetch-py
18# - build/deps.sh install-wedges-py
19#
20# TODO: Can we make most of them non-root deps? This requires rebuilding
21# containers, which requires podman.
22#
23# rm -r -f ~/wedge # would be better
24
25set -o nounset
26set -o pipefail
27set -o errexit
28
29REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
30
31source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
32source deps/from-apt.sh # PY3_BUILD_DEPS
33#source deps/podman.sh
34source devtools/run-task.sh # run-task
35source test/tsv-lib.sh # tsv-concat
36source web/table/html.sh # table-sort-{begin,end}
37
38# Also in build/dev-shell.sh
39USER_WEDGE_DIR=~/wedge/oils-for-unix.org
40ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
41
42readonly DEPS_SOURCE_DIR=_build/deps-source
43
44readonly RE2C_VERSION=3.0
45readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
46
47readonly CMARK_VERSION=0.29.0
48readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
49
50readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
51
52readonly PY2_VERSION=2.7.18
53readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
54
55readonly PY3_VERSION=3.10.4
56readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
57
58readonly BASH_VER=4.4 # don't clobber BASH_VERSION
59readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
60
61# Another version of bash to test
62readonly BASH5_VER=5.2.21
63readonly BASH5_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH5_VER.tar.gz"
64
65readonly DASH_VERSION=0.5.10.2
66readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
67
68readonly ZSH_VERSION=5.1.1
69readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
70
71readonly MKSH_VERSION=R52c
72readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
73
74readonly BUSYBOX_VERSION='1.35.0'
75readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
76
77readonly YASH_VERSION=2.49
78readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
79
80readonly MYPY_GIT_URL=https://github.com/python/mypy
81readonly MYPY_VERSION=0.780
82
83readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
84
85# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
86# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
87readonly PYFLAKES_VERSION=2.4.0
88#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
89# 2023-07: Mirrored to avoid network problem on broome during release
90readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
91
92readonly BLOATY_VERSION=1.1
93readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
94
95readonly UFTRACE_VERSION=0.13
96readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
97
98readonly SOUFFLE_VERSION=2.4.1
99readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
100
101log() {
102 echo "$@" >& 2
103}
104
105die() {
106 log "$0: fatal: $@"
107 exit 1
108}
109
110rm-oils-crap() {
111 ### When you want to start over
112
113 rm -r -f -v ~/wedge
114 sudo rm -r -f -v /wedge
115}
116
117# Note: git is an implicit dependency -- that's how we got the repo in the
118# first place!
119
120# python2-dev is no longer available on Debian 12
121# python-dev also seems gone
122#
123# wget: for fetching wedges (not on Debian by default!)
124# tree: tiny package that's useful for showing what we installed
125# g++: essential
126# libreadline-dev: needed for the build/prepare.sh Python build.
127# gawk: used by spec-runner.sh for the special match() function.
128# cmake: for cmark
129# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
130# as well
131readonly -a WEDGE_DEPS_DEBIAN=(
132 bzip2
133 wget
134 tree
135 gawk
136 g++
137 ninja-build
138 cmake
139 libreadline-dev
140 systemtap-sdt-dev
141
142 # for Souffle, flex and bison
143 #flex bison
144
145 "${PY3_BUILD_DEPS[@]}"
146)
147
148readonly -a WEDGE_DEPS_ALPINE=(
149 bzip2
150 xz
151
152 wget tree gawk
153
154 gcc g++
155 ninja-build
156 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
157 ninja-is-really-ninja
158 cmake
159
160 readline-dev
161 zlib-dev
162 libffi-dev
163 openssl-dev
164
165 ncurses-dev
166
167 # for Souffle, flex and bison
168 #flex bison
169)
170
171readonly -a WEDGE_DEPS_FEDORA=(
172
173 # Weird, Fedora doesn't have these by default!
174 hostname
175 tar
176 bzip2
177
178 # https://packages.fedoraproject.org/pkgs/wget/wget/
179 wget
180 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
181 tree
182 gawk
183
184 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
185 gcc gcc-c++
186
187 ninja-build
188 cmake
189
190 readline-devel
191
192 # Like PY3_BUILD_DEPS
193 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
194 zlib-devel
195 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
196 libffi-devel
197 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
198 openssl-devel
199
200 # For building zsh from source?
201 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
202 ncurses-devel
203 #libcap-devel
204
205 # still have a job control error compiling bash
206 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
207 # glibc-devel
208)
209
210install-debian-packages() {
211 ### Packages for build/py.sh all, building wedges, etc.
212
213 set -x # show what needs sudo
214
215 # pass -y for say gitpod
216 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
217 set +x
218
219 # maybe pass -y through
220 test/spec-bin.sh install-shells-with-apt "$@"
221}
222
223install-ubuntu-packages() {
224 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
225 install-debian-packages "$@"
226}
227
228wedge-deps-debian() {
229 # Install packages without prompt
230
231 # 2024-02 - there was an Ubuntu update, and we started needing this
232 sudo apt-get -y update
233
234 install-debian-packages -y
235}
236
237wedge-deps-fedora() {
238 # https://linuxconfig.org/install-development-tools-on-redhat-8
239 # Trying to get past compile errors
240 # sudo dnf group install --assumeyes 'Development Tools'
241
242 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
243}
244
245wedge-deps-alpine() {
246 # https://linuxconfig.org/install-development-tools-on-redhat-8
247 # Trying to get past compile errors
248 # sudo dnf group install --assumeyes 'Development Tools'
249
250 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
251}
252
253#
254# Unused patch, was experiment for Fedora
255#
256
257get-typed-ast-patch() {
258 curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
259}
260
261# Work around typed_ast bug:
262# https://github.com/python/typed_ast/issues/169
263#
264# Apply this patch
265# https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
266#
267# typed_ast is tarred up though
268patch-typed-ast() {
269 local package_dir=_cache/py3-libs
270 local patch=$PWD/deps/typed_ast.patch
271
272 pushd $package_dir
273 cat $patch
274 echo
275
276 local dir=typed_ast-1.4.3
277 local tar=typed_ast-1.4.3.tar.gz
278
279 echo OLD
280 ls -l $tar
281 echo
282
283 rm -r -f -v $dir
284 tar -x -z < $tar
285
286 pushd $dir
287 patch -p1 < $patch
288 popd
289 #find $dir
290
291 # Create a new one
292 tar --create --gzip --file $tar typed_ast-1.4.3
293
294 echo NEW
295 ls -l $tar
296 echo
297
298 popd
299}
300
301#
302# Fetch
303#
304
305download-to() {
306 local dir=$1
307 local url=$2
308 wget --no-clobber --directory-prefix "$dir" "$url"
309}
310
311maybe-extract() {
312 local wedge_dir=$1
313 local tar_name=$2
314 local out_dir=$3
315
316 if test -d "$wedge_dir/$out_dir"; then
317 log "Not extracting because $wedge_dir/$out_dir exists"
318 return
319 fi
320
321 local tar=$wedge_dir/$tar_name
322 case $tar_name in
323 *.gz|*.tgz) # mksh ends with .tgz
324 flag='--gzip'
325 ;;
326 *.bz2)
327 flag='--bzip2'
328 ;;
329 *.xz)
330 flag='--xz'
331 ;;
332 *)
333 die "tar with unknown extension: $tar_name"
334 ;;
335 esac
336
337 tar --extract $flag --file $tar --directory $wedge_dir
338}
339
340clone-mypy() {
341 ### replaces deps/from-git
342 local dest_dir=$1
343 local version=${2:-$MYPY_VERSION}
344
345 local dest=$dest_dir/mypy-$version
346 if test -d $dest; then
347 log "Not cloning because $dest exists"
348 return
349 fi
350
351 # v$VERSION is a tag, not a branch
352
353 # size optimization: --depth=1 --shallow-submodules
354 # https://git-scm.com/docs/git-clone
355
356 git clone --recursive --branch v$version \
357 --depth=1 --shallow-submodules \
358 $MYPY_GIT_URL $dest
359
360 # TODO: verify commit checksum
361}
362
363copy-source-medo() {
364 mkdir -p $DEPS_SOURCE_DIR
365
366 # Copy the whole tree, including the .treeptr files
367 cp --verbose --recursive --no-target-directory \
368 deps/source.medo/ $DEPS_SOURCE_DIR/
369}
370
371fetch-spec-bin() {
372 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
373 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" bash-$BASH_VER
374
375 download-to $DEPS_SOURCE_DIR/bash "$BASH5_URL"
376 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH5_URL)" bash-$BASH5_VER
377
378 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
379 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
380
381 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
382 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
383
384 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
385 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
386
387 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
388 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
389
390 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
391 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$YASH_VERSION
392
393 # Patch: this tarball doesn't follow the convention $name-$version
394 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
395 pushd $DEPS_SOURCE_DIR/mksh
396 mv -v mksh mksh-$MKSH_VERSION
397 popd
398 fi
399}
400
401fetch() {
402 local py_only=${1:-}
403
404 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
405 # would do: fetch compressed tarballs designated by .treeptr files, and
406 # expand them.
407
408 # _build/deps-source/
409 # re2c/
410 # WEDGE
411 # re2c-3.0/ # expanded .tar.xz file
412
413 copy-source-medo
414
415 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
416 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
417 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
418 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
419
420 if test -n "$py_only"; then
421 log "Fetched dependencies for 'build/py.sh'"
422 return
423 fi
424
425 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
426 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
427 pyflakes-$PYFLAKES_VERSION
428
429 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
430 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
431 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
432 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
433
434 fetch-spec-bin
435
436 # bloaty and uftrace are for benchmarks, in containers
437 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
438 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
439 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" bloaty-$BLOATY_VERSION
440 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" uftrace-$UFTRACE_VERSION
441
442 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
443
444 # It's also copied into a wedge in install-wedges.
445 clone-mypy $DEPS_SOURCE_DIR/mypy
446
447 if false; then
448 download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
449 maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
450 fi
451
452 if command -v tree > /dev/null; then
453 tree -L 2 $DEPS_SOURCE_DIR
454 fi
455}
456
457fetch-py() {
458 fetch py_only
459}
460
461mirror-pyflakes() {
462 ### Workaround for network error during release
463 scp \
464 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
465 oilshell.org:oilshell.org/blob/
466}
467
468wedge-exists() {
469 ### Does an installed wedge already exist?
470
471 local name=$1
472 local version=$2
473 local wedge_dir=${3:-/wedge/oils-for-unix.org}
474
475 local installed=$wedge_dir/pkg/$name/$version
476
477 if test -d $installed; then
478 log "$installed already exists"
479 return 0
480 else
481 return 1
482 fi
483}
484
485#
486# Install
487#
488
489# TODO: py3-libs needs to be a WEDGE, so that that you can run
490# 'wedge build deps/source.medo/py3-libs/' and then get it in
491#
492# _build/wedge/{absolute,relative} # which one?
493#
494# It needs a BUILD DEPENDENCY on:
495# - the python3 wedge, so you can do python3 -m pip install.
496# - the mypy repo, which has test-requirements.txt
497
498download-py3-libs() {
499 ### Download source/binary packages, AFTER python3 is installed
500
501 # Note that this is NOT source code; there is binary code, e.g. in
502 # lxml-*.whl
503
504 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
505 local py_package_dir=_cache/py3-libs
506 mkdir -p $py_package_dir
507
508 # Avoids a warning, but doesn't fix typed_ast
509 #python3 -m pip download -d $py_package_dir wheel
510
511 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
512 python3 -m pip download -d $py_package_dir pexpect
513}
514
515install-py3-libs-in-venv() {
516 local venv_dir=$1
517 local mypy_dir=$2 # This is a param for host build vs. container build
518 local package_dir=_cache/py3-libs
519
520 source $venv_dir/bin/activate # enter virtualenv
521
522 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
523 # conflicts with MyPy deps!
524 # "ERROR: pip's dependency resolver does not currently take into account all
525 # the packages that are installed."
526
527 # --find-links uses a "cache dir" for packages (weird flag name!)
528
529 # Avoids a warning, but doesn't fix typed_ast
530 #time python3 -m pip install --find-links $package_dir wheel
531
532 upgrade-typed-ast $mypy_dir/mypy-requirements.txt
533
534 # for mycpp/
535 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
536
537 # pexpect: for spec/stateful/*.py
538 time python3 -m pip install --find-links $package_dir pexpect
539}
540
541upgrade-typed-ast() {
542 local file=$1
543 sed -i 's/typed_ast.*/typed_ast==1.5.0/' $file
544}
545
546test-typed-ast() {
547 local dir=~/wedge/oils-for-unix.org/pkg/mypy/0.780
548
549 cp -v $dir/mypy-requirements.txt _tmp
550
551 local file=_tmp/mypy-requirements.txt
552 cat $file
553 #echo
554
555 # 1.5.0 fixed this bug
556 # https://github.com/python/typed_ast/issues/169
557
558 upgrade-typed-ast $file
559 echo
560 cat $file
561}
562
563install-py3-libs-from-cache() {
564
565 # As well as end users
566
567 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
568
569 local py3
570 py3=$(command -v python3)
571 case $py3 in
572 *wedge/oils-for-unix.org/*)
573 ;;
574 *)
575 die "python3 is '$py3', but expected it to be in a wedge"
576 ;;
577 esac
578
579 log "Ensuring pip is installed (interpreter $(command -v python3)"
580 python3 -m ensurepip
581
582 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
583 log "Creating venv in $venv_dir"
584
585 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
586 # the /wedge we just built
587 python3 -m venv $venv_dir
588
589 log "Installing MyPy deps in venv"
590
591 # Run in a subshell because it mutates shell state
592 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
593}
594
595install-py3-libs() {
596 ### Invoked by Dockerfile.cpp-small, etc.
597
598 download-py3-libs
599 install-py3-libs-from-cache
600}
601
602
603# zsh notes
604 # Fedora compiler error
605 # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
606 # fallback in Src/Modules/termcap.c that causes a compile error! It seems
607 # like ncurses-devel should fix this, but it doesn't
608 #
609 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
610 #
611 # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
612 # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
613 # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
614 #
615 # I think the ./configure is out of sync with the actual build?
616
617
618# TODO:
619# - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
620# already in each WEDGE file
621
622py-wedges() {
623 ### for build/py.sh all
624
625 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
626 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
627 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
628 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
629}
630
631cpp-wedges() {
632 ### for ninja / mycpp translation
633
634 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
635 echo mypy $MYPY_VERSION $USER_WEDGE_DIR
636
637 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
638 # separately for now
639 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
640}
641
642spec-bin-wedges() {
643 ### for test/spec-py.sh osh-all
644
645 echo dash $DASH_VERSION $USER_WEDGE_DIR
646 echo bash $BASH_VER $USER_WEDGE_DIR
647 echo bash $BASH5_VER $USER_WEDGE_DIR
648 echo mksh $MKSH_VERSION $USER_WEDGE_DIR
649 echo zsh $ZSH_VERSION $USER_WEDGE_DIR
650 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
651 echo yash $YASH_VERSION $USER_WEDGE_DIR
652}
653
654contributor-wedges() {
655 py-wedges
656 cpp-wedges
657 spec-bin-wedges
658}
659
660extra-wedges() {
661 # Contributors don't need uftrace, bloaty, and probably R-libs
662 # Although R-libs could be useful for benchmarks
663
664 # Test both outside the contianer, as well as inside?
665 echo uftrace $UFTRACE_VERSION $ROOT_WEDGE_DIR
666
667 #echo souffle $SOUFFLE_VERSION $USER_WEDGE_DIR
668}
669
670timestamp() {
671 date '+%H:%M:%S'
672}
673
674my-time-tsv() {
675 python3 benchmarks/time_.py \
676 --tsv \
677 --time-span --rusage \
678 "$@"
679}
680
681maybe-install-wedge() {
682 local name=$1
683 local version=$2
684 local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
685
686 local task_file=$WEDGE_LOG_DIR/$name-$version.task.tsv
687 local log_file=$WEDGE_LOG_DIR/$name-$version.log.txt
688
689 echo " TASK $(timestamp) $name $version > $log_file"
690
691 # python3 because it's OUTSIDE the container
692 # Separate columns that could be joined: number of files, total size
693 my-time-tsv --print-header \
694 --field xargs_slot \
695 --field wedge \
696 --field wedge_HREF \
697 --field version \
698 --output $task_file
699
700 if wedge-exists "$name" "$version" "$wedge_dir"; then
701 echo "CACHED $(timestamp) $name $version"
702 return
703 fi
704
705 local -a cmd=( deps/wedge.sh unboxed _build/deps-source/$name/ $version)
706
707 set +o errexit
708 my-time-tsv \
709 --field "$XARGS_SLOT" \
710 --field "$name" \
711 --field "$name-$version.log.txt" \
712 --field "$version" \
713 --append \
714 --output $task_file \
715 "${cmd[@]}" "$@" >$log_file 2>&1
716 local status=$?
717 set -o errexit
718
719 if test "$status" -eq 0; then
720 echo " OK $(timestamp) $name $version"
721 else
722 echo " FAIL $(timestamp) $name $version"
723 fi
724}
725
726dummy-task() {
727 ### For testing log capture
728 local name=$1
729 local version=$2
730
731 echo "Building $name $version"
732
733 # random float between 0 and 3
734 # weirdly we need a seed from bash
735 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
736 local secs
737 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
738
739 echo "sleep $secs"
740 sleep $secs
741
742 echo 'stdout'
743 log 'stderr'
744
745 if test $name = 'mksh'; then
746 echo "simulate failure for $name"
747 exit 2
748 fi
749}
750
751readonly WEDGE_LOG_DIR=_build/wedge/logs
752
753dummy-task-wrapper() {
754 # Similar to test/common.sh run-task-with-status, used by
755 # test/{spec,wild}-runner.sh
756 local name=$1
757 local version=$2
758
759 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
760 local log_file=$WEDGE_LOG_DIR/$name.log.txt
761
762 echo " TASK $(timestamp) $name $version > $log_file"
763
764 # python3 because it's OUTSIDE the container
765 # Separate columns that could be joined: number of files, total size
766 my-time-tsv --print-header \
767 --field xargs_slot \
768 --field wedge \
769 --field wedge_HREF \
770 --field version \
771 --output $task_file
772
773 my-time-tsv \
774 --field "$XARGS_SLOT" \
775 --field "$name" \
776 --field "$name.log.txt" \
777 --field "$version" \
778 --append \
779 --output $task_file \
780 $0 dummy-task "$@" >$log_file 2>&1 || true
781
782 echo " DONE $(timestamp) $name $version"
783}
784
785html-head() {
786 # python3 because we're outside containers
787 PYTHONPATH=. python3 doctools/html_head.py "$@"
788}
789
790index-html() {
791 local tasks_tsv=$1
792
793 local base_url='../../../web'
794 html-head --title 'Wedge Builds' \
795 "$base_url/ajax.js" \
796 "$base_url/table/table-sort.js" \
797 "$base_url/table/table-sort.css" \
798 "$base_url/base.css"\
799
800 table-sort-begin 'width60'
801
802 cat <<EOF
803 <p id="home-link">
804 <a href="/">oilshell.org</a>
805 </p>
806
807 <h1>Wedge Builds</h1>
808EOF
809
810 tsv2html3 $tasks_tsv
811
812 cat <<EOF
813 <p>
814 <a href="tasks.tsv">tasks.tsv</a>
815 </p>
816EOF
817
818 table-sort-end 'tasks' # ID for sorting
819}
820
821NPROC=$(nproc)
822#NPROC=1
823
824install-wedge-list() {
825 ### Reads task rows from stdin
826 local parallel=${1:-}
827
828 mkdir -p _build/wedge/logs
829
830 local -a flags
831 if test -n "$parallel"; then
832 log ""
833 log "=== Installing wedges with $NPROC jobs in parallel"
834 log ""
835 flags=( -P $NPROC )
836 else
837 log ""
838 log "=== Installing wedges serially"
839 log ""
840 fi
841
842 # Reads from stdin
843 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
844 #
845 # $name $version $wedge_dir
846 xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
847
848 #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
849}
850
851write-task-report() {
852 local tasks_tsv=_build/wedge/logs/tasks.tsv
853
854 python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
855 log "Wrote $tasks_tsv"
856
857 # TODO: version can be right-justified?
858 here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
859column_name type precision strftime
860status integer 0 -
861elapsed_secs float 1 -
862user_secs float 1 -
863start_time float 1 %H:%M:%S
864end_time float 1 %H:%M:%S
865sys_secs float 1 -
866max_rss_KiB integer 0 -
867xargs_slot integer 0 -
868wedge string 0 -
869wedge_HREF string 0 -
870version string 0 -
871EOF
872
873 index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
874 log "Wrote $WEDGE_LOG_DIR/index.html"
875}
876
877install-spec-bin-fast() {
878 spec-bin-wedges | install-wedge-list T
879 write-task-report
880}
881
882fake-py3-libs-wedge() {
883 local name=py3-libs
884 local version=$PY3_LIBS_VERSION
885
886 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
887 local log_file=$WEDGE_LOG_DIR/$name.log.txt
888
889 my-time-tsv --print-header \
890 --field xargs_slot \
891 --field wedge \
892 --field wedge_HREF \
893 --field version \
894 --output $task_file
895
896 # There is no xargs slot!
897 my-time-tsv \
898 --field "-1" \
899 --field "$name" \
900 --field "$name.log.txt" \
901 --field "$version" \
902 --append \
903 --output $task_file \
904 $0 install-py3-libs >$log_file 2>&1 || true
905
906 echo " FAKE $(timestamp) $name $version"
907}
908
909install-wedges-fast() {
910 local extra=${1:-}
911
912 # For contributor setup: we need to use this BEFORE running build/py.sh all
913 build/py.sh time-helper
914
915 echo " START $(timestamp)"
916
917 # Do all of them in parallel
918 if test -n "$extra"; then
919 { contributor-wedges; extra-wedges; } | install-wedge-list T
920 else
921 contributor-wedges | install-wedge-list T
922 fi
923
924 fake-py3-libs-wedge
925 echo " END $(timestamp)"
926
927 write-task-report
928}
929
930install-wedges-soil() {
931 install-wedges-fast extra
932}
933
934#
935# Unboxed wedge builds
936#
937
938uftrace-host() {
939 ### built on demand; run $0 first
940
941 # BUG: doesn't detect python3
942 # WEDGE tells me that it depends on pkg-config
943 # 'apt-get install pkgconf' gets it
944 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
945 deps/wedge.sh unboxed _build/deps-source/uftrace
946}
947
948bloaty-host() {
949 deps/wedge.sh unboxed _build/deps-source/bloaty
950}
951
952R-libs-host() {
953 deps/wedge.sh unboxed _build/deps-source/R-libs
954}
955
956#
957# Wedges built inside a container, for copying into a container
958#
959
960boxed-wedges() {
961 #### host _build/wedge/binary -> guest container /wedge or ~/wedge
962
963 #export-podman
964
965 # TODO:
966 #
967 # - Add equivalents of spec-bin
968 # - Use the same manifest as install-wedges-fast
969 # - so then you can delete the _build/wedge dir to re-run it
970 # - use xargs -n 1 so it's done serially
971
972 # - Do these lazily like we do in install-wedges-fast?
973
974 # We can test if the dir _build/wedge/binary/oils-for-unix.org/pkg/FOO exists
975 # if wedge-exists "$name" "$version" "$wedge_dir"; then
976 # echo "CACHED $(timestamp) $name $version"
977 # return
978 # fi
979
980 if false; then
981 deps/wedge.sh boxed deps/source.medo/time-helper
982 deps/wedge.sh boxed deps/source.medo/cmark/
983 deps/wedge.sh boxed deps/source.medo/re2c/
984 deps/wedge.sh boxed deps/source.medo/python3/
985 fi
986
987 if false; then
988 deps/wedge.sh boxed deps/source.medo/bloaty/
989 fi
990
991 if true; then
992 # build with debian-12, because soil-benchmarks2 is, because it has R
993 deps/wedge.sh boxed deps/source.medo/uftrace/ '' debian-12
994 # python2 needed everywhere
995 #deps/wedge.sh boxed deps/source.medo/python2/ '' debian-12
996
997 # TODO: build with debian-12
998 # Used in {benchmarks,benchmarks2,other-tests}
999 #deps/wedge.sh boxed deps/source.medo/R-libs/ '' debian-12
1000 fi
1001}
1002
1003boxed-spec-bin() {
1004 if true; then
1005 #deps/wedge.sh boxed deps/source.medo/bash '4.4'
1006 deps/wedge.sh boxed deps/source.medo/bash '5.2.21'
1007 fi
1008
1009 if false; then
1010 deps/wedge.sh boxed deps/source.medo/dash
1011 deps/wedge.sh boxed deps/source.medo/mksh
1012 fi
1013
1014 if false; then
1015 # Note: zsh requires libncursesw5-dev
1016 #deps/wedge.sh boxed deps/source.medo/zsh
1017
1018 #deps/wedge.sh boxed deps/source.medo/busybox
1019
1020 # Problem with out of tree build, as above. Skipping for now
1021 deps/wedge.sh boxed deps/source.medo/yash
1022 echo
1023 fi
1024}
1025
1026#
1027# Report
1028#
1029
1030commas() {
1031 # Wow I didn't know this :a trick
1032 #
1033 # OK this is a label and a loop, which makes sense. You can't do it with
1034 # pure regex.
1035 #
1036 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1037 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1038 sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
1039}
1040
1041wedge-sizes() {
1042 local tmp=_tmp/wedge-sizes.txt
1043
1044 # -b is --bytes, but use short flag for busybox compat
1045 du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
1046 { print $0 # print the line
1047 total_bytes += $1 # accumulate
1048 }
1049END { print total_bytes " TOTAL" }
1050' > $tmp
1051
1052 # printf justifies du output
1053 cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
1054 echo
1055
1056 #du -s --si /wedge/*/*/* ~/wedge/*/*/*
1057 #echo
1058}
1059
1060wedge-report() {
1061 # 4 levels deep shows the package
1062 if command -v tree > /dev/null; then
1063 tree -L 4 /wedge ~/wedge
1064 echo
1065 fi
1066
1067 wedge-sizes
1068
1069 local tmp=_tmp/wedge-manifest.txt
1070
1071 echo 'Biggest files'
1072 if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
1073 # busybox find doesn't have -printf
1074 echo 'find -printf failed'
1075 return
1076 fi
1077
1078 set +o errexit # ignore SIGPIPE
1079 sort -n --reverse $tmp | head -n 20 | commas
1080 set -o errexit
1081
1082 echo
1083
1084 # Show the most common file extensions
1085 #
1086 # I feel like we should be able to get rid of .a files? That's 92 MB, second
1087 # most common
1088 #
1089 # There are also duplicate .a files for Python -- should look at how distros
1090 # get rid of those
1091
1092 cat $tmp | python3 -c '
1093import os, sys, collections
1094
1095bytes = collections.Counter()
1096files = collections.Counter()
1097
1098for line in sys.stdin:
1099 size, path = line.split(None, 1)
1100 path = path.strip() # remove newline
1101 _, ext = os.path.splitext(path)
1102 size = int(size)
1103
1104 bytes[ext] += size
1105 files[ext] += 1
1106
1107#print(bytes)
1108#print(files)
1109
1110n = 20
1111
1112print("Most common file types")
1113for ext, count in files.most_common()[:n]:
1114 print("%10d %s" % (count, ext))
1115
1116print()
1117
1118print("Total bytes by file type")
1119for ext, total_bytes in bytes.most_common()[:n]:
1120 print("%10d %s" % (total_bytes, ext))
1121' | commas
1122}
1123
1124run-task "$@"