| 1 | #!/usr/bin/env bash
|
| 2 | #
|
| 3 | # Script for contributors to build dev dependencies -- packaged as cross-distro
|
| 4 | # "wedges". Tested in the Soil CI.
|
| 5 | #
|
| 6 | # Usage:
|
| 7 | # build/deps.sh <function name>
|
| 8 | #
|
| 9 | # Examples:
|
| 10 | # build/deps.sh fetch
|
| 11 | # build/deps.sh install-wedges # for both Python and C++
|
| 12 | # build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
|
| 13 | #
|
| 14 | # TODO: Do we need something faster, just python2, re2c, and cmark?
|
| 15 | #
|
| 16 | # - build/deps.sh fetch-py
|
| 17 | # - build/deps.sh install-wedges-py
|
| 18 | #
|
| 19 | # TODO: Can we make most of them non-root deps? This requires rebuilding
|
| 20 | # containers, which requires podman.
|
| 21 | #
|
| 22 | # rm -r -f ~/wedge # would be better
|
| 23 |
|
| 24 | set -o nounset
|
| 25 | set -o pipefail
|
| 26 | set -o errexit
|
| 27 |
|
| 28 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
| 29 |
|
| 30 | source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
|
| 31 | source deps/from-apt.sh # PY3_BUILD_DEPS
|
| 32 | #source deps/podman.sh
|
| 33 | source devtools/run-task.sh # run-task
|
| 34 | source test/tsv-lib.sh # tsv-concat
|
| 35 | source web/table/html.sh # table-sort-{begin,end}
|
| 36 |
|
| 37 | # Also in build/dev-shell.sh
|
| 38 | USER_WEDGE_DIR=~/wedge/oils-for-unix.org
|
| 39 | ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
|
| 40 |
|
| 41 | readonly DEPS_SOURCE_DIR=_build/deps-source
|
| 42 |
|
| 43 | readonly RE2C_VERSION=3.0
|
| 44 | readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
|
| 45 |
|
| 46 | readonly CMARK_VERSION=0.29.0
|
| 47 | readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
|
| 48 |
|
| 49 | readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
|
| 50 |
|
| 51 | readonly PY2_VERSION=2.7.18
|
| 52 | readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
|
| 53 |
|
| 54 | readonly PY3_VERSION=3.10.4
|
| 55 | readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
|
| 56 |
|
| 57 | readonly BASH_VER=4.4 # don't clobber BASH_VERSION
|
| 58 | readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
|
| 59 |
|
| 60 | # Another version of bash to test
|
| 61 | readonly BASH5_VER=5.2.21
|
| 62 | readonly BASH5_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH5_VER.tar.gz"
|
| 63 |
|
| 64 | readonly DASH_VERSION=0.5.10.2
|
| 65 | readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
|
| 66 |
|
| 67 | readonly ZSH_VERSION=5.1.1
|
| 68 | readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
|
| 69 |
|
| 70 | readonly MKSH_VERSION=R52c
|
| 71 | readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
|
| 72 |
|
| 73 | readonly BUSYBOX_VERSION='1.35.0'
|
| 74 | readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
|
| 75 |
|
| 76 | readonly YASH_VERSION=2.49
|
| 77 | readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
|
| 78 |
|
| 79 | readonly MYPY_GIT_URL=https://github.com/python/mypy
|
| 80 | readonly MYPY_VERSION=0.780
|
| 81 |
|
| 82 | readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
|
| 83 |
|
| 84 | # Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
|
| 85 | # https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
|
| 86 | readonly PYFLAKES_VERSION=2.4.0
|
| 87 | #readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
|
| 88 | # 2023-07: Mirrored to avoid network problem on broome during release
|
| 89 | readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
|
| 90 |
|
| 91 | readonly BLOATY_VERSION=1.1
|
| 92 | readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
|
| 93 |
|
| 94 | readonly UFTRACE_VERSION=0.13
|
| 95 | readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
|
| 96 |
|
| 97 | readonly SOUFFLE_VERSION=2.4.1
|
| 98 | readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
|
| 99 |
|
| 100 | log() {
|
| 101 | echo "$0: $@" >& 2
|
| 102 | }
|
| 103 |
|
| 104 | die() {
|
| 105 | log "$@"
|
| 106 | exit 1
|
| 107 | }
|
| 108 |
|
| 109 | rm-oils-crap() {
|
| 110 | ### When you want to start over
|
| 111 |
|
| 112 | rm -r -f -v ~/wedge
|
| 113 | sudo rm -r -f -v /wedge
|
| 114 | }
|
| 115 |
|
| 116 | # Note: git is an implicit dependency -- that's how we got the repo in the
|
| 117 | # first place!
|
| 118 |
|
| 119 | # python2-dev is no longer available on Debian 12
|
| 120 | # python-dev also seems gone
|
| 121 | #
|
| 122 | # wget: for fetching wedges (not on Debian by default!)
|
| 123 | # tree: tiny package that's useful for showing what we installed
|
| 124 | # g++: essential
|
| 125 | # libreadline-dev: needed for the build/prepare.sh Python build.
|
| 126 | # gawk: used by spec-runner.sh for the special match() function.
|
| 127 | # cmake: for cmark
|
| 128 | # PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
|
| 129 | # as well
|
| 130 | readonly -a WEDGE_DEPS_DEBIAN=(
|
| 131 | bzip2
|
| 132 | wget
|
| 133 | tree
|
| 134 | gawk
|
| 135 | g++
|
| 136 | ninja-build
|
| 137 | cmake
|
| 138 | libreadline-dev
|
| 139 | systemtap-sdt-dev
|
| 140 |
|
| 141 | # for Souffle, flex and bison
|
| 142 | #flex bison
|
| 143 |
|
| 144 | "${PY3_BUILD_DEPS[@]}"
|
| 145 | )
|
| 146 |
|
| 147 | readonly -a WEDGE_DEPS_ALPINE=(
|
| 148 | bzip2
|
| 149 | xz
|
| 150 |
|
| 151 | wget tree gawk
|
| 152 |
|
| 153 | gcc g++
|
| 154 | ninja-build
|
| 155 | # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
|
| 156 | ninja-is-really-ninja
|
| 157 | cmake
|
| 158 |
|
| 159 | readline-dev
|
| 160 | zlib-dev
|
| 161 | libffi-dev
|
| 162 | openssl-dev
|
| 163 |
|
| 164 | ncurses-dev
|
| 165 |
|
| 166 | # for Souffle, flex and bison
|
| 167 | #flex bison
|
| 168 | )
|
| 169 |
|
| 170 | readonly -a WEDGE_DEPS_FEDORA=(
|
| 171 |
|
| 172 | # Weird, Fedora doesn't have these by default!
|
| 173 | hostname
|
| 174 | tar
|
| 175 | bzip2
|
| 176 |
|
| 177 | # https://packages.fedoraproject.org/pkgs/wget/wget/
|
| 178 | wget
|
| 179 | # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
|
| 180 | tree
|
| 181 | gawk
|
| 182 |
|
| 183 | # https://packages.fedoraproject.org/pkgs/gcc/gcc/
|
| 184 | gcc gcc-c++
|
| 185 |
|
| 186 | ninja-build
|
| 187 | cmake
|
| 188 |
|
| 189 | readline-devel
|
| 190 |
|
| 191 | # Like PY3_BUILD_DEPS
|
| 192 | # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
|
| 193 | zlib-devel
|
| 194 | # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
|
| 195 | libffi-devel
|
| 196 | # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
|
| 197 | openssl-devel
|
| 198 |
|
| 199 | # For building zsh from source?
|
| 200 | # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
|
| 201 | ncurses-devel
|
| 202 | #libcap-devel
|
| 203 |
|
| 204 | # still have a job control error compiling bash
|
| 205 | # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
|
| 206 | # glibc-devel
|
| 207 | )
|
| 208 |
|
| 209 | install-debian-packages() {
|
| 210 | ### Packages for build/py.sh all, building wedges, etc.
|
| 211 |
|
| 212 | set -x # show what needs sudo
|
| 213 |
|
| 214 | # pass -y for say gitpod
|
| 215 | sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
|
| 216 | set +x
|
| 217 |
|
| 218 | # maybe pass -y through
|
| 219 | test/spec-bin.sh install-shells-with-apt "$@"
|
| 220 | }
|
| 221 |
|
| 222 | install-ubuntu-packages() {
|
| 223 | ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
|
| 224 | install-debian-packages "$@"
|
| 225 | }
|
| 226 |
|
| 227 | wedge-deps-debian() {
|
| 228 | # Install packages without prompt
|
| 229 |
|
| 230 | # 2024-02 - there was an Ubuntu update, and we started needing this
|
| 231 | sudo apt-get -y update
|
| 232 |
|
| 233 | install-debian-packages -y
|
| 234 | }
|
| 235 |
|
| 236 | wedge-deps-fedora() {
|
| 237 | # https://linuxconfig.org/install-development-tools-on-redhat-8
|
| 238 | # Trying to get past compile errors
|
| 239 | # sudo dnf group install --assumeyes 'Development Tools'
|
| 240 |
|
| 241 | sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
|
| 242 | }
|
| 243 |
|
| 244 | wedge-deps-alpine() {
|
| 245 | # https://linuxconfig.org/install-development-tools-on-redhat-8
|
| 246 | # Trying to get past compile errors
|
| 247 | # sudo dnf group install --assumeyes 'Development Tools'
|
| 248 |
|
| 249 | sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
|
| 250 | }
|
| 251 |
|
| 252 | #
|
| 253 | # Unused patch, was experiment for Fedora
|
| 254 | #
|
| 255 |
|
| 256 | get-typed-ast-patch() {
|
| 257 | curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
|
| 258 | }
|
| 259 |
|
| 260 | # Work around typed_ast bug:
|
| 261 | # https://github.com/python/typed_ast/issues/169
|
| 262 | #
|
| 263 | # Apply this patch
|
| 264 | # https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
|
| 265 | #
|
| 266 | # typed_ast is tarred up though
|
| 267 | patch-typed-ast() {
|
| 268 | local package_dir=_cache/py3-libs
|
| 269 | local patch=$PWD/deps/typed_ast.patch
|
| 270 |
|
| 271 | pushd $package_dir
|
| 272 | cat $patch
|
| 273 | echo
|
| 274 |
|
| 275 | local dir=typed_ast-1.4.3
|
| 276 | local tar=typed_ast-1.4.3.tar.gz
|
| 277 |
|
| 278 | echo OLD
|
| 279 | ls -l $tar
|
| 280 | echo
|
| 281 |
|
| 282 | rm -r -f -v $dir
|
| 283 | tar -x -z < $tar
|
| 284 |
|
| 285 | pushd $dir
|
| 286 | patch -p1 < $patch
|
| 287 | popd
|
| 288 | #find $dir
|
| 289 |
|
| 290 | # Create a new one
|
| 291 | tar --create --gzip --file $tar typed_ast-1.4.3
|
| 292 |
|
| 293 | echo NEW
|
| 294 | ls -l $tar
|
| 295 | echo
|
| 296 |
|
| 297 | popd
|
| 298 | }
|
| 299 |
|
| 300 | #
|
| 301 | # Fetch
|
| 302 | #
|
| 303 |
|
| 304 | download-to() {
|
| 305 | local dir=$1
|
| 306 | local url=$2
|
| 307 | wget --no-clobber --directory-prefix "$dir" "$url"
|
| 308 | }
|
| 309 |
|
| 310 | maybe-extract() {
|
| 311 | local wedge_dir=$1
|
| 312 | local tar_name=$2
|
| 313 | local out_dir=$3
|
| 314 |
|
| 315 | if test -d "$wedge_dir/$out_dir"; then
|
| 316 | log "Not extracting because $wedge_dir/$out_dir exists"
|
| 317 | return
|
| 318 | fi
|
| 319 |
|
| 320 | local tar=$wedge_dir/$tar_name
|
| 321 | case $tar_name in
|
| 322 | *.gz|*.tgz) # mksh ends with .tgz
|
| 323 | flag='--gzip'
|
| 324 | ;;
|
| 325 | *.bz2)
|
| 326 | flag='--bzip2'
|
| 327 | ;;
|
| 328 | *.xz)
|
| 329 | flag='--xz'
|
| 330 | ;;
|
| 331 | *)
|
| 332 | die "tar with unknown extension: $tar_name"
|
| 333 | ;;
|
| 334 | esac
|
| 335 |
|
| 336 | tar --extract $flag --file $tar --directory $wedge_dir
|
| 337 | }
|
| 338 |
|
| 339 | clone-mypy() {
|
| 340 | ### replaces deps/from-git
|
| 341 | local dest_dir=$1
|
| 342 | local version=${2:-$MYPY_VERSION}
|
| 343 |
|
| 344 | local dest=$dest_dir/mypy-$version
|
| 345 | if test -d $dest; then
|
| 346 | log "Not cloning because $dest exists"
|
| 347 | return
|
| 348 | fi
|
| 349 |
|
| 350 | # v$VERSION is a tag, not a branch
|
| 351 |
|
| 352 | # size optimization: --depth=1 --shallow-submodules
|
| 353 | # https://git-scm.com/docs/git-clone
|
| 354 |
|
| 355 | git clone --recursive --branch v$version \
|
| 356 | --depth=1 --shallow-submodules \
|
| 357 | $MYPY_GIT_URL $dest
|
| 358 |
|
| 359 | # TODO: verify commit checksum
|
| 360 | }
|
| 361 |
|
| 362 | copy-source-medo() {
|
| 363 | mkdir -p $DEPS_SOURCE_DIR
|
| 364 |
|
| 365 | # Copy the whole tree, including the .treeptr files
|
| 366 | cp --verbose --recursive --no-target-directory \
|
| 367 | deps/source.medo/ $DEPS_SOURCE_DIR/
|
| 368 | }
|
| 369 |
|
| 370 | fetch-spec-bin() {
|
| 371 | download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
|
| 372 | maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" bash-$BASH_VER
|
| 373 |
|
| 374 | download-to $DEPS_SOURCE_DIR/bash "$BASH5_URL"
|
| 375 | maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH5_URL)" bash-$BASH5_VER
|
| 376 |
|
| 377 | download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
|
| 378 | maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
|
| 379 |
|
| 380 | download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
|
| 381 | maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
|
| 382 |
|
| 383 | download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
|
| 384 | maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
|
| 385 |
|
| 386 | download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
|
| 387 | maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
|
| 388 |
|
| 389 | download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
|
| 390 | maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$YASH_VERSION
|
| 391 |
|
| 392 | # Patch: this tarball doesn't follow the convention $name-$version
|
| 393 | if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
|
| 394 | pushd $DEPS_SOURCE_DIR/mksh
|
| 395 | mv -v mksh mksh-$MKSH_VERSION
|
| 396 | popd
|
| 397 | fi
|
| 398 | }
|
| 399 |
|
| 400 | fetch() {
|
| 401 | local py_only=${1:-}
|
| 402 |
|
| 403 | # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
|
| 404 | # would do: fetch compressed tarballs designated by .treeptr files, and
|
| 405 | # expand them.
|
| 406 |
|
| 407 | # _build/deps-source/
|
| 408 | # re2c/
|
| 409 | # WEDGE
|
| 410 | # re2c-3.0/ # expanded .tar.xz file
|
| 411 |
|
| 412 | copy-source-medo
|
| 413 |
|
| 414 | download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
|
| 415 | download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
|
| 416 | maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
|
| 417 | maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
|
| 418 |
|
| 419 | if test -n "$py_only"; then
|
| 420 | log "Fetched dependencies for 'build/py.sh'"
|
| 421 | return
|
| 422 | fi
|
| 423 |
|
| 424 | download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
|
| 425 | maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
|
| 426 | pyflakes-$PYFLAKES_VERSION
|
| 427 |
|
| 428 | download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
|
| 429 | download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
|
| 430 | maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
|
| 431 | maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
|
| 432 |
|
| 433 | fetch-spec-bin
|
| 434 |
|
| 435 | # bloaty and uftrace are for benchmarks, in containers
|
| 436 | download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
|
| 437 | download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
|
| 438 | maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" bloaty-$BLOATY_VERSION
|
| 439 | maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" uftrace-$UFTRACE_VERSION
|
| 440 |
|
| 441 | # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
|
| 442 |
|
| 443 | # It's also copied into a wedge in install-wedges.
|
| 444 | clone-mypy $DEPS_SOURCE_DIR/mypy
|
| 445 |
|
| 446 | if false; then
|
| 447 | download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
|
| 448 | maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
|
| 449 | fi
|
| 450 |
|
| 451 | if command -v tree > /dev/null; then
|
| 452 | tree -L 2 $DEPS_SOURCE_DIR
|
| 453 | fi
|
| 454 | }
|
| 455 |
|
| 456 | fetch-py() {
|
| 457 | fetch py_only
|
| 458 | }
|
| 459 |
|
| 460 | mirror-pyflakes() {
|
| 461 | ### Workaround for network error during release
|
| 462 | scp \
|
| 463 | $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
|
| 464 | oilshell.org:oilshell.org/blob/
|
| 465 | }
|
| 466 |
|
| 467 | wedge-exists() {
|
| 468 | ### Does an installed wedge already exist?
|
| 469 |
|
| 470 | local name=$1
|
| 471 | local version=$2
|
| 472 | local wedge_dir=${3:-/wedge/oils-for-unix.org}
|
| 473 |
|
| 474 | local installed=$wedge_dir/pkg/$name/$version
|
| 475 |
|
| 476 | if test -d $installed; then
|
| 477 | log "$installed already exists"
|
| 478 | return 0
|
| 479 | else
|
| 480 | return 1
|
| 481 | fi
|
| 482 | }
|
| 483 |
|
| 484 | #
|
| 485 | # Install
|
| 486 | #
|
| 487 |
|
| 488 | # TODO: py3-libs needs to be a WEDGE, so that that you can run
|
| 489 | # 'wedge build deps/source.medo/py3-libs/' and then get it in
|
| 490 | #
|
| 491 | # _build/wedge/{absolute,relative} # which one?
|
| 492 | #
|
| 493 | # It needs a BUILD DEPENDENCY on:
|
| 494 | # - the python3 wedge, so you can do python3 -m pip install.
|
| 495 | # - the mypy repo, which has test-requirements.txt
|
| 496 |
|
| 497 | download-py3-libs() {
|
| 498 | ### Download source/binary packages, AFTER python3 is installed
|
| 499 |
|
| 500 | # Note that this is NOT source code; there is binary code, e.g. in
|
| 501 | # lxml-*.whl
|
| 502 |
|
| 503 | local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
|
| 504 | local py_package_dir=_cache/py3-libs
|
| 505 | mkdir -p $py_package_dir
|
| 506 |
|
| 507 | # Avoids a warning, but doesn't fix typed_ast
|
| 508 | #python3 -m pip download -d $py_package_dir wheel
|
| 509 |
|
| 510 | python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
|
| 511 | python3 -m pip download -d $py_package_dir pexpect
|
| 512 | }
|
| 513 |
|
| 514 | install-py3-libs-in-venv() {
|
| 515 | local venv_dir=$1
|
| 516 | local mypy_dir=$2 # This is a param for host build vs. container build
|
| 517 | local package_dir=_cache/py3-libs
|
| 518 |
|
| 519 | source $venv_dir/bin/activate # enter virtualenv
|
| 520 |
|
| 521 | # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
|
| 522 | # conflicts with MyPy deps!
|
| 523 | # "ERROR: pip's dependency resolver does not currently take into account all
|
| 524 | # the packages that are installed."
|
| 525 |
|
| 526 | # --find-links uses a "cache dir" for packages (weird flag name!)
|
| 527 |
|
| 528 | # Avoids a warning, but doesn't fix typed_ast
|
| 529 | #time python3 -m pip install --find-links $package_dir wheel
|
| 530 |
|
| 531 | # for mycpp/
|
| 532 | time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
|
| 533 |
|
| 534 | # pexpect: for spec/stateful/*.py
|
| 535 | time python3 -m pip install --find-links $package_dir pexpect
|
| 536 | }
|
| 537 |
|
| 538 | install-py3-libs-from-cache() {
|
| 539 |
|
| 540 | # As well as end users
|
| 541 |
|
| 542 | local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
|
| 543 |
|
| 544 | local py3
|
| 545 | py3=$(command -v python3)
|
| 546 | case $py3 in
|
| 547 | *wedge/oils-for-unix.org/*)
|
| 548 | ;;
|
| 549 | *)
|
| 550 | die "python3 is '$py3', but expected it to be in a wedge"
|
| 551 | ;;
|
| 552 | esac
|
| 553 |
|
| 554 | log "Ensuring pip is installed (interpreter $(command -v python3)"
|
| 555 | python3 -m ensurepip
|
| 556 |
|
| 557 | local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
|
| 558 | log "Creating venv in $venv_dir"
|
| 559 |
|
| 560 | # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
|
| 561 | # the /wedge we just built
|
| 562 | python3 -m venv $venv_dir
|
| 563 |
|
| 564 | log "Installing MyPy deps in venv"
|
| 565 |
|
| 566 | # Run in a subshell because it mutates shell state
|
| 567 | $0 install-py3-libs-in-venv $venv_dir $mypy_dir
|
| 568 | }
|
| 569 |
|
| 570 | install-py3-libs() {
|
| 571 | ### Invoked by Dockerfile.cpp-small, etc.
|
| 572 |
|
| 573 | download-py3-libs
|
| 574 | install-py3-libs-from-cache
|
| 575 | }
|
| 576 |
|
| 577 |
|
| 578 | # zsh notes
|
| 579 | # Fedora compiler error
|
| 580 | # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
|
| 581 | # fallback in Src/Modules/termcap.c that causes a compile error! It seems
|
| 582 | # like ncurses-devel should fix this, but it doesn't
|
| 583 | #
|
| 584 | # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
|
| 585 | #
|
| 586 | # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
|
| 587 | # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
|
| 588 | # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
|
| 589 | #
|
| 590 | # I think the ./configure is out of sync with the actual build?
|
| 591 |
|
| 592 |
|
| 593 | # TODO:
|
| 594 | # - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
|
| 595 | # already in each WEDGE file
|
| 596 |
|
| 597 | py-wedges() {
|
| 598 | ### for build/py.sh all
|
| 599 |
|
| 600 | echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
|
| 601 | echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
|
| 602 | echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
|
| 603 | echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
|
| 604 | }
|
| 605 |
|
| 606 | cpp-wedges() {
|
| 607 | ### for ninja / mycpp translation
|
| 608 |
|
| 609 | echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
|
| 610 | echo mypy $MYPY_VERSION $USER_WEDGE_DIR
|
| 611 |
|
| 612 | # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
|
| 613 | # separately for now
|
| 614 | #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
|
| 615 | }
|
| 616 |
|
| 617 | spec-bin-wedges() {
|
| 618 | ### for test/spec-py.sh osh-all
|
| 619 |
|
| 620 | echo dash $DASH_VERSION $USER_WEDGE_DIR
|
| 621 | echo bash $BASH_VER $USER_WEDGE_DIR
|
| 622 | echo bash $BASH5_VER $USER_WEDGE_DIR
|
| 623 | echo mksh $MKSH_VERSION $USER_WEDGE_DIR
|
| 624 | echo zsh $ZSH_VERSION $USER_WEDGE_DIR
|
| 625 | echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
|
| 626 | echo yash $YASH_VERSION $USER_WEDGE_DIR
|
| 627 | }
|
| 628 |
|
| 629 | contributor-wedges() {
|
| 630 | py-wedges
|
| 631 | cpp-wedges
|
| 632 | spec-bin-wedges
|
| 633 | }
|
| 634 |
|
| 635 | extra-wedges() {
|
| 636 | # Contributors don't need uftrace, bloaty, and probably R-libs
|
| 637 | # Although R-libs could be useful for benchmarks
|
| 638 |
|
| 639 | # Test both outside the contianer, as well as inside?
|
| 640 | echo uftrace $UFTRACE_VERSION $ROOT_WEDGE_DIR
|
| 641 |
|
| 642 | #echo souffle $SOUFFLE_VERSION $USER_WEDGE_DIR
|
| 643 | }
|
| 644 |
|
| 645 | timestamp() {
|
| 646 | date '+%H:%M:%S'
|
| 647 | }
|
| 648 |
|
| 649 | my-time-tsv() {
|
| 650 | python3 benchmarks/time_.py \
|
| 651 | --tsv \
|
| 652 | --time-span --rusage \
|
| 653 | "$@"
|
| 654 | }
|
| 655 |
|
| 656 | maybe-install-wedge() {
|
| 657 | local name=$1
|
| 658 | local version=$2
|
| 659 | local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
|
| 660 |
|
| 661 | local task_file=$WEDGE_LOG_DIR/$name-$version.task.tsv
|
| 662 | local log_file=$WEDGE_LOG_DIR/$name-$version.log.txt
|
| 663 |
|
| 664 | echo " TASK $(timestamp) $name $version > $log_file"
|
| 665 |
|
| 666 | # python3 because it's OUTSIDE the container
|
| 667 | # Separate columns that could be joined: number of files, total size
|
| 668 | my-time-tsv --print-header \
|
| 669 | --field xargs_slot \
|
| 670 | --field wedge \
|
| 671 | --field wedge_HREF \
|
| 672 | --field version \
|
| 673 | --output $task_file
|
| 674 |
|
| 675 | if wedge-exists "$name" "$version" "$wedge_dir"; then
|
| 676 | echo "CACHED $(timestamp) $name $version"
|
| 677 | return
|
| 678 | fi
|
| 679 |
|
| 680 | local -a cmd=( deps/wedge.sh unboxed _build/deps-source/$name/ $version)
|
| 681 |
|
| 682 | set +o errexit
|
| 683 | my-time-tsv \
|
| 684 | --field "$XARGS_SLOT" \
|
| 685 | --field "$name" \
|
| 686 | --field "$name-$version.log.txt" \
|
| 687 | --field "$version" \
|
| 688 | --append \
|
| 689 | --output $task_file \
|
| 690 | "${cmd[@]}" "$@" >$log_file 2>&1
|
| 691 | local status=$?
|
| 692 | set -o errexit
|
| 693 |
|
| 694 | if test "$status" -eq 0; then
|
| 695 | echo " OK $(timestamp) $name $version"
|
| 696 | else
|
| 697 | echo " FAIL $(timestamp) $name $version"
|
| 698 | fi
|
| 699 | }
|
| 700 |
|
| 701 | dummy-task() {
|
| 702 | ### For testing log capture
|
| 703 | local name=$1
|
| 704 | local version=$2
|
| 705 |
|
| 706 | echo "Building $name $version"
|
| 707 |
|
| 708 | # random float between 0 and 3
|
| 709 | # weirdly we need a seed from bash
|
| 710 | # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
|
| 711 | local secs
|
| 712 | secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
|
| 713 |
|
| 714 | echo "sleep $secs"
|
| 715 | sleep $secs
|
| 716 |
|
| 717 | echo 'stdout'
|
| 718 | log 'stderr'
|
| 719 |
|
| 720 | if test $name = 'mksh'; then
|
| 721 | echo "simulate failure for $name"
|
| 722 | exit 2
|
| 723 | fi
|
| 724 | }
|
| 725 |
|
| 726 | readonly WEDGE_LOG_DIR=_build/wedge/logs
|
| 727 |
|
| 728 | dummy-task-wrapper() {
|
| 729 | # Similar to test/common.sh run-task-with-status, used by
|
| 730 | # test/{spec,wild}-runner.sh
|
| 731 | local name=$1
|
| 732 | local version=$2
|
| 733 |
|
| 734 | local task_file=$WEDGE_LOG_DIR/$name.task.tsv
|
| 735 | local log_file=$WEDGE_LOG_DIR/$name.log.txt
|
| 736 |
|
| 737 | echo " TASK $(timestamp) $name $version > $log_file"
|
| 738 |
|
| 739 | # python3 because it's OUTSIDE the container
|
| 740 | # Separate columns that could be joined: number of files, total size
|
| 741 | my-time-tsv --print-header \
|
| 742 | --field xargs_slot \
|
| 743 | --field wedge \
|
| 744 | --field wedge_HREF \
|
| 745 | --field version \
|
| 746 | --output $task_file
|
| 747 |
|
| 748 | my-time-tsv \
|
| 749 | --field "$XARGS_SLOT" \
|
| 750 | --field "$name" \
|
| 751 | --field "$name.log.txt" \
|
| 752 | --field "$version" \
|
| 753 | --append \
|
| 754 | --output $task_file \
|
| 755 | $0 dummy-task "$@" >$log_file 2>&1 || true
|
| 756 |
|
| 757 | echo " DONE $(timestamp) $name $version"
|
| 758 | }
|
| 759 |
|
| 760 | html-head() {
|
| 761 | # python3 because we're outside containers
|
| 762 | PYTHONPATH=. python3 doctools/html_head.py "$@"
|
| 763 | }
|
| 764 |
|
| 765 | index-html() {
|
| 766 | local tasks_tsv=$1
|
| 767 |
|
| 768 | local base_url='../../../web'
|
| 769 | html-head --title 'Wedge Builds' \
|
| 770 | "$base_url/ajax.js" \
|
| 771 | "$base_url/table/table-sort.js" \
|
| 772 | "$base_url/table/table-sort.css" \
|
| 773 | "$base_url/base.css"\
|
| 774 |
|
| 775 | table-sort-begin 'width60'
|
| 776 |
|
| 777 | cat <<EOF
|
| 778 | <p id="home-link">
|
| 779 | <a href="/">oilshell.org</a>
|
| 780 | </p>
|
| 781 |
|
| 782 | <h1>Wedge Builds</h1>
|
| 783 | EOF
|
| 784 |
|
| 785 | tsv2html3 $tasks_tsv
|
| 786 |
|
| 787 | cat <<EOF
|
| 788 | <p>
|
| 789 | <a href="tasks.tsv">tasks.tsv</a>
|
| 790 | </p>
|
| 791 | EOF
|
| 792 |
|
| 793 | table-sort-end 'tasks' # ID for sorting
|
| 794 | }
|
| 795 |
|
| 796 | NPROC=$(nproc)
|
| 797 | #NPROC=1
|
| 798 |
|
| 799 | install-wedge-list() {
|
| 800 | ### Reads task rows from stdin
|
| 801 | local parallel=${1:-}
|
| 802 |
|
| 803 | mkdir -p _build/wedge/logs
|
| 804 |
|
| 805 | local -a flags
|
| 806 | if test -n "$parallel"; then
|
| 807 | log ""
|
| 808 | log "=== Installing wedges with $NPROC jobs in parallel"
|
| 809 | log ""
|
| 810 | flags=( -P $NPROC )
|
| 811 | else
|
| 812 | log ""
|
| 813 | log "=== Installing wedges serially"
|
| 814 | log ""
|
| 815 | fi
|
| 816 |
|
| 817 | # Reads from stdin
|
| 818 | # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
|
| 819 | #
|
| 820 | # $name $version $wedge_dir
|
| 821 | xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
|
| 822 |
|
| 823 | #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
|
| 824 | }
|
| 825 |
|
| 826 | write-task-report() {
|
| 827 | local tasks_tsv=_build/wedge/logs/tasks.tsv
|
| 828 |
|
| 829 | python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
|
| 830 | log "Wrote $tasks_tsv"
|
| 831 |
|
| 832 | # TODO: version can be right-justified?
|
| 833 | here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
|
| 834 | column_name type precision strftime
|
| 835 | status integer 0 -
|
| 836 | elapsed_secs float 1 -
|
| 837 | user_secs float 1 -
|
| 838 | start_time float 1 %H:%M:%S
|
| 839 | end_time float 1 %H:%M:%S
|
| 840 | sys_secs float 1 -
|
| 841 | max_rss_KiB integer 0 -
|
| 842 | xargs_slot integer 0 -
|
| 843 | wedge string 0 -
|
| 844 | wedge_HREF string 0 -
|
| 845 | version string 0 -
|
| 846 | EOF
|
| 847 |
|
| 848 | index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
|
| 849 | log "Wrote $WEDGE_LOG_DIR/index.html"
|
| 850 | }
|
| 851 |
|
| 852 | install-spec-bin-fast() {
|
| 853 | spec-bin-wedges | install-wedge-list T
|
| 854 | write-task-report
|
| 855 | }
|
| 856 |
|
| 857 | fake-py3-libs-wedge() {
|
| 858 | local name=py3-libs
|
| 859 | local version=$PY3_LIBS_VERSION
|
| 860 |
|
| 861 | local task_file=$WEDGE_LOG_DIR/$name.task.tsv
|
| 862 | local log_file=$WEDGE_LOG_DIR/$name.log.txt
|
| 863 |
|
| 864 | my-time-tsv --print-header \
|
| 865 | --field xargs_slot \
|
| 866 | --field wedge \
|
| 867 | --field wedge_HREF \
|
| 868 | --field version \
|
| 869 | --output $task_file
|
| 870 |
|
| 871 | # There is no xargs slot!
|
| 872 | my-time-tsv \
|
| 873 | --field "-1" \
|
| 874 | --field "$name" \
|
| 875 | --field "$name.log.txt" \
|
| 876 | --field "$version" \
|
| 877 | --append \
|
| 878 | --output $task_file \
|
| 879 | $0 install-py3-libs >$log_file 2>&1 || true
|
| 880 |
|
| 881 | echo " FAKE $(timestamp) $name $version"
|
| 882 | }
|
| 883 |
|
| 884 | install-wedges-fast() {
|
| 885 | local extra=${1:-}
|
| 886 |
|
| 887 | echo " START $(timestamp)"
|
| 888 |
|
| 889 | # Do all of them in parallel
|
| 890 | if test -n "$extra"; then
|
| 891 | { contributor-wedges; extra-wedges; } | install-wedge-list T
|
| 892 | else
|
| 893 | contributor-wedges | install-wedge-list T
|
| 894 | fi
|
| 895 |
|
| 896 | fake-py3-libs-wedge
|
| 897 | echo " END $(timestamp)"
|
| 898 |
|
| 899 | write-task-report
|
| 900 | }
|
| 901 |
|
| 902 | install-wedges-soil() {
|
| 903 | install-wedges-fast extra
|
| 904 | }
|
| 905 |
|
| 906 | #
|
| 907 | # Unboxed wedge builds
|
| 908 | #
|
| 909 |
|
| 910 | uftrace-host() {
|
| 911 | ### built on demand; run $0 first
|
| 912 |
|
| 913 | # BUG: doesn't detect python3
|
| 914 | # WEDGE tells me that it depends on pkg-config
|
| 915 | # 'apt-get install pkgconf' gets it
|
| 916 | # TODO: Should use python3 WEDGE instead of SYSTEM python3?
|
| 917 | deps/wedge.sh unboxed _build/deps-source/uftrace
|
| 918 | }
|
| 919 |
|
| 920 | bloaty-host() {
|
| 921 | deps/wedge.sh unboxed _build/deps-source/bloaty
|
| 922 | }
|
| 923 |
|
| 924 | R-libs-host() {
|
| 925 | deps/wedge.sh unboxed _build/deps-source/R-libs
|
| 926 | }
|
| 927 |
|
| 928 | #
|
| 929 | # Wedges built inside a container, for copying into a container
|
| 930 | #
|
| 931 |
|
| 932 | boxed-wedges() {
|
| 933 | #### host _build/wedge/binary -> guest container /wedge or ~/wedge
|
| 934 |
|
| 935 | #export-podman
|
| 936 |
|
| 937 | # TODO:
|
| 938 | #
|
| 939 | # - Add equivalents of spec-bin
|
| 940 | # - Use the same manifest as install-wedges-fast
|
| 941 | # - so then you can delete the _build/wedge dir to re-run it
|
| 942 | # - use xargs -n 1 so it's done serially
|
| 943 |
|
| 944 | # - Do these lazily like we do in install-wedges-fast?
|
| 945 |
|
| 946 | # We can test if the dir _build/wedge/binary/oils-for-unix.org/pkg/FOO exists
|
| 947 | # if wedge-exists "$name" "$version" "$wedge_dir"; then
|
| 948 | # echo "CACHED $(timestamp) $name $version"
|
| 949 | # return
|
| 950 | # fi
|
| 951 |
|
| 952 | if false; then
|
| 953 | deps/wedge.sh boxed deps/source.medo/time-helper
|
| 954 | deps/wedge.sh boxed deps/source.medo/cmark/
|
| 955 | deps/wedge.sh boxed deps/source.medo/re2c/
|
| 956 | deps/wedge.sh boxed deps/source.medo/python3/
|
| 957 | fi
|
| 958 |
|
| 959 | if false; then
|
| 960 | deps/wedge.sh boxed deps/source.medo/bloaty/
|
| 961 | fi
|
| 962 |
|
| 963 | if true; then
|
| 964 | # build with debian-12, because soil-benchmarks2 is, because it has R
|
| 965 | deps/wedge.sh boxed deps/source.medo/uftrace/ '' debian-12
|
| 966 | # python2 needed everywhere
|
| 967 | #deps/wedge.sh boxed deps/source.medo/python2/ '' debian-12
|
| 968 |
|
| 969 | # TODO: build with debian-12
|
| 970 | # Used in {benchmarks,benchmarks2,other-tests}
|
| 971 | #deps/wedge.sh boxed deps/source.medo/R-libs/ '' debian-12
|
| 972 | fi
|
| 973 | }
|
| 974 |
|
| 975 | boxed-spec-bin() {
|
| 976 | if true; then
|
| 977 | #deps/wedge.sh boxed deps/source.medo/bash '4.4'
|
| 978 | deps/wedge.sh boxed deps/source.medo/bash '5.2.21'
|
| 979 | fi
|
| 980 |
|
| 981 | if false; then
|
| 982 | deps/wedge.sh boxed deps/source.medo/dash
|
| 983 | deps/wedge.sh boxed deps/source.medo/mksh
|
| 984 | fi
|
| 985 |
|
| 986 | if false; then
|
| 987 | # Note: zsh requires libncursesw5-dev
|
| 988 | #deps/wedge.sh boxed deps/source.medo/zsh
|
| 989 |
|
| 990 | #deps/wedge.sh boxed deps/source.medo/busybox
|
| 991 |
|
| 992 | # Problem with out of tree build, as above. Skipping for now
|
| 993 | deps/wedge.sh boxed deps/source.medo/yash
|
| 994 | echo
|
| 995 | fi
|
| 996 | }
|
| 997 |
|
| 998 | #
|
| 999 | # Report
|
| 1000 | #
|
| 1001 |
|
| 1002 | commas() {
|
| 1003 | # Wow I didn't know this :a trick
|
| 1004 | #
|
| 1005 | # OK this is a label and a loop, which makes sense. You can't do it with
|
| 1006 | # pure regex.
|
| 1007 | #
|
| 1008 | # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
|
| 1009 | # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
|
| 1010 | sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
|
| 1011 | }
|
| 1012 |
|
| 1013 | wedge-sizes() {
|
| 1014 | local tmp=_tmp/wedge-sizes.txt
|
| 1015 |
|
| 1016 | # -b is --bytes, but use short flag for busybox compat
|
| 1017 | du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
|
| 1018 | { print $0 # print the line
|
| 1019 | total_bytes += $1 # accumulate
|
| 1020 | }
|
| 1021 | END { print total_bytes " TOTAL" }
|
| 1022 | ' > $tmp
|
| 1023 |
|
| 1024 | # printf justifies du output
|
| 1025 | cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
|
| 1026 | echo
|
| 1027 |
|
| 1028 | #du -s --si /wedge/*/*/* ~/wedge/*/*/*
|
| 1029 | #echo
|
| 1030 | }
|
| 1031 |
|
| 1032 | wedge-report() {
|
| 1033 | # 4 levels deep shows the package
|
| 1034 | if command -v tree > /dev/null; then
|
| 1035 | tree -L 4 /wedge ~/wedge
|
| 1036 | echo
|
| 1037 | fi
|
| 1038 |
|
| 1039 | wedge-sizes
|
| 1040 |
|
| 1041 | local tmp=_tmp/wedge-manifest.txt
|
| 1042 |
|
| 1043 | echo 'Biggest files'
|
| 1044 | if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
|
| 1045 | # busybox find doesn't have -printf
|
| 1046 | echo 'find -printf failed'
|
| 1047 | return
|
| 1048 | fi
|
| 1049 |
|
| 1050 | set +o errexit # ignore SIGPIPE
|
| 1051 | sort -n --reverse $tmp | head -n 20 | commas
|
| 1052 | set -o errexit
|
| 1053 |
|
| 1054 | echo
|
| 1055 |
|
| 1056 | # Show the most common file extensions
|
| 1057 | #
|
| 1058 | # I feel like we should be able to get rid of .a files? That's 92 MB, second
|
| 1059 | # most common
|
| 1060 | #
|
| 1061 | # There are also duplicate .a files for Python -- should look at how distros
|
| 1062 | # get rid of those
|
| 1063 |
|
| 1064 | cat $tmp | python3 -c '
|
| 1065 | import os, sys, collections
|
| 1066 |
|
| 1067 | bytes = collections.Counter()
|
| 1068 | files = collections.Counter()
|
| 1069 |
|
| 1070 | for line in sys.stdin:
|
| 1071 | size, path = line.split(None, 1)
|
| 1072 | path = path.strip() # remove newline
|
| 1073 | _, ext = os.path.splitext(path)
|
| 1074 | size = int(size)
|
| 1075 |
|
| 1076 | bytes[ext] += size
|
| 1077 | files[ext] += 1
|
| 1078 |
|
| 1079 | #print(bytes)
|
| 1080 | #print(files)
|
| 1081 |
|
| 1082 | n = 20
|
| 1083 |
|
| 1084 | print("Most common file types")
|
| 1085 | for ext, count in files.most_common()[:n]:
|
| 1086 | print("%10d %s" % (count, ext))
|
| 1087 |
|
| 1088 | print()
|
| 1089 |
|
| 1090 | print("Total bytes by file type")
|
| 1091 | for ext, total_bytes in bytes.most_common()[:n]:
|
| 1092 | print("%10d %s" % (total_bytes, ext))
|
| 1093 | ' | commas
|
| 1094 | }
|
| 1095 |
|
| 1096 | run-task "$@"
|