| 1 | #!/usr/bin/env bash
|
| 2 | #
|
| 3 | # Usage:
|
| 4 | # deps/from-apt.sh <function name>
|
| 5 |
|
| 6 | set -o nounset
|
| 7 | set -o pipefail
|
| 8 | set -o errexit
|
| 9 |
|
| 10 | # These are needed for bootstrapping pip in Python 3.10
|
| 11 | # (Also used by build/py.sh ubuntu-deps)
|
| 12 | #
|
| 13 | # For building Python 3.10 with working 'pip install'
|
| 14 | # libssl-dev: to download packages
|
| 15 | # libffi-dev: for working setuptools
|
| 16 | # zlib1g-dev: needed for 'import zlib'
|
| 17 | declare -a PY3_BUILD_DEPS=(libssl-dev libffi-dev zlib1g-dev)
|
| 18 |
|
| 19 | # for deps/from-R.sh
|
| 20 | declare -a R_BUILD_DEPS=(
|
| 21 | r-base-core # R interpreter
|
| 22 |
|
| 23 | # ICU for the R stringi package. This makes compilation faster; otherwise
|
| 24 | # it tries to compile it from source.
|
| 25 | # https://stringi.gagolewski.com/install.html
|
| 26 | libicu-dev
|
| 27 | )
|
| 28 |
|
| 29 | install-R() {
|
| 30 | ### For manual use OUTSIDE container
|
| 31 | apt-install "${R_BUILD_DEPS[@]}"
|
| 32 | }
|
| 33 |
|
| 34 | # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache
|
| 35 |
|
| 36 | # TODO: Use this for ALL images
|
| 37 | apt-install() {
|
| 38 | ### Helper to slim down images
|
| 39 |
|
| 40 | apt-get install -y --no-install-recommends "$@"
|
| 41 | }
|
| 42 |
|
| 43 | init-deb-cache() {
|
| 44 | rm -f /etc/apt/apt.conf.d/docker-clean
|
| 45 | echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
|
| 46 | }
|
| 47 |
|
| 48 | readonly -a BUILD_PACKAGES=(
|
| 49 | gcc
|
| 50 | g++ # re2c is C++
|
| 51 | make # to build re2c
|
| 52 |
|
| 53 | # for cmark
|
| 54 | cmake
|
| 55 |
|
| 56 | # cmake -G Ninja can be used
|
| 57 | ninja-build
|
| 58 |
|
| 59 | # For 'deps/wedge.sh unboxed-install'
|
| 60 | sudo
|
| 61 |
|
| 62 | # uftrace configure uses pkg-config to find python3 flags
|
| 63 | pkg-config
|
| 64 |
|
| 65 | # 2024-06: I think this is necessary for the boxed build of zsh
|
| 66 | # Not sure why the wedge-setup-debian VM build doesn't need it. Oh probably
|
| 67 | # because Github Actions pre-installs many packages for you.
|
| 68 | libncursesw5-dev
|
| 69 |
|
| 70 | # for USDT probes
|
| 71 | systemtap-sdt-dev
|
| 72 |
|
| 73 | # Dependencies for building our own Python3 wedge. Otherwise 'pip install'
|
| 74 | # won't work.
|
| 75 | # TODO: We should move 'pip install' to build time.
|
| 76 | "${PY3_BUILD_DEPS[@]}"
|
| 77 |
|
| 78 | # For installing R packages
|
| 79 | "${R_BUILD_DEPS[@]}"
|
| 80 | )
|
| 81 |
|
| 82 | layer-wedge-bootstrap-debian() {
|
| 83 | apt-get update
|
| 84 |
|
| 85 | # Pass aditional deps
|
| 86 | apt-install "${BUILD_PACKAGES[@]}" "$@"
|
| 87 | }
|
| 88 |
|
| 89 | layer-wedge-bootstrap-debian-10() {
|
| 90 | # Default packages
|
| 91 | layer-wedge-bootstrap-debian
|
| 92 | }
|
| 93 |
|
| 94 | layer-wedge-bootstrap-debian-12() {
|
| 95 | local -a uftrace_packages=(
|
| 96 | # uftrace configure detects with #include "Python.h"
|
| 97 | python3-dev
|
| 98 | # shared library for uftrace to do dlopen()
|
| 99 | # requires path in uftrace source
|
| 100 | libpython3.11
|
| 101 |
|
| 102 | #libpython3.7
|
| 103 | )
|
| 104 |
|
| 105 | layer-wedge-bootstrap-debian "${uftrace_packages[@]}"
|
| 106 | }
|
| 107 |
|
| 108 | layer-python-symlink() {
|
| 109 | ### A special layer for building CPython; done as root
|
| 110 | ln -s -f -v /usr/bin/python2 /usr/bin/python
|
| 111 | }
|
| 112 |
|
| 113 | layer-debian-10() {
|
| 114 | # Can't install packages in Debian without this
|
| 115 | apt-get update # uses /var/lib/apt
|
| 116 |
|
| 117 | # uses /var/cache/apt
|
| 118 | apt-install git python2
|
| 119 | }
|
| 120 |
|
| 121 | layer-debian-12() {
|
| 122 | # Can't install packages in Debian without this
|
| 123 | apt-get update # uses /var/lib/apt
|
| 124 |
|
| 125 | # uses /var/cache/apt
|
| 126 | # Soil's time-tsv3 can run under python3 too
|
| 127 | apt-install git python3
|
| 128 | }
|
| 129 |
|
| 130 | layer-locales() {
|
| 131 | apt-install locales
|
| 132 | # uncomment in a file
|
| 133 | sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
|
| 134 | locale-gen --purge en_US.UTF-8
|
| 135 | }
|
| 136 |
|
| 137 | test-image() {
|
| 138 | ### For testing builds, not run on CI
|
| 139 |
|
| 140 | apt-install build-essential "${PY3_BUILD_DEPS[@]}"
|
| 141 | }
|
| 142 |
|
| 143 | wild() {
|
| 144 | # for build/py.sh all
|
| 145 | local -a packages=(
|
| 146 | gcc # 'cc' for compiling Python extensions
|
| 147 | g++ # for C++ tarball
|
| 148 |
|
| 149 | python2-dev
|
| 150 | libreadline-dev
|
| 151 | curl # wait for cpp-tarball
|
| 152 | )
|
| 153 |
|
| 154 | apt-install "${packages[@]}"
|
| 155 | }
|
| 156 |
|
| 157 | dev-minimal() {
|
| 158 | local -a packages=(
|
| 159 | # TODO: remove
|
| 160 | python2-dev # for building Python extensions
|
| 161 | python-setuptools # Python 2, for flake8
|
| 162 | python-pip # flake8 typing
|
| 163 |
|
| 164 | gcc # for building Python extensions
|
| 165 | libreadline-dev
|
| 166 |
|
| 167 | python3-setuptools # mypy
|
| 168 | python3-pip
|
| 169 | # 2023-07: somehow this became necessary to pip3 install typed_ast, a MyPy
|
| 170 | # dep, which recently updated to version 1.5.5
|
| 171 | python3-dev
|
| 172 |
|
| 173 | # Note: osh-minimal task needs shells; testing WITHOUT spec-bin shells
|
| 174 | busybox-static mksh zsh
|
| 175 |
|
| 176 | gawk
|
| 177 |
|
| 178 | # 'ps' used by spec tests
|
| 179 | procps
|
| 180 | # for oil-spec task
|
| 181 | jq
|
| 182 | )
|
| 183 |
|
| 184 | apt-install "${packages[@]}"
|
| 185 | }
|
| 186 |
|
| 187 | pea() {
|
| 188 | # For installing MyPy
|
| 189 | # apt-install python3-pip
|
| 190 |
|
| 191 | echo 'None'
|
| 192 | }
|
| 193 |
|
| 194 | other-tests() {
|
| 195 | local -a packages=(
|
| 196 | libreadline-dev
|
| 197 | python2-dev # osh2oil needs build/py.sh minimal
|
| 198 |
|
| 199 | # Compilers for R. TODO: try removing after wedge
|
| 200 | gcc g++
|
| 201 |
|
| 202 | make # to build py27.grammar.marshal, ugh
|
| 203 |
|
| 204 | r-base-core
|
| 205 | )
|
| 206 |
|
| 207 | apt-install "${packages[@]}"
|
| 208 | }
|
| 209 |
|
| 210 | cpp-small() {
|
| 211 | local -a packages=(
|
| 212 | # for build/py.sh all
|
| 213 | libreadline-dev
|
| 214 | python2-dev
|
| 215 |
|
| 216 | # To compile Oil
|
| 217 | g++
|
| 218 | ninja-build
|
| 219 |
|
| 220 | # For 32-bit binaries with -m32
|
| 221 | gcc-multilib
|
| 222 | g++-multilib
|
| 223 |
|
| 224 | # For some tests
|
| 225 | gawk
|
| 226 |
|
| 227 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 228 | ca-certificates
|
| 229 |
|
| 230 | # for test/ltrace
|
| 231 | ltrace
|
| 232 |
|
| 233 | # for USDT probes
|
| 234 | systemtap-sdt-dev
|
| 235 | )
|
| 236 |
|
| 237 | apt-install "${packages[@]}"
|
| 238 | }
|
| 239 |
|
| 240 | benchmarks() {
|
| 241 | ### For benchmarks
|
| 242 |
|
| 243 | local -a packages=(
|
| 244 | # for build/py.sh all
|
| 245 | libreadline-dev
|
| 246 | python2-dev
|
| 247 |
|
| 248 | # To build Oils
|
| 249 | g++
|
| 250 | ninja-build
|
| 251 | make # to build R packages
|
| 252 |
|
| 253 | # to create _test/index.html
|
| 254 | gawk
|
| 255 |
|
| 256 | # for stable benchmarks. TODO: could move osh-parser cachegrind to benchmarks2
|
| 257 | valgrind
|
| 258 |
|
| 259 | # benchmarks compare system shells -- they don't use our spec-bin? In case
|
| 260 | # there are perf bugs caused by the build
|
| 261 | busybox-static mksh zsh
|
| 262 |
|
| 263 | # retrieving deps like benchmarks/osh-runtime -- TODO: move to build time
|
| 264 | wget
|
| 265 | bzip2 # extracting benchmarks/osh-runtime
|
| 266 | xz-utils
|
| 267 |
|
| 268 | # For analyzing benchmarks.
|
| 269 | r-base-core
|
| 270 |
|
| 271 | # pgrep used by test/stateful in interactive task
|
| 272 | # TODO: Could move both Python and C++ to their own image
|
| 273 | # That will be a good use case once we have
|
| 274 | procps
|
| 275 | )
|
| 276 |
|
| 277 | apt-install "${packages[@]}"
|
| 278 | }
|
| 279 |
|
| 280 | bloaty() {
|
| 281 | local -a packages=(
|
| 282 | g++ # for C++ tarball
|
| 283 | curl # wait for cpp-tarball
|
| 284 | )
|
| 285 |
|
| 286 | apt-install "${packages[@]}"
|
| 287 | }
|
| 288 |
|
| 289 | benchmarks2() {
|
| 290 | ### uftrace needs a Python plugin
|
| 291 |
|
| 292 | local -a packages=(
|
| 293 | curl # fetch C++ tarball
|
| 294 | g++ # build it
|
| 295 |
|
| 296 | # uftrace needs a Python 3 plugin
|
| 297 | # This is different than 'python3' or 'python3.11' -- it's only the shared
|
| 298 | # lib?
|
| 299 | libpython3.11
|
| 300 |
|
| 301 | # for stable benchmarks.
|
| 302 | valgrind
|
| 303 |
|
| 304 | # Analyze uftrace
|
| 305 | r-base-core
|
| 306 | )
|
| 307 |
|
| 308 | apt-install "${packages[@]}"
|
| 309 | }
|
| 310 |
|
| 311 | cpp-spec() {
|
| 312 | ### For cpp-spec
|
| 313 |
|
| 314 | local -a packages=(
|
| 315 | # for build/py.sh all
|
| 316 | libreadline-dev
|
| 317 | python2-dev
|
| 318 |
|
| 319 | # To build Oil
|
| 320 | g++
|
| 321 | ninja-build
|
| 322 |
|
| 323 | # to create _test/index.html
|
| 324 | gawk
|
| 325 |
|
| 326 | # spec tests use these
|
| 327 | procps
|
| 328 | jq
|
| 329 |
|
| 330 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 331 | ca-certificates
|
| 332 | )
|
| 333 |
|
| 334 | apt-install "${packages[@]}"
|
| 335 | }
|
| 336 |
|
| 337 | clang() {
|
| 338 | ### For cpp-coverage
|
| 339 |
|
| 340 | local -a packages=(
|
| 341 | # For build/py.sh minimal
|
| 342 | libreadline-dev
|
| 343 | python2-dev
|
| 344 |
|
| 345 | # Compile Oils
|
| 346 | g++
|
| 347 | ninja-build
|
| 348 |
|
| 349 | xz-utils # to extract Clang
|
| 350 |
|
| 351 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 352 | ca-certificates
|
| 353 | )
|
| 354 |
|
| 355 | apt-install "${packages[@]}"
|
| 356 | }
|
| 357 |
|
| 358 | ovm-tarball() {
|
| 359 | local -a packages=(
|
| 360 | # build/py.sh all
|
| 361 | libreadline-dev
|
| 362 | python2-dev
|
| 363 |
|
| 364 | # retrieving spec-bin -- TODO: move to build time
|
| 365 | wget
|
| 366 | # for wget https://. TODO: remove when the build is hermetic
|
| 367 | ca-certificates
|
| 368 |
|
| 369 | # when spec tests use 'time', dash falls back on 'time' command
|
| 370 | 'time'
|
| 371 |
|
| 372 | # TODO: probably can remove C++ compiler now that re2c is a wedge
|
| 373 | gcc
|
| 374 | g++
|
| 375 |
|
| 376 | # for cmark
|
| 377 | cmake
|
| 378 | # to build Python-2.7.13 (could be a wedge)
|
| 379 | make
|
| 380 |
|
| 381 | xz-utils # extract e.g. zsh/yash tarballs
|
| 382 | bzip2 # extract e.g. busybox tarball
|
| 383 |
|
| 384 | # for syscall measurements
|
| 385 | strace
|
| 386 |
|
| 387 | # used by test/spec-runner.sh
|
| 388 | gawk
|
| 389 | )
|
| 390 |
|
| 391 | apt-install "${packages[@]}"
|
| 392 | }
|
| 393 |
|
| 394 | app-tests() {
|
| 395 | local -a packages=(
|
| 396 | # build/py.sh all
|
| 397 | libreadline-dev
|
| 398 | python2-dev
|
| 399 |
|
| 400 | # retrieving spec-bin -- TODO: move to build time
|
| 401 | wget
|
| 402 | # for wget https://. TODO: remove when the build is hermetic
|
| 403 | ca-certificates
|
| 404 |
|
| 405 | curl # wait for cpp-tarball
|
| 406 |
|
| 407 | gcc
|
| 408 | g++ # for C++ tarball
|
| 409 |
|
| 410 | # to build ble.sh
|
| 411 | make
|
| 412 | # used by ble.sh
|
| 413 | gawk
|
| 414 | procps
|
| 415 |
|
| 416 | # for ble.sh contra
|
| 417 | libx11-dev
|
| 418 | libxft-dev
|
| 419 | libncursesw5-dev
|
| 420 | )
|
| 421 |
|
| 422 | apt-install "${packages[@]}"
|
| 423 | }
|
| 424 |
|
| 425 | if test $(basename $0) = 'from-apt.sh'; then
|
| 426 | "$@"
|
| 427 | fi
|