OILS / deps / wedge.sh View on Github | oilshell.org

513 lines, 270 significant
1#!/usr/bin/env bash
2#
3# Build a wedge.
4#
5# Usage:
6# deps/wedge.sh <function name>
7#
8# Containerized build:
9#
10# $0 build deps/source.medo/re2c/
11#
12# Host build, without containers:
13#
14# $0 unboxed deps/source.medo/re2c/
15#
16# Individual steps:
17#
18# $0 unboxed-make deps/source.medo/re2c/
19# $0 unboxed-install deps/source.medo/re2c/
20# $0 unboxed-smoke-test deps/source.medo/re2c/
21#
22# Host dir structure:
23#
24# ~/git/oilshell/oil
25# deps/
26# source.medo/ # Source Files
27# MEDO # points to silo
28# re2c.wedge.sh # later it will be re2c.wedge.hay
29# re2c-3.0.blob # .tar.gz file that you can 'medo sync'
30# re2c-3.1.blob
31# opaque.medo/ # Binary files, e.g. Clang
32# derived.medo/ # Saved output of 'wedge build'
33#
34# _build/ # Temp dirs and output
35# obj/ # for C++ / Ninja
36# deps-source/ # sync'd from deps/source.medo - should it be
37# # _build/wedge/source?
38# wedge/ # for containerized builds
39# tmp/ # build directory
40# boxed/ # output of containerized build
41# # TODO: rename from /binary/
42# logs/
43# smoke-test/ # current dir for smoke test
44
45# Every package ("wedge") has these dirs associated with it:
46#
47# 1. Dir with additional tests / files, near tarball and *.wedge.sh ($wedge_dir)
48# 2. Where it's extracted ($src_dir)
49# 3. The temp dir where you run ./configure --prefix; make; make install ($build_dir)
50# 4. The dir to install to ($install_dir)
51# 5. The temp dir where the smoke test is run
52
53# For Debian/Ubuntu
54
55# Note: xz-utils needed to extract, but medo should make that transparent?
56#
57# Container dir structure
58#
59# /home/uke/
60# tmp-mount/
61# _build/ # Build into this temp dir
62# deps-source/
63# re2c/
64# re2c-3.0.tar.xz
65# re2c-3.0/ # Extract it here
66# wedge/
67# re2c
68# /wedge/ # Output is mounted to oil/_mount/wedge-out
69# oilshell.org/
70# pkg/
71# re2c/
72# 3.0/
73# debug-info/ # Probably needs to be at an absolute path because of
74# # --debug-link
75# re2c/
76# 3.0/
77#
78# Then Dockerfile.wild does:
79#
80# COPY _build/wedge/binary/oils-for-unix.org/pkg/re2c/3.0 \
81# /wedge/oils-for-unix.org/pkg/re2c/3.0
82
83set -o nounset
84set -o pipefail
85set -o errexit
86
87REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
88readonly REPO_ROOT
89
90OILS_ABSOLUTE_ROOT='/wedge/oils-for-unix.org'
91
92# The user may build a wedge outside a container here
93OILS_RELATIVE_ROOT="$HOME/wedge/oils-for-unix.org"
94
95die() {
96 echo "$0: $@" >& 2
97 exit 1
98}
99
100#
101# Dirs
102#
103
104source-dir() {
105 if test -n "${WEDGE_TARBALL_NAME:-}"; then
106 # for Python-3.10.4 to override 'python3' package name
107 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_TARBALL_NAME-$WEDGE_VERSION"
108
109 else
110 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_NAME-$WEDGE_VERSION"
111 fi
112}
113
114build-dir() {
115 # call it tmp-build?
116 echo "$REPO_ROOT/_build/wedge/tmp/$WEDGE_NAME-$WEDGE_VERSION"
117}
118
119install-dir() {
120 local prefix
121 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
122 prefix=$OILS_ABSOLUTE_ROOT
123 else
124 prefix=$OILS_RELATIVE_ROOT
125 fi
126
127 # TODO: We want to support multiple versions of the same wedge
128 # So maybe we can provide
129 #
130 # WEDGE_VERSION_LIST='4.4 5.2'
131 #
132 # And then provide a flag to select them?
133
134 echo "$prefix/pkg/$WEDGE_NAME/$WEDGE_VERSION"
135}
136
137smoke-test-dir() {
138 echo "$REPO_ROOT/_build/wedge/smoke-test/$WEDGE_NAME-$WEDGE_VERSION"
139}
140
141load-wedge() {
142 ### source .wedge.sh file and ensure it conforms to protocol
143
144 local wedge_dir=$1
145 local version_requested=${2:-}
146
147 echo "Loading $wedge_dir"
148 echo
149
150 source $wedge_dir/WEDGE
151
152 echo " OK name: ${WEDGE_NAME?"$wedge_dir: WEDGE_NAME required"}"
153
154 # This WEDGE supports a single version.
155 if test -n "${WEDGE_VERSION:-}"; then
156 echo " -- single version: $WEDGE_VERSION"
157 fi
158
159 # Can validate version against this
160 if test -n "${WEDGE_VERSION_LIST:-}"; then
161 echo " -- version list: $WEDGE_VERSION_LIST"
162
163 if test -z "$version_requested"; then
164 die "FAIL Expected explicit version, one of: $WEDGE_VERSION_LIST"
165 fi
166
167 case "$WEDGE_VERSION_LIST" in
168 *"$version_requested"*)
169 echo " OK Setting WEDGE_VERSION to $version_requested"
170 WEDGE_VERSION=$version_requested
171 ;;
172 *)
173 die "FAIL Requested version $version_requested should be one of: $WEDGE_VERSION_LIST"
174 ;;
175 esac
176 fi
177
178 if test -n "${WEDGE_TARBALL_NAME:-}"; then
179 echo " -- tarball name: $WEDGE_TARBALL_NAME"
180 fi
181 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
182 echo ' -- WEDGE_IS_ABSOLUTE'
183 fi
184
185 # Python and R installation use the network
186 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
187 echo ' -- WEDGE_LEAKY_BUILD'
188 fi
189
190 if declare -f wedge-make; then
191 echo " OK wedge-make"
192 elif declare -f wedge-make-from-source-dir; then
193 echo " OK wedge-make-from-source-dir"
194 else
195 die "$wedge_dir: wedge-make(-from-source-dir) not declared"
196 fi
197
198 if declare -f wedge-install; then
199 echo " OK wedge-install"
200 elif declare -f wedge-make-from-source-dir; then
201 echo " OK wedge-install-from-source-dir"
202 else
203 die "$wedge_dir: wedge-install(-from-source-dir) not declared"
204 fi
205
206 # Just one function for now
207 for func in wedge-smoke-test; do
208 if declare -f $func > /dev/null; then
209 echo " OK $func"
210 else
211 die "$wedge_dir: $func not declared"
212 fi
213 done
214 echo
215
216 echo "Loaded $wedge_dir"
217 echo
218}
219
220_run-sourced-func() {
221 "$@"
222}
223
224#
225# Actions
226#
227
228validate() {
229 local wedge=$1
230 local version_requested=${2:-}
231
232 load-wedge $wedge "$version_requested"
233}
234
235unboxed-make() {
236 ### Build on the host
237
238 local wedge=$1 # e.g. re2c.wedge.sh
239 local version_requested=${2:-} # e.g. 5.2
240
241 load-wedge $wedge "$version_requested"
242
243 local source_dir
244 source_dir=$(source-dir)
245 echo " SRC $source_dir"
246
247 local build_dir
248 build_dir=$(build-dir)
249
250 # NOT created because it might require root permissions!
251 local install_dir
252 install_dir=$(install-dir)
253
254 rm -r -f -v $build_dir
255 mkdir -p $build_dir
256
257 if declare -f wedge-make-from-source-dir; then
258 # e.g. for yash, which can't build outside the source tree
259 pushd $source_dir
260 wedge-make-from-source-dir $source_dir $install_dir
261 popd
262 else
263 pushd $build_dir
264 wedge-make $source_dir $build_dir $install_dir
265 popd
266 fi
267}
268
269
270# https://www.gnu.org/prep/standards/html_node/Standard-Targets.html
271
272# Do not strip executables when installing them. This helps eventual
273# debugging that may be needed later, and nowadays disk space is cheap and
274# dynamic loaders typically ensure debug sections are not loaded during
275# normal execution. Users that need stripped binaries may invoke the
276# install-strip target to do that.
277
278_unboxed-install() {
279 local wedge=$1 # e.g. re2c.wedge.sh
280 local version_requested=${2:-} # e.g. 5.2
281
282 load-wedge $wedge "$version_requested"
283
284 local source_dir
285 source_dir=$(source-dir)
286
287 local build_dir
288 build_dir=$(build-dir)
289
290 local install_dir
291 install_dir=$(install-dir)
292 mkdir -p $install_dir
293
294 if declare -f wedge-make-from-source-dir; then
295 pushd $source_dir
296 wedge-install-from-source-dir $source_dir $install_dir
297 popd
298 else
299 # Note: install-dir needed for time-helper, but not others
300 #
301 # I think it would nicer to pushd $build_dir in most cases
302
303 wedge-install $build_dir $install_dir
304 fi
305}
306
307unboxed-install() {
308 local wedge=$1 # e.g. re2.wedge.sh
309
310 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
311 sudo $0 _unboxed-install "$@"
312 else
313 _unboxed-install "$@"
314 fi
315}
316
317unboxed-smoke-test() {
318 local wedge_dir=$1 # e.g. re2c/ with WEDGE
319 local version_requested=${2:-} # e.g. 5.2
320
321 load-wedge $wedge_dir "$version_requested"
322
323 local smoke_test_dir
324 smoke_test_dir=$(smoke-test-dir)
325 local install_dir
326 install_dir=$(install-dir)
327
328 echo ' SMOKE TEST'
329
330 local abs_wedge_dir
331 case $wedge_dir in
332 /*) # it's already absolute
333 abs_wedge_dir=$wedge_dir
334 ;;
335 *)
336 abs_wedge_dir=$PWD/$wedge_dir
337 ;;
338 esac
339
340 # TODO: To ensure a clean dir, it might be better to test that it does NOT
341 # exist first, and just make it. If it exists, then remove everything.
342
343 rm -r -f -v $smoke_test_dir
344 mkdir -p $smoke_test_dir
345
346 pushd $smoke_test_dir
347 set -x
348 wedge-smoke-test $install_dir $abs_wedge_dir
349 set +x
350 popd
351
352 echo ' OK'
353}
354
355unboxed-stats() {
356 local wedge=$1
357
358 load-wedge $wedge "$version_requested"
359
360 du --si -s $(source-dir)
361 echo
362
363 du --si -s $(build-dir)
364 echo
365
366 du --si -s $(install-dir)
367 echo
368}
369
370unboxed() {
371 local wedge_dir=$1
372
373 # Can override default version. Could be a flag since it's optional? But
374 # right now we always pass it.
375 local version_requested=${2:-}
376
377 # TODO:
378 # - Would be nice to export the logs somewhere
379
380 unboxed-make $wedge_dir "$version_requested"
381
382 unboxed-install $wedge_dir "$version_requested"
383
384 unboxed-smoke-test $wedge_dir "$version_requested"
385}
386
387readonly DEFAULT_DISTRO=debian-10 # Debian Buster
388
389DOCKER=${DOCKER:-docker}
390
391boxed() {
392 ### Build inside a container, and put output in a specific place.
393
394 # TODO: Specify the container OS, CPU like x86-64, etc.
395
396 local wedge=$1
397 local version_requested=${2:-}
398 local distro=${3:-$DEFAULT_DISTRO}
399
400 local bootstrap_image=oilshell/wedge-bootstrap-$distro
401
402 load-wedge $wedge "$version_requested"
403
404 # Permissions will be different, so we separate the two
405
406 local wedge_host_dir
407 local wedge_guest_dir
408 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
409 wedge_host_dir=_build/wedge/binary # TODO: rename to /absolute/
410 wedge_guest_dir=/wedge
411 else
412 wedge_host_dir=_build/wedge/relative
413 wedge_guest_dir=/home/uke0/wedge
414 fi
415
416 mkdir -v -p $wedge_host_dir
417
418 # TODO:
419 #
420 # Mount
421 # INPUTS: the PKG.wedge.sh, and the tarball
422 # CODE: this script: deps/wedge.sh
423 # OUTPUT: /wedge/oils-for-unix.org
424 # TODO: Also put logs and symbols somewhere
425
426 # Run unboxed-{build,install,smoke-test} INSIDE the container
427 local -a args=(
428 sh -c 'cd ~/oil; deps/wedge.sh unboxed "$1" "$2"'
429 dummy "$wedge" "$version_requested"
430 )
431
432 local -a docker_flags=()
433 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
434 :
435 else
436 # Disable network for hermetic builds. TODO: Add automated test
437 docker_flags=( --network none )
438 fi
439
440 # TODO:
441 # - Don't mount the whole REPO_ROOT
442 # - We want the bare minimum of files, for cache invalidation
443 # - Maybe make it read only
444 # - Bind mount WEDGE_DEPS='', space separated list of paths
445 # - py3-libs depends on python3 and mypy wedges!
446
447 # -E to preserve CONTAINERS_REGISTRIES_CONF
448 sudo -E $DOCKER run "${docker_flags[@]}" \
449 --mount "type=bind,source=$REPO_ROOT,target=/home/uke0/oil" \
450 --mount "type=bind,source=$PWD/$wedge_host_dir,target=$wedge_guest_dir" \
451 $bootstrap_image \
452 "${args[@]}"
453}
454
455smoke-test() {
456 local wedge_dir=$1
457 local wedge_out_dir=${2:-_build/wedge/binary} # TODO: rename to /boxed
458 local version_requested=${3:-}
459 local distro=${4:-$DEFAULT_DISTRO}
460 local debug_shell=${5:-}
461
462 load-wedge $wedge_dir "$version_requested"
463
464 local bootstrap_image=oilshell/wedge-bootstrap-$distro
465
466 local -a args=(
467 sh -c 'cd ~/oil; deps/wedge.sh unboxed-smoke-test $1' dummy "$wedge_dir"
468 )
469 local -a docker_flags=()
470 if test -n "$debug_shell"; then
471 docker_flags=( -i -t )
472 args=( "$debug_shell" )
473 fi
474
475 local wedge_mount_dir
476 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
477 wedge_mount_dir=/wedge
478 else
479 wedge_mount_dir=/home/uke0/wedge
480 fi
481
482 sudo $DOCKER run "${docker_flags[@]}" \
483 --network none \
484 --mount "type=bind,source=$REPO_ROOT,target=/home/uke0/oil" \
485 --mount "type=bind,source=$PWD/$wedge_out_dir,target=$wedge_mount_dir" \
486 $bootstrap_image \
487 "${args[@]}"
488}
489
490if [[ $# -eq 0 || $1 =~ ^(--help|-h)$ ]]; then
491 # A trick for help. TODO: Move this to a common file, and combine with help
492 # in test/spec.sh.
493
494 awk '
495 $0 ~ /^#/ { print $0 }
496 $0 !~ /^#/ { print ""; exit }
497 ' $0
498 exit
499fi
500
501case $1 in
502 validate|\
503 unboxed|\
504 unboxed-make|unboxed-install|_unboxed-install|\
505 unboxed-smoke-test|unboxed-stats|\
506 boxed|smoke-test)
507 "$@"
508 ;;
509
510 *)
511 die "$0: Invalid action '$1'"
512 ;;
513esac