1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Build a wedge.
|
4 | #
|
5 | # Usage:
|
6 | # deps/wedge.sh <function name>
|
7 | #
|
8 | # Containerized build:
|
9 | #
|
10 | # $0 build deps/source.medo/re2c/
|
11 | #
|
12 | # Host build, without containers:
|
13 | #
|
14 | # $0 unboxed deps/source.medo/re2c/
|
15 | #
|
16 | # Individual steps:
|
17 | #
|
18 | # $0 unboxed-make deps/source.medo/re2c/
|
19 | # $0 unboxed-install deps/source.medo/re2c/
|
20 | # $0 unboxed-smoke-test deps/source.medo/re2c/
|
21 | #
|
22 | # Host dir structure:
|
23 | #
|
24 | # ~/git/oilshell/oil
|
25 | # deps/
|
26 | # source.medo/ # Source Files
|
27 | # MEDO # points to silo
|
28 | # re2c.wedge.sh # later it will be re2c.wedge.hay
|
29 | # re2c-3.0.blob # .tar.gz file that you can 'medo sync'
|
30 | # re2c-3.1.blob
|
31 | # opaque.medo/ # Binary files, e.g. Clang
|
32 | # derived.medo/ # Saved output of 'wedge build'
|
33 | #
|
34 | # _build/ # Temp dirs and output
|
35 | # obj/ # for C++ / Ninja
|
36 | # deps-source/ # sync'd from deps/source.medo - should it be
|
37 | # # _build/wedge/source?
|
38 | # wedge/ # for containerized builds
|
39 | # tmp/ # build directory
|
40 | # boxed/ # output of containerized build
|
41 | # # TODO: rename from /binary/
|
42 | # logs/
|
43 | # smoke-test/ # current dir for smoke test
|
44 |
|
45 | # Every package ("wedge") has these dirs associated with it:
|
46 | #
|
47 | # 1. Dir with additional tests / files, near tarball and *.wedge.sh ($wedge_dir)
|
48 | # 2. Where it's extracted ($src_dir)
|
49 | # 3. The temp dir where you run ./configure --prefix; make; make install ($build_dir)
|
50 | # 4. The dir to install to ($install_dir)
|
51 | # 5. The temp dir where the smoke test is run
|
52 |
|
53 | # For Debian/Ubuntu
|
54 |
|
55 | # Note: xz-utils needed to extract, but medo should make that transparent?
|
56 | #
|
57 | # Container dir structure
|
58 | #
|
59 | # /home/uke/
|
60 | # tmp-mount/
|
61 | # _build/ # Build into this temp dir
|
62 | # deps-source/
|
63 | # re2c/
|
64 | # re2c-3.0.tar.xz
|
65 | # re2c-3.0/ # Extract it here
|
66 | # wedge/
|
67 | # re2c
|
68 | # /wedge/ # Output is mounted to oil/_mount/wedge-out
|
69 | # oilshell.org/
|
70 | # pkg/
|
71 | # re2c/
|
72 | # 3.0/
|
73 | # debug-info/ # Probably needs to be at an absolute path because of
|
74 | # # --debug-link
|
75 | # re2c/
|
76 | # 3.0/
|
77 | #
|
78 | # Then Dockerfile.wild does:
|
79 | #
|
80 | # COPY _build/wedge/binary/oils-for-unix.org/pkg/re2c/3.0 \
|
81 | # /wedge/oils-for-unix.org/pkg/re2c/3.0
|
82 |
|
83 | set -o nounset
|
84 | set -o pipefail
|
85 | set -o errexit
|
86 |
|
87 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
88 | readonly REPO_ROOT
|
89 |
|
90 | OILS_ABSOLUTE_ROOT='/wedge/oils-for-unix.org'
|
91 |
|
92 | # The user may build a wedge outside a container here
|
93 | OILS_RELATIVE_ROOT="$HOME/wedge/oils-for-unix.org"
|
94 |
|
95 | die() {
|
96 | echo "$0: $@" >& 2
|
97 | exit 1
|
98 | }
|
99 |
|
100 | #
|
101 | # Dirs
|
102 | #
|
103 |
|
104 | source-dir() {
|
105 | if test -n "${WEDGE_TARBALL_NAME:-}"; then
|
106 | # for Python-3.10.4 to override 'python3' package name
|
107 | echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_TARBALL_NAME-$WEDGE_VERSION"
|
108 |
|
109 | else
|
110 | echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_NAME-$WEDGE_VERSION"
|
111 | fi
|
112 | }
|
113 |
|
114 | build-dir() {
|
115 | # call it tmp-build?
|
116 | echo "$REPO_ROOT/_build/wedge/tmp/$WEDGE_NAME-$WEDGE_VERSION"
|
117 | }
|
118 |
|
119 | install-dir() {
|
120 | local prefix
|
121 | if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
|
122 | prefix=$OILS_ABSOLUTE_ROOT
|
123 | else
|
124 | prefix=$OILS_RELATIVE_ROOT
|
125 | fi
|
126 |
|
127 | # TODO: We want to support multiple versions of the same wedge
|
128 | # So maybe we can provide
|
129 | #
|
130 | # WEDGE_VERSION_LIST='4.4 5.2'
|
131 | #
|
132 | # And then provide a flag to select them?
|
133 |
|
134 | echo "$prefix/pkg/$WEDGE_NAME/$WEDGE_VERSION"
|
135 | }
|
136 |
|
137 | smoke-test-dir() {
|
138 | echo "$REPO_ROOT/_build/wedge/smoke-test/$WEDGE_NAME-$WEDGE_VERSION"
|
139 | }
|
140 |
|
141 | load-wedge() {
|
142 | ### source .wedge.sh file and ensure it conforms to protocol
|
143 |
|
144 | local wedge_dir=$1
|
145 | local version_requested=${2:-}
|
146 |
|
147 | echo "Loading $wedge_dir"
|
148 | echo
|
149 |
|
150 | source $wedge_dir/WEDGE
|
151 |
|
152 | echo " OK name: ${WEDGE_NAME?"$wedge_dir: WEDGE_NAME required"}"
|
153 |
|
154 | # This WEDGE supports a single version.
|
155 | if test -n "${WEDGE_VERSION:-}"; then
|
156 | echo " -- single version: $WEDGE_VERSION"
|
157 | fi
|
158 |
|
159 | # Can validate version against this
|
160 | if test -n "${WEDGE_VERSION_LIST:-}"; then
|
161 | echo " -- version list: $WEDGE_VERSION_LIST"
|
162 |
|
163 | if test -z "$version_requested"; then
|
164 | die "FAIL Expected explicit version, one of: $WEDGE_VERSION_LIST"
|
165 | fi
|
166 |
|
167 | case "$WEDGE_VERSION_LIST" in
|
168 | *"$version_requested"*)
|
169 | echo " OK Setting WEDGE_VERSION to $version_requested"
|
170 | WEDGE_VERSION=$version_requested
|
171 | ;;
|
172 | *)
|
173 | die "FAIL Requested version $version_requested should be one of: $WEDGE_VERSION_LIST"
|
174 | ;;
|
175 | esac
|
176 | fi
|
177 |
|
178 | if test -n "${WEDGE_TARBALL_NAME:-}"; then
|
179 | echo " -- tarball name: $WEDGE_TARBALL_NAME"
|
180 | fi
|
181 | if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
|
182 | echo ' -- WEDGE_IS_ABSOLUTE'
|
183 | fi
|
184 |
|
185 | # Python and R installation use the network
|
186 | if test -n "${WEDGE_LEAKY_BUILD:-}"; then
|
187 | echo ' -- WEDGE_LEAKY_BUILD'
|
188 | fi
|
189 |
|
190 | if declare -f wedge-make; then
|
191 | echo " OK wedge-make"
|
192 | elif declare -f wedge-make-from-source-dir; then
|
193 | echo " OK wedge-make-from-source-dir"
|
194 | else
|
195 | die "$wedge_dir: wedge-make(-from-source-dir) not declared"
|
196 | fi
|
197 |
|
198 | if declare -f wedge-install; then
|
199 | echo " OK wedge-install"
|
200 | elif declare -f wedge-make-from-source-dir; then
|
201 | echo " OK wedge-install-from-source-dir"
|
202 | else
|
203 | die "$wedge_dir: wedge-install(-from-source-dir) not declared"
|
204 | fi
|
205 |
|
206 | # Just one function for now
|
207 | for func in wedge-smoke-test; do
|
208 | if declare -f $func > /dev/null; then
|
209 | echo " OK $func"
|
210 | else
|
211 | die "$wedge_dir: $func not declared"
|
212 | fi
|
213 | done
|
214 | echo
|
215 |
|
216 | echo "Loaded $wedge_dir"
|
217 | echo
|
218 | }
|
219 |
|
220 | _run-sourced-func() {
|
221 | "$@"
|
222 | }
|
223 |
|
224 | #
|
225 | # Actions
|
226 | #
|
227 |
|
228 | validate() {
|
229 | local wedge=$1
|
230 | local version_requested=${2:-}
|
231 |
|
232 | load-wedge $wedge "$version_requested"
|
233 | }
|
234 |
|
235 | unboxed-make() {
|
236 | ### Build on the host
|
237 |
|
238 | local wedge=$1 # e.g. re2c.wedge.sh
|
239 | local version_requested=${2:-} # e.g. 5.2
|
240 |
|
241 | load-wedge $wedge "$version_requested"
|
242 |
|
243 | local source_dir
|
244 | source_dir=$(source-dir)
|
245 | echo " SRC $source_dir"
|
246 |
|
247 | local build_dir
|
248 | build_dir=$(build-dir)
|
249 |
|
250 | # NOT created because it might require root permissions!
|
251 | local install_dir
|
252 | install_dir=$(install-dir)
|
253 |
|
254 | rm -r -f -v $build_dir
|
255 | mkdir -p $build_dir
|
256 |
|
257 | if declare -f wedge-make-from-source-dir; then
|
258 | # e.g. for yash, which can't build outside the source tree
|
259 | pushd $source_dir
|
260 | wedge-make-from-source-dir $source_dir $install_dir
|
261 | popd
|
262 | else
|
263 | pushd $build_dir
|
264 | wedge-make $source_dir $build_dir $install_dir
|
265 | popd
|
266 | fi
|
267 | }
|
268 |
|
269 |
|
270 | # https://www.gnu.org/prep/standards/html_node/Standard-Targets.html
|
271 |
|
272 | # Do not strip executables when installing them. This helps eventual
|
273 | # debugging that may be needed later, and nowadays disk space is cheap and
|
274 | # dynamic loaders typically ensure debug sections are not loaded during
|
275 | # normal execution. Users that need stripped binaries may invoke the
|
276 | # install-strip target to do that.
|
277 |
|
278 | _unboxed-install() {
|
279 | local wedge=$1 # e.g. re2c.wedge.sh
|
280 | local version_requested=${2:-} # e.g. 5.2
|
281 |
|
282 | load-wedge $wedge "$version_requested"
|
283 |
|
284 | local source_dir
|
285 | source_dir=$(source-dir)
|
286 |
|
287 | local build_dir
|
288 | build_dir=$(build-dir)
|
289 |
|
290 | local install_dir
|
291 | install_dir=$(install-dir)
|
292 | mkdir -p $install_dir
|
293 |
|
294 | if declare -f wedge-make-from-source-dir; then
|
295 | pushd $source_dir
|
296 | wedge-install-from-source-dir $source_dir $install_dir
|
297 | popd
|
298 | else
|
299 | # Note: install-dir needed for time-helper, but not others
|
300 | #
|
301 | # I think it would nicer to pushd $build_dir in most cases
|
302 |
|
303 | wedge-install $build_dir $install_dir
|
304 | fi
|
305 | }
|
306 |
|
307 | unboxed-install() {
|
308 | local wedge=$1 # e.g. re2.wedge.sh
|
309 |
|
310 | if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
|
311 | sudo $0 _unboxed-install "$@"
|
312 | else
|
313 | _unboxed-install "$@"
|
314 | fi
|
315 | }
|
316 |
|
317 | unboxed-smoke-test() {
|
318 | local wedge_dir=$1 # e.g. re2c/ with WEDGE
|
319 | local version_requested=${2:-} # e.g. 5.2
|
320 |
|
321 | load-wedge $wedge_dir "$version_requested"
|
322 |
|
323 | local smoke_test_dir
|
324 | smoke_test_dir=$(smoke-test-dir)
|
325 | local install_dir
|
326 | install_dir=$(install-dir)
|
327 |
|
328 | echo ' SMOKE TEST'
|
329 |
|
330 | local abs_wedge_dir
|
331 | case $wedge_dir in
|
332 | /*) # it's already absolute
|
333 | abs_wedge_dir=$wedge_dir
|
334 | ;;
|
335 | *)
|
336 | abs_wedge_dir=$PWD/$wedge_dir
|
337 | ;;
|
338 | esac
|
339 |
|
340 | # TODO: To ensure a clean dir, it might be better to test that it does NOT
|
341 | # exist first, and just make it. If it exists, then remove everything.
|
342 |
|
343 | rm -r -f -v $smoke_test_dir
|
344 | mkdir -p $smoke_test_dir
|
345 |
|
346 | pushd $smoke_test_dir
|
347 | set -x
|
348 | wedge-smoke-test $install_dir $abs_wedge_dir
|
349 | set +x
|
350 | popd
|
351 |
|
352 | echo ' OK'
|
353 | }
|
354 |
|
355 | unboxed-stats() {
|
356 | local wedge=$1
|
357 |
|
358 | load-wedge $wedge "$version_requested"
|
359 |
|
360 | du --si -s $(source-dir)
|
361 | echo
|
362 |
|
363 | du --si -s $(build-dir)
|
364 | echo
|
365 |
|
366 | du --si -s $(install-dir)
|
367 | echo
|
368 | }
|
369 |
|
370 | unboxed() {
|
371 | local wedge_dir=$1
|
372 |
|
373 | # Can override default version. Could be a flag since it's optional? But
|
374 | # right now we always pass it.
|
375 | local version_requested=${2:-}
|
376 |
|
377 | # TODO:
|
378 | # - Would be nice to export the logs somewhere
|
379 |
|
380 | unboxed-make $wedge_dir "$version_requested"
|
381 |
|
382 | unboxed-install $wedge_dir "$version_requested"
|
383 |
|
384 | unboxed-smoke-test $wedge_dir "$version_requested"
|
385 | }
|
386 |
|
387 | readonly DEFAULT_DISTRO=debian-10 # Debian Buster
|
388 |
|
389 | DOCKER=${DOCKER:-docker}
|
390 |
|
391 | boxed() {
|
392 | ### Build inside a container, and put output in a specific place.
|
393 |
|
394 | # TODO: Specify the container OS, CPU like x86-64, etc.
|
395 |
|
396 | local wedge=$1
|
397 | local version_requested=${2:-}
|
398 | local distro=${3:-$DEFAULT_DISTRO}
|
399 |
|
400 | local bootstrap_image=oilshell/wedge-bootstrap-$distro
|
401 |
|
402 | load-wedge $wedge "$version_requested"
|
403 |
|
404 | # Permissions will be different, so we separate the two
|
405 |
|
406 | local wedge_host_dir
|
407 | local wedge_guest_dir
|
408 | if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
|
409 | wedge_host_dir=_build/wedge/binary # TODO: rename to /absolute/
|
410 | wedge_guest_dir=/wedge
|
411 | else
|
412 | wedge_host_dir=_build/wedge/relative
|
413 | wedge_guest_dir=/home/uke0/wedge
|
414 | fi
|
415 |
|
416 | mkdir -v -p $wedge_host_dir
|
417 |
|
418 | # TODO:
|
419 | #
|
420 | # Mount
|
421 | # INPUTS: the PKG.wedge.sh, and the tarball
|
422 | # CODE: this script: deps/wedge.sh
|
423 | # OUTPUT: /wedge/oils-for-unix.org
|
424 | # TODO: Also put logs and symbols somewhere
|
425 |
|
426 | # Run unboxed-{build,install,smoke-test} INSIDE the container
|
427 | local -a args=(
|
428 | sh -c 'cd ~/oil; deps/wedge.sh unboxed "$1" "$2"'
|
429 | dummy "$wedge" "$version_requested"
|
430 | )
|
431 |
|
432 | local -a docker_flags=()
|
433 | if test -n "${WEDGE_LEAKY_BUILD:-}"; then
|
434 | :
|
435 | else
|
436 | # Disable network for hermetic builds. TODO: Add automated test
|
437 | docker_flags=( --network none )
|
438 | fi
|
439 |
|
440 | # TODO:
|
441 | # - Don't mount the whole REPO_ROOT
|
442 | # - We want the bare minimum of files, for cache invalidation
|
443 | # - Maybe make it read only
|
444 | # - Bind mount WEDGE_DEPS='', space separated list of paths
|
445 | # - py3-libs depends on python3 and mypy wedges!
|
446 |
|
447 | # -E to preserve CONTAINERS_REGISTRIES_CONF
|
448 | sudo -E $DOCKER run "${docker_flags[@]}" \
|
449 | --mount "type=bind,source=$REPO_ROOT,target=/home/uke0/oil" \
|
450 | --mount "type=bind,source=$PWD/$wedge_host_dir,target=$wedge_guest_dir" \
|
451 | $bootstrap_image \
|
452 | "${args[@]}"
|
453 | }
|
454 |
|
455 | smoke-test() {
|
456 | local wedge_dir=$1
|
457 | local wedge_out_dir=${2:-_build/wedge/binary} # TODO: rename to /boxed
|
458 | local version_requested=${3:-}
|
459 | local distro=${4:-$DEFAULT_DISTRO}
|
460 | local debug_shell=${5:-}
|
461 |
|
462 | load-wedge $wedge_dir "$version_requested"
|
463 |
|
464 | local bootstrap_image=oilshell/wedge-bootstrap-$distro
|
465 |
|
466 | local -a args=(
|
467 | sh -c 'cd ~/oil; deps/wedge.sh unboxed-smoke-test $1' dummy "$wedge_dir"
|
468 | )
|
469 | local -a docker_flags=()
|
470 | if test -n "$debug_shell"; then
|
471 | docker_flags=( -i -t )
|
472 | args=( "$debug_shell" )
|
473 | fi
|
474 |
|
475 | local wedge_mount_dir
|
476 | if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
|
477 | wedge_mount_dir=/wedge
|
478 | else
|
479 | wedge_mount_dir=/home/uke0/wedge
|
480 | fi
|
481 |
|
482 | sudo $DOCKER run "${docker_flags[@]}" \
|
483 | --network none \
|
484 | --mount "type=bind,source=$REPO_ROOT,target=/home/uke0/oil" \
|
485 | --mount "type=bind,source=$PWD/$wedge_out_dir,target=$wedge_mount_dir" \
|
486 | $bootstrap_image \
|
487 | "${args[@]}"
|
488 | }
|
489 |
|
490 | if [[ $# -eq 0 || $1 =~ ^(--help|-h)$ ]]; then
|
491 | # A trick for help. TODO: Move this to a common file, and combine with help
|
492 | # in test/spec.sh.
|
493 |
|
494 | awk '
|
495 | $0 ~ /^#/ { print $0 }
|
496 | $0 !~ /^#/ { print ""; exit }
|
497 | ' $0
|
498 | exit
|
499 | fi
|
500 |
|
501 | case $1 in
|
502 | validate|\
|
503 | unboxed|\
|
504 | unboxed-make|unboxed-install|_unboxed-install|\
|
505 | unboxed-smoke-test|unboxed-stats|\
|
506 | boxed|smoke-test)
|
507 | "$@"
|
508 | ;;
|
509 |
|
510 | *)
|
511 | die "$0: Invalid action '$1'"
|
512 | ;;
|
513 | esac
|