1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Run continuous build tasks.
|
4 | #
|
5 | # Usage:
|
6 | # soil/worker.sh <function name>
|
7 |
|
8 | set -o nounset
|
9 | set -o pipefail
|
10 | set -o errexit
|
11 |
|
12 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd) # tsv-lib.sh uses this
|
13 | readonly REPO_ROOT
|
14 |
|
15 | source soil/common.sh
|
16 | source test/tsv-lib.sh
|
17 |
|
18 | py-all-and-ninja() {
|
19 | ### baseline for most tasks
|
20 |
|
21 | build/py.sh all
|
22 | ./NINJA-config.sh
|
23 | }
|
24 |
|
25 | ninja-config() {
|
26 | ./NINJA-config.sh
|
27 | }
|
28 |
|
29 | dummy-tasks() {
|
30 | ### Print tasks that execute quickly
|
31 |
|
32 | # (task_name, script, action, result_html)
|
33 | cat <<EOF
|
34 | os-info soil/diagnose.sh os-info -
|
35 | dump-env soil/diagnose.sh dump-env -
|
36 | EOF
|
37 | }
|
38 |
|
39 | raw-vm-tasks() {
|
40 | # The perf tool depends on a specific version of a kernel, so run it outside
|
41 | # a container.
|
42 |
|
43 | # (task_name, script, action, result_html)
|
44 | cat <<EOF
|
45 | os-info soil/diagnose.sh os-info -
|
46 | dump-env soil/diagnose.sh dump-env -
|
47 | perf-install benchmarks/perf.sh soil-install -
|
48 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
49 | test-install-tar devtools/release-native.sh test-install-tar -
|
50 | perf-profiles benchmarks/perf.sh soil-run _tmp/perf/index.html
|
51 | EOF
|
52 | }
|
53 |
|
54 | # Oh there is a large list of pre-installed software
|
55 | # https://github.com/actions/runner-images#available-images
|
56 | # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
|
57 | # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
|
58 | #
|
59 | # 1. System deps for building wedges - ninja, cmake, libreadline-dev, etc.
|
60 | # 2. fetch wedges - re2c, cmark, python2, python3, MyPy, pyflakes
|
61 | # - Python 3.10 desired for "pea"
|
62 | # 3. build them
|
63 | # 4. build Oils with them
|
64 |
|
65 | dev-setup-for() {
|
66 | local distro=$1
|
67 | # (task_name, script, action, result_html)
|
68 |
|
69 | cat <<EOF
|
70 | os-info soil/diagnose.sh os-info -
|
71 | dump-env soil/diagnose.sh dump-env -
|
72 | wedge-deps build/deps.sh wedge-deps-$distro -
|
73 | fetch build/deps.sh fetch -
|
74 | install-wedges build/deps.sh install-wedges-soil _build/wedge/logs/index.html
|
75 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
76 | smoke-test build/dev-setup-test.sh smoke-test -
|
77 | wedge-report build/deps.sh wedge-report -
|
78 | EOF
|
79 | }
|
80 |
|
81 | spec-bin-for() {
|
82 | local distro=$1
|
83 | cat <<EOF
|
84 | os-info soil/diagnose.sh os-info -
|
85 | dump-env soil/diagnose.sh dump-env -
|
86 | wedge-deps build/deps.sh wedge-deps-$distro -
|
87 | fetch build/deps.sh fetch -
|
88 | spec-bin build/deps.sh install-spec-bin-fast _build/wedge/logs/index.html
|
89 | EOF
|
90 | }
|
91 |
|
92 | dev-setup-debian-tasks() {
|
93 | # (task_name, script, action, result_html)
|
94 |
|
95 | dev-setup-for debian
|
96 | }
|
97 |
|
98 | dev-setup-fedora-tasks() {
|
99 | # (task_name, script, action, result_html)
|
100 |
|
101 | dev-setup-for fedora
|
102 | }
|
103 |
|
104 | dev-setup-alpine-tasks() {
|
105 | # (task_name, script, action, result_html)
|
106 |
|
107 | dev-setup-for alpine
|
108 | }
|
109 |
|
110 | pea-tasks() {
|
111 | ### Print tasks for the 'pea' build
|
112 |
|
113 | # We need a later version of Python 3 / MyPy both to type check and
|
114 | # to parse
|
115 |
|
116 | # Run py-source so we can type check generated code
|
117 | # We need to type check more than we translate
|
118 |
|
119 | # (task_name, script, action, result_html)
|
120 | cat <<EOF
|
121 | os-info soil/diagnose.sh os-info -
|
122 | dump-env soil/diagnose.sh dump-env -
|
123 | py-source build/py.sh py-source -
|
124 | check-types pea/TEST.sh check-types -
|
125 | run-tests pea/TEST.sh run-tests -
|
126 | parse-all pea/TEST.sh parse-all -
|
127 | EOF
|
128 | }
|
129 |
|
130 | dev-minimal-tasks() {
|
131 | ### Print tasks for the 'dev-minimal' build
|
132 |
|
133 | # repo overview is suggested by README.md
|
134 |
|
135 | # (task_name, script, action, result_html)
|
136 | cat <<EOF
|
137 | os-info soil/diagnose.sh os-info -
|
138 | dump-env soil/diagnose.sh dump-env -
|
139 | build-minimal build/py.sh minimal -
|
140 | repo-overview metrics/source-code.sh overview -
|
141 | lint test/lint.sh soil-run -
|
142 | asdl-types asdl/TEST.sh check-types -
|
143 | oil-types devtools/types.sh soil-run -
|
144 | unit test/unit.sh minimal _test/py-unit/
|
145 | lossless test/lossless.sh soil-run -
|
146 | parse-errors test/parse-errors.sh soil-run-py -
|
147 | runtime-errors test/runtime-errors.sh soil-run-py -
|
148 | ysh-parse-errors test/ysh-parse-errors.sh soil-run-py -
|
149 | ysh-runtime-errors test/ysh-runtime-errors.sh soil-run-py -
|
150 | ysh-every-string test/ysh-every-string.sh soil-run-py -
|
151 | ysh-large ysh/run.sh soil-run -
|
152 | j8-errors data_lang/j8-errors.sh soil-run-py -
|
153 | link-busybox-ash test/spec-bin.sh link-busybox-ash -
|
154 | osh-minimal test/spec-py.sh osh-minimal _tmp/spec/osh-minimal/index.html
|
155 | headless client/run.sh soil-run-py -
|
156 | EOF
|
157 | }
|
158 |
|
159 | interactive-tasks() {
|
160 | ### Print tasks for the 'interactive' build
|
161 |
|
162 | # TODO: also run interactive suite with osh-cpp
|
163 |
|
164 | # TODO: Why does the needs-terminal suite hang in Docker? It doesn't hang in an interactive Docker session.
|
165 | #
|
166 | # needs-terminal test/spec-py.sh needs-terminal-all _tmp/spec/needs-terminal-py/index.html
|
167 | #
|
168 | # https://oilshell.zulipchat.com/#narrow/stream/121539-oil-dev/topic/Spec.20Tests.20for.20Interactive.20Parsing
|
169 |
|
170 | cat <<EOF
|
171 | os-info soil/diagnose.sh os-info -
|
172 | dump-env soil/diagnose.sh dump-env -
|
173 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
174 | interactive-osh test/spec-py.sh interactive-osh _tmp/spec/interactive-osh/index.html
|
175 | nohup test/nohup.sh soil-run -
|
176 | process-table test/process-table.sh soil-run _tmp/process-table/index.html
|
177 | stateful test/stateful.sh soil-run _tmp/spec/stateful/index.html
|
178 | EOF
|
179 |
|
180 | }
|
181 |
|
182 | wild-tasks() {
|
183 | ### Print tasks for the 'wild' build
|
184 |
|
185 | # (task_name, script, action, result_html)
|
186 | cat <<EOF
|
187 | os-info soil/diagnose.sh os-info -
|
188 | dump-env soil/diagnose.sh dump-env -
|
189 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
190 | test-tar devtools/release-native.sh test-tar -
|
191 | linecount metrics/tarball.sh linecount-oils-cpp -
|
192 | wild test/wild.sh soil-run _tmp/wild-www/index.html
|
193 | EOF
|
194 | }
|
195 |
|
196 | benchmarks-tasks() {
|
197 | # (task_name, script, action, result_html)
|
198 |
|
199 | cat <<EOF
|
200 | os-info soil/diagnose.sh os-info -
|
201 | dump-env soil/diagnose.sh dump-env -
|
202 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
203 | dev-shell-test build/dev-shell-test.sh soil-run -
|
204 | id-test benchmarks/id-test.sh soil-run -
|
205 | osh-parser benchmarks/osh-parser.sh soil-run _tmp/osh-parser/index.html
|
206 | osh-runtime benchmarks/osh-runtime.sh soil-run _tmp/osh-runtime/index.html
|
207 | vm-baseline benchmarks/vm-baseline.sh soil-run _tmp/vm-baseline/index.html
|
208 | compute benchmarks/compute.sh soil-run _tmp/compute/index.html
|
209 | gc benchmarks/gc.sh soil-run _tmp/gc/index.html
|
210 | mycpp-benchmarks benchmarks/mycpp.sh soil-run _tmp/mycpp-examples/-wwz-index
|
211 | EOF
|
212 | }
|
213 |
|
214 | bloaty-tasks() {
|
215 | cat <<EOF
|
216 | os-info soil/diagnose.sh os-info -
|
217 | dump-env soil/diagnose.sh dump-env -
|
218 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
219 | test-tar devtools/release-native.sh test-tar -
|
220 | native-code metrics/native-code.sh oils-for-unix _tmp/metrics/oils-for-unix/
|
221 | EOF
|
222 | }
|
223 |
|
224 | benchmarks2-tasks() {
|
225 | # Note: id-test doesn't run in 'other-tests' because 'gawk' isn't in that image
|
226 | cat <<EOF
|
227 | os-info soil/diagnose.sh os-info -
|
228 | dump-env soil/diagnose.sh dump-env -
|
229 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
230 | test-tar devtools/release-native.sh test-tar -
|
231 | uftrace benchmarks/uftrace.sh soil-run _tmp/uftrace/index.html
|
232 | gc-cachegrind benchmarks/gc-cachegrind.sh soil-run _tmp/gc-cachegrind/index.html
|
233 | EOF
|
234 | }
|
235 |
|
236 | cpp-spec-tasks() {
|
237 | # (task_name, script, action, result_html)
|
238 |
|
239 | cat <<EOF
|
240 | os-info soil/diagnose.sh os-info -
|
241 | dump-env soil/diagnose.sh dump-env -
|
242 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
243 | oils-cpp-smoke build/native.sh soil-run -
|
244 | osh-all test/spec-cpp.sh osh-all _tmp/spec/osh-cpp/compare.html
|
245 | ysh-all test/spec-cpp.sh ysh-all _tmp/spec/ysh-cpp/compare.html
|
246 | ysh-py test/spec-py.sh ysh-all-serial _tmp/spec/ysh-py/index.html
|
247 | EOF
|
248 | }
|
249 |
|
250 | cpp-tarball-tasks() {
|
251 |
|
252 | # Note: build-times task requires _build/oils.sh
|
253 | # It's a bit redundant with test-tar
|
254 |
|
255 | cat <<EOF
|
256 | os-info soil/diagnose.sh os-info -
|
257 | dump-env soil/diagnose.sh dump-env -
|
258 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
259 | oils-cpp-smoke build/native.sh soil-run -
|
260 | make-tar devtools/release-native.sh make-tar _release/oils-for-unix.tar
|
261 | xshar-hello devtools/xshar.sh soil-run-hello _release/hello-xshar.xshar
|
262 | xshar-test-oils devtools/xshar.sh soil-run-test-oils _release/test-oils.xshar
|
263 | build-times build/native.sh measure-build-times -
|
264 | EOF
|
265 |
|
266 | # build-times is a good enough test
|
267 | # test-tar devtools/release-native.sh test-tar -
|
268 | #
|
269 | # Note: tarball is deployed outside the container
|
270 |
|
271 | }
|
272 |
|
273 | cpp-small-tasks() {
|
274 |
|
275 | # yaks could be moved to pea/ image once it has python2-dev
|
276 | cat <<EOF
|
277 | os-info soil/diagnose.sh os-info -
|
278 | dump-env soil/diagnose.sh dump-env -
|
279 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
280 | py-unit test/unit.sh all _test/py-unit/
|
281 | yaks yaks/TEST.sh soil-run -
|
282 | oils-cpp-smoke build/native.sh soil-run -
|
283 | cpp-unit test/cpp-unit.sh soil-run _test/-wwz-index
|
284 | headless client/run.sh soil-run-cpp -
|
285 | asan test/asan.sh soil-run -
|
286 | ltrace test/ltrace.sh soil-run -
|
287 | micro-syntax doctools/micro-syntax.sh soil-run -
|
288 | src-tree doctools/src-tree.sh soil-run _tmp/src-tree-www/index.html
|
289 | line-counts metrics/source-code.sh write-reports _tmp/metrics/line-counts/-wwz-index
|
290 | preprocessed metrics/source-code.sh preprocessed _tmp/metrics/preprocessed/-wwz-index
|
291 | mycpp-examples mycpp/TEST.sh soil-run _test/-wwz-index
|
292 | parse-errors test/parse-errors.sh soil-run-cpp -
|
293 | runtime-errors test/runtime-errors.sh soil-run-cpp -
|
294 | ysh-parse-errors test/ysh-parse-errors.sh soil-run-cpp -
|
295 | ysh-runtime-errors test/ysh-runtime-errors.sh soil-run-cpp -
|
296 | ysh-every-string test/ysh-every-string.sh soil-run-cpp -
|
297 | ysh-large ysh/run.sh soil-run-cpp -
|
298 | j8-errors data_lang/j8-errors.sh soil-run-cpp -
|
299 | houston-fp demo/houston-fp/run.sh soil-run -
|
300 | souffle-smoke-test test/souffle-smoke.sh soil-run -
|
301 | EOF
|
302 | }
|
303 |
|
304 | cpp-coverage-tasks() {
|
305 | # dep notes: hnode_asdl.h required by expr_asdl.h in mycpp/examples
|
306 |
|
307 | cat <<EOF
|
308 | os-info soil/diagnose.sh os-info -
|
309 | dump-env soil/diagnose.sh dump-env -
|
310 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
311 | extract-clang deps/from-binary.sh extract-clang-in-container -
|
312 | mycpp-unit-coverage mycpp/TEST.sh unit-test-coverage _test/clang-coverage/mycpp/html/index.html
|
313 | mycpp-examples-coverage mycpp/TEST.sh examples-coverage _test/clang-coverage/mycpp/examples/html/index.html
|
314 | cpp-coverage cpp/TEST.sh coverage _test/clang-coverage/cpp/html/index.html
|
315 | unified-coverage test/coverage.sh unified-report _test/clang-coverage/unified/html/index.html
|
316 | compare-gcc-clang metrics/native-code.sh compare-gcc-clang _tmp/metrics/compare-gcc-clang.txt
|
317 | EOF
|
318 | }
|
319 |
|
320 | ovm-tarball-tasks() {
|
321 | ### Print tasks for the 'ovm-tarball' build
|
322 |
|
323 | # notes:
|
324 | # - py-all needed to crawl dependencies to make tarball.
|
325 |
|
326 | # (task_name, script, action, result_html)
|
327 | cat <<EOF
|
328 | os-info soil/diagnose.sh os-info -
|
329 | dump-env soil/diagnose.sh dump-env -
|
330 | py-all build/py.sh all -
|
331 | syscall-by-code test/syscall.sh by-code _tmp/syscall/by-code.txt
|
332 | syscall-by-input test/syscall.sh by-input _tmp/syscall/by-input.txt
|
333 | osh-spec test/spec-py.sh osh-all-serial _tmp/spec/osh-py/index.html
|
334 | gold test/gold.sh soil-run -
|
335 | osh-usage test/osh-usage.sh soil-run -
|
336 | tools-deps test/tools-deps.sh soil-run -
|
337 | make-tarball devtools/release.sh py-tarball _release/oil.tar
|
338 | ysh-ovm-tarball test/spec-py.sh ysh-ovm-tarball _tmp/spec/ysh-py/index.html
|
339 | docs build/doc.sh soil-run _release/VERSION/index.html
|
340 | doc-metrics echo no-op _release/VERSION/doc/metrics.txt
|
341 | EOF
|
342 |
|
343 | # doc-metrics is a no-op, just for the link. Because soil-run just runs the
|
344 | # release, which creates metrics.
|
345 | }
|
346 |
|
347 | # Reuse ovm-tarball container
|
348 | app-tests-tasks() {
|
349 |
|
350 | cat <<EOF
|
351 | os-info soil/diagnose.sh os-info -
|
352 | dump-env soil/diagnose.sh dump-env -
|
353 | py-all build/py.sh all -
|
354 | ble-clone test/ble.sh clone -
|
355 | ble-build test/ble.sh build -
|
356 | ble-bash-suite test/ble.sh bash-suite -
|
357 | ble-test-osh-py test/ble.sh run-tests-osh-py -
|
358 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
359 | test-tar devtools/release-native.sh test-tar -
|
360 | ble-test-osh-cpp test/ble.sh run-tests-osh-cpp -
|
361 | EOF
|
362 |
|
363 | # This doesn't work
|
364 | # ble-test-osh-bash test/ble.sh run-tests-osh-bash -
|
365 | }
|
366 |
|
367 | # TODO: Most of these should be Ninja tasks.
|
368 | # Other tests:
|
369 | # find-test, xargs-test, pgen2-test, web/table/csv2html-test.sh
|
370 | # Probably should start using a shell test framework too.
|
371 | other-tests-tasks() {
|
372 | cat <<EOF
|
373 | os-info soil/diagnose.sh os-info -
|
374 | dump-env soil/diagnose.sh dump-env -
|
375 | build-minimal build/py.sh minimal -
|
376 | configure-test ./configure-test.sh soil_run -
|
377 | time-test benchmarks/time-test.sh soil-run -
|
378 | tsv-lib-test test/tsv-lib-test.sh soil-run -
|
379 | ysh-ify test/ysh-ify.sh soil-run -
|
380 | R-test devtools/R-test.sh soil-run -
|
381 | xargs-test test/other.sh xargs-test -
|
382 | csv2html-test test/other.sh csv2html-test -
|
383 | oil-python-symbols metrics/source-code.sh oil-python-symbols -
|
384 | opyc test/opyc.sh soil-run -
|
385 | opy-count-lines opy/soil.sh count-lines -
|
386 | test-gold opy/soil.sh test-gold -
|
387 | build-oil-repo opy/soil.sh build-oil-repo -
|
388 | regtest-compile opy/soil.sh regtest-compile -
|
389 | EOF
|
390 | }
|
391 |
|
392 | tests-todo() {
|
393 | ### More tests to add
|
394 | find . -name '_*' -a -prune -o -name '*-test.sh' -a -print
|
395 |
|
396 | # pgen2/pgen2-test.sh seems mostly broken
|
397 | }
|
398 |
|
399 | # Redefinition for quicker cloud debugging
|
400 | maybe-merge-tasks() {
|
401 | cat <<EOF
|
402 | os-info soil/diagnose.sh os-info -
|
403 | dump-env soil/diagnose.sh dump-env -
|
404 | maybe-merge soil/maybe-merge.sh soil-run -
|
405 | EOF
|
406 | }
|
407 |
|
408 | run-tasks() {
|
409 | ### Run the tasks on stdin and write _tmp/soil/INDEX.tsv.
|
410 | local job_name=$1
|
411 | local out_dir=$2 # should already exist
|
412 | local tty=$3
|
413 |
|
414 | mkdir -p $out_dir/logs
|
415 |
|
416 | # So we can run benchmarks/time_.py.
|
417 | # 2023-02-28: Images like soil-wild based off the new soil-common no longer
|
418 | # have 'cc'. Instead they use a wedge.
|
419 | if command -v cc > /dev/null; then
|
420 | build/py.sh time-helper
|
421 | else
|
422 | echo 'test time-tsv3'
|
423 | time-tsv3 -o /tmp/echo.tsv --append -- echo hi
|
424 |
|
425 | echo '/tmp/echo.tsv:'
|
426 | cat /tmp/echo.tsv
|
427 | fi
|
428 |
|
429 | # For the later deploy step to pick up
|
430 | date +%s > $out_dir/task-run-start-time.txt
|
431 |
|
432 | # This data can go on the dashboard index
|
433 | local tsv=$out_dir/INDEX.tsv
|
434 | rm -f $tsv
|
435 |
|
436 | local status
|
437 | local max_status=0
|
438 |
|
439 | while read task_name script action result_html; do
|
440 | log "--- task: $task_name ---"
|
441 |
|
442 | local log_path=$out_dir/logs/$task_name.txt
|
443 |
|
444 | # 15 minutes per task
|
445 | # One of the longest tasks is test/spec-cpp, which takes around 420 seconds
|
446 | # TODO: should have a configurable timeout
|
447 | local -a timeout=(timeout 900)
|
448 | local stdin_tty=''
|
449 |
|
450 | case $script in
|
451 | test/process-table.sh)
|
452 | # Workaround for weird interaction, see
|
453 | # $ test/process-table.sh timeout-issue
|
454 | timeout=()
|
455 | ;;
|
456 | test/nohup.sh)
|
457 | # Only run test/nohup.sh with TTY. For some reason build/py.sh all hangs
|
458 | # with $tty?
|
459 | stdin_tty=$tty
|
460 | ;;
|
461 | esac
|
462 |
|
463 | local -a argv=(
|
464 | time-tsv3 -o $tsv --append
|
465 | --field $task_name --field $script --field $action
|
466 | --field $result_html --
|
467 | "${timeout[@]}" "$script" "$action"
|
468 | )
|
469 |
|
470 | # Run task and save status
|
471 | set +o errexit
|
472 | if test -n "$stdin_tty"; then
|
473 | # explicitly connect TTY, e.g. for soil/interactive
|
474 | "${argv[@]}" > $log_path 2>&1 < $stdin_tty
|
475 | else
|
476 | "${argv[@]}" > $log_path 2>&1
|
477 | fi
|
478 | status=$?
|
479 | set -o errexit
|
480 |
|
481 | if test "$status" -gt "$max_status"; then
|
482 | max_status=$status
|
483 | fi
|
484 |
|
485 | # Show the last line
|
486 | echo
|
487 | tsv-row status elapsed task script action result_html
|
488 | tail -n 1 $tsv
|
489 | echo
|
490 | log "status=$status max_status=$max_status"
|
491 | done
|
492 |
|
493 | log '--- done ---'
|
494 | ls -l $out_dir
|
495 | wc -l $out_dir/logs/*
|
496 |
|
497 | # This suppressed the deployment of logs, which we don't want. So all our
|
498 | # Travis builds succeed? But then we can't use their failure notifications
|
499 | # (which might be OK).
|
500 | if false; then
|
501 | # exit with the maximum exit code.
|
502 | awk '
|
503 | BEGIN { max = 0 }
|
504 | { if ($1 > max) { max = $1 } }
|
505 | END { exit(max) }
|
506 | ' $tsv
|
507 | fi
|
508 |
|
509 | # To fail later. Important: this dir persists across jobs; it's NOT removed
|
510 | # by 'host-shim.sh job-reset'.
|
511 | mkdir -p _soil-jobs
|
512 |
|
513 | # Hack: Assign job_id and write it to the status file. Other jobs can poll
|
514 | # for completion of this job and access its resources.
|
515 |
|
516 | local job_id
|
517 | job_id="$(date +%Y-%m-%d__%H-%M-%S)"
|
518 |
|
519 | # e.g. _soil-jobs/dummy.status.txt
|
520 | echo "$max_status $job_id" > _soil-jobs/$job_name.status.txt
|
521 | }
|
522 |
|
523 | save-metadata() {
|
524 | ### Write metadata files to be saved as JSON
|
525 |
|
526 | # NOTE: host-shim.sh also writes image-pull-time.txt
|
527 |
|
528 | local job_name=$1
|
529 | local meta_dir=$2
|
530 |
|
531 | echo "$job_name" > $meta_dir/job-name.txt
|
532 |
|
533 | # command to show current branch
|
534 | # This does NOT work in detached HEAD! Travis puts the branch in an env
|
535 | # variable, but sr.ht doesn't.
|
536 | # git rev-parse --abbrev-ref HEAD > $meta_dir/git-branch.txt
|
537 |
|
538 | git log -n 1 --pretty='format:%H' > $meta_dir/commit-hash.txt
|
539 |
|
540 | # ISO 8601 format
|
541 | # Note: this can get confused with rebases. Two different commits can have
|
542 | # the same date.
|
543 | git log -n 1 --pretty='format:%aI' > $meta_dir/commit-date.txt
|
544 |
|
545 | git log -n 1 --pretty='format:%s' > $meta_dir/commit-line.txt # "subject"
|
546 | }
|
547 |
|
548 | disable-git-errors() {
|
549 |
|
550 | # 2023-02: The build started failing because of the permissions we set in
|
551 | # soil/host-shim.sh mount-perms.
|
552 | #
|
553 | # The issue is that the guest needs to be able to write to the Docker mount
|
554 | # of the repo. I think it may have been related to podman vs. Docker.
|
555 | # Should check if mount-perms is necessary in both places.
|
556 | #
|
557 | # git fails unless we have this workaround.
|
558 |
|
559 | # https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor
|
560 |
|
561 | # https://github.blog/2022-04-12-git-security-vulnerability-announced/
|
562 |
|
563 | #git config --global --add safe.directory '*'
|
564 |
|
565 | git config --global --add safe.directory /home/uke/oil
|
566 | }
|
567 |
|
568 | job-main() {
|
569 | local job_name=$1
|
570 |
|
571 | local out_dir=_tmp/soil
|
572 |
|
573 | # Report for debugging
|
574 | export EXTRA_MYCPP_ARGS='--stack-roots-warn 16'
|
575 |
|
576 | log-context 'job-main'
|
577 | mkdir -v -p $out_dir
|
578 | ls -l -d $out_dir
|
579 |
|
580 | disable-git-errors
|
581 |
|
582 | save-metadata $job_name $out_dir
|
583 |
|
584 | local captured
|
585 |
|
586 | set +o errexit
|
587 | captured=$(tty)
|
588 | status=$?
|
589 | set -o errexit
|
590 |
|
591 | if test $status -eq 0; then
|
592 | echo "TTY = $captured"
|
593 | local tty=$captured
|
594 | else
|
595 | echo "captured = $captured"
|
596 | local tty='' # clear the output
|
597 | fi
|
598 | echo
|
599 |
|
600 | ${job_name}-tasks | run-tasks $job_name $out_dir "$tty"
|
601 | }
|
602 |
|
603 | JOB-dummy() { job-main 'dummy'; }
|
604 | JOB-raw-vm() { job-main 'raw-vm'; }
|
605 | JOB-dev-setup-debian() { job-main 'dev-setup-debian'; }
|
606 | JOB-dev-setup-fedora() { job-main 'dev-setup-fedora'; }
|
607 | JOB-dev-setup-alpine() { job-main 'dev-setup-alpine'; }
|
608 |
|
609 | JOB-dev-minimal() { job-main 'dev-minimal'; }
|
610 | JOB-interactive() { job-main 'interactive'; }
|
611 |
|
612 | JOB-other-tests() { job-main 'other-tests'; }
|
613 |
|
614 | JOB-pea() { job-main 'pea'; }
|
615 |
|
616 | JOB-ovm-tarball() { job-main 'ovm-tarball'; }
|
617 | JOB-app-tests() { job-main 'app-tests'; }
|
618 |
|
619 | JOB-cpp-coverage() { job-main 'cpp-coverage'; }
|
620 | JOB-cpp-small() { job-main 'cpp-small'; }
|
621 | JOB-cpp-tarball() { job-main 'cpp-tarball'; }
|
622 | JOB-cpp-spec() { job-main 'cpp-spec'; }
|
623 |
|
624 | JOB-bloaty() { job-main 'bloaty'; }
|
625 | JOB-benchmarks() { job-main 'benchmarks'; }
|
626 | JOB-benchmarks2() { job-main 'benchmarks2'; }
|
627 |
|
628 | JOB-wild() { job-main 'wild'; }
|
629 |
|
630 | JOB-maybe-merge() { job-main 'maybe-merge'; }
|
631 |
|
632 | list-jobs() {
|
633 | # dev-setup-fedora for Fedora, disable
|
634 | compgen -A function | grep -- '^JOB-' | sed 's/^JOB-//g' | egrep -v 'maybe-merge|dev-setup-fedora|dev-setup-alpine'
|
635 | }
|
636 |
|
637 | "$@"
|