1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Run continuous build tasks.
|
4 | #
|
5 | # Usage:
|
6 | # soil/worker.sh <function name>
|
7 |
|
8 | set -o nounset
|
9 | set -o pipefail
|
10 | set -o errexit
|
11 |
|
12 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd) # tsv-lib.sh uses this
|
13 | readonly REPO_ROOT
|
14 |
|
15 | source soil/common.sh
|
16 | source test/tsv-lib.sh
|
17 |
|
18 | py-all-and-ninja() {
|
19 | ### baseline for most tasks
|
20 |
|
21 | build/py.sh all
|
22 | ./NINJA-config.sh
|
23 | }
|
24 |
|
25 | ninja-config() {
|
26 | ./NINJA-config.sh
|
27 | }
|
28 |
|
29 | dummy-tasks() {
|
30 | ### Print tasks that execute quickly
|
31 |
|
32 | # (task_name, script, action, result_html)
|
33 | cat <<EOF
|
34 | os-info soil/diagnose.sh os-info -
|
35 | dump-env soil/diagnose.sh dump-env -
|
36 | EOF
|
37 | }
|
38 |
|
39 | raw-vm-tasks() {
|
40 | # The perf tool depends on a specific version of a kernel, so run it outside
|
41 | # a container.
|
42 |
|
43 | # (task_name, script, action, result_html)
|
44 | cat <<EOF
|
45 | os-info soil/diagnose.sh os-info -
|
46 | dump-env soil/diagnose.sh dump-env -
|
47 | perf-install benchmarks/perf.sh soil-install -
|
48 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
49 | test-install-tar devtools/release-native.sh test-install-tar -
|
50 | perf-profiles benchmarks/perf.sh soil-run _tmp/perf/index.html
|
51 | EOF
|
52 | }
|
53 |
|
54 | # Oh there is a large list of pre-installed software
|
55 | # https://github.com/actions/runner-images#available-images
|
56 | # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
|
57 | # https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
|
58 | #
|
59 | # 1. System deps for building wedges - ninja, cmake, libreadline-dev, etc.
|
60 | # 2. fetch wedges - re2c, cmark, python2, python3, MyPy, pyflakes
|
61 | # - Python 3.10 desired for "pea"
|
62 | # 3. build them
|
63 | # 4. build Oils with them
|
64 |
|
65 | dev-setup-for() {
|
66 | local distro=$1
|
67 | # (task_name, script, action, result_html)
|
68 |
|
69 | cat <<EOF
|
70 | os-info soil/diagnose.sh os-info -
|
71 | dump-env soil/diagnose.sh dump-env -
|
72 | wedge-deps build/deps.sh wedge-deps-$distro -
|
73 | fetch build/deps.sh fetch -
|
74 | install-wedges build/deps.sh install-wedges-soil _build/wedge/logs/index.html
|
75 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
76 | smoke-test build/dev-setup-test.sh smoke-test -
|
77 | wedge-report build/deps.sh wedge-report -
|
78 | EOF
|
79 | }
|
80 |
|
81 | spec-bin-for() {
|
82 | local distro=$1
|
83 | cat <<EOF
|
84 | os-info soil/diagnose.sh os-info -
|
85 | dump-env soil/diagnose.sh dump-env -
|
86 | wedge-deps build/deps.sh wedge-deps-$distro -
|
87 | fetch build/deps.sh fetch -
|
88 | spec-bin build/deps.sh install-spec-bin-fast _build/wedge/logs/index.html
|
89 | EOF
|
90 | }
|
91 |
|
92 | dev-setup-debian-tasks() {
|
93 | # (task_name, script, action, result_html)
|
94 |
|
95 | dev-setup-for debian
|
96 | }
|
97 |
|
98 | dev-setup-fedora-tasks() {
|
99 | # (task_name, script, action, result_html)
|
100 |
|
101 | dev-setup-for fedora
|
102 | }
|
103 |
|
104 | dev-setup-alpine-tasks() {
|
105 | # (task_name, script, action, result_html)
|
106 |
|
107 | dev-setup-for alpine
|
108 | }
|
109 |
|
110 | pea-tasks() {
|
111 | ### Print tasks for the 'pea' build
|
112 |
|
113 | # We need a later version of Python 3 / MyPy both to type check and
|
114 | # to parse
|
115 |
|
116 | # Run py-source so we can type check generated code
|
117 | # We need to type check more than we translate
|
118 |
|
119 | # (task_name, script, action, result_html)
|
120 | cat <<EOF
|
121 | os-info soil/diagnose.sh os-info -
|
122 | dump-env soil/diagnose.sh dump-env -
|
123 | py-source build/py.sh py-source -
|
124 | check-types pea/TEST.sh check-types -
|
125 | run-tests pea/TEST.sh run-tests -
|
126 | parse-all pea/TEST.sh parse-all -
|
127 | EOF
|
128 | }
|
129 |
|
130 | dev-minimal-tasks() {
|
131 | ### Print tasks for the 'dev-minimal' build
|
132 |
|
133 | # repo overview is suggested by README.md
|
134 |
|
135 | # (task_name, script, action, result_html)
|
136 | cat <<EOF
|
137 | os-info soil/diagnose.sh os-info -
|
138 | dump-env soil/diagnose.sh dump-env -
|
139 | build-minimal build/py.sh minimal -
|
140 | repo-overview metrics/source-code.sh overview -
|
141 | lint test/lint.sh soil-run -
|
142 | asdl-types asdl/TEST.sh check-types -
|
143 | oil-types devtools/types.sh soil-run -
|
144 | unit test/unit.sh minimal _test/py-unit/
|
145 | lossless test/lossless.sh soil-run -
|
146 | parse-errors test/parse-errors.sh soil-run-py -
|
147 | runtime-errors test/runtime-errors.sh soil-run-py -
|
148 | ysh-parse-errors test/ysh-parse-errors.sh soil-run-py -
|
149 | ysh-runtime-errors test/ysh-runtime-errors.sh soil-run-py -
|
150 | ysh-every-string test/ysh-every-string.sh soil-run-py -
|
151 | ysh-large ysh/run.sh soil-run -
|
152 | j8-errors data_lang/j8-errors.sh soil-run-py -
|
153 | link-busybox-ash test/spec-bin.sh link-busybox-ash -
|
154 | osh-minimal test/spec-py.sh osh-minimal _tmp/spec/osh-minimal/index.html
|
155 | headless client/run.sh soil-run-py -
|
156 | stdlib-test stdlib/TEST.sh soil-run -
|
157 | EOF
|
158 | }
|
159 |
|
160 | interactive-tasks() {
|
161 | ### Print tasks for the 'interactive' build
|
162 |
|
163 | # TODO: also run interactive suite with osh-cpp
|
164 |
|
165 | # TODO: Why does the needs-terminal suite hang in Docker? It doesn't hang in an interactive Docker session.
|
166 | #
|
167 | # needs-terminal test/spec-py.sh needs-terminal-all _tmp/spec/needs-terminal-py/index.html
|
168 | #
|
169 | # https://oilshell.zulipchat.com/#narrow/stream/121539-oil-dev/topic/Spec.20Tests.20for.20Interactive.20Parsing
|
170 |
|
171 | cat <<EOF
|
172 | os-info soil/diagnose.sh os-info -
|
173 | dump-env soil/diagnose.sh dump-env -
|
174 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
175 | interactive-osh test/spec-py.sh interactive-osh _tmp/spec/interactive-osh/index.html
|
176 | nohup test/nohup.sh soil-run -
|
177 | process-table test/process-table.sh soil-run _tmp/process-table/index.html
|
178 | stateful test/stateful.sh soil-run _tmp/spec/stateful/index.html
|
179 | EOF
|
180 |
|
181 | }
|
182 |
|
183 | wild-tasks() {
|
184 | ### Print tasks for the 'wild' build
|
185 |
|
186 | # (task_name, script, action, result_html)
|
187 | cat <<EOF
|
188 | os-info soil/diagnose.sh os-info -
|
189 | dump-env soil/diagnose.sh dump-env -
|
190 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
191 | test-tar devtools/release-native.sh test-tar -
|
192 | linecount metrics/tarball.sh linecount-oils-cpp -
|
193 | wild test/wild.sh soil-run _tmp/wild-www/index.html
|
194 | EOF
|
195 | }
|
196 |
|
197 | benchmarks-tasks() {
|
198 | # (task_name, script, action, result_html)
|
199 |
|
200 | cat <<EOF
|
201 | os-info soil/diagnose.sh os-info -
|
202 | dump-env soil/diagnose.sh dump-env -
|
203 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
204 | dev-shell-test build/dev-shell-test.sh soil-run -
|
205 | id-test benchmarks/id-test.sh soil-run -
|
206 | osh-parser benchmarks/osh-parser.sh soil-run _tmp/osh-parser/index.html
|
207 | osh-runtime benchmarks/osh-runtime.sh soil-run _tmp/osh-runtime/index.html
|
208 | vm-baseline benchmarks/vm-baseline.sh soil-run _tmp/vm-baseline/index.html
|
209 | compute benchmarks/compute.sh soil-run _tmp/compute/index.html
|
210 | gc benchmarks/gc.sh soil-run _tmp/gc/index.html
|
211 | mycpp-benchmarks benchmarks/mycpp.sh soil-run _tmp/mycpp-examples/-wwz-index
|
212 | EOF
|
213 | }
|
214 |
|
215 | bloaty-tasks() {
|
216 | cat <<EOF
|
217 | os-info soil/diagnose.sh os-info -
|
218 | dump-env soil/diagnose.sh dump-env -
|
219 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
220 | test-tar devtools/release-native.sh test-tar -
|
221 | native-code metrics/native-code.sh oils-for-unix _tmp/metrics/oils-for-unix/
|
222 | EOF
|
223 | }
|
224 |
|
225 | benchmarks2-tasks() {
|
226 | # Note: id-test doesn't run in 'other-tests' because 'gawk' isn't in that image
|
227 | cat <<EOF
|
228 | os-info soil/diagnose.sh os-info -
|
229 | dump-env soil/diagnose.sh dump-env -
|
230 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
231 | test-tar devtools/release-native.sh test-tar -
|
232 | uftrace benchmarks/uftrace.sh soil-run _tmp/uftrace/index.html
|
233 | gc-cachegrind benchmarks/gc-cachegrind.sh soil-run _tmp/gc-cachegrind/index.html
|
234 | EOF
|
235 | }
|
236 |
|
237 | cpp-spec-tasks() {
|
238 | # (task_name, script, action, result_html)
|
239 |
|
240 | cat <<EOF
|
241 | os-info soil/diagnose.sh os-info -
|
242 | dump-env soil/diagnose.sh dump-env -
|
243 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
244 | oils-cpp-smoke build/native.sh soil-run -
|
245 | osh-all test/spec-cpp.sh osh-all _tmp/spec/osh-cpp/compare.html
|
246 | ysh-all test/spec-cpp.sh ysh-all _tmp/spec/ysh-cpp/compare.html
|
247 | ysh-py test/spec-py.sh ysh-all-serial _tmp/spec/ysh-py/index.html
|
248 | EOF
|
249 | }
|
250 |
|
251 | cpp-tarball-tasks() {
|
252 |
|
253 | # Note: build-times task requires _build/oils.sh
|
254 | # It's a bit redundant with test-tar
|
255 |
|
256 | cat <<EOF
|
257 | os-info soil/diagnose.sh os-info -
|
258 | dump-env soil/diagnose.sh dump-env -
|
259 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
260 | oils-cpp-smoke build/native.sh soil-run -
|
261 | make-tar devtools/release-native.sh make-tar _release/oils-for-unix.tar
|
262 | xshar-hello devtools/xshar.sh soil-run-hello _release/hello-xshar.xshar
|
263 | xshar-test-oils devtools/xshar.sh soil-run-test-oils _release/test-oils.xshar
|
264 | build-times build/native.sh measure-build-times -
|
265 | EOF
|
266 |
|
267 | # build-times is a good enough test
|
268 | # test-tar devtools/release-native.sh test-tar -
|
269 | #
|
270 | # Note: tarball is deployed outside the container
|
271 |
|
272 | }
|
273 |
|
274 | cpp-small-tasks() {
|
275 |
|
276 | # yaks could be moved to pea/ image once it has python2-dev
|
277 | cat <<EOF
|
278 | os-info soil/diagnose.sh os-info -
|
279 | dump-env soil/diagnose.sh dump-env -
|
280 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
281 | py-unit test/unit.sh all _test/py-unit/
|
282 | yaks yaks/TEST.sh soil-run -
|
283 | oils-cpp-smoke build/native.sh soil-run -
|
284 | cpp-unit test/cpp-unit.sh soil-run _test/-wwz-index
|
285 | headless client/run.sh soil-run-cpp -
|
286 | asan test/asan.sh soil-run -
|
287 | ltrace test/ltrace.sh soil-run -
|
288 | micro-syntax doctools/micro-syntax.sh soil-run -
|
289 | src-tree doctools/src-tree.sh soil-run _tmp/src-tree-www/index.html
|
290 | line-counts metrics/source-code.sh write-reports _tmp/metrics/line-counts/-wwz-index
|
291 | preprocessed metrics/source-code.sh preprocessed _tmp/metrics/preprocessed/-wwz-index
|
292 | mycpp-examples mycpp/TEST.sh soil-run _test/-wwz-index
|
293 | parse-errors test/parse-errors.sh soil-run-cpp -
|
294 | runtime-errors test/runtime-errors.sh soil-run-cpp -
|
295 | ysh-parse-errors test/ysh-parse-errors.sh soil-run-cpp -
|
296 | ysh-runtime-errors test/ysh-runtime-errors.sh soil-run-cpp -
|
297 | ysh-every-string test/ysh-every-string.sh soil-run-cpp -
|
298 | ysh-large ysh/run.sh soil-run-cpp -
|
299 | j8-errors data_lang/j8-errors.sh soil-run-cpp -
|
300 | houston-fp demo/houston-fp/run.sh soil-run -
|
301 | souffle-smoke-test test/souffle-smoke.sh soil-run -
|
302 | EOF
|
303 | }
|
304 |
|
305 | cpp-coverage-tasks() {
|
306 | # dep notes: hnode_asdl.h required by expr_asdl.h in mycpp/examples
|
307 |
|
308 | cat <<EOF
|
309 | os-info soil/diagnose.sh os-info -
|
310 | dump-env soil/diagnose.sh dump-env -
|
311 | py-all-and-ninja soil/worker.sh py-all-and-ninja -
|
312 | extract-clang deps/from-binary.sh extract-clang-in-container -
|
313 | mycpp-unit-coverage mycpp/TEST.sh unit-test-coverage _test/clang-coverage/mycpp/html/index.html
|
314 | mycpp-examples-coverage mycpp/TEST.sh examples-coverage _test/clang-coverage/mycpp/examples/html/index.html
|
315 | cpp-coverage cpp/TEST.sh coverage _test/clang-coverage/cpp/html/index.html
|
316 | unified-coverage test/coverage.sh unified-report _test/clang-coverage/unified/html/index.html
|
317 | compare-gcc-clang metrics/native-code.sh compare-gcc-clang _tmp/metrics/compare-gcc-clang.txt
|
318 | EOF
|
319 | }
|
320 |
|
321 | ovm-tarball-tasks() {
|
322 | ### Print tasks for the 'ovm-tarball' build
|
323 |
|
324 | # notes:
|
325 | # - py-all needed to crawl dependencies to make tarball.
|
326 |
|
327 | # (task_name, script, action, result_html)
|
328 | cat <<EOF
|
329 | os-info soil/diagnose.sh os-info -
|
330 | dump-env soil/diagnose.sh dump-env -
|
331 | py-all build/py.sh all -
|
332 | syscall-by-code test/syscall.sh by-code _tmp/syscall/by-code.txt
|
333 | syscall-by-input test/syscall.sh by-input _tmp/syscall/by-input.txt
|
334 | osh-spec test/spec-py.sh osh-all-serial _tmp/spec/osh-py/index.html
|
335 | gold test/gold.sh soil-run -
|
336 | osh-usage test/osh-usage.sh soil-run -
|
337 | tools-deps test/tools-deps.sh soil-run -
|
338 | make-tarball devtools/release.sh py-tarball _release/oil.tar
|
339 | ysh-ovm-tarball test/spec-py.sh ysh-ovm-tarball _tmp/spec/ysh-py/index.html
|
340 | docs build/doc.sh soil-run _release/VERSION/index.html
|
341 | doc-metrics echo no-op _release/VERSION/doc/metrics.txt
|
342 | EOF
|
343 |
|
344 | # doc-metrics is a no-op, just for the link. Because soil-run just runs the
|
345 | # release, which creates metrics.
|
346 | }
|
347 |
|
348 | # Reuse ovm-tarball container
|
349 | app-tests-tasks() {
|
350 |
|
351 | cat <<EOF
|
352 | os-info soil/diagnose.sh os-info -
|
353 | dump-env soil/diagnose.sh dump-env -
|
354 | py-all build/py.sh all -
|
355 | ble-clone test/ble.sh clone -
|
356 | ble-build test/ble.sh build -
|
357 | ble-bash-suite test/ble.sh bash-suite -
|
358 | ble-test-osh-py test/ble.sh run-tests-osh-py -
|
359 | wait-for-tarball soil/wait.sh for-cpp-tarball -
|
360 | test-tar devtools/release-native.sh test-tar -
|
361 | ble-test-osh-cpp test/ble.sh run-tests-osh-cpp -
|
362 | EOF
|
363 |
|
364 | # This doesn't work
|
365 | # ble-test-osh-bash test/ble.sh run-tests-osh-bash -
|
366 | }
|
367 |
|
368 | # TODO: Most of these should be Ninja tasks.
|
369 | # Other tests:
|
370 | # find-test, xargs-test, pgen2-test, web/table/csv2html-test.sh
|
371 | # Probably should start using a shell test framework too.
|
372 | other-tests-tasks() {
|
373 | cat <<EOF
|
374 | os-info soil/diagnose.sh os-info -
|
375 | dump-env soil/diagnose.sh dump-env -
|
376 | build-minimal build/py.sh minimal -
|
377 | configure-test ./configure-test.sh soil_run -
|
378 | time-test benchmarks/time-test.sh soil-run -
|
379 | tsv-lib-test test/tsv-lib-test.sh soil-run -
|
380 | ysh-ify test/ysh-ify.sh soil-run -
|
381 | R-test devtools/R-test.sh soil-run -
|
382 | xargs-test test/other.sh xargs-test -
|
383 | csv2html-test test/other.sh csv2html-test -
|
384 | oil-python-symbols metrics/source-code.sh oil-python-symbols -
|
385 | opyc test/opyc.sh soil-run -
|
386 | opy-count-lines opy/soil.sh count-lines -
|
387 | test-gold opy/soil.sh test-gold -
|
388 | build-oil-repo opy/soil.sh build-oil-repo -
|
389 | regtest-compile opy/soil.sh regtest-compile -
|
390 | EOF
|
391 | }
|
392 |
|
393 | tests-todo() {
|
394 | ### More tests to add
|
395 | find . -name '_*' -a -prune -o -name '*-test.sh' -a -print
|
396 |
|
397 | # pgen2/pgen2-test.sh seems mostly broken
|
398 | }
|
399 |
|
400 | # Redefinition for quicker cloud debugging
|
401 | maybe-merge-tasks() {
|
402 | cat <<EOF
|
403 | os-info soil/diagnose.sh os-info -
|
404 | dump-env soil/diagnose.sh dump-env -
|
405 | maybe-merge soil/maybe-merge.sh soil-run -
|
406 | EOF
|
407 | }
|
408 |
|
409 | run-tasks() {
|
410 | ### Run the tasks on stdin and write _tmp/soil/INDEX.tsv.
|
411 | local job_name=$1
|
412 | local out_dir=$2 # should already exist
|
413 | local tty=$3
|
414 |
|
415 | mkdir -p $out_dir/logs
|
416 |
|
417 | # So we can run benchmarks/time_.py.
|
418 | # 2023-02-28: Images like soil-wild based off the new soil-common no longer
|
419 | # have 'cc'. Instead they use a wedge.
|
420 | if command -v cc > /dev/null; then
|
421 | build/py.sh time-helper
|
422 | else
|
423 | echo 'test time-tsv3'
|
424 | time-tsv3 -o /tmp/echo.tsv --append -- echo hi
|
425 |
|
426 | echo '/tmp/echo.tsv:'
|
427 | cat /tmp/echo.tsv
|
428 | fi
|
429 |
|
430 | # For the later deploy step to pick up
|
431 | date +%s > $out_dir/task-run-start-time.txt
|
432 |
|
433 | # This data can go on the dashboard index
|
434 | local tsv=$out_dir/INDEX.tsv
|
435 | rm -f $tsv
|
436 |
|
437 | local status
|
438 | local max_status=0
|
439 |
|
440 | while read task_name script action result_html; do
|
441 | log "--- task: $task_name ---"
|
442 |
|
443 | local log_path=$out_dir/logs/$task_name.txt
|
444 |
|
445 | # 15 minutes per task
|
446 | # One of the longest tasks is test/spec-cpp, which takes around 420 seconds
|
447 | # TODO: should have a configurable timeout
|
448 | local -a timeout=(timeout 900)
|
449 | local stdin_tty=''
|
450 |
|
451 | case $script in
|
452 | test/process-table.sh)
|
453 | # Workaround for weird interaction, see
|
454 | # $ test/process-table.sh timeout-issue
|
455 | timeout=()
|
456 | ;;
|
457 | test/nohup.sh)
|
458 | # Only run test/nohup.sh with TTY. For some reason build/py.sh all hangs
|
459 | # with $tty?
|
460 | stdin_tty=$tty
|
461 | ;;
|
462 | esac
|
463 |
|
464 | local -a argv=(
|
465 | time-tsv3 -o $tsv --append
|
466 | --field $task_name --field $script --field $action
|
467 | --field $result_html --
|
468 | "${timeout[@]}" "$script" "$action"
|
469 | )
|
470 |
|
471 | # Run task and save status
|
472 | set +o errexit
|
473 | if test -n "$stdin_tty"; then
|
474 | # explicitly connect TTY, e.g. for soil/interactive
|
475 | "${argv[@]}" > $log_path 2>&1 < $stdin_tty
|
476 | else
|
477 | "${argv[@]}" > $log_path 2>&1
|
478 | fi
|
479 | status=$?
|
480 | set -o errexit
|
481 |
|
482 | if test "$status" -gt "$max_status"; then
|
483 | max_status=$status
|
484 | fi
|
485 |
|
486 | # Show the last line
|
487 | echo
|
488 | tsv-row status elapsed task script action result_html
|
489 | tail -n 1 $tsv
|
490 | echo
|
491 | log "status=$status max_status=$max_status"
|
492 | done
|
493 |
|
494 | log '--- done ---'
|
495 | ls -l $out_dir
|
496 | wc -l $out_dir/logs/*
|
497 |
|
498 | # This suppressed the deployment of logs, which we don't want. So all our
|
499 | # Travis builds succeed? But then we can't use their failure notifications
|
500 | # (which might be OK).
|
501 | if false; then
|
502 | # exit with the maximum exit code.
|
503 | awk '
|
504 | BEGIN { max = 0 }
|
505 | { if ($1 > max) { max = $1 } }
|
506 | END { exit(max) }
|
507 | ' $tsv
|
508 | fi
|
509 |
|
510 | # To fail later. Important: this dir persists across jobs; it's NOT removed
|
511 | # by 'host-shim.sh job-reset'.
|
512 | mkdir -p _soil-jobs
|
513 |
|
514 | # Hack: Assign job_id and write it to the status file. Other jobs can poll
|
515 | # for completion of this job and access its resources.
|
516 |
|
517 | local job_id
|
518 | job_id="$(date +%Y-%m-%d__%H-%M-%S)"
|
519 |
|
520 | # e.g. _soil-jobs/dummy.status.txt
|
521 | echo "$max_status $job_id" > _soil-jobs/$job_name.status.txt
|
522 | }
|
523 |
|
524 | save-metadata() {
|
525 | ### Write metadata files to be saved as JSON
|
526 |
|
527 | # NOTE: host-shim.sh also writes image-pull-time.txt
|
528 |
|
529 | local job_name=$1
|
530 | local meta_dir=$2
|
531 |
|
532 | echo "$job_name" > $meta_dir/job-name.txt
|
533 |
|
534 | # command to show current branch
|
535 | # This does NOT work in detached HEAD! Travis puts the branch in an env
|
536 | # variable, but sr.ht doesn't.
|
537 | # git rev-parse --abbrev-ref HEAD > $meta_dir/git-branch.txt
|
538 |
|
539 | git log -n 1 --pretty='format:%H' > $meta_dir/commit-hash.txt
|
540 |
|
541 | # ISO 8601 format
|
542 | # Note: this can get confused with rebases. Two different commits can have
|
543 | # the same date.
|
544 | git log -n 1 --pretty='format:%aI' > $meta_dir/commit-date.txt
|
545 |
|
546 | git log -n 1 --pretty='format:%s' > $meta_dir/commit-line.txt # "subject"
|
547 | }
|
548 |
|
549 | disable-git-errors() {
|
550 |
|
551 | # 2023-02: The build started failing because of the permissions we set in
|
552 | # soil/host-shim.sh mount-perms.
|
553 | #
|
554 | # The issue is that the guest needs to be able to write to the Docker mount
|
555 | # of the repo. I think it may have been related to podman vs. Docker.
|
556 | # Should check if mount-perms is necessary in both places.
|
557 | #
|
558 | # git fails unless we have this workaround.
|
559 |
|
560 | # https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor
|
561 |
|
562 | # https://github.blog/2022-04-12-git-security-vulnerability-announced/
|
563 |
|
564 | #git config --global --add safe.directory '*'
|
565 |
|
566 | git config --global --add safe.directory /home/uke/oil
|
567 | }
|
568 |
|
569 | job-main() {
|
570 | local job_name=$1
|
571 |
|
572 | local out_dir=_tmp/soil
|
573 |
|
574 | # Report for debugging
|
575 | export EXTRA_MYCPP_ARGS='--stack-roots-warn 16'
|
576 |
|
577 | log-context 'job-main'
|
578 | mkdir -v -p $out_dir
|
579 | ls -l -d $out_dir
|
580 |
|
581 | disable-git-errors
|
582 |
|
583 | save-metadata $job_name $out_dir
|
584 |
|
585 | local captured
|
586 |
|
587 | set +o errexit
|
588 | captured=$(tty)
|
589 | status=$?
|
590 | set -o errexit
|
591 |
|
592 | if test $status -eq 0; then
|
593 | echo "TTY = $captured"
|
594 | local tty=$captured
|
595 | else
|
596 | echo "captured = $captured"
|
597 | local tty='' # clear the output
|
598 | fi
|
599 | echo
|
600 |
|
601 | ${job_name}-tasks | run-tasks $job_name $out_dir "$tty"
|
602 | }
|
603 |
|
604 | JOB-dummy() { job-main 'dummy'; }
|
605 | JOB-raw-vm() { job-main 'raw-vm'; }
|
606 | JOB-dev-setup-debian() { job-main 'dev-setup-debian'; }
|
607 | JOB-dev-setup-fedora() { job-main 'dev-setup-fedora'; }
|
608 | JOB-dev-setup-alpine() { job-main 'dev-setup-alpine'; }
|
609 |
|
610 | JOB-dev-minimal() { job-main 'dev-minimal'; }
|
611 | JOB-interactive() { job-main 'interactive'; }
|
612 |
|
613 | JOB-other-tests() { job-main 'other-tests'; }
|
614 |
|
615 | JOB-pea() { job-main 'pea'; }
|
616 |
|
617 | JOB-ovm-tarball() { job-main 'ovm-tarball'; }
|
618 | JOB-app-tests() { job-main 'app-tests'; }
|
619 |
|
620 | JOB-cpp-coverage() { job-main 'cpp-coverage'; }
|
621 | JOB-cpp-small() { job-main 'cpp-small'; }
|
622 | JOB-cpp-tarball() { job-main 'cpp-tarball'; }
|
623 | JOB-cpp-spec() { job-main 'cpp-spec'; }
|
624 |
|
625 | JOB-bloaty() { job-main 'bloaty'; }
|
626 | JOB-benchmarks() { job-main 'benchmarks'; }
|
627 | JOB-benchmarks2() { job-main 'benchmarks2'; }
|
628 |
|
629 | JOB-wild() { job-main 'wild'; }
|
630 |
|
631 | JOB-maybe-merge() { job-main 'maybe-merge'; }
|
632 |
|
633 | list-jobs() {
|
634 | # dev-setup-fedora for Fedora, disable
|
635 | compgen -A function | grep -- '^JOB-' | sed 's/^JOB-//g' | egrep -v 'maybe-merge|dev-setup-fedora|dev-setup-alpine'
|
636 | }
|
637 |
|
638 | "$@"
|