OILS / soil / worker.sh View on Github | oilshell.org

638 lines, 208 significant
1#!/usr/bin/env bash
2#
3# Run continuous build tasks.
4#
5# Usage:
6# soil/worker.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd) # tsv-lib.sh uses this
13readonly REPO_ROOT
14
15source soil/common.sh
16source test/tsv-lib.sh
17
18py-all-and-ninja() {
19 ### baseline for most tasks
20
21 build/py.sh all
22 ./NINJA-config.sh
23}
24
25ninja-config() {
26 ./NINJA-config.sh
27}
28
29dummy-tasks() {
30 ### Print tasks that execute quickly
31
32 # (task_name, script, action, result_html)
33 cat <<EOF
34os-info soil/diagnose.sh os-info -
35dump-env soil/diagnose.sh dump-env -
36EOF
37}
38
39raw-vm-tasks() {
40 # The perf tool depends on a specific version of a kernel, so run it outside
41 # a container.
42
43 # (task_name, script, action, result_html)
44 cat <<EOF
45os-info soil/diagnose.sh os-info -
46dump-env soil/diagnose.sh dump-env -
47perf-install benchmarks/perf.sh soil-install -
48wait-for-tarball soil/wait.sh for-cpp-tarball -
49test-install-tar devtools/release-native.sh test-install-tar -
50perf-profiles benchmarks/perf.sh soil-run _tmp/perf/index.html
51EOF
52}
53
54# Oh there is a large list of pre-installed software
55# https://github.com/actions/runner-images#available-images
56# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
57# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
58#
59# 1. System deps for building wedges - ninja, cmake, libreadline-dev, etc.
60# 2. fetch wedges - re2c, cmark, python2, python3, MyPy, pyflakes
61# - Python 3.10 desired for "pea"
62# 3. build them
63# 4. build Oils with them
64
65dev-setup-for() {
66 local distro=$1
67 # (task_name, script, action, result_html)
68
69 cat <<EOF
70os-info soil/diagnose.sh os-info -
71dump-env soil/diagnose.sh dump-env -
72wedge-deps build/deps.sh wedge-deps-$distro -
73fetch build/deps.sh fetch -
74install-wedges build/deps.sh install-wedges-soil _build/wedge/logs/index.html
75py-all-and-ninja soil/worker.sh py-all-and-ninja -
76smoke-test build/dev-setup-test.sh smoke-test -
77wedge-report build/deps.sh wedge-report -
78EOF
79}
80
81spec-bin-for() {
82 local distro=$1
83 cat <<EOF
84os-info soil/diagnose.sh os-info -
85dump-env soil/diagnose.sh dump-env -
86wedge-deps build/deps.sh wedge-deps-$distro -
87fetch build/deps.sh fetch -
88spec-bin build/deps.sh install-spec-bin-fast _build/wedge/logs/index.html
89EOF
90}
91
92dev-setup-debian-tasks() {
93 # (task_name, script, action, result_html)
94
95 dev-setup-for debian
96}
97
98dev-setup-fedora-tasks() {
99 # (task_name, script, action, result_html)
100
101 dev-setup-for fedora
102}
103
104dev-setup-alpine-tasks() {
105 # (task_name, script, action, result_html)
106
107 dev-setup-for alpine
108}
109
110pea-tasks() {
111 ### Print tasks for the 'pea' build
112
113 # We need a later version of Python 3 / MyPy both to type check and
114 # to parse
115
116 # Run py-source so we can type check generated code
117 # We need to type check more than we translate
118
119 # (task_name, script, action, result_html)
120 cat <<EOF
121os-info soil/diagnose.sh os-info -
122dump-env soil/diagnose.sh dump-env -
123py-source build/py.sh py-source -
124check-types pea/TEST.sh check-types -
125run-tests pea/TEST.sh run-tests -
126parse-all pea/TEST.sh parse-all -
127EOF
128}
129
130dev-minimal-tasks() {
131 ### Print tasks for the 'dev-minimal' build
132
133 # repo overview is suggested by README.md
134
135 # (task_name, script, action, result_html)
136 cat <<EOF
137os-info soil/diagnose.sh os-info -
138dump-env soil/diagnose.sh dump-env -
139build-minimal build/py.sh minimal -
140repo-overview metrics/source-code.sh overview -
141lint test/lint.sh soil-run -
142asdl-types asdl/TEST.sh check-types -
143oil-types devtools/types.sh soil-run -
144unit test/unit.sh minimal _test/py-unit/
145lossless test/lossless.sh soil-run -
146parse-errors test/parse-errors.sh soil-run-py -
147runtime-errors test/runtime-errors.sh soil-run-py -
148ysh-parse-errors test/ysh-parse-errors.sh soil-run-py -
149ysh-runtime-errors test/ysh-runtime-errors.sh soil-run-py -
150ysh-every-string test/ysh-every-string.sh soil-run-py -
151ysh-large ysh/run.sh soil-run -
152j8-errors data_lang/j8-errors.sh soil-run-py -
153link-busybox-ash test/spec-bin.sh link-busybox-ash -
154osh-minimal test/spec-py.sh osh-minimal _tmp/spec/osh-minimal/index.html
155headless client/run.sh soil-run-py -
156stdlib-test stdlib/TEST.sh soil-run -
157EOF
158}
159
160interactive-tasks() {
161 ### Print tasks for the 'interactive' build
162
163 # TODO: also run interactive suite with osh-cpp
164
165# TODO: Why does the needs-terminal suite hang in Docker? It doesn't hang in an interactive Docker session.
166#
167# needs-terminal test/spec-py.sh needs-terminal-all _tmp/spec/needs-terminal-py/index.html
168#
169# https://oilshell.zulipchat.com/#narrow/stream/121539-oil-dev/topic/Spec.20Tests.20for.20Interactive.20Parsing
170
171 cat <<EOF
172os-info soil/diagnose.sh os-info -
173dump-env soil/diagnose.sh dump-env -
174py-all-and-ninja soil/worker.sh py-all-and-ninja -
175interactive-osh test/spec-py.sh interactive-osh _tmp/spec/interactive-osh/index.html
176nohup test/nohup.sh soil-run -
177process-table test/process-table.sh soil-run _tmp/process-table/index.html
178stateful test/stateful.sh soil-run _tmp/spec/stateful/index.html
179EOF
180
181}
182
183wild-tasks() {
184 ### Print tasks for the 'wild' build
185
186 # (task_name, script, action, result_html)
187 cat <<EOF
188os-info soil/diagnose.sh os-info -
189dump-env soil/diagnose.sh dump-env -
190wait-for-tarball soil/wait.sh for-cpp-tarball -
191test-tar devtools/release-native.sh test-tar -
192linecount metrics/tarball.sh linecount-oils-cpp -
193wild test/wild.sh soil-run _tmp/wild-www/index.html
194EOF
195}
196
197benchmarks-tasks() {
198 # (task_name, script, action, result_html)
199
200 cat <<EOF
201os-info soil/diagnose.sh os-info -
202dump-env soil/diagnose.sh dump-env -
203py-all-and-ninja soil/worker.sh py-all-and-ninja -
204dev-shell-test build/dev-shell-test.sh soil-run -
205id-test benchmarks/id-test.sh soil-run -
206osh-parser benchmarks/osh-parser.sh soil-run _tmp/osh-parser/index.html
207osh-runtime benchmarks/osh-runtime.sh soil-run _tmp/osh-runtime/index.html
208vm-baseline benchmarks/vm-baseline.sh soil-run _tmp/vm-baseline/index.html
209compute benchmarks/compute.sh soil-run _tmp/compute/index.html
210gc benchmarks/gc.sh soil-run _tmp/gc/index.html
211mycpp-benchmarks benchmarks/mycpp.sh soil-run _tmp/mycpp-examples/-wwz-index
212EOF
213}
214
215bloaty-tasks() {
216 cat <<EOF
217os-info soil/diagnose.sh os-info -
218dump-env soil/diagnose.sh dump-env -
219wait-for-tarball soil/wait.sh for-cpp-tarball -
220test-tar devtools/release-native.sh test-tar -
221native-code metrics/native-code.sh oils-for-unix _tmp/metrics/oils-for-unix/
222EOF
223}
224
225benchmarks2-tasks() {
226 # Note: id-test doesn't run in 'other-tests' because 'gawk' isn't in that image
227 cat <<EOF
228os-info soil/diagnose.sh os-info -
229dump-env soil/diagnose.sh dump-env -
230wait-for-tarball soil/wait.sh for-cpp-tarball -
231test-tar devtools/release-native.sh test-tar -
232uftrace benchmarks/uftrace.sh soil-run _tmp/uftrace/index.html
233gc-cachegrind benchmarks/gc-cachegrind.sh soil-run _tmp/gc-cachegrind/index.html
234EOF
235}
236
237cpp-spec-tasks() {
238 # (task_name, script, action, result_html)
239
240 cat <<EOF
241os-info soil/diagnose.sh os-info -
242dump-env soil/diagnose.sh dump-env -
243py-all-and-ninja soil/worker.sh py-all-and-ninja -
244oils-cpp-smoke build/native.sh soil-run -
245osh-all test/spec-cpp.sh osh-all _tmp/spec/osh-cpp/compare.html
246ysh-all test/spec-cpp.sh ysh-all _tmp/spec/ysh-cpp/compare.html
247ysh-py test/spec-py.sh ysh-all-serial _tmp/spec/ysh-py/index.html
248EOF
249}
250
251cpp-tarball-tasks() {
252
253 # Note: build-times task requires _build/oils.sh
254 # It's a bit redundant with test-tar
255
256 cat <<EOF
257os-info soil/diagnose.sh os-info -
258dump-env soil/diagnose.sh dump-env -
259py-all-and-ninja soil/worker.sh py-all-and-ninja -
260oils-cpp-smoke build/native.sh soil-run -
261make-tar devtools/release-native.sh make-tar _release/oils-for-unix.tar
262xshar-hello devtools/xshar.sh soil-run-hello _release/hello-xshar.xshar
263xshar-test-oils devtools/xshar.sh soil-run-test-oils _release/test-oils.xshar
264build-times build/native.sh measure-build-times -
265EOF
266
267# build-times is a good enough test
268# test-tar devtools/release-native.sh test-tar -
269#
270# Note: tarball is deployed outside the container
271
272}
273
274cpp-small-tasks() {
275
276 # yaks could be moved to pea/ image once it has python2-dev
277 cat <<EOF
278os-info soil/diagnose.sh os-info -
279dump-env soil/diagnose.sh dump-env -
280py-all-and-ninja soil/worker.sh py-all-and-ninja -
281py-unit test/unit.sh all _test/py-unit/
282yaks yaks/TEST.sh soil-run -
283oils-cpp-smoke build/native.sh soil-run -
284cpp-unit test/cpp-unit.sh soil-run _test/-wwz-index
285headless client/run.sh soil-run-cpp -
286asan test/asan.sh soil-run -
287ltrace test/ltrace.sh soil-run -
288micro-syntax doctools/micro-syntax.sh soil-run -
289src-tree doctools/src-tree.sh soil-run _tmp/src-tree-www/index.html
290line-counts metrics/source-code.sh write-reports _tmp/metrics/line-counts/-wwz-index
291preprocessed metrics/source-code.sh preprocessed _tmp/metrics/preprocessed/-wwz-index
292mycpp-examples mycpp/TEST.sh soil-run _test/-wwz-index
293parse-errors test/parse-errors.sh soil-run-cpp -
294runtime-errors test/runtime-errors.sh soil-run-cpp -
295ysh-parse-errors test/ysh-parse-errors.sh soil-run-cpp -
296ysh-runtime-errors test/ysh-runtime-errors.sh soil-run-cpp -
297ysh-every-string test/ysh-every-string.sh soil-run-cpp -
298ysh-large ysh/run.sh soil-run-cpp -
299j8-errors data_lang/j8-errors.sh soil-run-cpp -
300houston-fp demo/houston-fp/run.sh soil-run -
301souffle-smoke-test test/souffle-smoke.sh soil-run -
302EOF
303}
304
305cpp-coverage-tasks() {
306 # dep notes: hnode_asdl.h required by expr_asdl.h in mycpp/examples
307
308 cat <<EOF
309os-info soil/diagnose.sh os-info -
310dump-env soil/diagnose.sh dump-env -
311py-all-and-ninja soil/worker.sh py-all-and-ninja -
312extract-clang deps/from-binary.sh extract-clang-in-container -
313mycpp-unit-coverage mycpp/TEST.sh unit-test-coverage _test/clang-coverage/mycpp/html/index.html
314mycpp-examples-coverage mycpp/TEST.sh examples-coverage _test/clang-coverage/mycpp/examples/html/index.html
315cpp-coverage cpp/TEST.sh coverage _test/clang-coverage/cpp/html/index.html
316unified-coverage test/coverage.sh unified-report _test/clang-coverage/unified/html/index.html
317compare-gcc-clang metrics/native-code.sh compare-gcc-clang _tmp/metrics/compare-gcc-clang.txt
318EOF
319}
320
321ovm-tarball-tasks() {
322 ### Print tasks for the 'ovm-tarball' build
323
324 # notes:
325 # - py-all needed to crawl dependencies to make tarball.
326
327 # (task_name, script, action, result_html)
328 cat <<EOF
329os-info soil/diagnose.sh os-info -
330dump-env soil/diagnose.sh dump-env -
331py-all build/py.sh all -
332syscall-by-code test/syscall.sh by-code _tmp/syscall/by-code.txt
333syscall-by-input test/syscall.sh by-input _tmp/syscall/by-input.txt
334osh-spec test/spec-py.sh osh-all-serial _tmp/spec/osh-py/index.html
335gold test/gold.sh soil-run -
336osh-usage test/osh-usage.sh soil-run -
337tools-deps test/tools-deps.sh soil-run -
338make-tarball devtools/release.sh py-tarball _release/oil.tar
339ysh-ovm-tarball test/spec-py.sh ysh-ovm-tarball _tmp/spec/ysh-py/index.html
340docs build/doc.sh soil-run _release/VERSION/index.html
341doc-metrics echo no-op _release/VERSION/doc/metrics.txt
342EOF
343
344# doc-metrics is a no-op, just for the link. Because soil-run just runs the
345# release, which creates metrics.
346}
347
348# Reuse ovm-tarball container
349app-tests-tasks() {
350
351 cat <<EOF
352os-info soil/diagnose.sh os-info -
353dump-env soil/diagnose.sh dump-env -
354py-all build/py.sh all -
355ble-clone test/ble.sh clone -
356ble-build test/ble.sh build -
357ble-bash-suite test/ble.sh bash-suite -
358ble-test-osh-py test/ble.sh run-tests-osh-py -
359wait-for-tarball soil/wait.sh for-cpp-tarball -
360test-tar devtools/release-native.sh test-tar -
361ble-test-osh-cpp test/ble.sh run-tests-osh-cpp -
362EOF
363
364# This doesn't work
365# ble-test-osh-bash test/ble.sh run-tests-osh-bash -
366}
367
368# TODO: Most of these should be Ninja tasks.
369# Other tests:
370# find-test, xargs-test, pgen2-test, web/table/csv2html-test.sh
371# Probably should start using a shell test framework too.
372other-tests-tasks() {
373 cat <<EOF
374os-info soil/diagnose.sh os-info -
375dump-env soil/diagnose.sh dump-env -
376build-minimal build/py.sh minimal -
377configure-test ./configure-test.sh soil_run -
378time-test benchmarks/time-test.sh soil-run -
379tsv-lib-test test/tsv-lib-test.sh soil-run -
380ysh-ify test/ysh-ify.sh soil-run -
381R-test devtools/R-test.sh soil-run -
382xargs-test test/other.sh xargs-test -
383csv2html-test test/other.sh csv2html-test -
384oil-python-symbols metrics/source-code.sh oil-python-symbols -
385opyc test/opyc.sh soil-run -
386opy-count-lines opy/soil.sh count-lines -
387test-gold opy/soil.sh test-gold -
388build-oil-repo opy/soil.sh build-oil-repo -
389regtest-compile opy/soil.sh regtest-compile -
390EOF
391}
392
393tests-todo() {
394 ### More tests to add
395 find . -name '_*' -a -prune -o -name '*-test.sh' -a -print
396
397 # pgen2/pgen2-test.sh seems mostly broken
398}
399
400# Redefinition for quicker cloud debugging
401maybe-merge-tasks() {
402 cat <<EOF
403os-info soil/diagnose.sh os-info -
404dump-env soil/diagnose.sh dump-env -
405maybe-merge soil/maybe-merge.sh soil-run -
406EOF
407}
408
409run-tasks() {
410 ### Run the tasks on stdin and write _tmp/soil/INDEX.tsv.
411 local job_name=$1
412 local out_dir=$2 # should already exist
413 local tty=$3
414
415 mkdir -p $out_dir/logs
416
417 # So we can run benchmarks/time_.py.
418 # 2023-02-28: Images like soil-wild based off the new soil-common no longer
419 # have 'cc'. Instead they use a wedge.
420 if command -v cc > /dev/null; then
421 build/py.sh time-helper
422 else
423 echo 'test time-tsv3'
424 time-tsv3 -o /tmp/echo.tsv --append -- echo hi
425
426 echo '/tmp/echo.tsv:'
427 cat /tmp/echo.tsv
428 fi
429
430 # For the later deploy step to pick up
431 date +%s > $out_dir/task-run-start-time.txt
432
433 # This data can go on the dashboard index
434 local tsv=$out_dir/INDEX.tsv
435 rm -f $tsv
436
437 local status
438 local max_status=0
439
440 while read task_name script action result_html; do
441 log "--- task: $task_name ---"
442
443 local log_path=$out_dir/logs/$task_name.txt
444
445 # 15 minutes per task
446 # One of the longest tasks is test/spec-cpp, which takes around 420 seconds
447 # TODO: should have a configurable timeout
448 local -a timeout=(timeout 900)
449 local stdin_tty=''
450
451 case $script in
452 test/process-table.sh)
453 # Workaround for weird interaction, see
454 # $ test/process-table.sh timeout-issue
455 timeout=()
456 ;;
457 test/nohup.sh)
458 # Only run test/nohup.sh with TTY. For some reason build/py.sh all hangs
459 # with $tty?
460 stdin_tty=$tty
461 ;;
462 esac
463
464 local -a argv=(
465 time-tsv3 -o $tsv --append
466 --field $task_name --field $script --field $action
467 --field $result_html --
468 "${timeout[@]}" "$script" "$action"
469 )
470
471 # Run task and save status
472 set +o errexit
473 if test -n "$stdin_tty"; then
474 # explicitly connect TTY, e.g. for soil/interactive
475 "${argv[@]}" > $log_path 2>&1 < $stdin_tty
476 else
477 "${argv[@]}" > $log_path 2>&1
478 fi
479 status=$?
480 set -o errexit
481
482 if test "$status" -gt "$max_status"; then
483 max_status=$status
484 fi
485
486 # Show the last line
487 echo
488 tsv-row status elapsed task script action result_html
489 tail -n 1 $tsv
490 echo
491 log "status=$status max_status=$max_status"
492 done
493
494 log '--- done ---'
495 ls -l $out_dir
496 wc -l $out_dir/logs/*
497
498 # This suppressed the deployment of logs, which we don't want. So all our
499 # Travis builds succeed? But then we can't use their failure notifications
500 # (which might be OK).
501 if false; then
502 # exit with the maximum exit code.
503 awk '
504 BEGIN { max = 0 }
505 { if ($1 > max) { max = $1 } }
506 END { exit(max) }
507 ' $tsv
508 fi
509
510 # To fail later. Important: this dir persists across jobs; it's NOT removed
511 # by 'host-shim.sh job-reset'.
512 mkdir -p _soil-jobs
513
514 # Hack: Assign job_id and write it to the status file. Other jobs can poll
515 # for completion of this job and access its resources.
516
517 local job_id
518 job_id="$(date +%Y-%m-%d__%H-%M-%S)"
519
520 # e.g. _soil-jobs/dummy.status.txt
521 echo "$max_status $job_id" > _soil-jobs/$job_name.status.txt
522}
523
524save-metadata() {
525 ### Write metadata files to be saved as JSON
526
527 # NOTE: host-shim.sh also writes image-pull-time.txt
528
529 local job_name=$1
530 local meta_dir=$2
531
532 echo "$job_name" > $meta_dir/job-name.txt
533
534 # command to show current branch
535 # This does NOT work in detached HEAD! Travis puts the branch in an env
536 # variable, but sr.ht doesn't.
537 # git rev-parse --abbrev-ref HEAD > $meta_dir/git-branch.txt
538
539 git log -n 1 --pretty='format:%H' > $meta_dir/commit-hash.txt
540
541 # ISO 8601 format
542 # Note: this can get confused with rebases. Two different commits can have
543 # the same date.
544 git log -n 1 --pretty='format:%aI' > $meta_dir/commit-date.txt
545
546 git log -n 1 --pretty='format:%s' > $meta_dir/commit-line.txt # "subject"
547}
548
549disable-git-errors() {
550
551 # 2023-02: The build started failing because of the permissions we set in
552 # soil/host-shim.sh mount-perms.
553 #
554 # The issue is that the guest needs to be able to write to the Docker mount
555 # of the repo. I think it may have been related to podman vs. Docker.
556 # Should check if mount-perms is necessary in both places.
557 #
558 # git fails unless we have this workaround.
559
560 # https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor
561
562 # https://github.blog/2022-04-12-git-security-vulnerability-announced/
563
564 #git config --global --add safe.directory '*'
565
566 git config --global --add safe.directory /home/uke/oil
567}
568
569job-main() {
570 local job_name=$1
571
572 local out_dir=_tmp/soil
573
574 # Report for debugging
575 export EXTRA_MYCPP_ARGS='--stack-roots-warn 16'
576
577 log-context 'job-main'
578 mkdir -v -p $out_dir
579 ls -l -d $out_dir
580
581 disable-git-errors
582
583 save-metadata $job_name $out_dir
584
585 local captured
586
587 set +o errexit
588 captured=$(tty)
589 status=$?
590 set -o errexit
591
592 if test $status -eq 0; then
593 echo "TTY = $captured"
594 local tty=$captured
595 else
596 echo "captured = $captured"
597 local tty='' # clear the output
598 fi
599 echo
600
601 ${job_name}-tasks | run-tasks $job_name $out_dir "$tty"
602}
603
604JOB-dummy() { job-main 'dummy'; }
605JOB-raw-vm() { job-main 'raw-vm'; }
606JOB-dev-setup-debian() { job-main 'dev-setup-debian'; }
607JOB-dev-setup-fedora() { job-main 'dev-setup-fedora'; }
608JOB-dev-setup-alpine() { job-main 'dev-setup-alpine'; }
609
610JOB-dev-minimal() { job-main 'dev-minimal'; }
611JOB-interactive() { job-main 'interactive'; }
612
613JOB-other-tests() { job-main 'other-tests'; }
614
615JOB-pea() { job-main 'pea'; }
616
617JOB-ovm-tarball() { job-main 'ovm-tarball'; }
618JOB-app-tests() { job-main 'app-tests'; }
619
620JOB-cpp-coverage() { job-main 'cpp-coverage'; }
621JOB-cpp-small() { job-main 'cpp-small'; }
622JOB-cpp-tarball() { job-main 'cpp-tarball'; }
623JOB-cpp-spec() { job-main 'cpp-spec'; }
624
625JOB-bloaty() { job-main 'bloaty'; }
626JOB-benchmarks() { job-main 'benchmarks'; }
627JOB-benchmarks2() { job-main 'benchmarks2'; }
628
629JOB-wild() { job-main 'wild'; }
630
631JOB-maybe-merge() { job-main 'maybe-merge'; }
632
633list-jobs() {
634 # dev-setup-fedora for Fedora, disable
635 compgen -A function | grep -- '^JOB-' | sed 's/^JOB-//g' | egrep -v 'maybe-merge|dev-setup-fedora|dev-setup-alpine'
636}
637
638"$@"