OILS / devtools / release.sh View on Github | oilshell.org

812 lines, 388 significant
1#!/usr/bin/env bash
2#
3# The big Oils release process.
4#
5# Usage:
6# devtools/release.sh <function name>
7#
8# Steps:
9# edit oil-version.txt, build/doc.sh update-src-versions, bump devtools/release-note.sh
10# $0 make-release-branch
11# $0 two-tarballs # CPython, then oils-for-unix, which is INSTALLED
12# demo/osh-debug.sh osh-for-release: Start a shell to dogfood
13# build/cpython-defs.sh {oil-py-names,filter-methods}
14# (regenerate C source)
15#
16# Run on each machine:
17# $0 auto-machine1
18# $0 auto-machine2 ($0 dep-benchmarks first)
19#
20# In between:
21# [switch benchmarks-data repo] commit src/oil-for-unix-* and push to flanders.
22# TODO: Make sure benchmark-data directory is clean!
23#
24# Resume manual work
25#
26# Commit files to oilshell/benchmark-data repo and sync.
27# benchmarks/report.sh all
28# $0 deploy-tar # needed to publish tarball checksum in HTML
29# build/doc.sh run-for-release
30# $0 compress
31# devtools/release-version.sh git-changelog-$VERSION
32# devtools/release-version.sh announcement-$VERSION
33# MAYBE: ./local.sh test-release-tree if you want to preview it
34# $0 deploy-doc (makes releases.html)
35#
36# demo/osh-debug.sh analyze # see what you ran
37#
38# - Go to oilshell.org repo and do:
39# ./deploy.sh site # copy release
40# ./deploy.sh bump-index-version
41# make
42# ./deploy.sh site # copy new index
43# ./deploy.sh bump-release-version
44# - Go to oilshell.org__deploy and "git add release/$VERSION".
45# - git commit -a
46
47set -o nounset
48set -o pipefail
49set -o errexit
50
51shopt -s strict:all 2>/dev/null || true # dogfood for OSH
52
53REPO_ROOT=$(cd $(dirname $0)/.. ; pwd)
54OIL_VERSION=$(head -n 1 oil-version.txt)
55
56source devtools/common.sh # banner
57source benchmarks/common.sh # BENCHMARK_DATA_OILS, OSH_CPP_BENCHMARK_DATA
58 # redefines OIL_VERSION as readonly
59
60readonly OSH_RELEASE_BINARY=$REPO_ROOT/_tmp/oil-tar-test/oil-$OIL_VERSION/_bin/osh
61readonly YSH_RELEASE_BINARY=$REPO_ROOT/_tmp/oil-tar-test/oil-$OIL_VERSION/_bin/ysh
62
63log() {
64 echo "$@" 1>&2
65}
66
67make-release-branch() {
68 git checkout master
69 local name=release/$OIL_VERSION
70 git checkout -b $name
71 git push -u origin $name
72}
73
74ensure-smooth-build() {
75 # Stray files can mess up the unit tests
76 devtools/git.sh error-if-untracked
77
78 build/clean.sh
79
80 sudo -k; sudo true # clear and re-cache credentials
81
82 # Install with root privileges
83 _install
84}
85
86# For redoing a release. This is everything until you have to 'git pull' the
87# benchmark-data repo to make reports.
88#
89# PRECONDITION: $0 two-tarballs was run manually, which runs
90# ensure-smooth-build.
91auto-machine1() {
92 local resume=${1:-} # workaround for spec test flakiness bug
93 local resume2=${2:-} # skip past spec sanity check
94 local resume3=${3:-} # skip past metrics and wild tests
95 local resume4=${4:-} # skip past full spec tests
96
97 if test -z "$resume"; then
98 $0 build-and-test
99 fi
100
101 if test -z "$resume2"; then
102 _spec-sanity-check # just run a few spec tests
103 fi
104
105 if test -z "$resume3"; then
106 $0 metrics # this can catch bugs
107 test/wild.sh all
108 fi
109
110 if test -z "$resume4"; then
111 $0 spec-all # full spec test run
112 fi
113
114 $0 benchmark-run do_machine1
115}
116
117# Note: needs dep-benchmarks to run
118auto-machine2() {
119 ensure-smooth-build
120
121 # Note: this can't be done until we sync the oils-for-unix source from
122 # machine 1.
123 $0 benchmark-build
124 $0 benchmark-run
125}
126
127# TODO:
128# - enforce that there is a release/$VERSION branch?
129
130# oilshell.org__deploy/
131# releases.html
132# release/
133# $VERSION/
134# index.html # release page, from doc/release-index.md
135# oil-version.txt
136# release-date.txt
137# announcement.html # HTML redirect
138# changelog.html
139# doc/
140# index.html
141# ...
142# test/ # results
143# spec.wwz/
144# machine-lisa/
145# wild.wwz/
146# unit.wwz/
147# other.wwz/
148# gold.txt
149# parse-errors.txt
150# runtime-errors.txt
151# tools-deps.txt
152# osh-usage.txt
153# lossless.txt
154# tarball/ # log of building and running the tarball?
155# asan/ # spec tests or other?
156# # or it can be put under test/{spec,wild}
157# metrics.wwz/ # static metrics on source code?
158# line-counts/
159# nativedeps.txt (build/stats.sh line counts)
160# bytecode size, number of PyCodeObject
161# number of functions, classes, etc.?
162# bytecode/bundle size (binary size on x86_64 is in ovm-build.sh)
163# tarball size?
164# coverage.wwz/
165# unified/ # clang-coverage
166# benchmarks.wwz/
167# compute
168# osh-parser/
169# osh-runtime/
170# vm-baseline/
171# ...
172# startup/
173# download/ # What about native binaries?
174# 0.0.0/
175# oil-0.0.0.tar.xz
176
177_test-tarball() {
178 local name=${1:-hello}
179 local version=${2:-0.0.0}
180 local install=${3:-}
181
182 local tmp=_tmp/${name}-tar-test
183 rm -r -f $tmp
184 mkdir -p $tmp
185
186 pushd $tmp
187 tar --extract -z < ../../_release/$name-$version.tar.gz
188
189 cd $name-$version
190 ./configure
191
192 # Build the fast one for a test.
193 # TODO: Maybe edit the Makefile to change the top target.
194 local bin=_bin/${name}.ovm # not dbg
195 time make $bin
196 $bin --version
197
198 if test -n "$install"; then
199 sudo ./install
200 fi
201 popd
202}
203
204test-oil-tar() {
205 local install=${1:-} # non-empty to install
206 _test-tarball oil $(head -n 1 oil-version.txt) "$install"
207}
208
209_release-build() {
210 # NOTE: deps/from-tar.sh {configure,build}-python is assumed
211
212 # Build the oil tar
213 $0 oil
214
215 test-oil-tar
216
217 # For _spec-sanity-check
218 ln -s -f --no-target-directory -v oil.ovm $OSH_RELEASE_BINARY
219 ln -s -f --no-target-directory -v oil.ovm $YSH_RELEASE_BINARY
220}
221
222readonly HAVE_ROOT=1
223
224readonly -a MORE_TESTS=(
225 process-table
226 gold
227 ysh-ify
228 parse-errors runtime-errors
229 ysh-runtime-errors
230 ysh-parse-errors
231 ysh-every-string
232 lossless
233 osh-usage tools-deps
234 syscall
235)
236
237run-more-tests() {
238 for name in "${MORE_TESTS[@]}"; do
239 case $name in
240 gold)
241 if test -n "${OILS_HIJACK_SHEBANG:-}"; then
242 cat >&2 <<'EOF'
243=====
244WARNING: Skipping gold tests because $OILS_HIJACK_SHEBANG is set.'
245Run them manually with:
246
247 test/gold.sh run-for-release
248=====
249EOF
250 continue
251 fi
252 ;;
253 *)
254 banner "Test suite: $name"
255 ;;
256 esac
257
258 test/$name.sh run-for-release
259 done
260
261 ysh/run.sh run-for-release
262
263 data_lang/j8-errors.sh run-for-release
264}
265
266_spec-sanity-check() {
267 # Quick early test for _bin/osh and _bin/ysh
268
269 # TODO: Use --ovm-bin-dir
270 # Note: MAX_PROCS=1 prevents [#oil-dev > Random Spec Test Stoppages]
271 # Still need to fix that bug
272 MAX_PROCS=1 NUM_SPEC_TASKS=2 OSH_LIST="$OSH_RELEASE_BINARY" test/spec-py.sh osh-all
273 MAX_PROCS=1 NUM_SPEC_TASKS=2 YSH_LIST="$YSH_RELEASE_BINARY" test/spec-py.sh ysh-all
274}
275
276spec-all() {
277 ### Run all spec tests
278
279 test/stateful.sh soil-run # Same as CI
280
281 # Create the tests we're running
282 test/smoosh.sh make-spec
283
284 # TODO: Use --ovm-bin-dir
285 export OSH_LIST="$REPO_ROOT/bin/osh $OSH_RELEASE_BINARY"
286 export YSH_LIST="$REPO_ROOT/bin/ysh $YSH_RELEASE_BINARY"
287 test/spec-py.sh all-and-smoosh
288
289 # Build $OSH_CPP_BENCHMARK_DATA
290 _build-oils-benchmark-data
291
292 # TODO: Use --oils-cpp-bin-dir
293 # Collect and publish stats about the C++ translation.
294 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh osh-all
295 YSH_CC="$YSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh ysh-all
296}
297
298spec-cpp() {
299 ### For repair
300
301 # TODO: Use --oils-cpp-bin-dir
302
303 # Quick
304 # NUM_SPEC_TASKS=2 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh all
305 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh all
306}
307
308# For quickly debugging failures that don't happen in dev mode.
309spec-one() {
310 export OSH_LIST="$REPO_ROOT/bin/osh $OSH_RELEASE_BINARY"
311 export YSH_LIST="$REPO_ROOT/bin/ysh $YSH_RELEASE_BINARY"
312 test/spec.sh "$@"
313}
314
315build-and-test() {
316 ### Build tarballs and test them. And preliminaries like unit tests.
317
318 # TODO: Log this whole thing? Include logs with the /release/ page?
319
320 # Before doing anything
321 test/lint.sh soil-run
322
323 test/unit.sh run-for-release # Python unit tests
324
325 test/coverage.sh run-for-release # C++ unit tests
326
327 # App bundle
328 _release-build
329
330 # TODO: test oils-for-unix in Alpine chroot too.
331 # NOTE: Need test/alpine.sh download;extract;setup-dns,add-oil-build-deps,
332 # etc.
333 if test -n "$HAVE_ROOT"; then
334 # TODO: Factor out test/alpine.sh to test/chroot.sh
335 test/alpine.sh copy-tar '' oil
336 test/alpine.sh test-tar '' oil
337 fi
338
339 test/spec.sh smoke # Initial smoke test, slightly redundant.
340
341 run-more-tests
342}
343
344_install() {
345 test/spec-bin.sh install-shells-with-apt
346
347 # A subset of build/py.sh ubuntu-deps. (Do we need build-essential?)
348 #sudo apt install python-dev
349}
350
351_build-oils-benchmark-data() {
352 pushd $BENCHMARK_DATA_OILS
353 _build/oils.sh '' opt SKIP_REBUILD
354 _build/oils.sh '' dbg SKIP_REBUILD # for metrics/native-code.sh
355 popd
356}
357
358benchmark-build() {
359 ### Build function on machine 2.
360
361 build/clean.sh
362 if test -n "$HAVE_ROOT"; then
363 _install
364 fi
365 build/py.sh all
366 _release-build
367}
368
369# Run benchmarks with the binary built out of the tarball.
370benchmark-run() {
371 local do_machine1=${1:-}
372
373 _build-oils-benchmark-data
374 OSH_OVM=$OSH_RELEASE_BINARY benchmarks/auto.sh all "$do_machine1"
375}
376
377_compressed-tarball() {
378 local name=${1:-hello}
379 local version=${2:-0.0.0}
380
381 local in=_release/$name.tar
382 local out=_release/$name-$version.tar.gz
383
384 # Overwrite it to cause rebuild of oil.tar
385 build/stamp.sh write-release-date
386
387 #make -d -r $in # To debug
388 make $in
389 time gzip -c $in > $out
390 ls -l $out
391
392 # xz version is considerably smaller. 1.15 MB vs. 1.59 MB.
393 local out2=_release/$name-$version.tar.xz
394 time xz -c $in > $out2
395 ls -l $out2
396}
397
398oil() {
399 _compressed-tarball oil $OIL_VERSION
400}
401
402hello() {
403 _compressed-tarball hello $(head -n 1 build/testdata/hello-version.txt)
404}
405
406
407_link() {
408 ln -s -f -v --no-target-directory "$@"
409}
410
411compress() {
412 local root=$PWD/_release/VERSION/
413
414 log '--- more-tests'
415 local out="$root/more-tests.wwz"
416 pushd _tmp
417 time zip -r -q $out suite-logs unit syscall process-table
418 popd
419
420 # This has HTML reports, .profraw files, and logs of stdout, e.g.
421 # mycpp-unit/gc_heap_test.log
422 # About 1.5 MB
423 log "--- coverage"
424 local out="$root/test/coverage.wwz"
425 pushd _test/clang-coverage
426 # This also saves the logs
427 time zip -r -q $out .
428 popd
429
430 log "--- test/spec"
431 local out="$root/test/spec.wwz"
432 pushd _tmp/spec
433 time zip -r -q $out . # recursive, quiet
434 popd
435
436 log "--- test/wild"
437 local out="$root/test/wild.wwz"
438 pushd _tmp/wild-www
439 time zip -r -q $out . # recursive, quiet
440 popd
441
442 # NOTE: must be /pub/metrics.wwz so that relative URLs like
443 # ../../../web/line-counts.css work. The Soil UI also relies on such
444 # relative URLs.
445 log "--- metrics"
446 local out="$root/pub/metrics.wwz"
447 pushd _tmp/metrics
448 time zip -r -q $out . # recursive, quiet
449 popd
450
451 # Ditto: pub/src-tree.wwz lines up with URLs in Soil
452 log "--- src-tree"
453 local out="$root/pub/src-tree.wwz"
454 pushd _tmp/src-tree-www
455 time zip -r -q $out . # recursive, quiet
456 popd
457
458 compress-benchmarks
459
460 tree _release/VERSION
461}
462
463compress-benchmarks() {
464 local root=$PWD/_release/VERSION/
465 mkdir -p $root
466
467 log "--- benchmarks"
468
469 local out="$root/benchmarks.wwz"
470
471 # - For benchmarks that run on multiple machines, technically we only need
472 # index.html, but include stage1 and stage2.
473 # - For those that run on single machines, we also archive the raw/ dir.
474 # - Although benchmarks/compute is saved in oilshell/benchmark-data
475 # - Note: _tmp/uftrace/{raw,stage1} are big (hundreds of MB), so leave them
476 # out
477
478 pushd _tmp
479 find \
480 osh-parser/{stage1,stage2,index.html} \
481 osh-runtime/{stage1,stage2,index.html} \
482 vm-baseline/{stage1,stage2,index.html} \
483 ovm-build/{stage1,stage2,index.html} \
484 compute/{raw,stage1,stage2,index.html} \
485 gc/{raw,stage2,index.html} \
486 gc-cachegrind/{raw,stage2,index.html} \
487 mycpp-examples/{raw,stage2,index.html} \
488 uftrace/{stage2,index.html} \
489 -type f \
490 | xargs --verbose -- zip -q $out
491 popd
492}
493
494line-counts() {
495 local out_dir=$1 # should be an absolute path
496 mkdir -p $out_dir
497
498 # Counting directly from the build.
499 metrics/tarball.sh linecount-pydeps > $out_dir/pydeps.txt
500 metrics/tarball.sh linecount-nativedeps > $out_dir/nativedeps.txt
501 metrics/tarball.sh linecount-oils-cpp > $out_dir/oils-cpp.txt
502
503 metrics/source-code.sh write-reports $out_dir # for-translation and overview
504 metrics/source-code.sh cloc-report > $out_dir/cloc-report.txt
505
506 # goes to _tmp/metrics/preprocessed
507 metrics/source-code.sh preprocessed
508}
509
510metrics() {
511 local out=_tmp/metrics
512 mkdir -p $out
513
514 line-counts $PWD/$out/line-counts
515
516 # For another .wwz file
517 doctools/src-tree.sh soil-run
518
519 metrics/bytecode.sh run-for-release
520 metrics/native-code.sh run-for-release
521 build/cpython-defs.sh run-for-release
522
523 tree $out
524}
525
526deploy-doc() {
527 local deploy_repo='../oilshell.org__deploy'
528 local release_root_dir="$deploy_repo/release"
529 local release_dir="$release_root_dir/$OIL_VERSION"
530
531 cp -v -r --force --no-target-directory \
532 _release/VERSION/ $release_dir/
533
534 # Generate release index.
535 html-index $release_root_dir _tmp/releases.html
536 cp -v _tmp/releases.html $deploy_repo
537
538 tree -L 3 $release_root_dir
539
540 ls -l $deploy_repo/releases.html
541}
542
543readonly DOWNLOAD_DIR='../oilshell.org__deploy/download/'
544
545# Generating releases.html requires the old tarballs!
546sync-old-tar() {
547 local user=$1 # required username
548 rsync --archive --verbose \
549 $user@oilshell.org:oilshell.org/download/ $DOWNLOAD_DIR
550}
551
552# I think these aren't checked into git? They can just be managed separately?
553# Or should you check in the sha checksums? Those will be in releases.html,
554# but a CSV might be nice.
555deploy-tar() {
556 mkdir -p $DOWNLOAD_DIR
557
558 cp -v \
559 _release/oil-$OIL_VERSION.tar.* _release/oils-for-unix-$OIL_VERSION.tar.* \
560 $DOWNLOAD_DIR
561
562 ls -l $DOWNLOAD_DIR
563}
564
565#
566# Generate releases.html.
567#
568
569# Examples of similar release HTML pages:
570# - https://golang.org/dl/ - "Older versions", sha1 / sha256.
571# - Python has all point releases in chronological order, and then a separate
572# page for each changelog. There is too much boilerplate maybe?
573# - It has release notes before the downloads. Not sure I like that.
574# - node.js: https://nodejs.org/en/
575# - user agent detection for the right binary -- meh I don't want that
576# - Ruby: https://www.ruby-lang.org/en/downloads/releases/
577# - https://www.lua.org/download.html
578
579# Columns: Date / Version / Docs / / Files
580# Changelog .xz
581# Install
582# Docs/
583#
584# The files could be a separate page and separate table? I could provide
585# pre-built versions eventually? Linux static versions?
586
587# TODO: Each of these would be a good candidate for a data frame! Data vs.
588# presentation.
589
590# Simple UI:
591# - home page shows latest version (source release for now, binary release later?)
592# - link to Changelog, INSTALL, doc index
593# - or see all releases
594# - Grey out older releases?
595
596# TODO: Should be sorted by date? How to do that, with bash array? Or Awk?
597# $timestamp $version $timestamp file? And then sort -n I guess? Change
598# the release date format. It will use Unix timestamp (OK until 2038!)
599
600_html-index() {
601 local release_root_dir=$1 # the directory we want to make an index of
602
603 for entry in $release_root_dir/*; do
604 if ! test -d $entry; then
605 continue
606 fi
607 local dir=$entry
608
609 local version
610 version=$(head -n 1 $dir/oil-version.txt)
611 local release_date
612 release_date=$(head -n 1 $dir/release-date.txt)
613
614 log "-- $dir"
615 log "Version: $version"
616 log "Release Date: $release_date"
617 log ""
618
619 echo "$release_date $version"
620 done > _tmp/release-meta.txt
621
622 # Reverse sort by release date
623 sort -r _tmp/release-meta.txt > _tmp/sorted-releases.txt
624
625 while read date _ version; do
626 log "Release Date: $date"
627 log "Version: $version"
628
629 # anchor
630 cat <<EOF
631<tr>
632 <td>
633 <span class="date">$date</span>
634 </td>
635 <td>
636 <a name="$version"></a>
637 <span class="version-number">$version</span>
638 </td>
639 <td>
640 <p> <a href="release/$version/announcement.html">Announcement</a>
641 &nbsp; | &nbsp; <a href="release/$version/">Docs and Details</a>
642 </p>
643 </td>
644</tr>
645EOF
646
647 build/doc.sh tarball-links-row-html $version
648
649 cat <<EOF
650<tr>
651 <td colspan="3">
652 <div style="padding: 1em;" >
653 </div>
654 </td>
655</tr>
656
657EOF
658
659 done < _tmp/sorted-releases.txt
660}
661
662_releases-html-header() {
663 # TODO: use html-head here, and publish web/*.css somewhere outside of
664 # /release/$VERSION/? The list of all releases isn't versioned for obvious
665 # reasons. Other docs are in the oilshell.org repo using the all-2020.css
666 # bundle.
667
668 cat <<EOF
669<!DOCTYPE html>
670<html>
671 <head>
672 <meta name="viewport" content="width=device-width, initial-scale=1">
673 <title>Oils Releases</title>
674 <style>
675EOF
676
677 cat web/base.css
678 cat web/release-index.css
679
680cat <<EOF
681 h1 {
682 text-align: center;
683 }
684 </style>
685 </head>
686 <body class="width50">
687 <p id="home-link">
688 <a href="/">oilshell.org</a>
689 </p>
690 <h1>Oils Releases</h1>
691
692 <table class="release-table">
693EOF
694}
695
696html-index() {
697 local release_root_dir=$1
698 local out=${2:-_tmp/releases.html}
699
700 { _releases-html-header
701 _html-index $release_root_dir
702
703 cat <<EOF
704 </table>
705 </body>
706</html>
707EOF
708
709 } > $out
710
711 ls -l $out
712}
713
714# For quickly iterating on tarball size reductions.
715tarball-size() {
716 make clean-repo
717 make _bin/oil.ovm-dbg # faster way to build bytecode
718 oil # make tarball
719 test-oil-tar # Ctrl-C this, then run metrics/tarball.sh
720}
721
722dep-smoosh() {
723 local repo=~/git/languages/smoosh
724 if ! test -d $repo; then
725 local base_dir=$(dirname $repo)
726 mkdir -p $base_dir
727 pushd $base_dir
728 git clone git@github.com:mgree/smoosh.git
729 popd
730 fi
731}
732
733dep-benchmarks() {
734 ### Before auto-machine2
735
736 # 2023-07: Also need deps/from-tar.sh {configure,build}-cpython
737
738 benchmarks/osh-runtime.sh download
739 benchmarks/osh-runtime.sh extract
740
741 benchmarks/ovm-build.sh download
742 benchmarks/ovm-build.sh extract-other
743
744 # For ovm-build benchmark.
745 deps/from-binary.sh download-clang
746 deps/from-binary.sh extract-clang
747}
748
749more-release-deps() {
750 # List of deps that are NOT in soil/worker.sh here
751 # https://github.com/oilshell/oil/issues/926
752
753 # TODO: Make a container image for these.
754 if false; then
755 # TODO: Did this manually
756 # test/alpine.sh
757 # dep-alpine
758
759 # test/smoosh.sh
760 dep-smoosh
761
762 dep-benchmarks
763 fi
764}
765
766py-tarball() {
767 local in=_release/oil.tar
768 local out=_release/oil-$OIL_VERSION.tar.gz
769
770 make $in
771 time gzip -c $in > $out
772 ls -l $out
773
774 test-oil-tar
775}
776
777native-tarball() {
778 # oils-for-unix
779 devtools/release-native.sh make-tar
780 # Also install as root
781 devtools/release-native.sh extract-for-benchmarks INSTALL
782}
783
784two-tarballs() {
785 ### First step of release. Assume that CI passes
786
787 ensure-smooth-build
788
789 build/py.sh all
790 # "Base state" for repo scripts
791 ./NINJA-config.sh
792
793 py-tarball
794
795 native-tarball
796}
797
798upload-tmp() {
799 local tarball=$1
800 local user=$2
801
802 scp $tarball $user@oilshell.org:tmp/
803}
804
805sync-tmp() {
806 local user=$1
807 local dest=${2:-_tmp/candidates}
808 mkdir -p $dest
809 rsync --archive --verbose $user@oilshell.org:tmp/ $dest
810}
811
812"$@"