OILS / soil / web-worker.sh View on Github | oilshell.org

352 lines, 137 significant
1#!/usr/bin/env bash
2#
3# Functions to invoke soil/web remotely.
4#
5# soil/web is deployed manually, and then this runs at HEAD in the repo. Every
6# CI run has an up-to-date copy.
7#
8# Usage:
9# soil/web-worker.sh <function name>
10
11set -o nounset
12set -o pipefail
13set -o errexit
14
15REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
16
17source soil/common.sh
18source test/tsv-lib.sh # tsv2html
19source web/table/html.sh # table-sort-{begin,end}
20
21# ~/
22# soil-web/ # executable files
23# doctools/
24# html_head.py
25# soil/
26# web.py
27# web.sh
28# travis-ci.oilshell.org/ # served over HTTP
29# index.html
30# web/
31# base.css
32# soil.css
33# github-jobs/
34# index.html
35# 3619/ # $GITHUB_RUN_NUMBER
36# dev-minimal.wwz
37# cpp-small.wwz
38# srht-jobs/
39# index.html
40# 22/ # $JOB_ID
41# dev-minimal.wwz
42# 23 # $JOB_ID
43# cpp-small.wwz
44
45sshq() {
46 # Don't need commands module as I said here!
47 # http://www.oilshell.org/blog/2017/01/31.html
48 #
49 # This is Bernstein chaining through ssh.
50
51 ssh $SOIL_USER@$SOIL_HOST "$(printf '%q ' "$@")"
52}
53
54remote-rewrite-jobs-index() {
55 sshq soil-web/soil/web.sh rewrite-jobs-index "$@"
56}
57
58remote-cleanup-jobs-index() {
59 local prefix=$1
60 # clean it up for real!
61 sshq soil-web/soil/web.sh cleanup-jobs-index "$prefix" false
62}
63
64remote-cleanup-status-api() {
65 #sshq soil-web/soil/web.sh cleanup-status-api false
66 # 2024-07 - work around bug. The logic in soil/web.sh doesn't seem right
67 sshq soil-web/soil/web.sh cleanup-status-api true
68}
69
70my-scp() {
71 scp -o StrictHostKeyChecking=no "$@"
72}
73
74my-ssh() {
75 ssh -o StrictHostKeyChecking=no "$@"
76}
77
78scp-status-api() {
79 local run_id=${1:-TEST2-github-run-id}
80 local job_name=$2
81
82 local status_file="_soil-jobs/$job_name.status.txt"
83 local remote_path="$SOIL_REMOTE_DIR/status-api/github/$run_id/$job_name"
84
85 # We could make this one invocation of something like:
86 # cat $status_file | sshq soil/web.sh PUT $remote_path
87
88 my-ssh $SOIL_USER_HOST "mkdir -p $(dirname $remote_path)"
89
90 # the consumer should check if these are all zero
91 # note: the file gets RENAMED
92 my-scp $status_file "$SOIL_USER_HOST:$remote_path"
93}
94
95scp-results() {
96 # could also use Travis known_hosts addon?
97 local prefix=$1 # srht- or ''
98 shift
99
100 my-scp "$@" "$SOIL_USER_HOST:$SOIL_REMOTE_DIR/${prefix}jobs/"
101}
102
103# Dummy that doesn't depend on results
104deploy-test-wwz() {
105 set -x
106 local out_name="$(date +%Y-%m-%d__%H-%M-%S)_test"
107
108 local wwz=$out_name.wwz
109
110 cat >index.html <<EOF
111<a href="build/oil-manifest.txt">build/oil-manifest.txt</a> <br/>
112<a href="build/opy-manifest.txt">build/opy-manifest.txt</a> <br/>
113<a href="env.txt">env.txt</a> <br/>
114EOF
115
116 dump-env > env.txt
117
118 zip -q $wwz env.txt index.html build/*.txt
119
120 scp-results '' $wwz
121}
122
123format-wwz-index() {
124 ### What's displayed in $ID.wwz/index.html
125
126 local job_id=$1
127 local tsv=${2:-_tmp/soil/INDEX.tsv}
128
129 soil-html-head "$job_id.wwz"
130
131 cat <<EOF
132 <body class="width40">
133 <p id="home-link">
134 <a href="..">Up</a>
135 | <a href="/">Home</a>
136 | <a href="//oilshell.org/">oilshell.org</a>
137 </p>
138
139 <h1>$job_id.wwz</h1>
140EOF
141
142 echo '<ul>'
143 cat <<EOF
144 <li>
145 <a href="_tmp/soil/INDEX.tsv">_tmp/soil/INDEX.tsv</a>, also copied to
146 <a href="../$job_id.tsv">../$job_id.tsv</a>.
147 </li>
148 <li>
149 <a href="../$job_id.json">../$job_id.json</a>
150 </li>
151EOF
152
153 if test -f _tmp/soil/image.html; then
154 echo '
155 <li>
156 <a href="_tmp/soil/image.html">Container Image Stats</a>
157 </li>
158 '
159 fi
160
161 echo '</ul>'
162}
163
164format-image-stats() {
165 local soil_dir=${1:-_tmp/soil}
166 local web_base_url=${2:-'/web'} # for production
167
168 table-sort-html-head "Image Stats" $web_base_url
169
170 # prints <body>; make it wide for the shell commands
171 table-sort-begin "width60"
172
173 # TODO:
174 # - Format the TSV as an HTML table
175 # - Save the name and tag and show it
176
177 cat <<EOF
178 <p id="home-link">
179 <a href="/">Home</a>
180 | <a href="//oilshell.org/">oilshell.org</a>
181 </p>
182
183 <h1>Images Tagged</h1>
184
185 <a href="images-tagged.txt">images-tagged.txt</a> <br/>
186
187 <h1>Image Layers</h1>
188EOF
189
190 tsv2html3 $soil_dir/image-layers.tsv
191
192 # First column is number of bytes; ignore header
193 local total_bytes=$(awk '
194 { sum += $1 }
195 END { printf("%.1f", sum / 1000000) }
196 ' $soil_dir/image-layers.tsv)
197
198 echo "<p>Total Size: <b>$total_bytes MB</b></p>"
199
200
201 cat <<EOF
202 <h2>Raw Data</h2>
203
204 <a href="image-layers.txt">image-layers.txt</a> <br/>
205 <a href="image-layers.tsv">image-layers.tsv</a> <br/>
206 </body>
207</html>
208EOF
209
210 table-sort-end image-layers
211}
212
213make-job-wwz() {
214 local job_id=${1:-test-job}
215
216 local wwz=$job_id.wwz
217
218 # Doesn't exist when we're not using a container
219 if test -f _tmp/soil/image-layers.tsv; then
220 format-image-stats _tmp/soil > _tmp/soil/image.html
221 fi
222
223 format-wwz-index $job_id > index.html
224
225 # _tmp/soil: Logs are in _tmp, see soil/worker.sh
226 # web/ : spec test HTML references this.
227 # Note that that index references /web/{base,soil}.css, outside the .wwz
228 # osh-summary.html uses table-sort.js and ajax.js
229 #
230 # TODO:
231 # - Could move _tmp/{spec,stateful,syscall} etc. to _test
232 # - Create _tmp/benchmarks/{compute,gc,gc-cachegrind,osh-parser,mycpp-examples,...}
233 # - would require release/$VERSION/pub/benchmarks.wwz, like we have
234 # pub/metrics.wwz, for consistent links
235
236 zip -q -r $wwz \
237 index.html \
238 _build/wedge/logs \
239 _test \
240 _tmp/{soil,spec,src-tree-www,wild-www,stateful,process-table,syscall,benchmark-data,metrics,mycpp-examples,compute,gc,gc-cachegrind,perf,vm-baseline,osh-runtime,osh-parser,host-id,shell-id} \
241 _tmp/uftrace/{index.html,stage2} \
242 web/{base,src-tree,spec-tests,spec-cpp,line-counts,benchmarks,wild}.css web/ajax.js \
243 web/table/table-sort.{css,js} \
244 _release/oil*.tar _release/*.xshar _release/VERSION/
245}
246
247test-collect-json() {
248 soil/collect_json.py _tmp/soil PATH
249}
250
251deploy-job-results() {
252 ### Copy .wwz, .tsv, and .json to a new dir
253
254 local prefix=$1 # e.g. example.com/github-jobs/
255 local subdir=$2 # e.g. example.com/github-jobs/1234/ # make this dir
256 local job_name=$3 # e.g. example.com/github-jobs/1234/foo.wwz
257 shift 2
258 # rest of args are more env vars
259
260 # writes $job_name.wwz
261 make-job-wwz $job_name
262
263 # Debug permissions. When using docker rather than podman, these dirs can be
264 # owned by root and we can't write into them.
265 ls -l -d _tmp/soil
266 ls -l _tmp/soil
267
268 date +%s > _tmp/soil/task-deploy-start-time.txt
269
270 soil/collect_json.py _tmp/soil "$@" > $job_name.json
271
272 # So we don't have to unzip it
273 cp _tmp/soil/INDEX.tsv $job_name.tsv
274
275 local remote_dest_dir="$SOIL_REMOTE_DIR/${prefix}jobs/$subdir"
276 my-ssh $SOIL_USER_HOST "mkdir -p $remote_dest_dir"
277
278 # Do JSON last because that's what 'list-json' looks for
279 my-scp $job_name.{wwz,tsv,json} "$SOIL_USER_HOST:$remote_dest_dir"
280
281 log ''
282 log 'View CI results here:'
283 log ''
284 log "http://$SOIL_HOST/${prefix}jobs/$subdir/"
285 log "http://$SOIL_HOST/${prefix}jobs/$subdir/$job_name.wwz/"
286 log ''
287}
288
289publish-cpp-tarball() {
290 local prefix=${1:-'github-'} # e.g. example.com/github-jobs/
291
292 # Example of dir structure we need to cleanup:
293 #
294 # srht-jobs/
295 # git-$hash/
296 # index.html
297 # oils-for-unix.tar
298 # github-jobs/
299 # git-$hash/
300 # oils-for-unix.tar
301 #
302 # Algorithm
303 # 1. List all JSON, finding commit date and commit hash
304 # 2. Get the OLDEST commit dates, e.g. all except for 50
305 # 3. Delete all commit hash dirs not associated with them
306
307 # Fix subtle problem here !!!
308 shopt -s inherit_errexit
309
310 local git_commit_dir
311 git_commit_dir=$(git-commit-dir "$prefix")
312
313 my-ssh $SOIL_USER_HOST "mkdir -p $git_commit_dir"
314
315 # Do JSON last because that's what 'list-json' looks for
316
317 local tar=_release/oils-for-unix.tar
318
319 # Permission denied because of host/guest issue
320 #local tar_gz=$tar.gz
321 #gzip -c $tar > $tar_gz
322
323 # Avoid race condition
324 # Crappy UUID: seconds since epoch, plus PID
325 local timestamp
326 timestamp=$(date +%s)
327
328 local temp_name="tmp-$timestamp-$$.tar"
329
330 my-scp $tar "$SOIL_USER_HOST:$git_commit_dir/$temp_name"
331
332 my-ssh $SOIL_USER_HOST \
333 "mv -v $git_commit_dir/$temp_name $git_commit_dir/oils-for-unix.tar"
334
335 log 'Tarball:'
336 log ''
337 log "http://$git_commit_dir"
338}
339
340remote-event-job-done() {
341 ### "Client side" handler: a job calls this when it's done
342
343 log "remote-event-job-done"
344
345 # Deployed code dir
346 sshq soil-web/soil/web.sh event-job-done "$@"
347}
348
349filename=$(basename $0)
350if test $filename = 'web-worker.sh'; then
351 "$@"
352fi