OILS / benchmarks / osh-runtime.sh View on Github | oilshell.org

578 lines, 339 significant
1#!/usr/bin/env bash
2#
3# Test scripts found in the wild for both correctness and performance.
4#
5# Usage:
6# benchmarks/osh-runtime.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13
14source benchmarks/common.sh # tsv-concat
15source benchmarks/id.sh # print-job-id
16source test/common.sh
17source test/tsv-lib.sh # tsv-row
18
19readonly BASE_DIR=_tmp/osh-runtime
20
21# TODO: Move to ../oil_DEPS
22readonly TAR_DIR=$PWD/_deps/osh-runtime # Make it absolute
23
24#
25# Dependencies
26#
27
28readonly PY27_DIR=$PWD/Python-2.7.13
29
30# NOTE: Same list in oilshell.org/blob/run.sh.
31tarballs() {
32 cat <<EOF
33tcc-0.9.26.tar.bz2
34yash-2.46.tar.xz
35ocaml-4.06.0.tar.xz
36util-linux-2.40.tar.xz
37EOF
38}
39
40download() {
41 mkdir -p $TAR_DIR
42 tarballs | xargs -n 1 -I {} --verbose -- \
43 wget --no-clobber --directory $TAR_DIR 'https://www.oilshell.org/blob/testdata/{}'
44}
45
46extract() {
47 set -x
48 time for f in $TAR_DIR/*.{bz2,xz}; do
49 tar -x --directory $TAR_DIR --file $f
50 done
51 set +x
52
53 ls -l $TAR_DIR
54}
55
56#
57# Computation
58#
59
60run-tasks() {
61 local raw_out_dir=$1
62 raw_out_dir="$PWD/$raw_out_dir" # because we change dirs
63
64 # Bug fix for dynamic scoping!
65 local host_name sh_path workload
66
67 local task_id=0
68 while read -r host_name sh_path workload; do
69
70 log "*** $host_name $sh_path $workload $task_id"
71
72 local sh_run_path
73 case $sh_path in
74 /*) # Already absolute
75 sh_run_path=$sh_path
76 ;;
77 */*) # It's relative, so make it absolute
78 sh_run_path=$PWD/$sh_path
79 ;;
80 *) # 'dash' should remain 'dash'
81 sh_run_path=$sh_path
82 ;;
83 esac
84
85 local working_dir=''
86 local files_out_dir="$raw_out_dir/files-$task_id"
87 mkdir -v -p $files_out_dir
88
89 local save_new_files=''
90
91 local -a argv
92 case $workload in
93 hello-world)
94 argv=( testdata/osh-runtime/hello_world.sh )
95 ;;
96
97 bin-true)
98 argv=( testdata/osh-runtime/bin_true.sh )
99 ;;
100
101 abuild-print-help)
102 argv=( testdata/osh-runtime/abuild -h )
103 ;;
104
105 configure.cpython)
106 argv=( $PY27_DIR/configure )
107 working_dir=$files_out_dir
108 ;;
109
110 configure.util-linux)
111 # flag needed to avoid sqlite3 dep error message
112 argv=( $TAR_DIR/util-linux-2.40/configure --disable-liblastlog2 )
113 working_dir=$files_out_dir
114 ;;
115
116 configure.*)
117 argv=( ./configure )
118
119 local conf_dir
120 case $workload in
121 *.ocaml)
122 conf_dir='ocaml-4.06.0'
123 ;;
124 *.tcc)
125 conf_dir='tcc-0.9.26'
126 ;;
127 *.yash)
128 conf_dir='yash-2.46'
129 ;;
130 *)
131 die "Invalid workload $workload"
132 esac
133
134 # These are run in-tree?
135 working_dir=$TAR_DIR/$conf_dir
136 ;;
137
138 *)
139 die "Invalid workload $workload"
140 ;;
141 esac
142
143 local -a time_argv=(
144 time-tsv
145 --output "$raw_out_dir/times.tsv" --append
146 --rusage
147 --rusage-2
148 --field "$task_id"
149 --field "$host_name" --field "$sh_path"
150 --field "$workload"
151 -- "$sh_run_path" "${argv[@]}"
152 )
153
154 local stdout_file="$files_out_dir/STDOUT.txt"
155 local gc_stats_file="$raw_out_dir/gc-$task_id.txt"
156
157 # Maybe change dirs
158 if test -n "$working_dir"; then
159 pushd "$working_dir"
160 fi
161
162 if test -n "$save_new_files"; then
163 touch __TIMESTAMP
164 fi
165
166 # Run it, possibly with GC stats
167 case $sh_path in
168 *_bin/*/osh)
169 OILS_GC_STATS_FD=99 "${time_argv[@]}" > $stdout_file 99> $gc_stats_file
170 ;;
171 *)
172 "${time_argv[@]}" > $stdout_file
173 ;;
174 esac
175
176 if test -n "$save_new_files"; then
177 echo "COPYING to $files_out_dir"
178 find . -type f -newer __TIMESTAMP \
179 | xargs -I {} -- cp --verbose {} $files_out_dir
180 fi
181
182 # Restore dir
183 if test -n "$working_dir"; then
184 popd
185 fi
186
187 task_id=$((task_id + 1))
188 done
189}
190
191# Sorted by priority for test-oils.sh osh-runtime --num-shells 3
192
193readonly -a ALL_WORKLOADS=(
194 hello-world
195 bin-true
196
197 configure.cpython
198 configure.util-linux
199 configure.ocaml
200 configure.tcc
201 configure.yash
202
203 abuild-print-help
204)
205
206print-workloads() {
207 ### for help
208
209 for w in "${ALL_WORKLOADS[@]}"; do
210 echo " $w"
211 done
212}
213
214print-tasks() {
215 local host_name=$1
216 local osh_native=$2
217
218 if test -n "${QUICKLY:-}"; then
219 workloads=(
220 hello-world
221 bin-true
222 #configure.util-linux
223 #abuild-print-help
224 )
225 else
226 workloads=( "${ALL_WORKLOADS[@]}" )
227 fi
228
229 for sh_path in bash dash bin/osh $osh_native; do
230 for workload in "${workloads[@]}"; do
231 tsv-row $host_name $sh_path $workload
232 done
233 done
234}
235
236print-tasks-xshar() {
237 local host_name=$1
238 local osh_native=$2
239
240 local num_iters=${3:-1}
241 local num_shells=${4:-1}
242 local num_workloads=${5:-1}
243
244 for i in $(seq $num_iters); do
245
246 local s=0
247 for sh_path in $osh_native bash dash; do
248
249 local w=0
250 for workload in "${ALL_WORKLOADS[@]}"; do
251 tsv-row $host_name $sh_path $workload
252
253 w=$(( w + 1 )) # cut off at specified workloads
254 if test $w -eq $num_workloads; then
255 break
256 fi
257 done
258
259 s=$(( s + 1 )) # cut off as specified shells
260 if test $s -eq $num_shells; then
261 break
262 fi
263
264 done
265 done
266}
267
268test-print-tasks-xshar() {
269 print-tasks-xshar $(hostname) osh 1 1 1
270 echo
271 print-tasks-xshar $(hostname) osh 1 2 1
272 echo
273 print-tasks-xshar $(hostname) osh 1 2 2
274 echo
275 print-tasks-xshar $(hostname) osh 1 2 3
276 echo
277}
278
279run-tasks-wrapper() {
280 ### reads tasks from stdin
281
282 local host_name=$1 # 'no-host' or 'lenny'
283 local raw_out_dir=$2
284
285 mkdir -v -p $raw_out_dir
286
287 local tsv_out="$raw_out_dir/times.tsv"
288
289 # Write header of the TSV file that is appended to.
290 time-tsv -o $tsv_out --print-header \
291 --rusage \
292 --rusage-2 \
293 --field task_id \
294 --field host_name --field sh_path \
295 --field workload
296
297 # reads tasks from stdin
298 # run-tasks outputs 3 things: raw times.tsv, per-task STDOUT and files, and
299 # per-task GC stats
300 run-tasks $raw_out_dir
301
302 # Turn individual files into a TSV, adding host
303 benchmarks/gc_stats_to_tsv.py $raw_out_dir/gc-*.txt \
304 | tsv-add-const-column host_name "$host_name" \
305 > $raw_out_dir/gc_stats.tsv
306
307 cp -v _tmp/provenance.tsv $raw_out_dir
308}
309
310measure() {
311 ### For release and CI
312 local host_name=$1 # 'no-host' or 'lenny'
313 local raw_out_dir=$2 # _tmp/osh-runtime or ../../benchmark-data/osh-runtime
314 local osh_native=$3 # $OSH_CPP_NINJA_BUILD or $OSH_CPP_BENCHMARK_DATA
315
316 print-tasks "$host_name" "$osh_native" \
317 | run-tasks-wrapper "$host_name" "$raw_out_dir"
318}
319
320stage1() {
321 local base_dir=${1:-$BASE_DIR} # _tmp/osh-runtime or ../benchmark-data/osh-runtime
322 local single_machine=${2:-}
323
324 local out_dir=$BASE_DIR/stage1 # _tmp/osh-runtime
325 mkdir -p $out_dir
326
327 # Globs are in lexicographical order, which works for our dates.
328
329 local -a raw_times=()
330 local -a raw_gc_stats=()
331 local -a raw_provenance=()
332
333 if test -n "$single_machine"; then
334 local -a a=( $base_dir/raw.$single_machine.* )
335
336 raw_times+=( ${a[-1]}/times.tsv )
337 raw_gc_stats+=( ${a[-1]}/gc_stats.tsv )
338 raw_provenance+=( ${a[-1]}/provenance.tsv )
339
340 else
341 local -a a=( $base_dir/raw.$MACHINE1.* )
342 local -a b=( $base_dir/raw.$MACHINE2.* )
343
344 raw_times+=( ${a[-1]}/times.tsv ${b[-1]}/times.tsv )
345 raw_gc_stats+=( ${a[-1]}/gc_stats.tsv ${b[-1]}/gc_stats.tsv )
346 raw_provenance+=( ${a[-1]}/provenance.tsv ${b[-1]}/provenance.tsv )
347 fi
348
349 tsv-concat "${raw_times[@]}" > $out_dir/times.tsv
350
351 tsv-concat "${raw_gc_stats[@]}" > $out_dir/gc_stats.tsv
352
353 tsv-concat "${raw_provenance[@]}" > $out_dir/provenance.tsv
354}
355
356print-report() {
357 local in_dir=$1
358
359 benchmark-html-head 'OSH Runtime Performance'
360
361 cat <<EOF
362 <body class="width60">
363 <p id="home-link">
364 <a href="/">oilshell.org</a>
365 </p>
366EOF
367
368 cmark <<'EOF'
369## OSH Runtime Performance
370
371Source code: [benchmarks/osh-runtime.sh](https://github.com/oilshell/oil/tree/master/benchmarks/osh-runtime.sh)
372
373- [Elapsed Time](#elapsed-time)
374- [Minor Page Faults](#page-faults)
375- [Memory Usage](#memory-usage)
376- [GC Stats](#gc-stats)
377- [rusage Details](#rusage-details)
378- [More Details](#more-details)
379- [Shell and Host](#shell-and-host)
380
381[Raw files](-wwz-index)
382
383<a name="elapsed-time" />
384
385### Elapsed Time by Shell (milliseconds)
386
387Some benchmarks call many external tools, while some exercise the shell
388interpreter itself.
389EOF
390 tsv2html $in_dir/elapsed.tsv
391
392 cmark <<EOF
393<a name="page-faults" />
394
395### Minor Page Faults
396EOF
397
398 tsv2html $in_dir/page_faults.tsv
399
400 cmark <<EOF
401<a name="memory-usage" />
402
403### Memory Usage (Max Resident Set Size in MB)
404
405Memory usage is measured in MB (powers of 10), not MiB (powers of 2).
406EOF
407 tsv2html $in_dir/max_rss.tsv
408
409 cmark <<EOF
410<a name="gc-stats" />
411
412### GC Stats
413EOF
414 tsv2html $in_dir/gc_stats.tsv
415
416 cmark <<EOF
417<a name="rusage-details" />
418
419### rusage Details
420EOF
421 tsv2html $in_dir/details.tsv
422
423 cmark <<EOF
424<a name="more-details" />
425
426### More Details
427EOF
428 tsv2html $in_dir/details_io.tsv
429
430 cmark <<'EOF'
431<a name="shell-and-host" />
432
433### Shell and Host
434EOF
435 tsv2html $in_dir/shells.tsv
436 tsv2html $in_dir/hosts.tsv
437
438 cmark <<'EOF'
439
440 </body>
441</html>
442EOF
443}
444
445test-oils-run() {
446 local osh=$1
447 local job_id=$2
448 local host_name=$3
449
450 # flags passed by caller
451 local num_iters=${4:-1}
452 local num_shells=${5:-1}
453 local num_workloads=${6:-1}
454
455 local time_py=${XSHAR_DIR:-$REPO_ROOT}/benchmarks/time_.py
456 $time_py --tsv --rusage -- \
457 $osh -c 'echo "smoke test: hi from benchmarks/osh-runtime.sh"'
458
459 # Fresh build
460 rm -r -f -v $BASE_DIR _tmp/{shell,host}-id
461
462 # Write _tmp/provenance.* and _tmp/{host,shell}-id
463 shell-provenance-2 \
464 $host_name $job_id _tmp \
465 bash dash $osh
466
467 # e.g. 2024-05-01__10-11-12.ci-vm-name
468 local raw_out_dir="$BASE_DIR/raw"
469 mkdir -p $raw_out_dir
470
471 # Similar to 'measure', for soil-run and release
472 print-tasks-xshar $host_name $osh \
473 $num_iters $num_shells $num_workloads \
474 | tee $BASE_DIR/tasks.txt
475
476 run-tasks-wrapper $host_name $raw_out_dir < $BASE_DIR/tasks.txt
477 echo
478
479 # Note: 'stage1' in soil-run is a trivial concatenation, so we can create input for
480 # benchmarks/report.R. We don't need that here
481}
482
483soil-run() {
484 ### Run it on just this machine, and make a report
485
486 rm -r -f $BASE_DIR
487 mkdir -p $BASE_DIR
488
489 # TODO: This testdata should be baked into Docker image, or mounted
490 download
491 extract
492
493 # could add _bin/cxx-bumpleak/oils-for-unix, although sometimes it's slower
494 local -a osh_bin=( $OSH_CPP_NINJA_BUILD )
495 ninja "${osh_bin[@]}"
496
497 local single_machine='no-host'
498
499 local job_id
500 job_id=$(print-job-id)
501
502 # Write _tmp/provenance.* and _tmp/{host,shell}-id
503 shell-provenance-2 \
504 $single_machine $job_id _tmp \
505 bash dash bin/osh "${osh_bin[@]}"
506
507 local host_job_id="$single_machine.$job_id"
508 local raw_out_dir="$BASE_DIR/raw.$host_job_id"
509 mkdir -p $raw_out_dir $BASE_DIR/stage1
510
511 measure $single_machine $raw_out_dir $OSH_CPP_NINJA_BUILD
512
513 # Trivial concatenation for 1 machine
514 stage1 '' $single_machine
515
516 benchmarks/report.sh stage2 $BASE_DIR
517
518 benchmarks/report.sh stage3 $BASE_DIR
519}
520
521#
522# Debugging
523#
524
525compare-cpython() {
526 #local -a a=( ../benchmark-data/osh-runtime/*.lenny.2024* )
527 local -a a=( ../benchmark-data/osh-runtime/*.hoover.2024* )
528
529 # More of a diff here?
530 #local -a a=( ../benchmark-data/osh-runtime/*.broome.2023* )
531 # less diff here
532 #local -a a=( ../benchmark-data/osh-runtime/*.lenny.2023* )
533
534 local dir=${a[-1]}
535
536 echo $dir
537
538 head -n 1 $dir/times.tsv
539 fgrep 'configure.cpython' $dir/times.tsv
540
541 local bash_id=2
542 local dash_id=8
543 local osh_py_id=14
544 local osh_cpp_id=20
545
546 set +o errexit
547
548 local out_dir=_tmp/cpython-configure
549 mkdir -p $out_dir
550
551 echo 'bash vs. dash'
552 diff -u --recursive $dir/{files-2,files-8} > $out_dir/bash-vs-dash.txt
553 diffstat $out_dir/bash-vs-dash.txt
554 echo
555
556 echo 'bash vs. osh-py'
557 diff -u --recursive $dir/{files-2,files-14} > $out_dir/bash-vs-osh-py.txt
558 diffstat $out_dir/bash-vs-osh-py.txt
559 echo
560
561 echo 'bash vs. osh-cpp'
562 diff -u --recursive $dir/{files-2,files-20} > $out_dir/bash-vs-osh-cpp.txt
563 diffstat $out_dir/bash-vs-osh-cpp.txt
564 echo
565
566 return
567
568 diff -u $dir/{files-2,files-20}/STDOUT.txt
569 echo
570
571 diff -u $dir/{files-2,files-20}/pyconfig.h
572 echo
573
574 cdiff -u $dir/{files-2,files-20}/config.log
575 echo
576}
577
578"$@"