OILS / benchmarks / gc.sh View on Github | oilshell.org

725 lines, 408 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# benchmarks/gc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
11
12source benchmarks/common.sh # benchmark-html-head
13source benchmarks/cachegrind.sh # with-cachegrind
14source build/dev-shell.sh # R_LIBS_USER
15source test/tsv-lib.sh
16
17readonly BASE_DIR=_tmp/gc
18
19# duplicated in benchmarks/gc-cachegrind.sh
20readonly BASE_DIR_CACHEGRIND=_tmp/gc-cachegrind
21
22# See benchmarks/gperftools.sh. I think the Ubuntu package is very old
23
24download-tcmalloc() {
25 # TODO: move this to ../oil_DEPS ?
26 wget --directory _deps \
27 https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz
28
29 # Then ./configure; make; sudo make install
30 # installs in /usr/local/lib
31
32 # Note: there's a warning about libunwind -- maybe install that first. Does
33 # it only apply to CPU profiles?
34}
35
36debug-tcmalloc() {
37 touch mycpp/marksweep_heap.cc
38
39 # No evidence of difference
40 for bin in _bin/cxx-{opt,opt+tcmalloc}/osh; do
41 echo $bin
42 ninja $bin
43
44 ldd $bin
45 echo
46
47 ls -l $bin
48 echo
49
50 # Check what we're linking against
51 nm $bin | egrep -i 'malloc|calloc'
52 #wc -l
53 echo
54 done
55}
56
57install-m32() {
58 # needed to compile with -m32
59 sudo apt-get install gcc-multilib g++-multilib
60}
61
62max-rss() {
63 # %e is real time
64 /usr/bin/time --format '%e %M' -- "$@"
65}
66
67compare-m32() {
68 for bin in _bin/cxx-opt{,32}/osh; do
69 echo $bin
70 ninja $bin
71
72 ldd $bin
73 echo
74
75 file $bin
76 echo
77
78 ls -l $bin
79 echo
80
81 # 141136 KiB vs. 110924 KiB. Significant savings, but it's slower.
82 max-rss $bin --ast-format none -n benchmarks/testdata/configure-coreutils
83
84 done
85}
86
87banner() {
88 echo -----
89 echo "$@"
90}
91
92print-tasks() {
93 local -a workloads=(
94 parse.configure-coreutils
95 parse.configure-cpython
96 parse.abuild
97 ex.bashcomp-parse-help # only runs with bash
98 ex.abuild-print-help # bash / dash / zsh
99 ex.compute-fib # bash / dash / zsh
100 )
101
102 local -a shells=(
103 "bash$TAB-"
104 "dash$TAB-"
105 "zsh$TAB-"
106
107 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
108 "_bin/cxx-opt+bumproot/osh${TAB}mut"
109
110 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
111 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
112 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
113
114 # these have trivial GC stats
115 "_bin/cxx-opt/osh${TAB}mut+alloc"
116 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
117 # good GC stats
118 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
119 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
120 )
121
122 if test -n "${TCMALLOC:-}"; then
123 shells+=(
124 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc"
125 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free"
126 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free+gc"
127 )
128 fi
129
130 local id=0
131
132 for workload in "${workloads[@]}"; do
133 for shell in "${shells[@]}"; do
134 local row_part="$workload${TAB}$shell"
135
136 # Skip these rows
137 case $row_part in
138 "ex.bashcomp-parse-help${TAB}dash"*)
139 continue
140 ;;
141 "ex.bashcomp-parse-help${TAB}zsh"*)
142 continue
143 ;;
144 esac
145
146 local join_id="gc-$id"
147 local row="$join_id${TAB}$row_part"
148 echo "$row"
149
150 id=$((id + 1))
151
152 done
153
154 # Run a quick 10 tasks
155 if test -n "${QUICKLY:-}" && test $id -gt 10; then
156 break
157 fi
158 done
159}
160
161print-cachegrind-tasks() {
162 local -a workloads=(
163 # coreutils is on osh-parser
164 #parse.configure-coreutils
165
166 #parse.configure-cpython
167
168 # Faster tasks, like benchmarks/uftrace, which is instrumented
169 parse.abuild
170 ex.compute-fib
171 )
172
173 local -a shells=(
174 "bash${TAB}-"
175 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
176 "_bin/cxx-opt+bumproot/osh${TAB}mut"
177
178 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
179 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
180 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
181
182 "_bin/cxx-opt/osh${TAB}mut+alloc"
183 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
184 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
185 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
186 )
187
188 local id=0
189 for workload in "${workloads[@]}"; do
190 for shell in "${shells[@]}"; do
191 local row_part="$workload${TAB}$shell"
192
193 local join_id="cachegrind-$id"
194 local row="$join_id${TAB}$row_part"
195 echo "$row"
196
197 id=$((id + 1))
198 done
199 done
200 #print-tasks | egrep 'configure-coreutils' | egrep osh
201}
202
203
204readonly BIG_THRESHOLD=$(( 1 * 1000 * 1000 * 1000 )) # 1 B
205
206run-tasks() {
207 local tsv_out=$1
208 local mode=${2:-time}
209
210 while read -r join_id task sh_path shell_runtime_opts; do
211
212 # Parse different files
213 case $task in
214 parse.configure-coreutils)
215 data_file='benchmarks/testdata/configure-coreutils'
216 ;;
217 parse.configure-cpython)
218 data_file='Python-2.7.13/configure'
219 ;;
220 parse.abuild)
221 data_file='benchmarks/testdata/abuild'
222 ;;
223 esac
224
225 # Construct argv for each task
226 local -a argv
227 case $task in
228 parse.*)
229 argv=( -n $data_file )
230
231 case $sh_path in
232 _bin/*/osh)
233 argv=( --ast-format none "${argv[@]}" )
234 ;;
235 esac
236 ;;
237
238 ex.bashcomp-parse-help)
239 argv=( benchmarks/parse-help/pure-excerpt.sh parse_help_file
240 benchmarks/parse-help/clang.txt )
241 ;;
242
243 ex.abuild-print-help)
244 argv=( testdata/osh-runtime/abuild -h )
245 ;;
246
247 ex.compute-fib)
248 # fewer iterations when instrumented
249 local iters
250 if test $mode = time; then
251 iters=100
252 else
253 iters=10
254 fi
255
256 argv=( benchmarks/compute/fib.sh $iters 44 )
257 ;;
258
259 *)
260 die "Invalid task $task"
261 ;;
262 esac
263
264 echo $join_id $task $sh_path $shell_runtime_opts
265
266 argv=( $sh_path "${argv[@]}" )
267 #echo + "${argv[@]}"
268 #set -x
269
270 if test $mode = cachegrind; then
271 # Add prefix
272 argv=( $0 with-cachegrind $BASE_DIR_CACHEGRIND/raw/$join_id.txt "${argv[@]}" )
273 fi
274
275 # Wrap in a command that writes one row of a TSV
276 # Note: for cachegrind, we need the join ID, but the --rusage is meaningless
277 local -a instrumented=(
278 time-tsv -o $tsv_out --append
279 --rusage
280 --field "$join_id" --field "$task" --field "$sh_path"
281 --field "$shell_runtime_opts"
282 -- "${argv[@]}"
283 )
284
285 # Run with the right environment variables
286
287 case $shell_runtime_opts in
288 -)
289 "${instrumented[@]}" > /dev/null
290 ;;
291 mut)
292 OILS_GC_STATS=1 \
293 "${instrumented[@]}" > /dev/null
294 ;;
295 mut+alloc)
296 # disable GC with big threshold
297 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD \
298 "${instrumented[@]}" > /dev/null
299 ;;
300 mut+alloc+free)
301 # do a single GC on exit
302 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD OILS_GC_ON_EXIT=1 \
303 "${instrumented[@]}" > /dev/null
304 ;;
305 mut+alloc+free+gc)
306 # Default configuration
307 #
308 # Save the GC stats here. None of the other runtime options are that
309 # interesting.
310
311 if test $mode = 'time' && test $sh_path != _bin/cxx-opt+nopool/osh; then
312 OILS_GC_STATS_FD=99 \
313 "${instrumented[@]}" > /dev/null 99>$BASE_DIR/raw/$join_id.txt
314 else
315 "${instrumented[@]}" > /dev/null
316 fi
317 ;;
318 mut+alloc+free+gc+exit)
319 # also GC on exit
320 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
321 "${instrumented[@]}" > /dev/null
322 ;;
323
324 *)
325 die "Invalid shell runtime opts $shell_runtime_opts"
326 ;;
327 esac
328
329 done
330
331 # TODO: OILS_GC_STATS_FD and tsv_column_from_files.py
332}
333
334fd-demo() {
335 local out=_tmp/gc/demo.txt
336
337 local bin=_bin/cxx-dbg/oils-for-unix
338 ninja $bin
339
340 # Hm you can't do $fd>out.txt, but that's OK
341 local fd=99
342
343 OILS_GC_STATS_FD=$fd 99>$out \
344 $bin --ast-format none -n benchmarks/testdata/configure
345
346 ls -l $out
347 cat $out
348}
349
350more-variants() {
351 # TODO: could revive this
352
353 case $compare_more in
354 (*m32*)
355 # Surprisingly, -m32 is SLOWER, even though it allocates less.
356 # My guess is because less work is going into maintaining this code path in
357 # GCC.
358
359 # 223 ms
360 # 61.9 MB bytes allocated
361 local bin=_bin/cxx-opt32/oils-for-unix
362 OILS_GC_THRESHOLD=$big_threshold \
363 run-osh $tsv_out $bin 'm32 mutator+malloc' $file
364
365 # 280 ms
366 OILS_GC_STATS=1 \
367 run-osh $tsv_out $bin 'm32 mutator+malloc+free+gc' $file
368 ;;
369 esac
370
371 # Show log of GC
372 case $compare_more in
373 (*gcverbose*)
374 local bin=_bin/cxx-gcverbose/oils-for-unix
375 # 280 ms
376 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
377 run-osh $tsv_out $bin 'gcverbose mutator+malloc+free+gc' $file
378 ;;
379 esac
380
381 if command -v pretty-tsv; then
382 pretty-tsv $tsv_out
383 fi
384}
385
386build-binaries() {
387 if true; then
388
389 soil/cpp-tarball.sh build-like-ninja \
390 opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}
391
392 else
393
394 # Old Ninja build
395 local -a bin=( _bin/cxx-opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}/osh )
396
397 if test -n "${TCMALLOC:-}"; then
398 bin+=( _bin/cxx-opt+tcmalloc/osh )
399 fi
400 ninja "${bin[@]}"
401 fi
402}
403
404measure-all() {
405 build-binaries
406
407 local tsv_out=${1:-$BASE_DIR/raw/times.tsv}
408 mkdir -p $(dirname $tsv_out)
409
410 # Make the header
411 time-tsv -o $tsv_out --print-header \
412 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
413
414 time print-tasks | run-tasks $tsv_out
415
416 if command -v pretty-tsv; then
417 pretty-tsv $tsv_out
418 fi
419}
420
421measure-cachegrind() {
422 build-binaries
423
424 local tsv_out=${1:-$BASE_DIR_CACHEGRIND/raw/times.tsv}
425
426 mkdir -p $(dirname $tsv_out)
427
428 # Make the header
429 time-tsv -o $tsv_out --print-header \
430 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
431
432 print-cachegrind-tasks | run-tasks $tsv_out cachegrind
433
434 # TODO: join cachegrind columns
435
436 if command -v pretty-tsv; then
437 pretty-tsv $tsv_out
438 fi
439}
440
441print-report() {
442 local in_dir=$1
443
444 benchmark-html-head 'Memory Management Overhead'
445
446 cat <<EOF
447 <body class="width60">
448 <p id="home-link">
449 <a href="/">oilshell.org</a>
450 </p>
451EOF
452
453 cmark << 'EOF'
454## Memory Management Overhead
455
456Source code: [oil/benchmarks/gc.sh](https://github.com/oilshell/oil/tree/master/benchmarks/gc.sh)
457EOF
458
459 cmark << 'EOF'
460### GC Stats
461
462EOF
463
464 tsv2html $in_dir/gc_stats.tsv
465
466 cmark << 'EOF'
467
468- Underlying data: [stage2/gc_stats.tsv](stage2/gc_stats.tsv)
469- More columns: [stage1/gc_stats.tsv](stage1/gc_stats.tsv)
470
471### Resource Usage
472
473#### parse.configure-cpython
474
475EOF
476
477 tsv2html $in_dir/parse.configure-cpython.tsv
478
479 cmark << 'EOF'
480#### parse.configure-coreutils
481
482Parsing the autoconf-generated `configure` script from GNU coreutils.
483
484Note that unlike other shells, `osh -n` retains all nodes on purpose. (See the
485[parser benchmark](../osh-parser/index.html)).
486
487EOF
488
489 tsv2html $in_dir/parse.configure-coreutils.tsv
490
491 cmark <<'EOF'
492#### parse.abuild
493
494Parsing `abuild` from Alpine Linux.
495EOF
496
497 tsv2html $in_dir/parse.abuild.tsv
498
499 cmark <<'EOF'
500#### ex.compute-fib
501
502A synthetic benchmark for POSIX shell arithmetic.
503EOF
504
505 tsv2html $in_dir/ex.compute-fib.tsv
506
507 cmark <<'EOF'
508#### ex.bashcomp-parse-help
509
510A realistic `bash-completion` workload.
511EOF
512
513 tsv2html $in_dir/ex.bashcomp-parse-help.tsv
514
515 cmark <<'EOF'
516#### ex.abuild-print-help
517
518Running `abuild -h` from Alpine Linux.
519
520EOF
521
522 tsv2html $in_dir/ex.abuild-print-help.tsv
523
524 cmark << 'EOF'
525- Underlying data: [stage2/times.tsv](stage2/times.tsv)
526EOF
527
528 cat <<EOF
529
530 </body>
531</html>
532EOF
533}
534
535make-report() {
536 mkdir -p $BASE_DIR/{stage1,stage2}
537
538 # Concatenate tiny files
539 benchmarks/gc_stats_to_tsv.py $BASE_DIR/raw/gc-*.txt \
540 > $BASE_DIR/stage1/gc_stats.tsv
541
542 # Make TSV files
543 benchmarks/report.R gc $BASE_DIR $BASE_DIR/stage2
544
545 # Make HTML
546 benchmarks/report.sh stage3 $BASE_DIR
547}
548
549soil-run() {
550 ### Run in soil/benchmarks
551
552 measure-all
553
554 make-report
555}
556
557#
558# Misc Tests
559#
560
561gc-parse-smoke() {
562 local variant=${1:-opt}
563 local file=${2:-configure}
564
565 local bin=_bin/cxx-$variant/osh
566 ninja $bin
567
568 # OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
569 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
570 $bin --ast-format none -n $file
571
572 # No leaks
573 # OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 $bin -n -c '('
574}
575
576gc-parse-big() {
577 local variant=${1:-opt}
578
579 gc-parse-smoke $variant benchmarks/testdata/configure-coreutils
580}
581
582gc-run-smoke() {
583 local variant=${1:-opt}
584
585 local bin=_bin/cxx-$variant/oils-for-unix
586 ninja $bin
587
588 # expose a bug with printf
589 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=500 OILS_GC_ON_EXIT=1 \
590 $bin -c 'for i in $(seq 100); do printf "%s\\n" "-- $i"; done'
591}
592
593gc-run-oil() {
594 ### Run some scripts from the repo
595
596 local variant=${1:-opt}
597
598 local bin=_bin/cxx-$variant/oils-for-unix
599 ninja $bin
600
601 local i=0
602 for script in */*.sh; do
603 case $script in
604 (build/clean.sh|build/common.sh|build/dev.sh)
605 # Top level does something!
606 echo "=== SKIP $script"
607 continue
608 ;;
609 esac
610
611 echo
612 echo "=== ($i) $script"
613
614 # Just run the top level, which (hopefully) does nothing
615 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
616 $bin $script
617
618 i=$((i + 1))
619 if test $i -gt 60; then
620 break
621 fi
622 done
623}
624
625gc-run-big() {
626 local variant=${1:-opt}
627
628 local target=_bin/cxx-$variant/oils-for-unix
629 ninja $target
630
631 local osh=$REPO_ROOT/$target
632
633 local dir=_tmp/gc-run-big
634 rm -r -f -v $dir
635 mkdir -v -p $dir
636
637 pushd $dir
638 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=100000 OILS_GC_ON_EXIT=1 \
639 $osh ../../Python-2.7.13/configure
640 popd
641}
642
643run-verbose() {
644 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
645 /usr/bin/time --format '*** MAX RSS KiB = %M' -- \
646 "$@"
647}
648
649# This hit the 24-bit object ID limitation in 2.5 seconds
650# Should be able to run indefinitely.
651run-for-a-long-time() {
652 local bin=_bin/cxx-opt/osh
653 ninja $bin
654 run-verbose $bin benchmarks/compute/fib.sh 10000
655
656 # time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 _bin/cxx-opt/osh benchmarks/compute/fib.sh 10000
657}
658
659while-loop() {
660 local i=0
661 while test $i -lt 10000; do
662 if ((i % 1000 == 0)) ; then
663 echo $i
664 fi
665 i=$((i + 1))
666 continue # BUG: skipped GC point
667 done
668}
669
670for-loop() {
671 for i in $(seq 10000); do
672 if ((i % 1000 == 0)) ; then
673 echo $i
674 fi
675 continue
676 done
677}
678
679recurse() {
680 local n=${1:-3000}
681
682 if ((n % 100 == 0)) ; then
683 echo $n
684 fi
685
686 if test $n = 0; then
687 return
688 fi
689
690 recurse $((n - 1))
691}
692
693test-loops() {
694 ### Regression for leak
695
696 local bin=_bin/cxx-opt/osh
697 ninja $bin
698
699 run-verbose $bin $0 recurse
700 echo
701
702 run-verbose $bin $0 while-loop
703 echo
704
705 run-verbose $bin $0 for-loop
706}
707
708expand-loop() {
709 local n=$1
710
711 local bin=_bin/cxx-opt/osh
712 ninja $bin
713
714 set -x
715 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
716 $bin -c "for i in {1..$n}; do echo \$i; done > /dev/null"
717 set +x
718}
719
720test-brace-exp() {
721 expand-loop 330000
722 expand-loop 340000
723}
724
725"$@"