OILS / devtools / test-oils.sh View on Github | oilshell.org

360 lines, 174 significant
1#!/usr/bin/env bash
2#
3# Main file for test-oils.xshar
4#
5# Usage:
6# devtools/test-oils.sh <function name>
7#
8# It will contain
9#
10# _release/
11# oils-for-unix.tar
12# benchmarks/
13# time-helper.c
14# osh-runtime.sh
15#
16# It will run benchmarks, and then upload a TSV file to a server.
17#
18# The TSV file will be labeled with
19#
20# - git commit that created the xshar file (in oilshell/oil)
21# - date
22# - label: github actions / sourcehut
23# - and then we'll also have provenance and system info
24# - machine name, OS, CPUs, etc.
25
26set -o nounset
27set -o pipefail
28set -o errexit
29
30source benchmarks/id.sh
31source test/common.sh # die
32
33OILS_VERSION=$(head -n 1 oil-version.txt)
34
35REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
36
37FLAG_num_iters=1 # iterations
38FLAG_num_shells=1
39FLAG_num_workloads=1
40FLAG_upload=''
41FLAG_subdir=''
42FLAG_url=http://travis-ci.oilshell.org/wwup.cgi
43FLAG_dry_run=''
44
45readonly XSHAR_NAME=test-oils.xshar # hard-coded for now
46
47print-help() {
48 # Other flags:
49 #
50 # --host host to upload to?
51 # --auth allow writing to host
52 # --no-taskset disable taskset?
53
54 cat <<EOF
55Usage: $XSHAR_NAME ACTION FLAGS*
56
57This is a self-extracting, executable shell archive (xshar) to test Oils. It
58contains:
59
60 1. The oils-for-unix tarball, which can be compiled on any machine
61 2. Test / benchmark scripts and their dependencies
62 3. This harness, which compiles the tarball, accepts flags, runs benchmarks.
63
64Actions:
65
66 osh-runtime (currently the only one)
67
68 Flags:
69 -n --num-iters (default: 1) Run everything this number of times.
70
71 -s --num-shells Run the first N of osh, bash, dash
72
73 -w --num-workloads Run the first N of the workloads
74
75 -u --upload Upload .wwz results to the CI server
76
77 --subdir Subdirectory for the wwz. results
78 default: git-\$commit
79
80 --url Upload to this URL
81 default: $FLAG_url
82
83 --dry-run Print out tasks, but don't execute them
84
85 Workloads:
86EOF
87 benchmarks/osh-runtime.sh print-workloads
88 cat <<EOF
89
90Example:
91
92 $XSHAR_NAME osh-runtime --num-iters 2 --num-shells 2 --num-workloads 3
93
94will run this benchmark matrix 2 times:
95
96 (osh, bash) X (hello-world, bin-true, configure-cpython)
97
98xshar runtime dependencies:
99
100 - base64, tar, gzip - for unpacking the payload
101
102osh-runtime dependencies:
103
104 - C++ compiler, for oils-for-unix and benchmarks/time-helper.c
105 - On OS X, the compiler comes with XCode
106 - python2 for - benchmarks/time_.py, tsv*.py, gc_stats*.py
107 - zip for creating the .wwz payload
108 - curl for uploading it via HTTP
109EOF
110}
111
112print-version() {
113 echo "$XSHAR_NAME was built from git commit ${XSHAR_GIT_COMMIT:-?}"
114}
115
116parse-flags-osh-runtime() {
117 ### Sets global vars FLAG_*
118
119 while test $# -ne 0; do
120 case "$1" in
121 -v|--version)
122 print-version
123 exit
124 ;;
125 -h|--help)
126 print-help
127 exit
128 ;;
129
130 -n|--num-iters)
131 if test $# -eq 1; then
132 die "-n / --num-iters requires an argument"
133 fi
134 shift
135 FLAG_num_iters=$1
136 ;;
137
138 -s|--num-shells)
139 if test $# -eq 1; then
140 die "-s / --num-shells requires an argument"
141 fi
142 shift
143 FLAG_num_shells=$1
144 ;;
145
146 -w|--num-workloads)
147 if test $# -eq 1; then
148 die "-w / --num-workloads requires an argument"
149 fi
150 shift
151 FLAG_num_workloads=$1
152 ;;
153
154 -u|--upload)
155 FLAG_upload=T
156 ;;
157
158 --subdir)
159 if test $# -eq 1; then
160 die "--subdir requires an argument"
161 fi
162 shift
163 FLAG_subdir=$1
164 ;;
165
166 --url)
167 if test $# -eq 1; then
168 die "--url requires an argument"
169 fi
170 shift
171 FLAG_url=$1
172 ;;
173
174 --dry-run)
175 FLAG_dry_run=T
176 ;;
177
178 *)
179 die "Invalid flag '$1'"
180 ;;
181 esac
182 shift
183 done
184}
185
186osh-runtime() {
187 # $XSHAR_DIR looks like like $REPO_ROOT
188
189 parse-flags-osh-runtime "$@"
190 echo num_iters=$FLAG_num_iters
191 echo num_shells=$FLAG_num_shells
192 echo num_workloads=$FLAG_num_workloads
193
194 local time_py="${XSHAR_DIR:-$REPO_ROOT}/benchmarks/time_.py"
195 build/py.sh time-helper
196
197 # Extract and compile the tarball
198 # Similar to devtools/release-native.sh test-tar
199 local tmp=_tmp/oils-tar
200 mkdir -p $tmp
201
202 pushd $tmp
203 tar -x < ../../_release/oils-for-unix.tar
204
205 pushd oils-for-unix-$OILS_VERSION
206 build/native.sh tarball-demo
207
208 local osh=$PWD/_bin/cxx-opt-sh/osh
209
210 # Smoke test
211 $time_py --tsv --rusage -- \
212 $osh -c 'echo "smoke test: osh and time_.py"'
213
214 popd
215 popd
216
217 local job_id host_name
218 # this form used by benchmarks to name raw TSV
219 job_id=$(print-job-id) # benchmarks/id.sh
220 host_name=$(hostname)
221
222
223 if test -n "$FLAG_dry_run"; then
224 echo
225 echo '--- Tasks that would be run ---'
226 echo
227
228 benchmarks/osh-runtime.sh print-tasks-xshar $host_name $osh \
229 $FLAG_num_iters $FLAG_num_shells $FLAG_num_workloads
230
231 echo
232
233 return
234 fi
235
236 benchmarks/osh-runtime.sh test-oils-run $osh $job_id $host_name \
237 $FLAG_num_iters $FLAG_num_shells $FLAG_num_workloads
238
239 # /uuu/
240 # osh-runtime/
241 # $FLAG_subdir/ # Human-readable label
242 # git-$hash/ # For PRs
243 # $(date).$machine.wwz/
244 # osh-runtime/
245 # times.tsv
246 # gc_stats.tsv
247 # provenance.tsv
248 # shell-id/
249 # host-id/
250
251 if test -n "$FLAG_upload"; then
252
253 if ! command -v zip >/dev/null; then
254 echo "$0: zip command not found. It's needed to upload benchmark results."
255 # For now, don't return 1, because the CI will fail
256 return 1
257 fi
258
259 local wwz_name="${job_id}.${host_name}.wwz"
260
261 pushd _tmp
262 # Only zip metadata files in osh-runtime, so we don't serve untrusted stuff
263 zip -r $wwz_name \
264 osh-runtime/*.txt osh-runtime/raw/*.tsv shell-id/ host-id/
265 popd
266
267 local wwz_path=_tmp/$wwz_name
268 unzip -l $wwz_path
269
270 local subdir
271 if test -n "$FLAG_subdir"; then
272 subdir=$FLAG_subdir
273 else
274 if test -n "${XSHAR_GIT_COMMIT:-}"; then
275 local suffix=${XSHAR_GIT_COMMIT:0:8} # like benchmarks/id.sh
276 subdir="git-$suffix"
277 else
278 subdir=unknown
279 fi
280 fi
281
282 curl --verbose \
283 --form "payload-type=osh-runtime" \
284 --form "subdir=$subdir" \
285 --form "wwz=@$wwz_path" \
286 $FLAG_url
287 fi
288
289 # TODO: upload these with curl
290 #
291 # _tmp/
292 # osh-runtime/
293 # shell-id/
294 # host-id/
295 #
296 # Either as individual files:
297 #
298 # curl \
299 # --form client=$XSHAR_NAME \
300 # --form file1=@_tmp/foo \
301 # --form file2=@_tmp/bar \
302 # http://travis-ci.oilshell.org/untrusted/
303 #
304 # Or as a .wwz or .zip file?
305 #
306 # wwup can either write the .wwz literally, or it can unpack it with zipfile
307 # module. (Beware of file system traversal issues!)
308 #
309 # Then enhance devtools/src_tree.py to make a nicer view of the tree,
310 # including TSV files rendered as HTML.
311}
312
313demo() {
314 ### Show how we compile the code
315
316 local time_py="$PWD/benchmarks/time_.py"
317
318 build/py.sh time-helper
319
320 # Extract and compile the tarball
321 # Similar to devtools/release-native.sh test-tar
322
323 local tmp=_tmp/xshar-demo
324 mkdir -p $tmp
325
326 pushd $tmp
327 tar -x < ../../_release/oils-for-unix.tar
328
329 pushd oils-for-unix-$OILS_VERSION
330 build/native.sh tarball-demo
331
332 local osh=$PWD/_bin/cxx-opt-sh/osh
333
334 $time_py --tsv --rusage -o demo.tsv -- \
335 $osh -c 'sleep 0.1; echo "hi from osh"'
336 cat demo.tsv
337
338 popd
339
340 popd
341
342 #time OILS_GC_STATS=1 $osh Python-2.7.13/configure
343}
344
345if test $# -eq 0; then
346 print-help
347else
348 case "$1" in
349 -v|--version)
350 print-version
351 exit
352 ;;
353 -h|--help)
354 print-help
355 exit
356 ;;
357 esac
358
359 "$@"
360fi