OILS / metrics / source-code.sh View on Github | oilshell.org

583 lines, 319 significant
1#!/usr/bin/env bash
2#
3# Count lines of code in various ways.
4#
5# Usage:
6# metrics/source-code.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd $(dirname $0)/.. && pwd) # tsv-lib.sh uses this
13readonly REPO_ROOT
14
15source test/common.sh
16source test/tsv-lib.sh
17
18filter-py() {
19 grep -E -v '__init__.py$|_gen.py|_test.py|_tests.py|NINJA_subgraph.py$'
20}
21
22readonly -a OSH_ASDL=( {frontend,core}/*.asdl )
23
24oils-files() {
25 # what's in the runtime
26 osh-files
27 ysh-files
28 data-lang-files
29 tools-files
30}
31
32# OSH and common
33osh-files() {
34 # Exclude:
35 # - line_input.c because I didn't write it. It still should be minimized.
36 # - code generators
37 # - test library
38
39 ls bin/oils_for_unix.py {osh,core,frontend}/*.py builtin/*_osh.py \
40 pyext/*.c */*.pyi \
41 "${OSH_ASDL[@]}" \
42 | filter-py | grep -E -v 'posixmodule.c$|line_input.c$|_gen.py$|test_lib.py$|os.pyi$'
43}
44
45# cloc doesn't understand ASDL files.
46# Use a wc-like format, filtering out blank lines and comments.
47asdl-cloc() {
48 python -c '
49import sys
50
51total = 0
52for path in sys.argv[1:]:
53 num_lines = 0
54 with open(path) as f:
55 for line in f:
56 line = line.strip()
57 if not line or line.startswith("#"):
58 continue
59 num_lines += 1
60
61 print "%5d %s" % (num_lines, path)
62 total += num_lines
63
64print "%5d %s" % (total, "total")
65' "$@"
66}
67
68cloc-report() {
69 echo '(non-blank non-comment lines)'
70 echo
71
72 echo 'OSH'
73 echo
74 osh-files | xargs cloc --quiet "$@"
75 echo
76 echo
77
78 echo 'YSH'
79 echo
80 ysh-files | xargs cloc --quiet "$@"
81 echo
82 echo
83
84 echo 'Data Languages'
85 echo
86 data-lang-files | xargs cloc --quiet "$@"
87 echo
88 echo
89
90 echo 'Tools'
91 echo
92 tools-files | xargs cloc --quiet "$@"
93 echo
94 echo
95
96 echo 'ASDL SCHEMAS (non-blank non-comment lines)'
97 asdl-cloc "${OSH_ASDL[@]}" data_lang/*.asdl
98 echo
99 echo
100
101 # NOTE: --csv option could be parsed into HTML.
102 # Or just sum with asdl-cloc!
103
104 echo 'Hand-Written C++ code (non-blank non-comment lines)'
105 echo
106 { cpp-binding-files; mycpp-runtime-files; } | xargs cloc --quiet "$@"
107}
108
109preprocessed() {
110 ./NINJA-config.sh
111
112 # Clang has slightly fewer lines, but it's not on the CI machine
113 #local -a files=(_build/preprocessed/{cxx,clang}-{dbg,opt}.txt)
114
115 local -a files=(_build/preprocessed/cxx-{dbg,opt}.txt)
116
117 ninja "${files[@]}"
118
119 # Publish with release and show and CI
120
121 local dir=_tmp/metrics/preprocessed
122 mkdir -p $dir
123 cp -v "${files[@]}" $dir
124
125 head -n 100 $dir/*.txt
126}
127
128#
129# Two variants of the $count function: text and html
130#
131
132category-text() {
133 local header=$1
134 local comment=$2
135
136 echo "$header"
137 # omit comment
138
139 # stdin is the files
140 xargs wc -l | sort --numeric
141 echo
142}
143
144# This is overly clever ...
145shopt -s lastpipe
146SECTION_ID=0 # mutable global
147
148category-html() {
149 # TODO: Don't use wc -l, and just count and sum the lines yourself
150
151 xargs wc -l | metrics/line_counts.py $((++SECTION_ID)) "$@"
152}
153
154#
155# Functions That Count
156#
157
158# Note this style is OVERLY ABSTRACT, but it's hard to do better in shell. We
159# want to parameterize over text and HTML. In Oils I think we would use this:
160#
161# proc p1 {
162# category 'OSH (and common libraries)' {
163# comment = 'This is the input'
164# osh-files | read --lines :files
165# }
166# }
167#
168# This produces a series of dicts that looks like
169# { name: 'OSH ...', comment: "This ...", files: %(one two three) }
170#
171# Then we iterate over the categories and produce text or HTML.
172
173osh-counts() {
174 local count=$1
175 shift
176
177 osh-files | $count \
178 'OSH (and common libraries)' \
179 'This is the input to the translators, written in statically-typed Python. Note that bash is at least 140K lines of code, and OSH implements a large part of bash and more.' \
180 "$@"
181}
182
183ysh-files() {
184 ls ysh/*.{py,pgen2} builtin/{func,method}*.py builtin/*_ysh.py | filter-py
185}
186
187ysh-counts() {
188 local count=$1
189 shift
190
191 ysh-files | $count \
192 'YSH' 'Expression grammar, parser, evaluator, etc.' "$@"
193}
194
195data-lang-files() {
196 ls data_lang/*.asdl
197 ls data_lang/*.py | filter-py
198 ls data_lang/*.{c,h} | egrep -v '_test' # exclude j8_test_lib as well
199}
200
201data-lang-counts() {
202 local count=$1
203 shift
204
205 data-lang-files | $count \
206 'Data Languages' 'JSON, J8 Notation, ...' "$@"
207}
208
209tools-files() {
210 ls tools/*.py | filter-py
211}
212
213tools-counts() {
214 local count=$1
215 shift
216
217 tools-files | $count \
218 'Tools' '' "$@"
219}
220
221cpp-binding-files() {
222 ls cpp/*.{cc,h} | egrep -v '_test.cc'
223}
224
225mycpp-runtime-files() {
226 ls mycpp/*.{cc,h} | egrep -v '_test.cc|bump_leak_heap'
227}
228
229cpp-counts() {
230 local count=$1
231 shift
232
233 cpp-binding-files | $count \
234 'Hand-written C++ Code' \
235 'Includes OS bindings. Small C++ files like cpp/osh_arith_parse.{cc,h} correspond to larger Python files like osh/arith_parse.py.' \
236 "$@"
237
238 # Remove code that isn't "in production"
239 mycpp-runtime-files | $count \
240 'Garbage-Collected Runtime' \
241 'Uses a fork-friendly Mark-Sweep collector.' \
242 "$@"
243
244 ls mycpp/*_test.cc cpp/*_test.cc | $count \
245 'Unit tests in C++' \
246 'The goal is to make the spec tests pass, but unit tests are helpful too.' \
247 "$@"
248
249 ls NINJA*.sh */NINJA*.py build/ninja*.{sh,py} | $count \
250 'Incremental C++ Build' '' "$@"
251}
252
253gen-cpp-counts() {
254 local count=$1
255 shift
256
257 # NOTE: this excludes .re2c.h file
258 ls _gen/*/*.{cc,h} | $count \
259 'Generated C++ Code' \
260 'mycpp generates the big file _gen/bin/oils-for-unix.mycpp.cc. Other programs like Zephyr ASDL and re2c generate other files.' \
261 "$@"
262}
263
264mycpp-counts() {
265 local count=$1
266 shift
267
268 ls mycpp/*.py | grep -v 'NINJA_subgraph.py' | filter-py | $count \
269 'mycpp Translator' \
270 "This prototype uses the MyPy frontend to translate statically-typed Python to C++. The generated code calls a small runtime which implements things like List[T], Dict[K, V], and Python's len()." \
271 "$@"
272
273 ls mycpp/examples/*.py | $count \
274 'mycpp Test Data' \
275 'Small Python examples that translate to C++, compile, and run.' \
276 "$@"
277}
278
279code-generator-counts() {
280 local count=$1
281 shift
282
283 ls asdl/*.py | filter-py | grep -v -E 'arith_|tdop|_demo' | $count \
284 'Zephyr ASDL' \
285 'A DSL for algebraic data types, borrowed from Python. Oils is the most strongly typed Bourne shell implementation!' \
286 "$@"
287
288 ls pgen2/*.py | filter-py | $count \
289 'pgen2 Parser Generator' \
290 'An LL(1) parser generator used to parse YSH expressions. Also borrowed from CPython.' \
291 "$@"
292
293 ls */*_gen.py | $count \
294 'Other Code Generators' \
295 'In order to make Oils statically typed, we had to abandon Python reflection and use C++ source code generation instead. The lexer, flag definitions, and constants can be easily compiled to C++.' \
296 "$@"
297
298 ls yaks/*.py | filter-py | $count \
299 'Yaks' \
300 'Experimental replacement for mycpp' \
301 "$@"
302}
303
304spec-gold-counts() {
305 local count=$1
306 shift
307
308 ls spec/*.test.sh | $count \
309 'Spec Tests' \
310 'A comprehensive test suite that compares OSH against other shells. If OSH passes these tests in BOTH Python and C++, it means that the translation works.' \
311 "$@"
312
313 ls test/gold/*.sh | $count \
314 'Gold Tests' \
315 'Another suite that tests shells "from the outside". Instead of making explicit assertions, we verify that OSH behaves like bash.' \
316 "$@"
317}
318
319#
320# Top Level Summaries
321#
322
323_for-translation() {
324 local count=$1
325 shift
326
327 mycpp-counts $count "$@"
328
329 code-generator-counts $count "$@"
330
331 cpp-counts $count "$@"
332
333 osh-counts $count "$@"
334
335 ysh-counts $count "$@"
336
337 data-lang-counts $count "$@"
338
339 tools-counts $count "$@"
340
341 spec-gold-counts $count "$@"
342
343 gen-cpp-counts $count "$@"
344}
345
346_overview() {
347 local count=$1
348 shift
349
350 osh-counts $count "$@"
351
352 ysh-counts $count "$@"
353
354 data-lang-counts $count "$@"
355
356 tools-counts $count "$@"
357
358 ls stdlib/*.ysh | $count \
359 "YSH stdlib" '' "$@"
360
361 ls pylib/*.py | filter-py | $count \
362 "Code Borrowed from Python's stdlib" '' "$@"
363
364 spec-gold-counts $count "$@"
365
366 test/unit.sh files-to-count | $count \
367 'Python Unit Tests' '' "$@"
368
369 ls test/*.{sh,py,R} | filter-py | grep -v jsontemplate.py | $count \
370 'Other Shell Tests' '' "$@"
371
372 ls */TEST.sh | $count \
373 'Test Automation' '' "$@"
374
375 mycpp-counts $count "$@"
376
377 code-generator-counts $count "$@"
378
379 cpp-counts $count "$@"
380
381 # Leaving off gen-cpp-counts since that requires a C++ build
382
383 ls build/*.{mk,sh,py,c} Makefile configure install \
384 | filter-py | egrep -v 'NINJA|TEST' | $count \
385 'Build Automation' '' "$@"
386
387 ls devtools/release*.sh | $count \
388 'Release Automation' '' "$@"
389
390 ls soil/*.{sh,py} | $count \
391 'Soil: Multi-cloud CI with containers' '' "$@"
392
393 ls benchmarks/*.{sh,py,R} | $count \
394 'Benchmarks' '' "$@"
395
396 ls metrics/*.{sh,R} | $count \
397 'Metrics' '' "$@"
398
399 ls _devbuild/gen/*.py | $count \
400 'Generated Python Code' \
401 'For the Python App Bundle.' \
402 "$@"
403
404 ls {doctools,lazylex}/*.py doctools/*.{h,cc} | filter-py | $count \
405 'Doc Tools' '' "$@"
406
407 ls web/*.js web/*/*.{js,py} | $count \
408 'Web' '' "$@"
409}
410
411for-translation() {
412 _for-translation category-text
413}
414
415overview() {
416 _overview category-text
417}
418
419print-files() {
420 xargs -n 1 -- echo
421}
422
423overview-list() {
424 _overview print-files
425}
426
427#
428# HTML Versions
429#
430
431html-head() {
432 PYTHONPATH=. doctools/html_head.py "$@"
433}
434
435metrics-html-head() {
436 local title="$1"
437
438 local base_url='../../../web'
439
440 html-head --title "$title" "$base_url/base.css" "$base_url/table/table-sort.css" "$base_url/line-counts.css"
441}
442
443counts-html() {
444 local name=$1
445 local title=$2
446
447 local tmp_dir=_tmp/metrics/line-counts/$name
448
449 rm -r -f -v $tmp_dir >& 2
450 mkdir -v -p $tmp_dir >& 2
451
452 tsv-row category category_HREF total_lines num_files > $tmp_dir/INDEX.tsv
453
454 echo $'column_name\ttype
455category\tstring
456category_HREF\tstring
457total_lines\tinteger
458num_files\tinteger' >$tmp_dir/INDEX.schema.tsv
459
460 # Generate the HTML
461 "_$name" category-html $tmp_dir
462
463 metrics-html-head "$title"
464 echo ' <body class="width40">'
465
466 echo "<h1>$title</h1>"
467
468 tsv2html $tmp_dir/INDEX.tsv
469
470 echo '<hr/>'
471
472 echo '<h2>Related Documents</h2>
473 <p>The <a href="https://www.oilshell.org/release/latest/doc/README.html">README for oilshell/oil</a>
474 has another overview of the repository.
475 </p>'
476
477 # All the parts
478 cat $tmp_dir/*.html
479
480 echo ' </body>'
481 echo '</html>'
482}
483
484for-translation-html() {
485 local title='Overview: Translating Oils to C++'
486 counts-html for-translation "$title"
487}
488
489overview-html() {
490 local title='Overview of Oils Code'
491 counts-html overview "$title"
492}
493
494write-reports() {
495 local out_dir=${1:-_tmp/metrics/line-counts}
496
497 mkdir -v -p $out_dir
498
499 for-translation-html > $out_dir/for-translation.html
500
501 overview-html > $out_dir/overview.html
502
503 ls -l $out_dir
504}
505
506#
507# Misc
508#
509
510# count instructions, for fun
511instructions() {
512 # http://pepijndevos.nl/2016/08/24/x86-instruction-distribution.html
513
514 local bin=_build/oil/ovm-opt.stripped
515 objdump -d $bin | cut -f3 | grep -oE "^[a-z]+" | hist
516}
517
518hist() {
519 sort | uniq -c | sort -n
520}
521
522stdlib-imports() {
523 oil-osh-files | xargs grep --no-filename '^import' | hist
524}
525
526imports() {
527 oil-osh-files | xargs grep --no-filename -w import | hist
528}
529
530imports-not-at-top() {
531 oil-osh-files | xargs grep -n -w import | awk -F : ' $2 > 100'
532}
533
534# For the compiler, see what's at the top level.
535top-level() {
536 grep '^[a-zA-Z]' {core,osh}/*.py \
537 | grep -v '_test.py' \
538 | egrep -v ':import|from|class|def' # note: colon is from grep output
539}
540
541_python-symbols() {
542 local main=$1
543 local name=$2
544 local out_dir=$3
545
546 mkdir -p $out_dir
547 local out=${out_dir}/${name}-symbols.txt
548
549 # To debug what version we're running eci
550 /usr/bin/env python2 -V
551 echo
552
553 # Run this from the repository root.
554 PYTHONPATH='.:vendor/' CALLGRAPH=1 $main | tee $out
555
556 wc -l $out
557 echo
558 echo "Wrote $out"
559}
560
561oil-python-symbols() {
562 local out_dir=${1:-_tmp/opy-test}
563 _python-symbols bin/oil.py oil $out_dir
564}
565
566old-style-classes() {
567 oil-python-symbols | grep -v '<'
568}
569
570# Some of these are "abstract classes" like ChildStateChange
571NotImplementedError() {
572 grep NotImplementedError */*.py
573}
574
575py-ext() {
576 # for the py-source build
577 # 35 imports
578 osh-files | xargs -- egrep 'import (fanos|libc|line_input|posix_|yajl)'
579}
580
581if test $(basename $0) = 'source-code.sh'; then
582 "$@"
583fi