1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Count lines of code in various ways.
|
4 | #
|
5 | # Usage:
|
6 | # metrics/source-code.sh <function name>
|
7 |
|
8 | set -o nounset
|
9 | set -o pipefail
|
10 | set -o errexit
|
11 |
|
12 | REPO_ROOT=$(cd $(dirname $0)/.. && pwd) # tsv-lib.sh uses this
|
13 | readonly REPO_ROOT
|
14 |
|
15 | source test/common.sh
|
16 | source test/tsv-lib.sh
|
17 |
|
18 | filter-py() {
|
19 | grep -E -v '__init__.py$|_gen.py|_test.py|_tests.py|NINJA_subgraph.py$'
|
20 | }
|
21 |
|
22 | readonly -a OSH_ASDL=( {frontend,core}/*.asdl )
|
23 |
|
24 | oils-files() {
|
25 | # what's in the runtime
|
26 | osh-files
|
27 | ysh-files
|
28 | data-lang-files
|
29 | tools-files
|
30 | }
|
31 |
|
32 | # OSH and common
|
33 | osh-files() {
|
34 | # Exclude:
|
35 | # - line_input.c because I didn't write it. It still should be minimized.
|
36 | # - code generators
|
37 | # - test library
|
38 |
|
39 | ls bin/oils_for_unix.py {osh,core,frontend}/*.py builtin/*_osh.py \
|
40 | pyext/*.c */*.pyi \
|
41 | "${OSH_ASDL[@]}" \
|
42 | | filter-py | grep -E -v 'posixmodule.c$|line_input.c$|_gen.py$|test_lib.py$|os.pyi$'
|
43 | }
|
44 |
|
45 | # cloc doesn't understand ASDL files.
|
46 | # Use a wc-like format, filtering out blank lines and comments.
|
47 | asdl-cloc() {
|
48 | python -c '
|
49 | import sys
|
50 |
|
51 | total = 0
|
52 | for path in sys.argv[1:]:
|
53 | num_lines = 0
|
54 | with open(path) as f:
|
55 | for line in f:
|
56 | line = line.strip()
|
57 | if not line or line.startswith("#"):
|
58 | continue
|
59 | num_lines += 1
|
60 |
|
61 | print "%5d %s" % (num_lines, path)
|
62 | total += num_lines
|
63 |
|
64 | print "%5d %s" % (total, "total")
|
65 | ' "$@"
|
66 | }
|
67 |
|
68 | cloc-report() {
|
69 | echo '(non-blank non-comment lines)'
|
70 | echo
|
71 |
|
72 | echo 'OSH'
|
73 | echo
|
74 | osh-files | xargs cloc --quiet "$@"
|
75 | echo
|
76 | echo
|
77 |
|
78 | echo 'YSH'
|
79 | echo
|
80 | ysh-files | xargs cloc --quiet "$@"
|
81 | echo
|
82 | echo
|
83 |
|
84 | echo 'Data Languages'
|
85 | echo
|
86 | data-lang-files | xargs cloc --quiet "$@"
|
87 | echo
|
88 | echo
|
89 |
|
90 | echo 'Tools'
|
91 | echo
|
92 | tools-files | xargs cloc --quiet "$@"
|
93 | echo
|
94 | echo
|
95 |
|
96 | echo 'ASDL SCHEMAS (non-blank non-comment lines)'
|
97 | asdl-cloc "${OSH_ASDL[@]}" data_lang/*.asdl
|
98 | echo
|
99 | echo
|
100 |
|
101 | # NOTE: --csv option could be parsed into HTML.
|
102 | # Or just sum with asdl-cloc!
|
103 |
|
104 | echo 'Hand-Written C++ code (non-blank non-comment lines)'
|
105 | echo
|
106 | { cpp-binding-files; mycpp-runtime-files; } | xargs cloc --quiet "$@"
|
107 | }
|
108 |
|
109 | preprocessed() {
|
110 | ./NINJA-config.sh
|
111 |
|
112 | # Clang has slightly fewer lines, but it's not on the CI machine
|
113 | #local -a files=(_build/preprocessed/{cxx,clang}-{dbg,opt}.txt)
|
114 |
|
115 | local -a files=(_build/preprocessed/cxx-{dbg,opt}.txt)
|
116 |
|
117 | ninja "${files[@]}"
|
118 |
|
119 | # Publish with release and show and CI
|
120 |
|
121 | local dir=_tmp/metrics/preprocessed
|
122 | mkdir -p $dir
|
123 | cp -v "${files[@]}" $dir
|
124 |
|
125 | head -n 100 $dir/*.txt
|
126 | }
|
127 |
|
128 | #
|
129 | # Two variants of the $count function: text and html
|
130 | #
|
131 |
|
132 | category-text() {
|
133 | local header=$1
|
134 | local comment=$2
|
135 |
|
136 | echo "$header"
|
137 | # omit comment
|
138 |
|
139 | # stdin is the files
|
140 | xargs wc -l | sort --numeric
|
141 | echo
|
142 | }
|
143 |
|
144 | # This is overly clever ...
|
145 | shopt -s lastpipe
|
146 | SECTION_ID=0 # mutable global
|
147 |
|
148 | category-html() {
|
149 | # TODO: Don't use wc -l, and just count and sum the lines yourself
|
150 |
|
151 | xargs wc -l | metrics/line_counts.py $((++SECTION_ID)) "$@"
|
152 | }
|
153 |
|
154 | #
|
155 | # Functions That Count
|
156 | #
|
157 |
|
158 | # Note this style is OVERLY ABSTRACT, but it's hard to do better in shell. We
|
159 | # want to parameterize over text and HTML. In Oils I think we would use this:
|
160 | #
|
161 | # proc p1 {
|
162 | # category 'OSH (and common libraries)' {
|
163 | # comment = 'This is the input'
|
164 | # osh-files | read --lines :files
|
165 | # }
|
166 | # }
|
167 | #
|
168 | # This produces a series of dicts that looks like
|
169 | # { name: 'OSH ...', comment: "This ...", files: %(one two three) }
|
170 | #
|
171 | # Then we iterate over the categories and produce text or HTML.
|
172 |
|
173 | osh-counts() {
|
174 | local count=$1
|
175 | shift
|
176 |
|
177 | osh-files | $count \
|
178 | 'OSH (and common libraries)' \
|
179 | 'This is the input to the translators, written in statically-typed Python. Note that bash is at least 140K lines of code, and OSH implements a large part of bash and more.' \
|
180 | "$@"
|
181 | }
|
182 |
|
183 | ysh-files() {
|
184 | ls ysh/*.{py,pgen2} builtin/{func,method}*.py builtin/*_ysh.py | filter-py
|
185 | }
|
186 |
|
187 | ysh-counts() {
|
188 | local count=$1
|
189 | shift
|
190 |
|
191 | ysh-files | $count \
|
192 | 'YSH' 'Expression grammar, parser, evaluator, etc.' "$@"
|
193 | }
|
194 |
|
195 | data-lang-files() {
|
196 | ls data_lang/*.asdl
|
197 | ls data_lang/*.py | filter-py
|
198 | ls data_lang/*.{c,h} | egrep -v '_test' # exclude j8_test_lib as well
|
199 | }
|
200 |
|
201 | data-lang-counts() {
|
202 | local count=$1
|
203 | shift
|
204 |
|
205 | data-lang-files | $count \
|
206 | 'Data Languages' 'JSON, J8 Notation, ...' "$@"
|
207 | }
|
208 |
|
209 | tools-files() {
|
210 | ls tools/*.py | filter-py
|
211 | }
|
212 |
|
213 | tools-counts() {
|
214 | local count=$1
|
215 | shift
|
216 |
|
217 | tools-files | $count \
|
218 | 'Tools' '' "$@"
|
219 | }
|
220 |
|
221 | cpp-binding-files() {
|
222 | ls cpp/*.{cc,h} | egrep -v '_test.cc'
|
223 | }
|
224 |
|
225 | mycpp-runtime-files() {
|
226 | ls mycpp/*.{cc,h} | egrep -v '_test.cc|bump_leak_heap'
|
227 | }
|
228 |
|
229 | cpp-counts() {
|
230 | local count=$1
|
231 | shift
|
232 |
|
233 | cpp-binding-files | $count \
|
234 | 'Hand-written C++ Code' \
|
235 | 'Includes OS bindings. Small C++ files like cpp/osh_arith_parse.{cc,h} correspond to larger Python files like osh/arith_parse.py.' \
|
236 | "$@"
|
237 |
|
238 | # Remove code that isn't "in production"
|
239 | mycpp-runtime-files | $count \
|
240 | 'Garbage-Collected Runtime' \
|
241 | 'Uses a fork-friendly Mark-Sweep collector.' \
|
242 | "$@"
|
243 |
|
244 | ls mycpp/*_test.cc cpp/*_test.cc | $count \
|
245 | 'Unit tests in C++' \
|
246 | 'The goal is to make the spec tests pass, but unit tests are helpful too.' \
|
247 | "$@"
|
248 |
|
249 | ls NINJA*.sh */NINJA*.py build/ninja*.{sh,py} | $count \
|
250 | 'Incremental C++ Build' '' "$@"
|
251 | }
|
252 |
|
253 | gen-cpp-counts() {
|
254 | local count=$1
|
255 | shift
|
256 |
|
257 | # NOTE: this excludes .re2c.h file
|
258 | ls _gen/*/*.{cc,h} | $count \
|
259 | 'Generated C++ Code' \
|
260 | 'mycpp generates the big file _gen/bin/oils-for-unix.mycpp.cc. Other programs like Zephyr ASDL and re2c generate other files.' \
|
261 | "$@"
|
262 | }
|
263 |
|
264 | mycpp-counts() {
|
265 | local count=$1
|
266 | shift
|
267 |
|
268 | ls mycpp/*.py | grep -v 'NINJA_subgraph.py' | filter-py | $count \
|
269 | 'mycpp Translator' \
|
270 | "This prototype uses the MyPy frontend to translate statically-typed Python to C++. The generated code calls a small runtime which implements things like List[T], Dict[K, V], and Python's len()." \
|
271 | "$@"
|
272 |
|
273 | ls mycpp/examples/*.py | $count \
|
274 | 'mycpp Test Data' \
|
275 | 'Small Python examples that translate to C++, compile, and run.' \
|
276 | "$@"
|
277 | }
|
278 |
|
279 | code-generator-counts() {
|
280 | local count=$1
|
281 | shift
|
282 |
|
283 | ls asdl/*.py | filter-py | grep -v -E 'arith_|tdop|_demo' | $count \
|
284 | 'Zephyr ASDL' \
|
285 | 'A DSL for algebraic data types, borrowed from Python. Oils is the most strongly typed Bourne shell implementation!' \
|
286 | "$@"
|
287 |
|
288 | ls pgen2/*.py | filter-py | $count \
|
289 | 'pgen2 Parser Generator' \
|
290 | 'An LL(1) parser generator used to parse YSH expressions. Also borrowed from CPython.' \
|
291 | "$@"
|
292 |
|
293 | ls */*_gen.py | $count \
|
294 | 'Other Code Generators' \
|
295 | 'In order to make Oils statically typed, we had to abandon Python reflection and use C++ source code generation instead. The lexer, flag definitions, and constants can be easily compiled to C++.' \
|
296 | "$@"
|
297 |
|
298 | ls yaks/*.py | filter-py | $count \
|
299 | 'Yaks' \
|
300 | 'Experimental replacement for mycpp' \
|
301 | "$@"
|
302 | }
|
303 |
|
304 | spec-gold-counts() {
|
305 | local count=$1
|
306 | shift
|
307 |
|
308 | ls spec/*.test.sh | $count \
|
309 | 'Spec Tests' \
|
310 | 'A comprehensive test suite that compares OSH against other shells. If OSH passes these tests in BOTH Python and C++, it means that the translation works.' \
|
311 | "$@"
|
312 |
|
313 | ls test/gold/*.sh | $count \
|
314 | 'Gold Tests' \
|
315 | 'Another suite that tests shells "from the outside". Instead of making explicit assertions, we verify that OSH behaves like bash.' \
|
316 | "$@"
|
317 | }
|
318 |
|
319 | #
|
320 | # Top Level Summaries
|
321 | #
|
322 |
|
323 | _for-translation() {
|
324 | local count=$1
|
325 | shift
|
326 |
|
327 | mycpp-counts $count "$@"
|
328 |
|
329 | code-generator-counts $count "$@"
|
330 |
|
331 | cpp-counts $count "$@"
|
332 |
|
333 | osh-counts $count "$@"
|
334 |
|
335 | ysh-counts $count "$@"
|
336 |
|
337 | data-lang-counts $count "$@"
|
338 |
|
339 | tools-counts $count "$@"
|
340 |
|
341 | spec-gold-counts $count "$@"
|
342 |
|
343 | gen-cpp-counts $count "$@"
|
344 | }
|
345 |
|
346 | _overview() {
|
347 | local count=$1
|
348 | shift
|
349 |
|
350 | osh-counts $count "$@"
|
351 |
|
352 | ysh-counts $count "$@"
|
353 |
|
354 | data-lang-counts $count "$@"
|
355 |
|
356 | tools-counts $count "$@"
|
357 |
|
358 | ls stdlib/*.ysh | $count \
|
359 | "YSH stdlib" '' "$@"
|
360 |
|
361 | ls pylib/*.py | filter-py | $count \
|
362 | "Code Borrowed from Python's stdlib" '' "$@"
|
363 |
|
364 | spec-gold-counts $count "$@"
|
365 |
|
366 | test/unit.sh files-to-count | $count \
|
367 | 'Python Unit Tests' '' "$@"
|
368 |
|
369 | ls test/*.{sh,py,R} | filter-py | grep -v jsontemplate.py | $count \
|
370 | 'Other Shell Tests' '' "$@"
|
371 |
|
372 | ls */TEST.sh | $count \
|
373 | 'Test Automation' '' "$@"
|
374 |
|
375 | mycpp-counts $count "$@"
|
376 |
|
377 | code-generator-counts $count "$@"
|
378 |
|
379 | cpp-counts $count "$@"
|
380 |
|
381 | # Leaving off gen-cpp-counts since that requires a C++ build
|
382 |
|
383 | ls build/*.{mk,sh,py,c} Makefile configure install \
|
384 | | filter-py | egrep -v 'NINJA|TEST' | $count \
|
385 | 'Build Automation' '' "$@"
|
386 |
|
387 | ls devtools/release*.sh | $count \
|
388 | 'Release Automation' '' "$@"
|
389 |
|
390 | ls soil/*.{sh,py} | $count \
|
391 | 'Soil: Multi-cloud CI with containers' '' "$@"
|
392 |
|
393 | ls benchmarks/*.{sh,py,R} | $count \
|
394 | 'Benchmarks' '' "$@"
|
395 |
|
396 | ls metrics/*.{sh,R} | $count \
|
397 | 'Metrics' '' "$@"
|
398 |
|
399 | ls _devbuild/gen/*.py | $count \
|
400 | 'Generated Python Code' \
|
401 | 'For the Python App Bundle.' \
|
402 | "$@"
|
403 |
|
404 | ls {doctools,lazylex}/*.py doctools/*.{h,cc} | filter-py | $count \
|
405 | 'Doc Tools' '' "$@"
|
406 |
|
407 | ls web/*.js web/*/*.{js,py} | $count \
|
408 | 'Web' '' "$@"
|
409 | }
|
410 |
|
411 | for-translation() {
|
412 | _for-translation category-text
|
413 | }
|
414 |
|
415 | overview() {
|
416 | _overview category-text
|
417 | }
|
418 |
|
419 | print-files() {
|
420 | xargs -n 1 -- echo
|
421 | }
|
422 |
|
423 | overview-list() {
|
424 | _overview print-files
|
425 | }
|
426 |
|
427 | #
|
428 | # HTML Versions
|
429 | #
|
430 |
|
431 | html-head() {
|
432 | PYTHONPATH=. doctools/html_head.py "$@"
|
433 | }
|
434 |
|
435 | metrics-html-head() {
|
436 | local title="$1"
|
437 |
|
438 | local base_url='../../../web'
|
439 |
|
440 | html-head --title "$title" "$base_url/base.css" "$base_url/table/table-sort.css" "$base_url/line-counts.css"
|
441 | }
|
442 |
|
443 | counts-html() {
|
444 | local name=$1
|
445 | local title=$2
|
446 |
|
447 | local tmp_dir=_tmp/metrics/line-counts/$name
|
448 |
|
449 | rm -r -f -v $tmp_dir >& 2
|
450 | mkdir -v -p $tmp_dir >& 2
|
451 |
|
452 | tsv-row category category_HREF total_lines num_files > $tmp_dir/INDEX.tsv
|
453 |
|
454 | echo $'column_name\ttype
|
455 | category\tstring
|
456 | category_HREF\tstring
|
457 | total_lines\tinteger
|
458 | num_files\tinteger' >$tmp_dir/INDEX.schema.tsv
|
459 |
|
460 | # Generate the HTML
|
461 | "_$name" category-html $tmp_dir
|
462 |
|
463 | metrics-html-head "$title"
|
464 | echo ' <body class="width40">'
|
465 |
|
466 | echo "<h1>$title</h1>"
|
467 |
|
468 | tsv2html $tmp_dir/INDEX.tsv
|
469 |
|
470 | echo '<hr/>'
|
471 |
|
472 | echo '<h2>Related Documents</h2>
|
473 | <p>The <a href="https://www.oilshell.org/release/latest/doc/README.html">README for oilshell/oil</a>
|
474 | has another overview of the repository.
|
475 | </p>'
|
476 |
|
477 | # All the parts
|
478 | cat $tmp_dir/*.html
|
479 |
|
480 | echo ' </body>'
|
481 | echo '</html>'
|
482 | }
|
483 |
|
484 | for-translation-html() {
|
485 | local title='Overview: Translating Oils to C++'
|
486 | counts-html for-translation "$title"
|
487 | }
|
488 |
|
489 | overview-html() {
|
490 | local title='Overview of Oils Code'
|
491 | counts-html overview "$title"
|
492 | }
|
493 |
|
494 | write-reports() {
|
495 | local out_dir=${1:-_tmp/metrics/line-counts}
|
496 |
|
497 | mkdir -v -p $out_dir
|
498 |
|
499 | for-translation-html > $out_dir/for-translation.html
|
500 |
|
501 | overview-html > $out_dir/overview.html
|
502 |
|
503 | ls -l $out_dir
|
504 | }
|
505 |
|
506 | #
|
507 | # Misc
|
508 | #
|
509 |
|
510 | # count instructions, for fun
|
511 | instructions() {
|
512 | # http://pepijndevos.nl/2016/08/24/x86-instruction-distribution.html
|
513 |
|
514 | local bin=_build/oil/ovm-opt.stripped
|
515 | objdump -d $bin | cut -f3 | grep -oE "^[a-z]+" | hist
|
516 | }
|
517 |
|
518 | hist() {
|
519 | sort | uniq -c | sort -n
|
520 | }
|
521 |
|
522 | stdlib-imports() {
|
523 | oil-osh-files | xargs grep --no-filename '^import' | hist
|
524 | }
|
525 |
|
526 | imports() {
|
527 | oil-osh-files | xargs grep --no-filename -w import | hist
|
528 | }
|
529 |
|
530 | imports-not-at-top() {
|
531 | oil-osh-files | xargs grep -n -w import | awk -F : ' $2 > 100'
|
532 | }
|
533 |
|
534 | # For the compiler, see what's at the top level.
|
535 | top-level() {
|
536 | grep '^[a-zA-Z]' {core,osh}/*.py \
|
537 | | grep -v '_test.py' \
|
538 | | egrep -v ':import|from|class|def' # note: colon is from grep output
|
539 | }
|
540 |
|
541 | _python-symbols() {
|
542 | local main=$1
|
543 | local name=$2
|
544 | local out_dir=$3
|
545 |
|
546 | mkdir -p $out_dir
|
547 | local out=${out_dir}/${name}-symbols.txt
|
548 |
|
549 | # To debug what version we're running eci
|
550 | /usr/bin/env python2 -V
|
551 | echo
|
552 |
|
553 | # Run this from the repository root.
|
554 | PYTHONPATH='.:vendor/' CALLGRAPH=1 $main | tee $out
|
555 |
|
556 | wc -l $out
|
557 | echo
|
558 | echo "Wrote $out"
|
559 | }
|
560 |
|
561 | oil-python-symbols() {
|
562 | local out_dir=${1:-_tmp/opy-test}
|
563 | _python-symbols bin/oil.py oil $out_dir
|
564 | }
|
565 |
|
566 | old-style-classes() {
|
567 | oil-python-symbols | grep -v '<'
|
568 | }
|
569 |
|
570 | # Some of these are "abstract classes" like ChildStateChange
|
571 | NotImplementedError() {
|
572 | grep NotImplementedError */*.py
|
573 | }
|
574 |
|
575 | py-ext() {
|
576 | # for the py-source build
|
577 | # 35 imports
|
578 | osh-files | xargs -- egrep 'import (fanos|libc|line_input|posix_|yajl)'
|
579 | }
|
580 |
|
581 | if test $(basename $0) = 'source-code.sh'; then
|
582 | "$@"
|
583 | fi
|