1 | #!/usr/bin/env python
|
2 | from __future__ import print_function
|
3 | """
|
4 | wild_report.py
|
5 | """
|
6 |
|
7 | import json
|
8 | import os
|
9 | import sys
|
10 |
|
11 | import jsontemplate
|
12 |
|
13 | # JSON Template Evaluation:
|
14 | #
|
15 | # - {.if}{.or} is confusing
|
16 | # I think there is even a bug with {.if}{.else}{.end} -- it accepts it but
|
17 | # doesn't do the right thing!
|
18 | # - {.if test} does work though, but it took me awhile to remember that or
|
19 | # - I forgot about {.link?} too
|
20 | # even find it in the source code. I don't like this separate predicate
|
21 | # language. Could just be PHP-ish I guess.
|
22 | # - Predicates are a little annoying.
|
23 | # - Lack of location information on undefined variables is annoying. It spews
|
24 | # a big stack trace.
|
25 | # - The styles thing seems awkward. Copied from srcbook.
|
26 | # - I don't have {total_secs|%.3f} , but the
|
27 | # LookupChain/DictRegistry/CallableRegistry thing is quite onerous.
|
28 | #
|
29 | # Good parts:
|
30 | # Just making one big dict is pretty nice.
|
31 |
|
32 | T = jsontemplate.Template
|
33 |
|
34 | F = {
|
35 | 'commas': lambda n: '{:,}'.format(n),
|
36 | #'urlesc': urllib.quote_plus,
|
37 | }
|
38 |
|
39 | def MakeHtmlGroup(title_str, body_str):
|
40 | """Make a group of templates that we can expand with a common style."""
|
41 | return {
|
42 | 'TITLE': T(title_str, default_formatter='html', more_formatters=F),
|
43 | 'BODY': T(body_str, default_formatter='html', more_formatters=F),
|
44 | 'NAV': NAV_TEMPLATE,
|
45 | }
|
46 |
|
47 | BODY_STYLE = jsontemplate.Template("""\
|
48 | <!DOCTYPE html>
|
49 | <html>
|
50 | <head>
|
51 | <title>{.template TITLE}</title>
|
52 |
|
53 | <script type="text/javascript" src="{base_url}../../web/ajax.js"></script>
|
54 | <script type="text/javascript" src="{base_url}../../web/table/table-sort.js"></script>
|
55 | <link rel="stylesheet" type="text/css" href="{base_url}../../web/table/table-sort.css" />
|
56 | <link rel="stylesheet" type="text/css" href="{base_url}../../web/wild.css" />
|
57 | </head>
|
58 |
|
59 | <body onload="initPage(gUrlHash, gTables, gTableStates, kStatusElem);"
|
60 | onhashchange="onHashChange(gUrlHash, gTableStates, kStatusElem);">
|
61 | <p id="status"></p>
|
62 |
|
63 | <p style="text-align: right"><a href="/">oilshell.org</a></p>
|
64 | {.template NAV}
|
65 |
|
66 | {.template BODY}
|
67 | </body>
|
68 |
|
69 | </html>
|
70 | """, default_formatter='html')
|
71 |
|
72 | # NOTE: {.link} {.or id?} {.or} {.end} doesn't work? That is annoying.
|
73 | NAV_TEMPLATE = jsontemplate.Template("""\
|
74 | {.section nav}
|
75 | <p id="nav">
|
76 | {.repeated section @}
|
77 | {.link?}
|
78 | <a href="{link|htmltag}">{anchor}</a>
|
79 | {.or}
|
80 | {anchor}
|
81 | {.end}
|
82 | {.alternates with}
|
83 | /
|
84 | {.end}
|
85 | </p>
|
86 | {.end}
|
87 | """, default_formatter='html')
|
88 |
|
89 |
|
90 | PAGE_TEMPLATES = {}
|
91 |
|
92 | # One is used for sort order. One is used for alignment.
|
93 | # type="string"
|
94 | # should we use the column css class as the sort order? Why not?
|
95 |
|
96 | # NOTES on columns:
|
97 | # - The col is used to COLOR the column when it's being sorted by
|
98 | # - But it can't be use to align text right. See
|
99 | # https://stackoverflow.com/questions/1238115/using-text-align-center-in-colgroup
|
100 | # - type="number" is used in table-sort.js for the sort order.
|
101 | # - We use CSS classes on individual cells like <td class="name"> to align
|
102 | # columns. That seems to be the only way to do it?
|
103 |
|
104 | PAGE_TEMPLATES['LISTING'] = MakeHtmlGroup(
|
105 | 'WILD/{rel_path} - Parsing and Translating Shell Scripts with Oil',
|
106 | """\
|
107 |
|
108 | {.section subtree_stats}
|
109 | <div id="summary">
|
110 | <ul>
|
111 | {.parse_failed?}
|
112 | <li>
|
113 | Attempted to parse <b>{num_files|commas}</b> shell scripts totalling
|
114 | <b>{num_lines|commas}</b> lines.
|
115 | </li>
|
116 | <li>
|
117 | Failed to parse <b>{parse_failed|commas}</b> scripts, leaving
|
118 | <b>{lines_parsed|commas}</b> lines parsed in <b>{parse_proc_secs}</b> seconds
|
119 | (<b>{lines_per_sec}</b> lines/sec).
|
120 | </li>
|
121 | {.or}
|
122 | <li>
|
123 | Successfully parsed <b>{num_files|commas}</b> shell scripts totalling
|
124 | <b>{num_lines|commas}</b> lines
|
125 | in <b>{parse_proc_secs}</b> seconds
|
126 | (<b>{lines_per_sec}</b> lines/sec).
|
127 | </li>
|
128 | {.end}
|
129 |
|
130 | <li><b>{osh2oil_failed|commas}</b> OSH-to-Oil translations failed.</li>
|
131 | </ul>
|
132 | </div>
|
133 |
|
134 | <p></p>
|
135 | {.end}
|
136 |
|
137 | {.section dirs}
|
138 | <table id="dirs">
|
139 | <colgroup> <!-- for table-sort.js -->
|
140 | <col type="number">
|
141 | <col type="number">
|
142 | <col type="number">
|
143 | <col type="number">
|
144 | <col type="number">
|
145 | <col type="number">
|
146 | <col type="number">
|
147 | <col type="case-insensitive">
|
148 | </colgroup>
|
149 | <thead>
|
150 | <tr>
|
151 | <td>Files</td>
|
152 | <td>Max Lines</td>
|
153 | <td>Total Lines</td>
|
154 | <!-- <td>Lines Parsed</td> -->
|
155 | <td>Parse Failures</td>
|
156 | <td>Max Parse Time (secs)</td>
|
157 | <td>Total Parse Time (secs)</td>
|
158 | <td>Translation Failures</td>
|
159 | <td class="name">Directory</td>
|
160 | </tr>
|
161 | </thead>
|
162 | <tbody>
|
163 | {.repeated section @}
|
164 | <tr>
|
165 | <td>{num_files|commas}</td>
|
166 | <td>{max_lines|commas}</td>
|
167 | <td>{num_lines|commas}</td>
|
168 | <!-- <td>{lines_parsed|commas}</td> -->
|
169 | {.parse_failed?}
|
170 | <td class="fail">{parse_failed|commas}</td>
|
171 | {.or}
|
172 | <td class="ok">{parse_failed|commas}</td>
|
173 | {.end}
|
174 | <td>{max_parse_secs}</td>
|
175 | <td>{parse_proc_secs}</td>
|
176 |
|
177 | {.osh2oil_failed?}
|
178 | <!-- <td class="fail">{osh2oil_failed|commas}</td> -->
|
179 | <td>{osh2oil_failed|commas}</td>
|
180 | {.or}
|
181 | <!-- <td class="ok">{osh2oil_failed|commas}</td> -->
|
182 | <td>{osh2oil_failed|commas}</td>
|
183 | {.end}
|
184 |
|
185 | <td class="name">
|
186 | <a href="{name|htmltag}/index.html">{name|html}/</a>
|
187 | </td>
|
188 | </tr>
|
189 | {.end}
|
190 | </tbody>
|
191 | </table>
|
192 | {.end}
|
193 |
|
194 | <p>
|
195 | </p>
|
196 |
|
197 | {.section files}
|
198 | <table id="files">
|
199 | <colgroup> <!-- for table-sort.js -->
|
200 | <col type="case-insensitive">
|
201 | <col type="number">
|
202 | <col type="case-insensitive">
|
203 | <col type="number">
|
204 | <col type="case-insensitive">
|
205 | <col type="case-insensitive">
|
206 | </colgroup>
|
207 | <thead>
|
208 | <tr>
|
209 | <td>Side By Side</td>
|
210 | <td>Lines</td>
|
211 | <td>Parsed?</td>
|
212 | <td>Parse Process Time (secs)</td>
|
213 | <td>Translated?</td>
|
214 | <td class="name">Filename</td>
|
215 | </tr>
|
216 | </thead>
|
217 | <tbody>
|
218 | {.repeated section @}
|
219 | <tr>
|
220 | <td>
|
221 | <a href="{base_url}osh-to-oil.html#{rel_path|htmltag}/{name|htmltag}">view</a>
|
222 | </td>
|
223 | <td>{num_lines|commas}</td>
|
224 | <td>
|
225 | {.parse_failed?}
|
226 | <a class="fail" href="#stderr_parse_{name}">FAIL</a>
|
227 | <td>{parse_proc_secs}</td>
|
228 | {.or}
|
229 | <a class="ok" href="{name}__ast.html">OK</a>
|
230 | <td>{parse_proc_secs}</td>
|
231 | {.end}
|
232 | </td>
|
233 |
|
234 | <td>
|
235 | {.osh2oil_failed?}
|
236 | <!-- <a class="fail" href="#stderr_osh2oil_{name}">FAIL</a> -->
|
237 | FAIL
|
238 | {.or}
|
239 | <!-- <a class="ok" href="{name}__oil.txt">OK</a> -->
|
240 | OK
|
241 | {.end}
|
242 | </td>
|
243 | <td class="name">
|
244 | <a href="{name|htmltag}.txt">{name|html}</a>
|
245 | </td>
|
246 | </tr>
|
247 | {.end}
|
248 | </tbody>
|
249 | </table>
|
250 | {.end}
|
251 |
|
252 | {.if test empty}
|
253 | <i>(empty dir)</i>
|
254 | {.end}
|
255 |
|
256 | {.section stderr}
|
257 | <h2>stderr</h2>
|
258 |
|
259 | <table id="stderr">
|
260 |
|
261 | {.repeated section @}
|
262 | <tr>
|
263 | <td>
|
264 | <a name="stderr_{action}_{name|htmltag}"></a>
|
265 | {.if test parsing}
|
266 | Parsing {name|html}
|
267 | {.or}
|
268 | Translating {name|html}
|
269 | {.end}
|
270 | </td>
|
271 | <td>
|
272 | <pre>
|
273 | {contents|html}
|
274 | </pre>
|
275 | </td>
|
276 | <tr/>
|
277 | {.end}
|
278 |
|
279 | </table>
|
280 | {.end}
|
281 |
|
282 | <!-- page globals -->
|
283 | <script type="text/javascript">
|
284 | var gUrlHash = new UrlHash(location.hash);
|
285 | var gTableStates = {};
|
286 | var kStatusElem = document.getElementById('status');
|
287 |
|
288 | var gTables = [];
|
289 | var e1 = document.getElementById('dirs');
|
290 | var e2 = document.getElementById('files');
|
291 |
|
292 | // If no hash, "redirect" to a state where we sort ascending by dir name and
|
293 | // filename. TODO: These column numbers are a bit fragile.
|
294 | var params = [];
|
295 | if (e1) {
|
296 | gTables.push(e1);
|
297 | params.push('t:dirs=8a');
|
298 | }
|
299 | if (e2) {
|
300 | gTables.push(e2);
|
301 | params.push('t:files=7a');
|
302 | }
|
303 |
|
304 | function initPage(urlHash, gTables, tableStates, statusElem) {
|
305 | makeTablesSortable(urlHash, gTables, tableStates);
|
306 | /* Disable for now, this seems odd? Think about mutability of gUrlHash.
|
307 | if (location.hash === '') {
|
308 | document.location = '#' + params.join('&');
|
309 | gUrlHash = new UrlHash(location.hash);
|
310 | }
|
311 | */
|
312 | updateTables(urlHash, tableStates, statusElem);
|
313 | }
|
314 |
|
315 | function onHashChange(urlHash, tableStates, statusElem) {
|
316 | updateTables(urlHash, tableStates, statusElem);
|
317 | }
|
318 | </script>
|
319 | """)
|
320 |
|
321 |
|
322 | def log(msg, *args):
|
323 | if msg:
|
324 | msg = msg % args
|
325 | print(msg, file=sys.stderr)
|
326 |
|
327 |
|
328 | class DirNode:
|
329 | """Entry in the file system tree."""
|
330 |
|
331 | def __init__(self):
|
332 | self.files = {} # filename -> stats for success/failure, time, etc.
|
333 | self.dirs = {} # subdir name -> Dir object
|
334 |
|
335 | self.subtree_stats = {} # name -> value
|
336 |
|
337 | # show all the non-empty stderr here?
|
338 | # __osh2oil.stderr.txt
|
339 | # __parse.stderr.txt
|
340 | self.stderr = []
|
341 |
|
342 |
|
343 | def UpdateNodes(node, path_parts, file_stats):
|
344 | """
|
345 | Create a file node and update the stats of all its descendants in the FS
|
346 | tree.
|
347 | """
|
348 | first = path_parts[0]
|
349 | rest = path_parts[1:]
|
350 |
|
351 | for name, value in file_stats.iteritems():
|
352 | # Sum numerical properties, but not strings
|
353 | if isinstance(value, int) or isinstance(value, float):
|
354 | if name in node.subtree_stats:
|
355 | node.subtree_stats[name] += value
|
356 | else:
|
357 | # NOTE: Could be int or float!!!
|
358 | node.subtree_stats[name] = value
|
359 |
|
360 | # Calculate maximums
|
361 | m = node.subtree_stats.get('max_parse_secs', 0.0)
|
362 | node.subtree_stats['max_parse_secs'] = max(m, file_stats['parse_proc_secs'])
|
363 |
|
364 | m = node.subtree_stats.get('max_lines', 0) # integer
|
365 | node.subtree_stats['max_lines'] = max(m, file_stats['num_lines'])
|
366 |
|
367 | if rest: # update an intermediate node
|
368 | if first in node.dirs:
|
369 | child = node.dirs[first]
|
370 | else:
|
371 | child = DirNode()
|
372 | node.dirs[first] = child
|
373 |
|
374 | UpdateNodes(child, rest, file_stats)
|
375 | else:
|
376 | # Include stderr if non-empty, or if FAILED
|
377 | parse_stderr = file_stats.pop('parse_stderr')
|
378 | if parse_stderr or file_stats['parse_failed']:
|
379 | node.stderr.append({
|
380 | 'parsing': True,
|
381 | 'action': 'parse',
|
382 | 'name': first,
|
383 | 'contents': parse_stderr,
|
384 | })
|
385 | osh2oil_stderr = file_stats.pop('osh2oil_stderr')
|
386 |
|
387 | # Concentrating on parsing failures for now.
|
388 |
|
389 | #if osh2oil_stderr or file_stats['osh2oil_failed']:
|
390 | # node.stderr.append({
|
391 | # 'parsing': False,
|
392 | # 'action': 'osh2oil',
|
393 | # 'name': first,
|
394 | # 'contents': osh2oil_stderr,
|
395 | # })
|
396 |
|
397 | # Attach to this dir
|
398 | node.files[first] = file_stats
|
399 |
|
400 |
|
401 | def DebugPrint(node, indent=0):
|
402 | """Debug print."""
|
403 | ind = indent * ' '
|
404 | #print('FILES', node.files.keys())
|
405 | for name in node.files:
|
406 | print('%s%s - %s' % (ind, name, node.files[name]))
|
407 | for name, child in node.dirs.iteritems():
|
408 | print('%s%s/ - %s' % (ind, name, child.subtree_stats))
|
409 | DebugPrint(child, indent=indent+1)
|
410 |
|
411 |
|
412 | def WriteJsonFiles(node, out_dir):
|
413 | """Write a index.json file for every directory."""
|
414 | path = os.path.join(out_dir, 'index.json')
|
415 | with open(path, 'w') as f:
|
416 | raise AssertionError # fix dir_totals
|
417 | d = {'files': node.files, 'dirs': node.dir_totals}
|
418 | json.dump(d, f)
|
419 |
|
420 | log('Wrote %s', path)
|
421 |
|
422 | for name, child in node.dirs.iteritems():
|
423 | WriteJsonFiles(child, os.path.join(out_dir, name))
|
424 |
|
425 |
|
426 | def _MakeNav(rel_path):
|
427 | assert not rel_path.startswith('/'), rel_path
|
428 | assert not rel_path.endswith('/'), rel_path
|
429 | # Get rid of ['']
|
430 | parts = ['WILD'] + [p for p in rel_path.split('/') if p]
|
431 | data = []
|
432 | n = len(parts)
|
433 | for i, p in enumerate(parts):
|
434 | if i == n - 1:
|
435 | link = None # Current page shouldn't have link
|
436 | else:
|
437 | link = '../' * (n - 1 - i) + 'index.html'
|
438 | data.append({'anchor': p, 'link': link})
|
439 | return data
|
440 |
|
441 |
|
442 | def _Lower(s):
|
443 | return s.lower()
|
444 |
|
445 |
|
446 | def WriteHtmlFiles(node, out_dir, rel_path='', base_url=''):
|
447 | """Write a index.html file for every directory.
|
448 |
|
449 | NOTE:
|
450 | - osh-to-oil.html lives at $base_url
|
451 | - table-sort.js lives at $base_url/../table-sort.js
|
452 |
|
453 | wild/
|
454 | table-sort.js
|
455 | table-sort.css
|
456 | www/
|
457 | index.html
|
458 | osh-to-oil.html
|
459 |
|
460 | wild/
|
461 | table-sort.js
|
462 | table-sort.css
|
463 | wild.wwz/ # Zip file
|
464 | index.html
|
465 | osh-to-oil.html
|
466 |
|
467 | wwz latency is subject to caching headers.
|
468 | """
|
469 | path = os.path.join(out_dir, 'index.html')
|
470 | with open(path, 'w') as f:
|
471 | files = []
|
472 | for name in sorted(node.files, key=_Lower):
|
473 | stats = node.files[name]
|
474 | entry = dict(stats)
|
475 | entry['name'] = name
|
476 | # TODO: This should be internal time
|
477 | lines_per_sec = entry['lines_parsed'] / entry['parse_proc_secs']
|
478 | entry['lines_per_sec'] = '%.1f' % lines_per_sec
|
479 | files.append(entry)
|
480 |
|
481 | dirs = []
|
482 | for name in sorted(node.dirs, key=_Lower):
|
483 | entry = dict(node.dirs[name].subtree_stats)
|
484 | entry['name'] = name
|
485 | # TODO: This should be internal time
|
486 | lines_per_sec = entry['lines_parsed'] / entry['parse_proc_secs']
|
487 | entry['lines_per_sec'] = '%.1f' % lines_per_sec
|
488 | dirs.append(entry)
|
489 |
|
490 | # TODO: Is there a way to make this less redundant?
|
491 | st = node.subtree_stats
|
492 | try:
|
493 | lines_per_sec = st['lines_parsed'] / st['parse_proc_secs']
|
494 | st['lines_per_sec'] = '%.1f' % lines_per_sec
|
495 | except KeyError:
|
496 | # This usually there were ZERO files.
|
497 | print(node, st, repr(rel_path), file=sys.stderr)
|
498 | raise
|
499 |
|
500 | data = {
|
501 | 'rel_path': rel_path,
|
502 | 'subtree_stats': node.subtree_stats, # redundant totals
|
503 | 'files': files,
|
504 | 'dirs': dirs,
|
505 | 'base_url': base_url,
|
506 | 'stderr': node.stderr,
|
507 | 'nav': _MakeNav(rel_path),
|
508 | }
|
509 |
|
510 | group = PAGE_TEMPLATES['LISTING']
|
511 | body = BODY_STYLE.expand(data, group=group)
|
512 | f.write(body)
|
513 |
|
514 | log('Wrote %s', path)
|
515 |
|
516 | # Recursive
|
517 | for name, child in node.dirs.iteritems():
|
518 | child_out = os.path.join(out_dir, name)
|
519 | child_rel = os.path.join(rel_path, name)
|
520 | child_base = base_url + '../'
|
521 | WriteHtmlFiles(child, child_out, rel_path=child_rel, base_url=child_base)
|
522 |
|
523 |
|
524 | def main(argv):
|
525 | action = argv[1]
|
526 |
|
527 | if action == 'summarize-dirs':
|
528 | # lines and size, oops
|
529 |
|
530 | # TODO: Need read the manifest instead, and then go by dirname() I guess
|
531 | # I guess it is a BFS so you can just assume?
|
532 | # os.path.dirname() on the full path?
|
533 | # Or maybe you need the output files?
|
534 |
|
535 | root_node = DirNode()
|
536 |
|
537 | # Collect work into dirs
|
538 | for line in sys.stdin:
|
539 | #d = line.strip()
|
540 | proj, abs_path, rel_path = line.split()
|
541 | #print proj, '-', abs_path, '-', rel_path
|
542 |
|
543 | def _ReadTaskFile(path):
|
544 | try:
|
545 | with open(path) as f:
|
546 | parts = f.read().split()
|
547 | status, secs = parts
|
548 | except ValueError as e:
|
549 | log('ERROR reading %s: %s', path, e)
|
550 | raise
|
551 | # Turn it into pass/fail
|
552 | num_failed = 1 if int(status) >= 1 else 0
|
553 | return num_failed, float(secs)
|
554 |
|
555 | raw_base = os.path.join('_tmp/wild/raw', proj, rel_path)
|
556 | st = {}
|
557 |
|
558 | # TODO:
|
559 | # - Open stderr to get internal time
|
560 |
|
561 | parse_task_path = raw_base + '__parse.task.txt'
|
562 | st['parse_failed'], st['parse_proc_secs'] = _ReadTaskFile(
|
563 | parse_task_path)
|
564 |
|
565 | with open(raw_base + '__parse.stderr.txt') as f:
|
566 | st['parse_stderr'] = f.read()
|
567 |
|
568 | osh2oil_task_path = raw_base + '__osh2oil.task.txt'
|
569 | st['osh2oil_failed'], st['osh2oil_proc_secs'] = _ReadTaskFile(
|
570 | osh2oil_task_path)
|
571 |
|
572 | with open(raw_base + '__osh2oil.stderr.txt') as f:
|
573 | st['osh2oil_stderr'] = f.read()
|
574 |
|
575 | wc_path = raw_base + '__wc.txt'
|
576 | with open(wc_path) as f:
|
577 | st['num_lines'] = int(f.read().split()[0])
|
578 | # For lines per second calculation
|
579 | st['lines_parsed'] = 0 if st['parse_failed'] else st['num_lines']
|
580 |
|
581 | st['num_files'] = 1
|
582 |
|
583 | path_parts = proj.split('/') + rel_path.split('/')
|
584 | #print path_parts
|
585 | UpdateNodes(root_node, path_parts, st)
|
586 |
|
587 | # Debug print
|
588 | #DebugPrint(root_node)
|
589 | #WriteJsonFiles(root_node, '_tmp/wild/www')
|
590 |
|
591 | WriteHtmlFiles(root_node, '_tmp/wild/www')
|
592 |
|
593 | else:
|
594 | raise RuntimeError('Invalid action %r' % action)
|
595 |
|
596 |
|
597 | if __name__ == '__main__':
|
598 | try:
|
599 | main(sys.argv)
|
600 | except RuntimeError as e:
|
601 | print('FATAL: %s' % e, file=sys.stderr)
|
602 | sys.exit(1)
|