OILS / doctools / src_tree.py View on Github | oilshell.org

470 lines, 237 significant
1#!/usr/bin/env python2
2"""src_tree.py: Publish a directory tree as HTML.
3
4TODO:
5
6- dir listing:
7 - should have columns
8 - or add line counts, and file counts?
9 - render README.md - would be nice
10
11- Could use JSON Template {.template} like test/wild_report.py
12 - for consistent header and all that
13
14AUTO
15
16- overview.html and for-translation.html should link to these files, not Github
17"""
18from __future__ import print_function
19
20import json
21import os
22import shutil
23import sys
24
25from doctools.util import log
26from doctools import html_head
27from test import wild_report
28from vendor import jsontemplate
29
30T = jsontemplate.Template
31
32
33def DetectType(path):
34
35 # Most support moved to src-tree.sh and micro-syntax
36
37 if path.endswith('.test.sh'):
38 return 'spec'
39
40 else:
41 return 'other'
42
43
44def Breadcrumb(rel_path, out_f, is_file=False):
45 offset = -1 if is_file else 0
46 data = wild_report.MakeNav(rel_path, root_name='OILS', offset=offset)
47 out_f.write(wild_report.NAV_TEMPLATE.expand({'nav': data}))
48
49
50# CSS class .line has white-space: pre
51
52# To avoid copy-paste problem, you could try the <div> solutions like this:
53# https://gitlab.com/gitlab-examples/python-getting-started/-/blob/master/manage.py?ref_type=heads
54
55# Note: we are compressing some stuff
56
57ROW_T = T("""\
58<tr>
59 <td class=num>{line_num}</td>
60 <td id=L{line_num}>
61 <span class="line {.section line_class}{@}{.end}">{line}</span>
62 </td>
63</tr>
64""",
65 default_formatter='html')
66
67LISTING_T = T("""\
68{.section dirs}
69<h1>Dirs</h1>
70<div id="dirs" class="listing">
71 {.repeated section @}
72 <a href="{name|htmltag}/index.html">{name|html}/</a> <br/>
73 {.end}
74</div>
75{.end}
76
77{.section files}
78<h1>Files</h1>
79<div id="files" class="listing">
80 {.repeated section @}
81 <a href="{url|htmltag}">{anchor|html}</a> <br/>
82 {.end}
83</div>
84{.end}
85
86</body>
87""")
88
89FILE_COUNTS_T = T("""\
90<div id="file-counts"> {num_lines} lines, {num_sig_lines} significant </div>
91""",
92 default_formatter='html')
93
94
95def SpecFiles(pairs, attrs_f):
96
97 for i, (path, html_out) in enumerate(pairs):
98 #log(path)
99
100 try:
101 os.makedirs(os.path.dirname(html_out))
102 except OSError:
103 pass
104
105 with open(path) as in_f, open(html_out, 'w') as out_f:
106 title = path
107
108 # How deep are we?
109 n = path.count('/') + 2
110 base_dir = '/'.join(['..'] * n)
111
112 #css_urls = ['%s/web/base.css' % base_dir, '%s/web/src-tree.css' % base_dir]
113 css_urls = ['%s/web/src-tree.css' % base_dir]
114
115 html_head.Write(out_f, title, css_urls=css_urls)
116
117 out_f.write('''
118 <body class="">
119 <div id="home-link">
120 <a href="https://github.com/oilshell/oil/blob/master/%s">View on Github</a>
121 |
122 <a href="/">oilshell.org</a>
123 </div>
124 <table>
125 ''' % path)
126
127 file_type = DetectType(path)
128
129 line_num = 1 # 1-based
130 for line in in_f:
131 if line.endswith('\n'):
132 line = line[:-1]
133
134 # Write line numbers
135 row = {'line_num': line_num, 'line': line}
136
137 s = line.lstrip()
138
139 if file_type == 'spec':
140 if s.startswith('####'):
141 row['line_class'] = 'spec-comment'
142 elif s.startswith('#'):
143 row['line_class'] = 'comm'
144
145 out_f.write(ROW_T.expand(row))
146
147 line_num += 1
148
149 # could be parsed by 'dirs'
150 print('%s lines=%d' % (path, line_num), file=attrs_f)
151
152 out_f.write('''
153 </table>
154 </body>
155 </html>''')
156
157 return i + 1
158
159
160def ReadFragments(in_f):
161 while True:
162 path = ReadNetString(in_f)
163 if path is None:
164 break
165
166 html_frag = ReadNetString(in_f)
167 if html_frag is None:
168 raise RuntimeError('Expected 2nd record (HTML fragment)')
169
170 s = ReadNetString(in_f)
171 if s is None:
172 raise RuntimeError('Expected 3rd record (file summary)')
173
174 summary = json.loads(s)
175
176 yield path, html_frag, summary
177
178
179def WriteHtmlFragments(in_f, out_dir, attrs_f=sys.stdout):
180
181 i = 0
182 for rel_path, html_frag, summary in ReadFragments(in_f):
183 html_size = len(html_frag)
184 if html_size > 300000:
185 out_path = os.path.join(out_dir, rel_path)
186 try:
187 os.makedirs(os.path.dirname(out_path))
188 except OSError:
189 pass
190
191 shutil.copyfile(rel_path, out_path)
192
193 # Attrs are parsed by MakeTree(), and then used by WriteDirsHtml().
194 # So we can print the right link.
195 print('%s raw=1' % rel_path, file=attrs_f)
196
197 file_size = os.path.getsize(rel_path)
198 log('Big HTML fragment of %.1f KB', float(html_size) / 1000)
199 log('Copied %s -> %s, %.1f KB', rel_path, out_path,
200 float(file_size) / 1000)
201
202 continue
203
204 html_out = os.path.join(out_dir, rel_path + '.html')
205
206 try:
207 os.makedirs(os.path.dirname(html_out))
208 except OSError:
209 pass
210
211 with open(html_out, 'w') as out_f:
212 title = rel_path
213
214 # How deep are we?
215 n = rel_path.count('/') + 2
216 base_dir = '/'.join(['..'] * n)
217
218 #css_urls = ['%s/web/base.css' % base_dir, '%s/web/src-tree.css' % base_dir]
219 css_urls = ['%s/web/src-tree.css' % base_dir]
220 html_head.Write(out_f, title, css_urls=css_urls)
221
222 out_f.write('''
223 <body class="">
224 <p>
225 ''')
226 Breadcrumb(rel_path, out_f, is_file=True)
227
228 out_f.write('''
229 <span id="home-link">
230 <a href="https://github.com/oilshell/oil/blob/master/%s">View on Github</a>
231 |
232 <a href="/">oilshell.org</a>
233 </span>
234 </p>
235 ''' % rel_path)
236
237 out_f.write(FILE_COUNTS_T.expand(summary))
238
239 out_f.write('<table>')
240 out_f.write(html_frag)
241
242 print('%s lines=%d' % (rel_path, summary['num_lines']),
243 file=attrs_f)
244
245 out_f.write('''
246 </table>
247 </body>
248 </html>''')
249
250 i += 1
251
252 log('Wrote %d HTML fragments', i)
253
254
255class DirNode:
256 """Entry in the file system tree.
257
258 Similar to test/wild_report.py
259 """
260
261 def __init__(self):
262 self.files = {} # filename -> attrs dict
263 self.dirs = {} # subdir name -> DirNode object
264
265 # Can accumulate total lines here
266 self.subtree_stats = {} # name -> value
267
268
269def DebugPrint(node, indent=0):
270 """Pretty-print our tree data structure."""
271 ind = indent * ' '
272 #print('FILES', node.files.keys())
273 for name in node.files:
274 print('%s%s - %s' % (ind, name, node.files[name]))
275
276 for name, child in node.dirs.iteritems():
277 print('%s%s/ - %s' % (ind, name, child.subtree_stats))
278 DebugPrint(child, indent=indent + 1)
279
280
281def UpdateNodes(node, path_parts, attrs):
282 """Similar to test/wild_report.py."""
283
284 first = path_parts[0]
285 rest = path_parts[1:]
286
287 if rest: # update an intermediate node
288 if first in node.dirs:
289 child = node.dirs[first]
290 else:
291 child = DirNode()
292 node.dirs[first] = child
293
294 UpdateNodes(child, rest, attrs)
295 # TODO: Update subtree_stats
296
297 else:
298 # leaf node
299 node.files[first] = attrs
300
301
302def MakeTree(stdin, root_node):
303 """Reads a stream of lines Each line contains a path and key=value attrs.
304
305 - Doesn't handle filenames with spaces
306 - Doesn't handle empty dirs that are leaves (since only files are first
307 class)
308 """
309 for line in sys.stdin:
310 parts = line.split()
311 path = parts[0]
312
313 # Examples:
314 # {'lines': '345'}
315 # {'raw': '1'}
316 attrs = {}
317 for part in parts[1:]:
318 k, v = part.split('=')
319 attrs[k] = v
320
321 path_parts = path.split('/')
322 UpdateNodes(root_node, path_parts, attrs)
323
324
325def WriteDirsHtml(node, out_dir, rel_path='', base_url=''):
326 #log('WriteDirectory %s %s %s', out_dir, rel_path, base_url)
327
328 files = []
329 for name in sorted(node.files):
330 attrs = node.files[name]
331
332 # Big files are raw, e.g. match.re2c.h and syntax_asdl.py
333 url = name if attrs.get('raw') else '%s.html' % name
334 f = {'url': url, 'anchor': name}
335 files.append(f)
336
337 dirs = []
338 for name in sorted(node.dirs):
339 dirs.append({'name': name})
340
341 data = {'files': files, 'dirs': dirs}
342 body = LISTING_T.expand(data)
343
344 path = os.path.join(out_dir, 'index.html')
345 with open(path, 'w') as f:
346
347 title = '%s - Listing' % rel_path
348 prefix = '%s../..' % base_url
349 css_urls = ['%s/web/base.css' % prefix, '%s/web/src-tree.css' % prefix]
350 html_head.Write(f, title, css_urls=css_urls)
351
352 f.write('''
353 <body>
354 <p>
355 ''')
356 Breadcrumb(rel_path, f)
357
358 f.write('''
359 <span id="home-link">
360 <a href="/">oilshell.org</a>
361 </span>
362 </p>
363 ''')
364
365 f.write(body)
366
367 f.write('</html>')
368
369 # Recursive
370 for name, child in node.dirs.iteritems():
371 child_out = os.path.join(out_dir, name)
372 child_rel = os.path.join(rel_path, name)
373 child_base = base_url + '../'
374 WriteDirsHtml(child,
375 child_out,
376 rel_path=child_rel,
377 base_url=child_base)
378
379
380def ReadNetString(in_f):
381
382 digits = []
383 for i in xrange(10): # up to 10 digits
384 c = in_f.read(1)
385 if c == '':
386 return None # EOF
387
388 if c == ':':
389 break
390
391 if not c.isdigit():
392 raise RuntimeError('Bad byte %r' % c)
393
394 digits.append(c)
395
396 if c != ':':
397 raise RuntimeError('Expected colon, got %r' % c)
398
399 n = int(''.join(digits))
400
401 s = in_f.read(n)
402 if len(s) != n:
403 raise RuntimeError('Expected %d bytes, got %d' % (n, len(s)))
404
405 c = in_f.read(1)
406 if c != ',':
407 raise RuntimeError('Expected comma, got %r' % c)
408
409 return s
410
411
412def main(argv):
413 action = argv[1]
414
415 if action == 'spec-files':
416 # Policy for _tmp/spec/osh-minimal/foo.test.html
417 # This just changes the HTML names?
418
419 out_dir = argv[2]
420 spec_names = argv[3:]
421
422 pairs = []
423 for name in spec_names:
424 src = 'spec/%s.test.sh' % name
425 html_out = os.path.join(out_dir, '%s.test.html' % name)
426 pairs.append((src, html_out))
427
428 attrs_f = sys.stdout
429 n = SpecFiles(pairs, attrs_f)
430 log('%s: Wrote %d HTML files -> %s', os.path.basename(sys.argv[0]), n,
431 out_dir)
432
433 elif action == 'smoosh-file':
434 # TODO: Should fold this generated code into the source tree, and run in CI
435
436 in_path = argv[2]
437 out_path = argv[3]
438 pairs = [(in_path, out_path)]
439
440 attrs_f = sys.stdout
441 n = SpecFiles(pairs, attrs_f)
442 log('%s: %s -> %s', os.path.basename(sys.argv[0]), in_path, out_path)
443
444 elif action == 'write-html-fragments':
445
446 out_dir = argv[2]
447 WriteHtmlFragments(sys.stdin, out_dir)
448
449 elif action == 'dirs':
450 # stdin: a bunch of merged ATTRs file?
451
452 # We load them, and write a whole tree?
453 out_dir = argv[2]
454
455 # I think we make a big data structure here
456
457 root_node = DirNode()
458 MakeTree(sys.stdin, root_node)
459
460 if 0:
461 DebugPrint(root_node)
462
463 WriteDirsHtml(root_node, out_dir)
464
465 else:
466 raise RuntimeError('Invalid action %r' % action)
467
468
469if __name__ == '__main__':
470 main(sys.argv)