OILS / opy / _regtest / src / web / table / csv2html.py View on Github | oilshell.org

369 lines, 228 significant
1#!/usr/bin/env python
2from __future__ import print_function
3"""
4csv2html.py
5
6Usage:
7 csv2html.py foo.csv
8
9Attempts to read foo_schema.csv. If not it assumes everything is a string.
10
11Things it handles:
12
13- table-sort.js integration <colgroup>
14 - <table id="foo"> for making columns sortable
15 - for choosing the comparator to use!
16 - for highlighting on sort
17- static / visual
18 - Aligning right for number, left for strings.
19 - highlighting NA numbers in red (only if it's considered a number)
20 - formatting numbers to a certain precision
21 - or displaying them as percentages
22 - changing CSV headers like 'elapsed_ms' to 'elapsed ms'
23 - Accepting a column with a '_HREF' suffix to make an HTML link
24 - We could have something like type:
25 string/anchor:shell-id
26 string/href:shell-id
27 - But the simple _HREF suffix is simpler. Easier to write R code for.
28
29Implementation notes:
30- To align right: need a class on every cell, e.g. "num". Can't do it through
31 <colgroup>.
32- To color, can use <colgroup>. table-sort.js needs this.
33
34TODO:
35 Does it make sense to implement <rowspan> and <colspan> ? It's nice for
36 visualization.
37"""
38
39import cgi
40import csv
41import optparse
42import os
43import re
44import sys
45
46
47def log(msg, *args):
48 if args:
49 msg = msg % args
50 print(msg, file=sys.stderr)
51
52
53class NullSchema:
54 def VerifyColumnNames(self, col_names):
55 pass
56
57 def IsNumeric(self, col_name):
58 return False
59
60 def ColumnIndexIsNumeric(self, index):
61 return False
62
63 def ColumnIndexIsInteger(self, index):
64 return False
65
66 def ColumnIndexHasHref(self, index):
67 return False
68
69
70INTEGER_TYPES = ('integer',)
71
72# for sorting, right-justification
73NUMERIC_TYPES = ('double', 'number') + INTEGER_TYPES
74
75
76class Schema:
77 def __init__(self, rows):
78 schema_col_names = rows[0]
79 assert 'column_name' in schema_col_names, schema_col_names
80 assert 'type' in schema_col_names, schema_col_names
81
82 # Schema columns
83 s_cols = {}
84 s_cols['column_name'] = []
85 s_cols['type'] = []
86 s_cols['precision'] = []
87 for row in rows[1:]:
88 for i, cell in enumerate(row):
89 name = schema_col_names[i]
90 s_cols[name].append(cell)
91
92 self.type_lookup = dict(
93 (name, t) for (name, t) in
94 zip(s_cols['column_name'], s_cols['type']))
95
96 # NOTE: it's OK if precision is missing.
97 self.precision_lookup = dict(
98 (name, p) for (name, p) in
99 zip(s_cols['column_name'], s_cols['precision']))
100
101 #log('SCHEMA %s', schema_col_names)
102 #log('type_lookup %s', self.type_lookup)
103 #log('precision_lookup %s', self.precision_lookup)
104
105 self.col_names = None
106 self.col_has_href = None
107
108 def VerifyColumnNames(self, col_names):
109 """Assert that the column names we got are all in the schema."""
110 for name in col_names:
111 log('%s : %s', name, self.type_lookup[name])
112
113 n = len(col_names)
114 self.col_has_href = [False] * n
115 for i in xrange(n-1):
116 this_name, next_name= col_names[i], col_names[i+1]
117 if this_name + '_HREF' == next_name:
118 self.col_has_href[i] = True
119
120 log('href: %s', self.col_has_href)
121 self.col_names = col_names
122
123 def IsNumeric(self, col_name):
124 return self.type_lookup[col_name] in NUMERIC_TYPES
125
126 def ColumnIndexIsNumeric(self, index):
127 col_name = self.col_names[index]
128 return self.IsNumeric(col_name)
129
130 def ColumnIndexIsInteger(self, index):
131 col_name = self.col_names[index]
132 return self.type_lookup[col_name] in INTEGER_TYPES
133
134 def ColumnIndexHasHref(self, index):
135 """
136 Is the next one?
137 """
138 return self.col_has_href[index]
139
140 def ColumnPrecision(self, index):
141 col_name = self.col_names[index]
142 return self.precision_lookup.get(col_name, 1) # default is arbitrary
143
144
145def PrintRow(row, schema):
146 """Print a CSV row as HTML, using the given formatting.
147
148 Returns:
149 An array of booleans indicating whether each cell is a number.
150 """
151 i = 0
152 n = len(row)
153 while True:
154 if i == n:
155 break
156
157 cell = row[i]
158 css_classes = []
159 cell_str = cell # by default, we don't touch it
160
161 if schema.ColumnIndexIsInteger(i):
162 css_classes.append('num') # right justify
163
164 try:
165 cell_int = int(cell)
166 except ValueError:
167 pass # NA?
168 else:
169 # commas AND floating point
170 cell_str = '{:,}'.format(cell_int)
171
172 # Look up by index now?
173 elif schema.ColumnIndexIsNumeric(i):
174 css_classes.append('num') # right justify
175
176 try:
177 cell_float = float(cell)
178 except ValueError:
179 pass # NA
180 else:
181 # commas AND floating point to a given precision
182 precision = schema.ColumnPrecision(i)
183 cell_str = '{0:,.{precision}f}'.format(cell_float, precision=precision)
184
185 # Percentage
186 #cell_str = '{:.1f}%'.format(cell_float * 100)
187
188 # Special CSS class for R NA values.
189 if cell.strip() == 'NA':
190 css_classes.append('na') # make it red
191
192 if css_classes:
193 print(' <td class="{}">'.format(' '.join(css_classes)), end=' ')
194 else:
195 print(' <td>', end=' ')
196
197 # Advance to next row if it's an _HREF.
198 if schema.ColumnIndexHasHref(i):
199 i += 1
200 href = row[i]
201 s = '<a href="%s">%s</a>' % (cgi.escape(href), cgi.escape(cell_str))
202 else:
203 s = cgi.escape(cell_str)
204
205 print(s, end=' ')
206 print('</td>')
207
208 i += 1
209
210
211def PrintColGroup(col_names, schema):
212 """Print HTML colgroup element, used for JavaScript sorting."""
213 print(' <colgroup>')
214 for i, col in enumerate(col_names):
215 if col.endswith('_HREF'):
216 continue
217
218 # CSS class is used for sorting
219 if schema.IsNumeric(col):
220 css_class = 'number'
221 else:
222 css_class = 'case-insensitive'
223
224 # NOTE: id is a comment only; not used
225 print(' <col id="{}" type="{}" />'.format(col, css_class))
226 print(' </colgroup>')
227
228
229def PrintTable(css_id, schema, col_names, rows, css_class_pattern):
230 if css_class_pattern:
231 css_class, r = css_class_pattern.split(None, 2)
232 cell_regex = re.compile(r)
233 else:
234 css_class = None
235 cell_regex = None
236
237 print('<table id="%s">' % css_id)
238 print(' <thead>')
239 print(' <tr>')
240 for i, col in enumerate(col_names):
241 if col.endswith('_HREF'):
242 continue
243 heading_str = cgi.escape(col.replace('_', ' '))
244 if schema.ColumnIndexIsNumeric(i):
245 print(' <td class="num">%s</td>' % heading_str)
246 else:
247 print(' <td>%s</td>' % heading_str)
248 print(' </tr>')
249 print(' </thead>')
250
251 print(' <tbody>')
252 for row in rows:
253
254 # TODO: There should be a special column called CSS_CLASS. Output that
255 # from R.
256 row_class = ''
257 if cell_regex:
258 for cell in row:
259 if cell_regex.search(cell):
260 row_class = 'class="%s"' % css_class
261 break
262
263 print(' <tr {}>'.format(row_class))
264
265 PrintRow(row, schema)
266 print(' </tr>')
267 print(' </tbody>')
268
269 PrintColGroup(col_names, schema)
270
271 print('</table>')
272
273
274def ReadFile(f, tsv=False):
275 """Read the CSV file, returning the column names and rows."""
276
277 if tsv:
278 c = csv.reader(f, delimiter='\t', doublequote=False,
279 quoting=csv.QUOTE_NONE)
280 else:
281 c = csv.reader(f)
282
283 # The first row of the CSV is assumed to be a header. The rest are data.
284 col_names = []
285 rows = []
286 for i, row in enumerate(c):
287 if i == 0:
288 col_names = row
289 continue
290 rows.append(row)
291 return col_names, rows
292
293
294def CreateOptionsParser():
295 p = optparse.OptionParser()
296
297 # We are taking a path, and not using stdin, because we read it twice.
298 p.add_option(
299 '--schema', dest='schema', metavar="PATH", type='str',
300 help='Path to the schema.')
301 p.add_option(
302 '--tsv', dest='tsv', default=False, action='store_true',
303 help='Read input in TSV format')
304 p.add_option(
305 '--css-class-pattern', dest='css_class_pattern', type='str',
306 help='A string of the form CSS_CLASS:PATTERN. If the cell contents '
307 'matches the pattern, then apply the given CSS class. '
308 'Example: osh:^osh')
309 return p
310
311
312def main(argv):
313 (opts, argv) = CreateOptionsParser().parse_args(argv[1:])
314
315 try:
316 csv_path = argv[0]
317 except IndexError:
318 raise RuntimeError('Expected CSV filename.')
319
320 schema = None
321 if opts.schema:
322 try:
323 schema_f = open(opts.schema)
324 except IOError as e:
325 raise RuntimeError('Error opening schema: %s' % e)
326 else:
327 if csv_path.endswith('.csv'):
328 schema_path = csv_path.replace('.csv', '.schema.csv')
329 elif csv_path.endswith('.tsv'):
330 schema_path = csv_path.replace('.tsv', '.schema.tsv')
331 else:
332 raise AssertionError(csv_path)
333
334 log('schema path %s', schema_path)
335 try:
336 schema_f = open(schema_path)
337 except IOError:
338 schema_f = None # allowed to have no schema
339
340 if schema_f:
341 if opts.tsv:
342 r = csv.reader(schema_f, delimiter='\t', doublequote=False,
343 quoting=csv.QUOTE_NONE)
344 else:
345 r = csv.reader(schema_f)
346
347 schema = Schema(list(r))
348 else:
349 schema = NullSchema()
350 # Default string schema
351
352 log('schema %s', schema)
353
354 with open(csv_path) as f:
355 col_names, rows = ReadFile(f, opts.tsv)
356
357 schema.VerifyColumnNames(col_names)
358
359 filename = os.path.basename(csv_path)
360 css_id, _ = os.path.splitext(filename)
361 PrintTable(css_id, schema, col_names, rows, opts.css_class_pattern)
362
363
364if __name__ == '__main__':
365 try:
366 main(sys.argv)
367 except RuntimeError as e:
368 print('FATAL: %s' % e, file=sys.stderr)
369 sys.exit(1)