1 | #!/usr/bin/env python
|
2 | """
|
3 | virtual_memory.py
|
4 | """
|
5 |
|
6 | import csv
|
7 | import os
|
8 | import sys
|
9 | import re
|
10 |
|
11 | # VmSize, VmData might be interesting too.
|
12 | METRIC_RE = re.compile('^(VmPeak|VmRSS):\s*(\d+)')
|
13 |
|
14 |
|
15 | def main(argv):
|
16 | action = argv[1]
|
17 |
|
18 | if action == 'baseline':
|
19 | input_dirs = argv[2:]
|
20 |
|
21 | out = csv.writer(sys.stdout)
|
22 | out.writerow(
|
23 | ('host', 'shell_name', 'shell_hash', 'metric_name', 'metric_value'))
|
24 |
|
25 | # Dir name looks like "$host.$job_id"
|
26 | for input_dir in input_dirs:
|
27 | d = os.path.basename(input_dir)
|
28 | host, job_id = d.split('.')
|
29 | for name in os.listdir(input_dir):
|
30 | n, _ = os.path.splitext(name)
|
31 | shell_name, shell_hash = n.split('-')
|
32 | path = os.path.join(input_dir, name)
|
33 | with open(path) as f:
|
34 | for line in f:
|
35 | m = METRIC_RE.match(line)
|
36 | if m:
|
37 | name, value = m.groups()
|
38 | row = (host, shell_name, shell_hash, name, value)
|
39 | out.writerow(row)
|
40 |
|
41 | elif action == 'osh-parser':
|
42 | input_dirs = argv[2:]
|
43 | out = csv.writer(sys.stdout)
|
44 | HEADER = (
|
45 | 'host', 'shell_name', 'shell_hash', 'filename', 'metric_name',
|
46 | 'metric_value')
|
47 | out.writerow(HEADER)
|
48 |
|
49 | for input_dir in input_dirs:
|
50 | d = os.path.basename(input_dir)
|
51 | host, job_id, _ = d.split('.')
|
52 | for name in os.listdir(input_dir):
|
53 | n, _ = os.path.splitext(name)
|
54 | shell_id, filename = n.split('__')
|
55 | shell_name, shell_hash = shell_id.split('-')
|
56 |
|
57 | path = os.path.join(input_dir, name)
|
58 | with open(path) as f:
|
59 | for line in f:
|
60 | m = METRIC_RE.match(line)
|
61 | if m:
|
62 | name, value = m.groups()
|
63 | row = (host, shell_name, shell_hash, filename, name, value)
|
64 | out.writerow(row)
|
65 |
|
66 | elif action == 'osh-runtime':
|
67 | # NOTE: This is mostly a copy/paste of osh-parser.
|
68 |
|
69 | input_dirs = argv[2:]
|
70 | out = csv.writer(sys.stdout)
|
71 | HEADER = (
|
72 | 'host', 'shell_name', 'shell_hash', 'task_arg', 'event', 'metric_name',
|
73 | 'metric_value')
|
74 | out.writerow(HEADER)
|
75 |
|
76 | for input_dir in input_dirs:
|
77 | d = os.path.basename(input_dir)
|
78 | host, job_id, _ = d.split('.')
|
79 | for name in os.listdir(input_dir):
|
80 | n, _ = os.path.splitext(name)
|
81 | shell_id, task_arg, event = n.split('__')
|
82 | shell_name, shell_hash = shell_id.split('-')
|
83 |
|
84 | path = os.path.join(input_dir, name)
|
85 | with open(path) as f:
|
86 | for line in f:
|
87 | m = METRIC_RE.match(line)
|
88 | if m:
|
89 | name, value = m.groups()
|
90 | row = (host, shell_name, shell_hash, task_arg, event, name,
|
91 | value)
|
92 | out.writerow(row)
|
93 |
|
94 | else:
|
95 | raise RuntimeError('Invalid action %r' % action)
|
96 |
|
97 |
|
98 |
|
99 | if __name__ == '__main__':
|
100 | try:
|
101 | main(sys.argv)
|
102 | except RuntimeError as e:
|
103 | print >>sys.stderr, 'FATAL: %s' % e
|
104 | sys.exit(1)
|