1 | #!/usr/bin/python3 |
---|
2 | """ |
---|
3 | Python Script to convert output from mutilate to rmit like output |
---|
4 | """ |
---|
5 | import argparse |
---|
6 | import json |
---|
7 | import locale |
---|
8 | import os |
---|
9 | import re |
---|
10 | import sys |
---|
11 | |
---|
12 | locale.setlocale( locale.LC_ALL, 'en_US.UTF-8' ) |
---|
13 | |
---|
14 | parser = argparse.ArgumentParser(description='Python Script to convert output from mutilate to rmit like output') |
---|
15 | parser.add_argument('--out', nargs='?', type=argparse.FileType('w'), default=sys.stdout) |
---|
16 | parser.add_argument('--var', nargs='?', type=str, default='Target QPS') |
---|
17 | try: |
---|
18 | options = parser.parse_args() |
---|
19 | except: |
---|
20 | print('ERROR: invalid arguments', file=sys.stderr) |
---|
21 | parser.print_help(sys.stderr) |
---|
22 | sys.exit(1) |
---|
23 | |
---|
24 | thisdir = os.getcwd() |
---|
25 | dirs = os.listdir( thisdir ) |
---|
26 | |
---|
27 | names = ['fibre', 'forall', 'vanilla'] |
---|
28 | names_re = '|'.join(names) |
---|
29 | |
---|
30 | def precentile(line): |
---|
31 | fields = line.split() |
---|
32 | |
---|
33 | try: |
---|
34 | lat50s = fields[6] |
---|
35 | lat99s = fields[9] |
---|
36 | except: |
---|
37 | raise Warning("Warning: \"{}\"! insufficient fields".format(line)) |
---|
38 | |
---|
39 | try: |
---|
40 | lat50 = locale.atof(lat50s) |
---|
41 | lat99 = locale.atof(lat99s) |
---|
42 | except: |
---|
43 | raise Warning("Warning: \"{}\" \"{}\"! can't convert to float".format(lat50s, lat99s)) |
---|
44 | |
---|
45 | return lat50, lat99 |
---|
46 | |
---|
47 | def want0(line): |
---|
48 | line = line.strip() |
---|
49 | if not line.endswith("= 0 (0.0%)"): |
---|
50 | raise Warning("Warning: \"{}\"! should be 0".format(line)) |
---|
51 | |
---|
52 | def extract(filename, out): |
---|
53 | with open(filename, "r") as file: |
---|
54 | lines = file.readlines() |
---|
55 | |
---|
56 | warns = [] |
---|
57 | |
---|
58 | for line in lines: |
---|
59 | try: |
---|
60 | if line.startswith("read"): |
---|
61 | rlat50, rlat99 = precentile(line) |
---|
62 | |
---|
63 | elif line.startswith("update"): |
---|
64 | ulat50, ulat99 = precentile(line) |
---|
65 | |
---|
66 | elif line.startswith("Total QPS"): |
---|
67 | match = re.search("Total QPS = ([0-9,\.]+)", line) |
---|
68 | if match: |
---|
69 | try: |
---|
70 | qps = locale.atof(match[1]) |
---|
71 | except: |
---|
72 | raise Warning("Warning: \"{}\" can't convert qps to float".format(match[1])) |
---|
73 | else: |
---|
74 | raise Warning("Warning: \"{}\" line unreadable".format(line)) |
---|
75 | |
---|
76 | if line.startswith("Misses") or line.startswith("Skipped TXs"): |
---|
77 | want0(line) |
---|
78 | except Warning as w: |
---|
79 | warns.append(str(w)) |
---|
80 | |
---|
81 | try: |
---|
82 | out['Actual QPS'] = qps |
---|
83 | except: |
---|
84 | warns.append("Warning: No total QPS") |
---|
85 | |
---|
86 | try: |
---|
87 | out['Median Read Latency'] = rlat50 |
---|
88 | out['Tail Read Latency'] = rlat99 |
---|
89 | except: |
---|
90 | warns.append("Warning: no read latencies") |
---|
91 | |
---|
92 | try: |
---|
93 | out['Median Update Latency'] = ulat50 |
---|
94 | out['Tail Update Latency'] = ulat99 |
---|
95 | except: |
---|
96 | warns.append("Warning: no update latencies") |
---|
97 | |
---|
98 | return warns |
---|
99 | |
---|
100 | |
---|
101 | data = [] |
---|
102 | |
---|
103 | for filename in dirs: |
---|
104 | f = os.path.join( thisdir, filename ) |
---|
105 | # checking if it is a file |
---|
106 | if os.path.isfile(f): |
---|
107 | match = re.search("({})\.([0-9]+)\.([0-9]+)".format(names_re), filename) |
---|
108 | try: |
---|
109 | series = match[1] |
---|
110 | rate = match[2] |
---|
111 | rep = match[3] |
---|
112 | except: |
---|
113 | continue |
---|
114 | |
---|
115 | d = { options.var : int(rate) } |
---|
116 | |
---|
117 | w = extract( f, d ) |
---|
118 | |
---|
119 | data.append([series, "memcached {}".format(series), d]) |
---|
120 | if w: |
---|
121 | print("{} {} {}\n{}\n".format(series, rate, rep, '\n'.join(w))) |
---|
122 | |
---|
123 | options.out.write(json.dumps(data)) |
---|
124 | options.out.flush() |
---|
125 | options.out.write("\n") |
---|