]>
Commit | Line | Data |
---|---|---|
223e47cc LB |
1 | #!/usr/bin/env python |
2 | import re, string, sys, os, time, math | |
3 | ||
4 | DEBUG = 0 | |
5 | ||
6 | (tp, exp) = ('compile', 'exec') | |
7 | ||
8 | def parse(file): | |
9 | f = open(file, 'r') | |
10 | d = f.read() | |
11 | ||
12 | # Cleanup weird stuff | |
13 | d = re.sub(r',\d+:\d', '', d) | |
14 | ||
15 | r = re.findall(r'TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n', d) | |
16 | ||
17 | test = {} | |
18 | fname = '' | |
19 | for t in r: | |
20 | if DEBUG: | |
21 | print t | |
22 | ||
23 | if t[0] == 'PASS' or t[0] == 'FAIL' : | |
24 | tmp = t[2].split('llvm-test/') | |
25 | ||
26 | if DEBUG: | |
27 | print tmp | |
28 | ||
29 | if len(tmp) == 2: | |
30 | fname = tmp[1].strip('\r\n') | |
31 | else: | |
32 | fname = tmp[0].strip('\r\n') | |
33 | ||
34 | if not test.has_key(fname): | |
35 | test[fname] = {} | |
36 | ||
37 | test[fname][t[1] + ' state'] = t[0] | |
38 | test[fname][t[1] + ' time'] = float('nan') | |
39 | else : | |
40 | try: | |
41 | n = t[0].split('RESULT-')[1] | |
42 | ||
43 | if DEBUG: | |
44 | print "n == ", n; | |
45 | ||
46 | if n == 'compile-success': | |
47 | test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n')) | |
48 | ||
49 | elif n == 'exec-success': | |
50 | test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n')) | |
51 | if DEBUG: | |
52 | print test[fname][string.replace(n, '-success', '')] | |
53 | ||
54 | else : | |
55 | # print "ERROR!" | |
56 | sys.exit(1) | |
57 | ||
58 | except: | |
59 | continue | |
60 | ||
61 | return test | |
62 | ||
63 | # Diff results and look for regressions. | |
64 | def diffResults(d_old, d_new): | |
65 | regressions = {} | |
66 | passes = {} | |
67 | removed = '' | |
68 | ||
69 | for x in ['compile state', 'compile time', 'exec state', 'exec time']: | |
70 | regressions[x] = '' | |
71 | passes[x] = '' | |
72 | ||
73 | for t in sorted(d_old.keys()) : | |
74 | if d_new.has_key(t): | |
75 | ||
76 | # Check if the test passed or failed. | |
77 | for x in ['compile state', 'compile time', 'exec state', 'exec time']: | |
78 | ||
79 | if not d_old[t].has_key(x) and not d_new[t].has_key(x): | |
80 | continue | |
81 | ||
82 | if d_old[t].has_key(x): | |
83 | if d_new[t].has_key(x): | |
84 | ||
85 | if d_old[t][x] == 'PASS': | |
86 | if d_new[t][x] != 'PASS': | |
87 | regressions[x] += t + "\n" | |
88 | else: | |
89 | if d_new[t][x] == 'PASS': | |
90 | passes[x] += t + "\n" | |
91 | ||
92 | else : | |
93 | regressions[x] += t + "\n" | |
94 | ||
95 | if x == 'compile state' or x == 'exec state': | |
96 | continue | |
97 | ||
98 | # For execution time, if there is no result it's a fail. | |
99 | if not d_old[t].has_key(x) and not d_new[t].has_key(x): | |
100 | continue | |
101 | elif not d_new[t].has_key(x): | |
102 | regressions[x] += t + "\n" | |
103 | elif not d_old[t].has_key(x): | |
104 | passes[x] += t + "\n" | |
105 | ||
106 | if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): | |
107 | continue | |
108 | ||
109 | elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]): | |
110 | passes[x] += t + "\n" | |
111 | ||
112 | elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): | |
113 | regressions[x] += t + ": NaN%\n" | |
114 | ||
115 | if d_new[t][x] > d_old[t][x] and d_old[t][x] > 0.0 and \ | |
116 | (d_new[t][x] - d_old[t][x]) / d_old[t][x] > .05: | |
117 | regressions[x] += t + ": " + "{0:.1f}".format(100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x]) + "%\n" | |
118 | ||
119 | else : | |
120 | removed += t + "\n" | |
121 | ||
122 | if len(regressions['compile state']) != 0: | |
123 | print 'REGRESSION: Compilation Failed' | |
124 | print regressions['compile state'] | |
125 | ||
126 | if len(regressions['exec state']) != 0: | |
127 | print 'REGRESSION: Execution Failed' | |
128 | print regressions['exec state'] | |
129 | ||
130 | if len(regressions['compile time']) != 0: | |
131 | print 'REGRESSION: Compilation Time' | |
132 | print regressions['compile time'] | |
133 | ||
134 | if len(regressions['exec time']) != 0: | |
135 | print 'REGRESSION: Execution Time' | |
136 | print regressions['exec time'] | |
137 | ||
138 | if len(passes['compile state']) != 0: | |
139 | print 'NEW PASSES: Compilation' | |
140 | print passes['compile state'] | |
141 | ||
142 | if len(passes['exec state']) != 0: | |
143 | print 'NEW PASSES: Execution' | |
144 | print passes['exec state'] | |
145 | ||
146 | if len(removed) != 0: | |
147 | print 'REMOVED TESTS' | |
148 | print removed | |
149 | ||
150 | # Main | |
151 | if len(sys.argv) < 3 : | |
152 | print 'Usage:', sys.argv[0], '<old log> <new log>' | |
153 | sys.exit(-1) | |
154 | ||
155 | d_old = parse(sys.argv[1]) | |
156 | d_new = parse(sys.argv[2]) | |
157 | ||
158 | diffResults(d_old, d_new) |