| 19 | |
| 20 | |
| 21 | def main(args): |
| 22 | benchmark = '' |
| 23 | benchmarker = '' |
| 24 | # the first line has: [<empty string>, benchmarker name 1, ..] |
| 25 | # other lines have: [benchmark name, result 1 , ..] |
| 26 | matrix = [] |
| 27 | |
| 28 | for line in open(args[0], encoding='utf-8').readlines(): |
| 29 | line = line.strip() |
| 30 | if line.startswith('test_'): |
| 31 | benchmark = line.split(' ')[0][5:] |
| 32 | # print('benchmark:', benchmark) |
| 33 | if len(matrix) == 0: |
| 34 | # the first line has [(free space), benchmarker name 1, ..] |
| 35 | matrix += [[' ']] |
| 36 | # other lines have [benchmark name, result 1, ..] |
| 37 | matrix += [[benchmark]] |
| 38 | elif line.startswith('Running benchmarker'): |
| 39 | benchmarker = line.split(':')[-1].strip() |
| 40 | if benchmarker not in matrix[0]: |
| 41 | matrix[0] += [benchmarker] |
| 42 | # print('benchmarker:', benchmarker) |
| 43 | elif line.startswith(benchmarker + ':'): |
| 44 | parts = line.split(' ') |
| 45 | mean = float(parts[2]) |
| 46 | median = float(parts[7]) |
| 47 | noise = float(parts[13][:-2]) |
| 48 | if noise > 5: |
| 49 | print('warning: noisy! (%s: %f%%)' % (benchmark + '.' + benchmarker, noise)) |
| 50 | if abs(mean - median) / mean > 0.05: |
| 51 | print('warning: mean and median diverge! (%s: %f vs %f)' % (benchmark + '.' + benchmarker, mean, median)) |
| 52 | # print(benchmark, benchmarker, mean, median, noise) |
| 53 | matrix[-1] += [median] |
| 54 | |
| 55 | # normalize results |
| 56 | for line in matrix[1:]: |
| 57 | if len(line) >= 2: |
| 58 | base = line[1] |
| 59 | for i in range(1, len(line)): |
| 60 | line[i] /= base |
| 61 | |
| 62 | col0_width = max(len(r[0]) for r in matrix) |
| 63 | |
| 64 | # filter results |
| 65 | result = [] |
| 66 | for i, row in enumerate(matrix): |
| 67 | if len(row) != len(matrix[0]): |
| 68 | print('warning: not enough results, skipping row:', row[0]) |
| 69 | else: |
| 70 | line = '%*s ' % (col0_width, row[0]) |
| 71 | if i == 0: |
| 72 | line += '\t'.join([str(x) for x in row[1:]]) |
| 73 | else: |
| 74 | line += '\t'.join(['%.3f' % x for x in row[1:]]) |
| 75 | result.append(line) |
| 76 | |
| 77 | # print results |
| 78 | print() |