ceph/src/boost/libs/metaparse/tools/benchmark/benchmark.py

   1 #!/usr/bin/python
   2 """Utility to benchmark the generated source files"""
   3
   4 # Copyright Abel Sinkovics (abel@sinkovics.hu) 2016.
   5 # Distributed under the Boost Software License, Version 1.0.
   6 #    (See accompanying file LICENSE_1_0.txt or copy at
   7 #          http://www.boost.org/LICENSE_1_0.txt)
   8
   9 import argparse
  10 import os
  11 import subprocess
  12 import json
  13 import math
  14 import platform
  15 import matplotlib
  16 import random
  17 import re
  18 import time
  19 import psutil
  20 import PIL
  21
  22 matplotlib.use('Agg')
  23 import matplotlib.pyplot  # pylint:disable=I0011,C0411,C0412,C0413
  24
  25
  26 def benchmark_command(cmd, progress):
  27     """Benchmark one command execution"""
  28     full_cmd = '/usr/bin/time --format="%U %M" {0}'.format(cmd)
  29     print '{0:6.2f}% Running {1}'.format(100.0 * progress, full_cmd)
  30     (_, err) = subprocess.Popen(
  31         ['/bin/bash', '-c', full_cmd],
  32         stdin=subprocess.PIPE,
  33         stdout=subprocess.PIPE,
  34         stderr=subprocess.PIPE
  35     ).communicate('')
  36
  37     values = err.strip().split(' ')
  38     if len(values) == 2:
  39         try:
  40             return (float(values[0]), float(values[1]))
  41         except:  # pylint:disable=I0011,W0702
  42             pass  # Handled by the code after the "if"
  43
  44     print err
  45     raise Exception('Error during benchmarking')
  46
  47
  48 def benchmark_file(
  49         filename, compiler, include_dirs, (progress_from, progress_to),
  50         iter_count):
  51     """Benchmark one file"""
  52     time_sum = 0
  53     mem_sum = 0
  54     for nth_run in xrange(0, iter_count):
  55         (time_spent, mem_used) = benchmark_command(
  56             '{0} -std=c++11 {1} -c {2}'.format(
  57                 compiler,
  58                 ' '.join('-I{0}'.format(i) for i in include_dirs),
  59                 filename
  60             ),
  61             (
  62                 progress_to * nth_run + progress_from * (iter_count - nth_run)
  63             ) / iter_count
  64         )
  65         os.remove(os.path.splitext(os.path.basename(filename))[0] + '.o')
  66         time_sum = time_sum + time_spent
  67         mem_sum = mem_sum + mem_used
  68
  69     return {
  70         "time": time_sum / iter_count,
  71         "memory": mem_sum / (iter_count * 1024)
  72     }
  73
  74
  75 def compiler_info(compiler):
  76     """Determine the name + version of the compiler"""
  77     (out, err) = subprocess.Popen(
  78         ['/bin/bash', '-c', '{0} -v'.format(compiler)],
  79         stdin=subprocess.PIPE,
  80         stdout=subprocess.PIPE,
  81         stderr=subprocess.PIPE
  82     ).communicate('')
  83
  84     gcc_clang = re.compile('(gcc|clang) version ([0-9]+(\\.[0-9]+)*)')
  85
  86     for line in (out + err).split('\n'):
  87         mtch = gcc_clang.search(line)
  88         if mtch:
  89             return mtch.group(1) + ' ' + mtch.group(2)
  90
  91     return compiler
  92
  93
  94 def string_char(char):
  95     """Turn the character into one that can be part of a filename"""
  96     return '_' if char in [' ', '~', '(', ')', '/', '\\'] else char
  97
  98
  99 def make_filename(string):
 100     """Turn the string into a filename"""
 101     return ''.join(string_char(c) for c in string)
 102
 103
 104 def files_in_dir(path, extension):
 105     """Enumartes the files in path with the given extension"""
 106     ends = '.{0}'.format(extension)
 107     return (f for f in os.listdir(path) if f.endswith(ends))
 108
 109
 110 def format_time(seconds):
 111     """Format a duration"""
 112     minute = 60
 113     hour = minute * 60
 114     day = hour * 24
 115     week = day * 7
 116
 117     result = []
 118     for name, dur in [
 119             ('week', week), ('day', day), ('hour', hour),
 120             ('minute', minute), ('second', 1)
 121     ]:
 122         if seconds > dur:
 123             value = seconds // dur
 124             result.append(
 125                 '{0} {1}{2}'.format(int(value), name, 's' if value > 1 else '')
 126             )
 127             seconds = seconds % dur
 128     return ' '.join(result)
 129
 130
 131 def benchmark(src_dir, compiler, include_dirs, iter_count):
 132     """Do the benchrmarking"""
 133
 134     files = list(files_in_dir(src_dir, 'cpp'))
 135     random.shuffle(files)
 136
 137     started_at = time.time()
 138     result = {}
 139     for filename in files:
 140         progress = len(result)
 141         result[filename] = benchmark_file(
 142             os.path.join(src_dir, filename),
 143             compiler,
 144             include_dirs,
 145             (float(progress) / len(files), float(progress + 1) / len(files)),
 146             iter_count
 147         )
 148         elapsed = time.time() - started_at
 149         total = float(len(files) * elapsed) / len(result)
 150         print 'Elapsed time: {0}, Remaining time: {1}'.format(
 151             format_time(elapsed),
 152             format_time(total - elapsed)
 153         )
 154     return result
 155
 156
 157 def plot(values, mode_names, title, (xlabel, ylabel), out_file):
 158     """Plot a diagram"""
 159     matplotlib.pyplot.clf()
 160     for mode, mode_name in mode_names.iteritems():
 161         vals = values[mode]
 162         matplotlib.pyplot.plot(
 163             [x for x, _ in vals],
 164             [y for _, y in vals],
 165             label=mode_name
 166         )
 167     matplotlib.pyplot.title(title)
 168     matplotlib.pyplot.xlabel(xlabel)
 169     matplotlib.pyplot.ylabel(ylabel)
 170     if len(mode_names) > 1:
 171         matplotlib.pyplot.legend()
 172     matplotlib.pyplot.savefig(out_file)
 173
 174
 175 def mkdir_p(path):
 176     """mkdir -p path"""
 177     try:
 178         os.makedirs(path)
 179     except OSError:
 180         pass
 181
 182
 183 def configs_in(src_dir):
 184     """Enumerate all configs in src_dir"""
 185     for filename in files_in_dir(src_dir, 'json'):
 186         with open(os.path.join(src_dir, filename), 'rb') as in_f:
 187             yield json.load(in_f)
 188
 189
 190 def byte_to_gb(byte):
 191     """Convert bytes to GB"""
 192     return byte / (1024.0 * 1024 * 1024)
 193
 194
 195 def join_images(img_files, out_file):
 196     """Join the list of images into the out file"""
 197     images = [PIL.Image.open(f) for f in img_files]
 198     joined = PIL.Image.new(
 199         'RGB',
 200         (sum(i.size[0] for i in images), max(i.size[1] for i in images))
 201     )
 202     left = 0
 203     for img in images:
 204         joined.paste(im=img, box=(left, 0))
 205         left = left + img.size[0]
 206     joined.save(out_file)
 207
 208
 209 def plot_temp_diagrams(config, results, temp_dir):
 210     """Plot temporary diagrams"""
 211     display_name = {
 212         'time': 'Compilation time (s)',
 213         'memory': 'Compiler memory usage (MB)',
 214     }
 215
 216     files = config['files']
 217     img_files = []
 218     for measured in ['time', 'memory']:
 219         mpts = sorted(int(k) for k in files.keys())
 220         img_files.append(os.path.join(temp_dir, '_{0}.png'.format(measured)))
 221         plot(
 222             {
 223                 m: [(x, results[files[str(x)][m]][measured]) for x in mpts]
 224                 for m in config['modes'].keys()
 225             },
 226             config['modes'],
 227             display_name[measured],
 228             (config['x_axis_label'], display_name[measured]),
 229             img_files[-1]
 230         )
 231     return img_files
 232
 233
 234 def plot_diagram(config, results, images_dir, out_filename):
 235     """Plot one diagram"""
 236     img_files = plot_temp_diagrams(config, results, images_dir)
 237     join_images(img_files, out_filename)
 238     for img_file in img_files:
 239         os.remove(img_file)
 240
 241
 242 def plot_diagrams(results, configs, compiler, out_dir):
 243     """Plot all diagrams specified by the configs"""
 244     compiler_fn = make_filename(compiler)
 245     total = psutil.virtual_memory().total  # pylint:disable=I0011,E1101
 246     memory = int(math.ceil(byte_to_gb(total)))
 247
 248     images_dir = os.path.join(out_dir, 'images')
 249
 250     for config in configs:
 251         out_prefix = '{0}_{1}'.format(config['name'], compiler_fn)
 252
 253         plot_diagram(
 254             config,
 255             results,
 256             images_dir,
 257             os.path.join(images_dir, '{0}.png'.format(out_prefix))
 258         )
 259
 260         with open(
 261             os.path.join(out_dir, '{0}.qbk'.format(out_prefix)),
 262             'wb'
 263         ) as out_f:
 264             qbk_content = """{0}
 265 Measured on a {2} host with {3} GB memory. Compiler used: {4}.
 266
 267 [$images/metaparse/{1}.png [width 100%]]
 268 """.format(config['desc'], out_prefix, platform.platform(), memory, compiler)
 269             out_f.write(qbk_content)
 270
 271
 272 def main():
 273     """The main function of the script"""
 274     desc = 'Benchmark the files generated by generate.py'
 275     parser = argparse.ArgumentParser(description=desc)
 276     parser.add_argument(
 277         '--src',
 278         dest='src_dir',
 279         default='generated',
 280         help='The directory containing the sources to benchmark'
 281     )
 282     parser.add_argument(
 283         '--out',
 284         dest='out_dir',
 285         default='../../doc',
 286         help='The output directory'
 287     )
 288     parser.add_argument(
 289         '--include',
 290         dest='include',
 291         default='include',
 292         help='The directory containing the headeres for the benchmark'
 293     )
 294     parser.add_argument(
 295         '--boost_headers',
 296         dest='boost_headers',
 297         default='../../../..',
 298         help='The directory containing the Boost headers (the boost directory)'
 299     )
 300     parser.add_argument(
 301         '--compiler',
 302         dest='compiler',
 303         default='g++',
 304         help='The compiler to do the benchmark with'
 305     )
 306     parser.add_argument(
 307         '--repeat_count',
 308         dest='repeat_count',
 309         type=int,
 310         default=5,
 311         help='How many times a measurement should be repeated.'
 312     )
 313
 314     args = parser.parse_args()
 315
 316     compiler = compiler_info(args.compiler)
 317     results = benchmark(
 318         args.src_dir,
 319         args.compiler,
 320         [args.include, args.boost_headers],
 321         args.repeat_count
 322     )
 323
 324     plot_diagrams(results, configs_in(args.src_dir), compiler, args.out_dir)
 325
 326
 327 if __name__ == '__main__':
 328     main()