ceph/src/dmclock/benchmark/data_parser.py

   1 #!/usr/bin/env python
   2
   3 class DataPoint:
   4   def __init__(self):
   5     self.nserver = 0;
   6     self.nclient = 0;
   7     self.heap_type = 0;
   8     self.total_time_to_add_req = 0;
   9     self.total_time_to_complete_req = 0;
  10     self.config = ''
  11
  12   def set_name(self, config, heap_type):
  13     self.config = config;
  14     self.heap_type = heap_type
  15
  16   def get_conig(self):
  17     import re
  18     return re.split(r"/|\.", self.config)[1]
  19
  20   def __str__(self):
  21     return "s:%d, c:%d,h:%d,config:%s"%(self.nserver, self.nclient, self.heap_type, self.config);
  22 # end DataPoint
  23
  24
  25 def isFloat(elem):
  26  try:
  27   float(elem)
  28   return True
  29  except ValueError:
  30   return False
  31 #end isFloat
  32
  33
  34 def parse_config_params(fname):
  35   nclient = 0;
  36   nserver = 0;
  37   # read config file property
  38   with open(fname, 'r') as f:
  39     for line in f:
  40       line = line.strip('\n \t')
  41       if not line: continue;
  42       if line.startswith("client_count"):
  43         nclient += int(line.split('=')[-1]);
  44       if line.startswith("server_count"):
  45         nserver += int(line.split('=')[-1]);
  46   # end of file
  47   return [nserver, nclient];
  48 # parse_config_params
  49
  50 def make_aggregate_data_point(dps, config, heap_type):
  51     # create new aggregate point
  52     dp = DataPoint();
  53     # set set and k_way_heap property
  54     dp.set_name(config, heap_type);
  55
  56     num_run = 0
  57     for _dp in dps:
  58       if _dp.config == config and _dp.heap_type == heap_type:
  59         # print _dp, config, heap_type
  60         dp.nserver =_dp.nserver
  61         dp.nclient = _dp.nclient
  62         num_run                       += 1
  63         dp.total_time_to_add_req      += _dp.total_time_to_add_req
  64         dp.total_time_to_complete_req += _dp.total_time_to_complete_req
  65
  66     # average
  67     dp.total_time_to_add_req      /= num_run;
  68     dp.total_time_to_complete_req /= num_run
  69     #print dp
  70     return dp;
  71
  72 def parse_data_points(filename):
  73   dps = []; #data-points
  74   dp = None;
  75   state = 0;
  76   configs = {}
  77   k_ways  = {}
  78
  79   with open(filename, 'r') as f:
  80     for line in f:
  81       line = line.strip('\n \t')
  82       if not line: continue;
  83
  84       # file_name:1:configs/dmc_sim_8_6.conf
  85       if line.startswith("file_name"):
  86         if dp:
  87           dps.append(dp);
  88           state = 0;
  89
  90         # new data-point
  91         dp = DataPoint();
  92         parts = line.split(':')
  93         fname = parts[-1];
  94         dp.heap_type = int(parts[1]);
  95         if dp.heap_type not in k_ways:
  96           k_ways[dp.heap_type] = 1;
  97
  98         # add to the dictionary
  99         configs[fname] = 1;
 100
 101         dp.config = fname;
 102         params = parse_config_params(fname)
 103         dp.nserver = params[0];
 104         dp.nclient = params[-1];
 105
 106       elif line.startswith("average"):  # take last 2 averages
 107         r = [float(s) for s in line.split(' ') if isFloat(s)]
 108         state +=1;
 109         #print r, dp #if isFloat(s)
 110         if state == 3:
 111           dp.total_time_to_add_req = r[0]
 112         elif state == 4:
 113           dp.total_time_to_complete_req = r[0]
 114         else: pass
 115
 116       else:
 117         pass;
 118   # final entry
 119   dps.append(dp)
 120
 121   # compute average of multiple runs
 122   dps_avg = []
 123   for config in configs:
 124     data_per_config = []
 125     for k in k_ways:
 126       aggr_dp = make_aggregate_data_point(dps, config , k);
 127       data_per_config.append(aggr_dp);
 128     dps_avg.append(data_per_config);
 129   # end for
 130   return dps_avg;
 131 # end parse_data_points
 132
 133
 134 def create_header(num_cols):
 135   fields = ['nserver_nclient(config_file)','add_req', 'complete_req'];
 136   header = fields[0]
 137   #write add_req_{1, ...}
 138   for i in range(num_cols):
 139     header = '%s %s_%i'%(header, fields[1], i+2)
 140   #write complete_req_{1, ...}
 141   for i in range(num_cols):
 142     header = '%s %s_%i'%(header, fields[2], i+2)
 143   # new-line
 144   header = '%s\n'%(header)
 145   return header
 146 # end create_header
 147
 148
 149 def create_data_line(aggr_dp):
 150   # get common info
 151   dp = aggr_dp[0]
 152   data_line = "s:%d_c:%d "%(dp.nserver, dp.nclient);
 153   # get the point-count
 154   num_cols = len(aggr_dp);
 155   # write add_req_{1, ...}
 156   for i in range(num_cols):
 157     data_line = '%s %f'%(data_line, aggr_dp[i].total_time_to_add_req)
 158   # write complete_req_{1, ...}
 159   for i in range(num_cols):
 160     data_line = '%s %f'%(data_line, aggr_dp[i].total_time_to_complete_req)
 161   # new-line
 162   data_line = '%s\n'%(data_line)
 163   return data_line
 164 # end create_data_line
 165
 166
 167 def make_data(filename):
 168   # write the aggregated point in space separated file
 169   dps = parse_data_points(filename);
 170   if not len(dps) : return
 171   print "total points: ", len(dps)
 172   # open file
 173   with open('%s.dat'%(filename), 'w+') as f:
 174     # write header
 175     f.write(create_header(len(dps[0])));
 176     # write data-line
 177     for aggr_dp in dps:
 178         f.write(create_data_line(aggr_dp));
 179
 180
 181 def main(output_file):
 182   print output_file
 183   make_data(output_file);
 184
 185 import sys
 186 if __name__ == "__main__":
 187   file_name="result"
 188   if len(sys.argv) > 1:
 189     file_name=sys.argv[1].strip()
 190   main(file_name)
 191