ceph/src/boost/libs/predef/tools/ci/build_log.py

   1 #!/usr/bin/env python
   2
   3 # Copyright 2008 Rene Rivera
   4 # Distributed under the Boost Software License, Version 1.0.
   5 # (See accompanying file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
   6
   7 import re
   8 import optparse
   9 import time
  10 import xml.dom.minidom
  11 import xml.dom.pulldom
  12 from xml.sax.saxutils import unescape, escape
  13 import os.path
  14 from pprint import pprint
  15 from __builtin__ import exit
  16
  17 class BuildOutputXMLParsing(object):
  18     '''
  19     XML parsing utilities for dealing with the Boost Build output
  20     XML format.
  21     '''
  22
  23     def get_child_data( self, root, tag = None, id = None, name = None, strip = False, default = None ):
  24         return self.get_data(self.get_child(root,tag=tag,id=id,name=name),strip=strip,default=default)
  25
  26     def get_data( self, node, strip = False, default = None ):
  27         data = None
  28         if node:
  29             data_node = None
  30             if not data_node:
  31                 data_node = self.get_child(node,tag='#text')
  32             if not data_node:
  33                 data_node = self.get_child(node,tag='#cdata-section')
  34             data = ""
  35             while data_node:
  36                 data += data_node.data
  37                 data_node = data_node.nextSibling
  38                 if data_node:
  39                     if data_node.nodeName != '#text' \
  40                         and data_node.nodeName != '#cdata-section':
  41                         data_node = None
  42         if not data:
  43             data = default
  44         else:
  45             if strip:
  46                 data = data.strip()
  47         return data
  48
  49     def get_child( self, root, tag = None, id = None, name = None, type = None ):
  50         return self.get_sibling(root.firstChild,tag=tag,id=id,name=name,type=type)
  51
  52     def get_sibling( self, sibling, tag = None, id = None, name = None, type = None ):
  53         n = sibling
  54         while n:
  55             found = True
  56             if type and found:
  57                 found = found and type == n.nodeType
  58             if tag and found:
  59                 found = found and tag == n.nodeName
  60             if (id or name) and found:
  61                 found = found and n.nodeType == xml.dom.Node.ELEMENT_NODE
  62             if id and found:
  63                 if n.hasAttribute('id'):
  64                     found = found and n.getAttribute('id') == id
  65                 else:
  66                     found = found and n.hasAttribute('id') and n.getAttribute('id') == id
  67             if name and found:
  68                 found = found and n.hasAttribute('name') and n.getAttribute('name') == name
  69             if found:
  70                 return n
  71             n = n.nextSibling
  72         return None
  73
  74 class BuildOutputProcessor(BuildOutputXMLParsing):
  75
  76     def __init__(self, inputs):
  77         self.test = {}
  78         self.target_to_test = {}
  79         self.target = {}
  80         self.parent = {}
  81         self.timestamps = []
  82         for input in inputs:
  83             self.add_input(input)
  84
  85     def add_input(self, input):
  86         '''
  87         Add a single build XML output file to our data.
  88         '''
  89         events = xml.dom.pulldom.parse(input)
  90         context = []
  91         for (event,node) in events:
  92             if event == xml.dom.pulldom.START_ELEMENT:
  93                 context.append(node)
  94                 if node.nodeType == xml.dom.Node.ELEMENT_NODE:
  95                     x_f = self.x_name_(*context)
  96                     if x_f:
  97                         events.expandNode(node)
  98                         # expanding eats the end element, hence walking us out one level
  99                         context.pop()
 100                         # call handler
 101                         (x_f[1])(node)
 102             elif event == xml.dom.pulldom.END_ELEMENT:
 103                 context.pop()
 104
 105     def x_name_(self, *context, **kwargs):
 106         node = None
 107         names = [ ]
 108         for c in context:
 109             if c:
 110                 if not isinstance(c,xml.dom.Node):
 111                     suffix = '_'+c.replace('-','_').replace('#','_')
 112                 else:
 113                     suffix = '_'+c.nodeName.replace('-','_').replace('#','_')
 114                     node = c
 115                 names.append('x')
 116                 names = map(lambda x: x+suffix,names)
 117         if node:
 118             for name in names:
 119                 if hasattr(self,name):
 120                     return (name,getattr(self,name))
 121         return None
 122
 123     def x_build_test(self, node):
 124         '''
 125         Records the initial test information that will eventually
 126         get expanded as we process the rest of the results.
 127         '''
 128         test_node = node
 129         test_name = test_node.getAttribute('name')
 130         test_target = self.get_child_data(test_node,tag='target',strip=True)
 131         ## print ">>> %s %s" %(test_name,test_target)
 132         self.test[test_name] = {
 133             'library' : "/".join(test_name.split('/')[0:-1]),
 134             'test-name' : test_name.split('/')[-1],
 135             'test-type' : test_node.getAttribute('type').lower(),
 136             'test-program' : self.get_child_data(test_node,tag='source',strip=True),
 137             'target' : test_target,
 138             'info' : self.get_child_data(test_node,tag='info',strip=True),
 139             'dependencies' : [],
 140             'actions' : [],
 141             }
 142         # Add a lookup for the test given the test target.
 143         self.target_to_test[self.test[test_name]['target']] = test_name
 144         return None
 145
 146     def x_build_targets_target( self, node ):
 147         '''
 148         Process the target dependency DAG into an ancestry tree so we can look up
 149         which top-level library and test targets specific build actions correspond to.
 150         '''
 151         target_node = node
 152         name = self.get_child_data(target_node,tag='name',strip=True)
 153         path = self.get_child_data(target_node,tag='path',strip=True)
 154         jam_target = self.get_child_data(target_node,tag='jam-target',strip=True)
 155         #~ Map for jam targets to virtual targets.
 156         self.target[jam_target] = {
 157             'name' : name,
 158             'path' : path
 159             }
 160         #~ Create the ancestry.
 161         dep_node = self.get_child(self.get_child(target_node,tag='dependencies'),tag='dependency')
 162         while dep_node:
 163             child = self.get_data(dep_node,strip=True)
 164             child_jam_target = '<p%s>%s' % (path,child.split('//',1)[1])
 165             self.parent[child_jam_target] = jam_target
 166             dep_node = self.get_sibling(dep_node.nextSibling,tag='dependency')
 167         return None
 168
 169     def x_build_action( self, node ):
 170         '''
 171         Given a build action log, process into the corresponding test log and
 172         specific test log sub-part.
 173         '''
 174         action_node = node
 175         name = self.get_child(action_node,tag='name')
 176         if name:
 177             name = self.get_data(name)
 178             #~ Based on the action, we decide what sub-section the log
 179             #~ should go into.
 180             action_type = None
 181             if re.match('[^%]+%[^.]+[.](compile)',name):
 182                 action_type = 'compile'
 183             elif re.match('[^%]+%[^.]+[.](link|archive)',name):
 184                 action_type = 'link'
 185             elif re.match('[^%]+%testing[.](capture-output)',name):
 186                 action_type = 'run'
 187             elif re.match('[^%]+%testing[.](expect-failure|expect-success)',name):
 188                 action_type = 'result'
 189             else:
 190                 # TODO: Enable to see what other actions can be included in the test results.
 191                 # action_type = None
 192                 action_type = 'other'
 193             #~ print "+   [%s] %s %s :: %s" %(action_type,name,'','')
 194             if action_type:
 195                 #~ Get the corresponding test.
 196                 (target,test) = self.get_test(action_node,type=action_type)
 197                 #~ Skip action that have no corresponding test as they are
 198                 #~ regular build actions and don't need to show up in the
 199                 #~ regression results.
 200                 if not test:
 201                     ##print "??? [%s] %s %s :: %s" %(action_type,name,target,test)
 202                     return None
 203                 ##print "+++ [%s] %s %s :: %s" %(action_type,name,target,test)
 204                 #~ Collect some basic info about the action.
 205                 action = {
 206                     'command' : self.get_action_command(action_node,action_type),
 207                     'output' : self.get_action_output(action_node,action_type),
 208                     'info' : self.get_action_info(action_node,action_type)
 209                     }
 210                 #~ For the test result status we find the appropriate node
 211                 #~ based on the type of test. Then adjust the result status
 212                 #~ accordingly. This makes the result status reflect the
 213                 #~ expectation as the result pages post processing does not
 214                 #~ account for this inversion.
 215                 action['type'] = action_type
 216                 if action_type == 'result':
 217                     if re.match(r'^compile',test['test-type']):
 218                         action['type'] = 'compile'
 219                     elif re.match(r'^link',test['test-type']):
 220                         action['type'] = 'link'
 221                     elif re.match(r'^run',test['test-type']):
 222                         action['type'] = 'run'
 223                 #~ The result sub-part we will add this result to.
 224                 if action_node.getAttribute('status') == '0':
 225                     action['result'] = 'succeed'
 226                 else:
 227                     action['result'] = 'fail'
 228                 # Add the action to the test.
 229                 test['actions'].append(action)
 230                 # Set the test result if this is the result action for the test.
 231                 if action_type == 'result':
 232                     test['result'] = action['result']
 233         return None
 234
 235     def x_build_timestamp( self, node ):
 236         '''
 237         The time-stamp goes to the corresponding attribute in the result.
 238         '''
 239         self.timestamps.append(self.get_data(node).strip())
 240         return None
 241
 242     def get_test( self, node, type = None ):
 243         '''
 244         Find the test corresponding to an action. For testing targets these
 245         are the ones pre-declared in the --dump-test option. For libraries
 246         we create a dummy test as needed.
 247         '''
 248         jam_target = self.get_child_data(node,tag='jam-target')
 249         base = self.target[jam_target]['name']
 250         target = jam_target
 251         while target in self.parent:
 252             target = self.parent[target]
 253         #~ print "--- TEST: %s ==> %s" %(jam_target,target)
 254         #~ main-target-type is a precise indicator of what the build target is
 255         #~ originally meant to be.
 256         #main_type = self.get_child_data(self.get_child(node,tag='properties'),
 257         #    name='main-target-type',strip=True)
 258         main_type = None
 259         if main_type == 'LIB' and type:
 260             lib = self.target[target]['name']
 261             if not lib in self.test:
 262                 self.test[lib] = {
 263                     'library' : re.search(r'libs/([^/]+)',lib).group(1),
 264                     'test-name' : os.path.basename(lib),
 265                     'test-type' : 'lib',
 266                     'test-program' : os.path.basename(lib),
 267                     'target' : lib
 268                     }
 269             test = self.test[lib]
 270         else:
 271             target_name_ = self.target[target]['name']
 272             if self.target_to_test.has_key(target_name_):
 273                 test = self.test[self.target_to_test[target_name_]]
 274             else:
 275                 test = None
 276         return (base,test)
 277
 278     #~ The command executed for the action. For run actions we omit the command
 279     #~ as it's just noise.
 280     def get_action_command( self, action_node, action_type ):
 281         if action_type != 'run':
 282             return self.get_child_data(action_node,tag='command')
 283         else:
 284             return ''
 285
 286     #~ The command output.
 287     def get_action_output( self, action_node, action_type ):
 288         return self.get_child_data(action_node,tag='output',default='')
 289
 290     #~ Some basic info about the action.
 291     def get_action_info( self, action_node, action_type ):
 292         info = {}
 293         #~ The jam action and target.
 294         info['name'] = self.get_child_data(action_node,tag='name')
 295         info['path'] = self.get_child_data(action_node,tag='path')
 296         #~ The timing of the action.
 297         info['time-start'] = action_node.getAttribute('start')
 298         info['time-end'] = action_node.getAttribute('end')
 299         info['time-user'] = action_node.getAttribute('user')
 300         info['time-system'] = action_node.getAttribute('system')
 301         #~ Testing properties.
 302         test_info_prop = self.get_child_data(self.get_child(action_node,tag='properties'),name='test-info')
 303         info['always_show_run_output'] = test_info_prop == 'always_show_run_output'
 304         #~ And for compiles some context that may be hidden if using response files.
 305         if action_type == 'compile':
 306             info['define'] = []
 307             define = self.get_child(self.get_child(action_node,tag='properties'),name='define')
 308             while define:
 309                 info['define'].append(self.get_data(define,strip=True))
 310                 define = self.get_sibling(define.nextSibling,name='define')
 311         return info
 312
 313 class BuildConsoleSummaryReport(object):
 314
 315     HEADER = '\033[35m\033[1m'
 316     INFO = '\033[34m'
 317     OK = '\033[32m'
 318     WARNING = '\033[33m'
 319     FAIL = '\033[31m'
 320     ENDC = '\033[0m'
 321
 322     def __init__(self, bop, opt):
 323         self.bop = bop
 324
 325     def generate(self):
 326         self.summary_info = {
 327             'total' : 0,
 328             'success' : 0,
 329             'failed' : [],
 330             }
 331         self.header_print("======================================================================")
 332         self.print_test_log()
 333         self.print_summary()
 334         self.header_print("======================================================================")
 335
 336     @property
 337     def failed(self):
 338         return len(self.summary_info['failed']) > 0
 339
 340     def print_test_log(self):
 341         self.header_print("Tests run..")
 342         self.header_print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
 343         for k in sorted(self.bop.test.keys()):
 344             test = self.bop.test[k]
 345             if len(test['actions']) > 0:
 346                 self.summary_info['total'] += 1
 347                 ##print ">>>> {0}".format(test['test-name'])
 348                 if 'result' in test:
 349                     succeed = test['result'] == 'succeed'
 350                 else:
 351                     succeed = test['actions'][-1]['result'] == 'succeed'
 352                 if succeed:
 353                     self.summary_info['success'] += 1
 354                 else:
 355                     self.summary_info['failed'].append(test)
 356                 if succeed:
 357                     self.ok_print("[PASS] {0}",k)
 358                 else:
 359                     self.fail_print("[FAIL] {0}",k)
 360                 for action in test['actions']:
 361                     self.print_action(succeed, action)
 362
 363     def print_action(self, test_succeed, action):
 364         '''
 365         Print the detailed info of failed or always print tests.
 366         '''
 367         #self.info_print(">>> {0}",action.keys())
 368         if not test_succeed or action['info']['always_show_run_output']:
 369             output = action['output'].strip()
 370             if output != "":
 371                 p = self.fail_print if action['result'] == 'fail' else self.p_print
 372                 self.info_print("")
 373                 self.info_print("({0}) {1}",action['info']['name'],action['info']['path'])
 374                 p("")
 375                 p("{0}",action['command'].strip())
 376                 p("")
 377                 for line in output.splitlines():
 378                     p("{0}",line.encode('utf-8'))
 379
 380     def print_summary(self):
 381         self.header_print("")
 382         self.header_print("Testing summary..")
 383         self.header_print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
 384         self.p_print("Total: {0}",self.summary_info['total'])
 385         self.p_print("Success: {0}",self.summary_info['success'])
 386         if self.failed:
 387             self.fail_print("Failed: {0}",len(self.summary_info['failed']))
 388             for test in self.summary_info['failed']:
 389                 self.fail_print("  {0}/{1}",test['library'],test['test-name'])
 390
 391     def p_print(self, format, *args, **kargs):
 392         print format.format(*args,**kargs)
 393
 394     def info_print(self, format, *args, **kargs):
 395         print self.INFO+format.format(*args,**kargs)+self.ENDC
 396
 397     def header_print(self, format, *args, **kargs):
 398         print self.HEADER+format.format(*args,**kargs)+self.ENDC
 399
 400     def ok_print(self, format, *args, **kargs):
 401         print self.OK+format.format(*args,**kargs)+self.ENDC
 402
 403     def warn_print(self, format, *args, **kargs):
 404         print self.WARNING+format.format(*args,**kargs)+self.ENDC
 405
 406     def fail_print(self, format, *args, **kargs):
 407         print self.FAIL+format.format(*args,**kargs)+self.ENDC
 408
 409 class Main(object):
 410
 411     def __init__(self,args=None):
 412         op = optparse.OptionParser(
 413             usage="%prog [options] input+")
 414         op.add_option( '--output',
 415             help="type of output to generate" )
 416         ( opt, inputs ) = op.parse_args(args)
 417         bop = BuildOutputProcessor(inputs)
 418         output = None
 419         if opt.output == 'console':
 420             output = BuildConsoleSummaryReport(bop, opt)
 421         if output:
 422             output.generate()
 423             self.failed = output.failed
 424
 425 if __name__ == '__main__':
 426     m = Main()
 427     if m.failed:
 428         exit(-1)