tests/test-runner/cmd/test-runner.py

   1 #!/usr/bin/python
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2013 by Delphix. All rights reserved.
  16 #
  17
  18 import ConfigParser
  19 import os
  20 import logging
  21 from datetime import datetime
  22 from optparse import OptionParser
  23 from pwd import getpwnam
  24 from pwd import getpwuid
  25 from select import select
  26 from subprocess import PIPE
  27 from subprocess import Popen
  28 from sys import argv
  29 from sys import maxint
  30 from sys import exit
  31 from threading import Timer
  32 from time import time
  33
  34 BASEDIR = '/var/tmp/test_results'
  35 TESTDIR = '/usr/share/zfs/'
  36 KILL = 'kill'
  37 TRUE = 'true'
  38 SUDO = 'sudo'
  39
  40
  41 class Result(object):
  42     total = 0
  43     runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0}
  44
  45     def __init__(self):
  46         self.starttime = None
  47         self.returncode = None
  48         self.runtime = ''
  49         self.stdout = []
  50         self.stderr = []
  51         self.result = ''
  52
  53     def done(self, proc, killed):
  54         """
  55         Finalize the results of this Cmd.
  56         """
  57         Result.total += 1
  58         m, s = divmod(time() - self.starttime, 60)
  59         self.runtime = '%02d:%02d' % (m, s)
  60         self.returncode = proc.returncode
  61         if killed:
  62             self.result = 'KILLED'
  63             Result.runresults['KILLED'] += 1
  64         elif self.returncode is 0:
  65             self.result = 'PASS'
  66             Result.runresults['PASS'] += 1
  67         elif self.returncode is 4:
  68             self.result = 'SKIP'
  69             Result.runresults['SKIP'] += 1
  70         elif self.returncode is not 0:
  71             self.result = 'FAIL'
  72             Result.runresults['FAIL'] += 1
  73
  74
  75 class Output(object):
  76     """
  77     This class is a slightly modified version of the 'Stream' class found
  78     here: http://goo.gl/aSGfv
  79     """
  80     def __init__(self, stream):
  81         self.stream = stream
  82         self._buf = ''
  83         self.lines = []
  84
  85     def fileno(self):
  86         return self.stream.fileno()
  87
  88     def read(self, drain=0):
  89         """
  90         Read from the file descriptor. If 'drain' set, read until EOF.
  91         """
  92         while self._read() is not None:
  93             if not drain:
  94                 break
  95
  96     def _read(self):
  97         """
  98         Read up to 4k of data from this output stream. Collect the output
  99         up to the last newline, and append it to any leftover data from a
 100         previous call. The lines are stored as a (timestamp, data) tuple
 101         for easy sorting/merging later.
 102         """
 103         fd = self.fileno()
 104         buf = os.read(fd, 4096)
 105         if not buf:
 106             return None
 107         if '\n' not in buf:
 108             self._buf += buf
 109             return []
 110
 111         buf = self._buf + buf
 112         tmp, rest = buf.rsplit('\n', 1)
 113         self._buf = rest
 114         now = datetime.now()
 115         rows = tmp.split('\n')
 116         self.lines += [(now, r) for r in rows]
 117
 118
 119 class Cmd(object):
 120     verified_users = []
 121
 122     def __init__(self, pathname, outputdir=None, timeout=None, user=None):
 123         self.pathname = pathname
 124         self.outputdir = outputdir or 'BASEDIR'
 125         self.timeout = timeout
 126         self.user = user or ''
 127         self.killed = False
 128         self.result = Result()
 129
 130         if self.timeout is None:
 131             self.timeout = 60
 132
 133     def __str__(self):
 134         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % (
 135             self.pathname, self.outputdir, self.timeout, self.user)
 136
 137     def kill_cmd(self, proc):
 138         """
 139         Kill a running command due to timeout, or ^C from the keyboard. If
 140         sudo is required, this user was verified previously.
 141         """
 142         self.killed = True
 143         do_sudo = len(self.user) != 0
 144         signal = '-TERM'
 145
 146         cmd = [SUDO, KILL, signal, str(proc.pid)]
 147         if not do_sudo:
 148             del cmd[0]
 149
 150         try:
 151             kp = Popen(cmd)
 152             kp.wait()
 153         except:
 154             pass
 155
 156     def update_cmd_privs(self, cmd, user):
 157         """
 158         If a user has been specified to run this Cmd and we're not already
 159         running as that user, prepend the appropriate sudo command to run
 160         as that user.
 161         """
 162         me = getpwuid(os.getuid())
 163
 164         if not user or user is me:
 165             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 166                 cmd += '.ksh'
 167             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 168                 cmd += '.sh'
 169             return cmd
 170
 171         if not os.path.isfile(cmd):
 172             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 173                 cmd += '.ksh'
 174             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 175                 cmd += '.sh'
 176
 177         ret = '%s -E -u %s %s' % (SUDO, user, cmd)
 178         return ret.split(' ')
 179
 180     def collect_output(self, proc):
 181         """
 182         Read from stdout/stderr as data becomes available, until the
 183         process is no longer running. Return the lines from the stdout and
 184         stderr Output objects.
 185         """
 186         out = Output(proc.stdout)
 187         err = Output(proc.stderr)
 188         res = []
 189         while proc.returncode is None:
 190             proc.poll()
 191             res = select([out, err], [], [], .1)
 192             for fd in res[0]:
 193                 fd.read()
 194         for fd in res[0]:
 195             fd.read(drain=1)
 196
 197         return out.lines, err.lines
 198
 199     def run(self, options):
 200         """
 201         This is the main function that runs each individual test.
 202         Determine whether or not the command requires sudo, and modify it
 203         if needed. Run the command, and update the result object.
 204         """
 205         if options.dryrun is True:
 206             print self
 207             return
 208
 209         privcmd = self.update_cmd_privs(self.pathname, self.user)
 210         try:
 211             old = os.umask(0)
 212             if not os.path.isdir(self.outputdir):
 213                 os.makedirs(self.outputdir, mode=0777)
 214             os.umask(old)
 215         except OSError, e:
 216             fail('%s' % e)
 217
 218         self.result.starttime = time()
 219         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
 220         # Allow a special timeout value of 0 to mean infinity
 221         if int(self.timeout) == 0:
 222             self.timeout = maxint
 223         t = Timer(int(self.timeout), self.kill_cmd, [proc])
 224
 225         try:
 226             t.start()
 227             self.result.stdout, self.result.stderr = self.collect_output(proc)
 228         except KeyboardInterrupt:
 229             self.kill_cmd(proc)
 230             fail('\nRun terminated at user request.')
 231         finally:
 232             t.cancel()
 233
 234         self.result.done(proc, self.killed)
 235
 236     def skip(self):
 237         """
 238         Initialize enough of the test result that we can log a skipped
 239         command.
 240         """
 241         Result.total += 1
 242         Result.runresults['SKIP'] += 1
 243         self.result.stdout = self.result.stderr = []
 244         self.result.starttime = time()
 245         m, s = divmod(time() - self.result.starttime, 60)
 246         self.result.runtime = '%02d:%02d' % (m, s)
 247         self.result.result = 'SKIP'
 248
 249     def log(self, logger, options):
 250         """
 251         This function is responsible for writing all output. This includes
 252         the console output, the logfile of all results (with timestamped
 253         merged stdout and stderr), and for each test, the unmodified
 254         stdout/stderr/merged in it's own file.
 255         """
 256         if logger is None:
 257             return
 258
 259         logname = getpwuid(os.getuid()).pw_name
 260         user = ' (run as %s)' % (self.user if len(self.user) else logname)
 261         msga = 'Test: %s%s ' % (self.pathname, user)
 262         msgb = '[%s] [%s]' % (self.result.runtime, self.result.result)
 263         pad = ' ' * (80 - (len(msga) + len(msgb)))
 264
 265         # If -q is specified, only print a line for tests that didn't pass.
 266         # This means passing tests need to be logged as DEBUG, or the one
 267         # line summary will only be printed in the logfile for failures.
 268         if not options.quiet:
 269             logger.info('%s%s%s' % (msga, pad, msgb))
 270         elif self.result.result is not 'PASS':
 271             logger.info('%s%s%s' % (msga, pad, msgb))
 272         else:
 273             logger.debug('%s%s%s' % (msga, pad, msgb))
 274
 275         lines = self.result.stdout + self.result.stderr
 276         for dt, line in sorted(lines):
 277             logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line))
 278
 279         if len(self.result.stdout):
 280             with open(os.path.join(self.outputdir, 'stdout'), 'w') as out:
 281                 for _, line in self.result.stdout:
 282                     os.write(out.fileno(), '%s\n' % line)
 283         if len(self.result.stderr):
 284             with open(os.path.join(self.outputdir, 'stderr'), 'w') as err:
 285                 for _, line in self.result.stderr:
 286                     os.write(err.fileno(), '%s\n' % line)
 287         if len(self.result.stdout) and len(self.result.stderr):
 288             with open(os.path.join(self.outputdir, 'merged'), 'w') as merged:
 289                 for _, line in sorted(lines):
 290                     os.write(merged.fileno(), '%s\n' % line)
 291
 292
 293 class Test(Cmd):
 294     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
 295              'post_user']
 296
 297     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 298                  pre=None, pre_user=None, post=None, post_user=None):
 299         super(Test, self).__init__(pathname, outputdir, timeout, user)
 300         self.pre = pre or ''
 301         self.pre_user = pre_user or ''
 302         self.post = post or ''
 303         self.post_user = post_user or ''
 304
 305     def __str__(self):
 306         post_user = pre_user = ''
 307         if len(self.pre_user):
 308             pre_user = ' (as %s)' % (self.pre_user)
 309         if len(self.post_user):
 310             post_user = ' (as %s)' % (self.post_user)
 311         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \
 312                "%s%s\nUser: %s\n" % (
 313                    self.pathname, self.outputdir,
 314                    self.timeout, self.pre, pre_user, self.post, post_user,
 315                    self.user)
 316
 317     def verify(self, logger):
 318         """
 319         Check the pre/post scripts, user and Test. Omit the Test from this
 320         run if there are any problems.
 321         """
 322         files = [self.pre, self.pathname, self.post]
 323         users = [self.pre_user, self.user, self.post_user]
 324
 325         for f in [f for f in files if len(f)]:
 326             if not verify_file(f):
 327                 logger.info("Warning: Test '%s' not added to this run because"
 328                             " it failed verification." % f)
 329                 return False
 330
 331         for user in [user for user in users if len(user)]:
 332             if not verify_user(user, logger):
 333                 logger.info("Not adding Test '%s' to this run." %
 334                             self.pathname)
 335                 return False
 336
 337         return True
 338
 339     def run(self, logger, options):
 340         """
 341         Create Cmd instances for the pre/post scripts. If the pre script
 342         doesn't pass, skip this Test. Run the post script regardless.
 343         """
 344         pretest = Cmd(self.pre, outputdir=os.path.join(self.outputdir,
 345                       os.path.basename(self.pre)), timeout=self.timeout,
 346                       user=self.pre_user)
 347         test = Cmd(self.pathname, outputdir=self.outputdir,
 348                    timeout=self.timeout, user=self.user)
 349         posttest = Cmd(self.post, outputdir=os.path.join(self.outputdir,
 350                        os.path.basename(self.post)), timeout=self.timeout,
 351                        user=self.post_user)
 352
 353         cont = True
 354         if len(pretest.pathname):
 355             pretest.run(options)
 356             cont = pretest.result.result is 'PASS'
 357             pretest.log(logger, options)
 358
 359         if cont:
 360             test.run(options)
 361         else:
 362             test.skip()
 363
 364         test.log(logger, options)
 365
 366         if len(posttest.pathname):
 367             posttest.run(options)
 368             posttest.log(logger, options)
 369
 370
 371 class TestGroup(Test):
 372     props = Test.props + ['tests']
 373
 374     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 375                  pre=None, pre_user=None, post=None, post_user=None,
 376                  tests=None):
 377         super(TestGroup, self).__init__(pathname, outputdir, timeout, user,
 378                                         pre, pre_user, post, post_user)
 379         self.tests = tests or []
 380
 381     def __str__(self):
 382         post_user = pre_user = ''
 383         if len(self.pre_user):
 384             pre_user = ' (as %s)' % (self.pre_user)
 385         if len(self.post_user):
 386             post_user = ' (as %s)' % (self.post_user)
 387         return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \
 388             "Pre: %s%s\nPost: %s%s\nUser: %s\n" % (
 389                 self.pathname, self.outputdir, self.tests, self.timeout,
 390                 self.pre, pre_user, self.post, post_user, self.user)
 391
 392     def verify(self, logger):
 393         """
 394         Check the pre/post scripts, user and tests in this TestGroup. Omit
 395         the TestGroup entirely, or simply delete the relevant tests in the
 396         group, if that's all that's required.
 397         """
 398         # If the pre or post scripts are relative pathnames, convert to
 399         # absolute, so they stand a chance of passing verification.
 400         if len(self.pre) and not os.path.isabs(self.pre):
 401             self.pre = os.path.join(self.pathname, self.pre)
 402         if len(self.post) and not os.path.isabs(self.post):
 403             self.post = os.path.join(self.pathname, self.post)
 404
 405         auxfiles = [self.pre, self.post]
 406         users = [self.pre_user, self.user, self.post_user]
 407
 408         for f in [f for f in auxfiles if len(f)]:
 409             if self.pathname != os.path.dirname(f):
 410                 logger.info("Warning: TestGroup '%s' not added to this run. "
 411                             "Auxiliary script '%s' exists in a different "
 412                             "directory." % (self.pathname, f))
 413                 return False
 414
 415             if not verify_file(f):
 416                 logger.info("Warning: TestGroup '%s' not added to this run. "
 417                             "Auxiliary script '%s' failed verification." %
 418                             (self.pathname, f))
 419                 return False
 420
 421         for user in [user for user in users if len(user)]:
 422             if not verify_user(user, logger):
 423                 logger.info("Not adding TestGroup '%s' to this run." %
 424                             self.pathname)
 425                 return False
 426
 427         # If one of the tests is invalid, delete it, log it, and drive on.
 428         for test in self.tests:
 429             if not verify_file(os.path.join(self.pathname, test)):
 430                 del self.tests[self.tests.index(test)]
 431                 logger.info("Warning: Test '%s' removed from TestGroup '%s' "
 432                             "because it failed verification." %
 433                             (test, self.pathname))
 434
 435         return len(self.tests) is not 0
 436
 437     def run(self, logger, options):
 438         """
 439         Create Cmd instances for the pre/post scripts. If the pre script
 440         doesn't pass, skip all the tests in this TestGroup. Run the post
 441         script regardless.
 442         """
 443         pretest = Cmd(self.pre, outputdir=os.path.join(self.outputdir,
 444                       os.path.basename(self.pre)), timeout=self.timeout,
 445                       user=self.pre_user)
 446         posttest = Cmd(self.post, outputdir=os.path.join(self.outputdir,
 447                        os.path.basename(self.post)), timeout=self.timeout,
 448                        user=self.post_user)
 449
 450         cont = True
 451         if len(pretest.pathname):
 452             pretest.run(options)
 453             cont = pretest.result.result is 'PASS'
 454             pretest.log(logger, options)
 455
 456         for fname in self.tests:
 457             test = Cmd(os.path.join(self.pathname, fname),
 458                        outputdir=os.path.join(self.outputdir, fname),
 459                        timeout=self.timeout, user=self.user)
 460             if cont:
 461                 test.run(options)
 462             else:
 463                 test.skip()
 464
 465             test.log(logger, options)
 466
 467         if len(posttest.pathname):
 468             posttest.run(options)
 469             posttest.log(logger, options)
 470
 471
 472 class TestRun(object):
 473     props = ['quiet', 'outputdir']
 474
 475     def __init__(self, options):
 476         self.tests = {}
 477         self.testgroups = {}
 478         self.starttime = time()
 479         self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
 480         self.outputdir = os.path.join(options.outputdir, self.timestamp)
 481         self.logger = self.setup_logging(options)
 482         self.defaults = [
 483             ('outputdir', BASEDIR),
 484             ('quiet', False),
 485             ('timeout', 60),
 486             ('user', ''),
 487             ('pre', ''),
 488             ('pre_user', ''),
 489             ('post', ''),
 490             ('post_user', '')
 491         ]
 492
 493     def __str__(self):
 494         s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
 495         s += 'TESTS:\n'
 496         for key in sorted(self.tests.keys()):
 497             s += '%s%s' % (self.tests[key].__str__(), '\n')
 498         s += 'TESTGROUPS:\n'
 499         for key in sorted(self.testgroups.keys()):
 500             s += '%s%s' % (self.testgroups[key].__str__(), '\n')
 501         return s
 502
 503     def addtest(self, pathname, options):
 504         """
 505         Create a new Test, and apply any properties that were passed in
 506         from the command line. If it passes verification, add it to the
 507         TestRun.
 508         """
 509         test = Test(pathname)
 510         for prop in Test.props:
 511             setattr(test, prop, getattr(options, prop))
 512
 513         if test.verify(self.logger):
 514             self.tests[pathname] = test
 515
 516     def addtestgroup(self, dirname, filenames, options):
 517         """
 518         Create a new TestGroup, and apply any properties that were passed
 519         in from the command line. If it passes verification, add it to the
 520         TestRun.
 521         """
 522         if dirname not in self.testgroups:
 523             testgroup = TestGroup(dirname)
 524             for prop in Test.props:
 525                 setattr(testgroup, prop, getattr(options, prop))
 526
 527             # Prevent pre/post scripts from running as regular tests
 528             for f in [testgroup.pre, testgroup.post]:
 529                 if f in filenames:
 530                     del filenames[filenames.index(f)]
 531
 532             self.testgroups[dirname] = testgroup
 533             self.testgroups[dirname].tests = sorted(filenames)
 534
 535             testgroup.verify(self.logger)
 536
 537     def read(self, logger, options):
 538         """
 539         Read in the specified runfile, and apply the TestRun properties
 540         listed in the 'DEFAULT' section to our TestRun. Then read each
 541         section, and apply the appropriate properties to the Test or
 542         TestGroup. Properties from individual sections override those set
 543         in the 'DEFAULT' section. If the Test or TestGroup passes
 544         verification, add it to the TestRun.
 545         """
 546         config = ConfigParser.RawConfigParser()
 547         if not len(config.read(options.runfile)):
 548             fail("Coulnd't read config file %s" % options.runfile)
 549
 550         for opt in TestRun.props:
 551             if config.has_option('DEFAULT', opt):
 552                 setattr(self, opt, config.get('DEFAULT', opt))
 553         self.outputdir = os.path.join(self.outputdir, self.timestamp)
 554
 555         for section in config.sections():
 556             if 'tests' in config.options(section):
 557                 if os.path.isdir(section):
 558                     pathname = section
 559                 elif os.path.isdir(os.path.join(options.testdir, section)):
 560                     pathname = os.path.join(options.testdir, section)
 561                 else:
 562                     pathname = section
 563
 564                 testgroup = TestGroup(os.path.abspath(pathname))
 565                 for prop in TestGroup.props:
 566                     try:
 567                         setattr(testgroup, prop, config.get('DEFAULT', prop))
 568                         setattr(testgroup, prop, config.get(section, prop))
 569                     except ConfigParser.NoOptionError:
 570                         pass
 571
 572                 # Repopulate tests using eval to convert the string to a list
 573                 testgroup.tests = eval(config.get(section, 'tests'))
 574
 575                 if testgroup.verify(logger):
 576                     self.testgroups[section] = testgroup
 577             else:
 578                 test = Test(section)
 579                 for prop in Test.props:
 580                     try:
 581                         setattr(test, prop, config.get('DEFAULT', prop))
 582                         setattr(test, prop, config.get(section, prop))
 583                     except ConfigParser.NoOptionError:
 584                         pass
 585                 if test.verify(logger):
 586                     self.tests[section] = test
 587
 588     def write(self, options):
 589         """
 590         Create a configuration file for editing and later use. The
 591         'DEFAULT' section of the config file is created from the
 592         properties that were specified on the command line. Tests are
 593         simply added as sections that inherit everything from the
 594         'DEFAULT' section. TestGroups are the same, except they get an
 595         option including all the tests to run in that directory.
 596         """
 597
 598         defaults = dict([(prop, getattr(options, prop)) for prop, _ in
 599                         self.defaults])
 600         config = ConfigParser.RawConfigParser(defaults)
 601
 602         for test in sorted(self.tests.keys()):
 603             config.add_section(test)
 604
 605         for testgroup in sorted(self.testgroups.keys()):
 606             config.add_section(testgroup)
 607             config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
 608
 609         try:
 610             with open(options.template, 'w') as f:
 611                 return config.write(f)
 612         except IOError:
 613             fail('Could not open \'%s\' for writing.' % options.template)
 614
 615     def complete_outputdirs(self, options):
 616         """
 617         Collect all the pathnames for Tests, and TestGroups. Work
 618         backwards one pathname component at a time, to create a unique
 619         directory name in which to deposit test output. Tests will be able
 620         to write output files directly in the newly modified outputdir.
 621         TestGroups will be able to create one subdirectory per test in the
 622         outputdir, and are guaranteed uniqueness because a group can only
 623         contain files in one directory. Pre and post tests will create a
 624         directory rooted at the outputdir of the Test or TestGroup in
 625         question for their output.
 626         """
 627         done = False
 628         components = 0
 629         tmp_dict = dict(self.tests.items() + self.testgroups.items())
 630         total = len(tmp_dict)
 631         base = self.outputdir
 632
 633         while not done:
 634             l = []
 635             components -= 1
 636             for testfile in tmp_dict.keys():
 637                 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
 638                 if uniq not in l:
 639                     l.append(uniq)
 640                     tmp_dict[testfile].outputdir = os.path.join(base, uniq)
 641                 else:
 642                     break
 643             done = total == len(l)
 644
 645     def setup_logging(self, options):
 646         """
 647         Two loggers are set up here. The first is for the logfile which
 648         will contain one line summarizing the test, including the test
 649         name, result, and running time. This logger will also capture the
 650         timestamped combined stdout and stderr of each run. The second
 651         logger is optional console output, which will contain only the one
 652         line summary. The loggers are initialized at two different levels
 653         to facilitate segregating the output.
 654         """
 655         if options.dryrun is True:
 656             return
 657
 658         testlogger = logging.getLogger(__name__)
 659         testlogger.setLevel(logging.DEBUG)
 660
 661         if options.cmd is not 'wrconfig':
 662             try:
 663                 old = os.umask(0)
 664                 os.makedirs(self.outputdir, mode=0777)
 665                 os.umask(old)
 666             except OSError, e:
 667                 fail('%s' % e)
 668             filename = os.path.join(self.outputdir, 'log')
 669
 670             logfile = logging.FileHandler(filename)
 671             logfile.setLevel(logging.DEBUG)
 672             logfilefmt = logging.Formatter('%(message)s')
 673             logfile.setFormatter(logfilefmt)
 674             testlogger.addHandler(logfile)
 675
 676         cons = logging.StreamHandler()
 677         cons.setLevel(logging.INFO)
 678         consfmt = logging.Formatter('%(message)s')
 679         cons.setFormatter(consfmt)
 680         testlogger.addHandler(cons)
 681
 682         return testlogger
 683
 684     def run(self, options):
 685         """
 686         Walk through all the Tests and TestGroups, calling run().
 687         """
 688         try:
 689             os.chdir(self.outputdir)
 690         except OSError:
 691             fail('Could not change to directory %s' % self.outputdir)
 692         for test in sorted(self.tests.keys()):
 693             self.tests[test].run(self.logger, options)
 694         for testgroup in sorted(self.testgroups.keys()):
 695             self.testgroups[testgroup].run(self.logger, options)
 696
 697     def summary(self):
 698         if Result.total is 0:
 699             return
 700
 701         print '\nResults Summary'
 702         for key in Result.runresults.keys():
 703             if Result.runresults[key] is not 0:
 704                 print '%s\t% 4d' % (key, Result.runresults[key])
 705
 706         m, s = divmod(time() - self.starttime, 60)
 707         h, m = divmod(m, 60)
 708         print '\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)
 709         print 'Percent passed:\t%.1f%%' % (
 710             (float(Result.runresults['PASS']) / float(Result.total)) * 100)
 711         print 'Log directory:\t%s' % self.outputdir
 712
 713
 714 def verify_file(pathname):
 715     """
 716     Verify that the supplied pathname is an executable regular file.
 717     """
 718     if os.path.isdir(pathname) or os.path.islink(pathname):
 719         return False
 720
 721     for ext in '', '.ksh', '.sh':
 722         script_path = pathname + ext
 723         if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
 724             return True
 725
 726     return False
 727
 728
 729 def verify_user(user, logger):
 730     """
 731     Verify that the specified user exists on this system, and can execute
 732     sudo without being prompted for a password.
 733     """
 734     testcmd = [SUDO, '-n', '-u', user, TRUE]
 735
 736     if user in Cmd.verified_users:
 737         return True
 738
 739     try:
 740         getpwnam(user)
 741     except KeyError:
 742         logger.info("Warning: user '%s' does not exist.", user)
 743         return False
 744
 745     p = Popen(testcmd)
 746     p.wait()
 747     if p.returncode is not 0:
 748         logger.info("Warning: user '%s' cannot use passwordless sudo.", user)
 749         return False
 750     else:
 751         Cmd.verified_users.append(user)
 752
 753     return True
 754
 755
 756 def find_tests(testrun, options):
 757     """
 758     For the given list of pathnames, add files as Tests. For directories,
 759     if do_groups is True, add the directory as a TestGroup. If False,
 760     recursively search for executable files.
 761     """
 762
 763     for p in sorted(options.pathnames):
 764         if os.path.isdir(p):
 765             for dirname, _, filenames in os.walk(p):
 766                 if options.do_groups:
 767                     testrun.addtestgroup(dirname, filenames, options)
 768                 else:
 769                     for f in sorted(filenames):
 770                         testrun.addtest(os.path.join(dirname, f), options)
 771         else:
 772             testrun.addtest(p, options)
 773
 774
 775 def fail(retstr, ret=1):
 776     print '%s: %s' % (argv[0], retstr)
 777     exit(ret)
 778
 779
 780 def options_cb(option, opt_str, value, parser):
 781     path_options = ['runfile', 'outputdir', 'template', 'testdir']
 782
 783     if option.dest is 'runfile' and '-w' in parser.rargs or \
 784        option.dest is 'template' and '-c' in parser.rargs:
 785         fail('-c and -w are mutually exclusive.')
 786
 787     if opt_str in parser.rargs:
 788         fail('%s may only be specified once.' % opt_str)
 789
 790     if option.dest is 'runfile':
 791         parser.values.cmd = 'rdconfig'
 792     if option.dest is 'template':
 793         parser.values.cmd = 'wrconfig'
 794
 795     setattr(parser.values, option.dest, value)
 796     if option.dest in path_options:
 797         setattr(parser.values, option.dest, os.path.abspath(value))
 798
 799
 800 def parse_args():
 801     parser = OptionParser()
 802     parser.add_option('-c', action='callback', callback=options_cb,
 803                       type='string', dest='runfile', metavar='runfile',
 804                       help='Specify tests to run via config file.')
 805     parser.add_option('-d', action='store_true', default=False, dest='dryrun',
 806                       help='Dry run. Print tests, but take no other action.')
 807     parser.add_option('-g', action='store_true', default=False,
 808                       dest='do_groups', help='Make directories TestGroups.')
 809     parser.add_option('-o', action='callback', callback=options_cb,
 810                       default=BASEDIR, dest='outputdir', type='string',
 811                       metavar='outputdir', help='Specify an output directory.')
 812     parser.add_option('-i', action='callback', callback=options_cb,
 813                       default=TESTDIR, dest='testdir', type='string',
 814                       metavar='testdir', help='Specify a test directory.')
 815     parser.add_option('-p', action='callback', callback=options_cb,
 816                       default='', dest='pre', metavar='script',
 817                       type='string', help='Specify a pre script.')
 818     parser.add_option('-P', action='callback', callback=options_cb,
 819                       default='', dest='post', metavar='script',
 820                       type='string', help='Specify a post script.')
 821     parser.add_option('-q', action='store_true', default=False, dest='quiet',
 822                       help='Silence on the console during a test run.')
 823     parser.add_option('-t', action='callback', callback=options_cb, default=60,
 824                       dest='timeout', metavar='seconds', type='int',
 825                       help='Timeout (in seconds) for an individual test.')
 826     parser.add_option('-u', action='callback', callback=options_cb,
 827                       default='', dest='user', metavar='user', type='string',
 828                       help='Specify a different user name to run as.')
 829     parser.add_option('-w', action='callback', callback=options_cb,
 830                       default=None, dest='template', metavar='template',
 831                       type='string', help='Create a new config file.')
 832     parser.add_option('-x', action='callback', callback=options_cb, default='',
 833                       dest='pre_user', metavar='pre_user', type='string',
 834                       help='Specify a user to execute the pre script.')
 835     parser.add_option('-X', action='callback', callback=options_cb, default='',
 836                       dest='post_user', metavar='post_user', type='string',
 837                       help='Specify a user to execute the post script.')
 838     (options, pathnames) = parser.parse_args()
 839
 840     if not options.runfile and not options.template:
 841         options.cmd = 'runtests'
 842
 843     if options.runfile and len(pathnames):
 844         fail('Extraneous arguments.')
 845
 846     options.pathnames = [os.path.abspath(path) for path in pathnames]
 847
 848     return options
 849
 850
 851 def main(args):
 852     options = parse_args()
 853     testrun = TestRun(options)
 854
 855     if options.cmd is 'runtests':
 856         find_tests(testrun, options)
 857     elif options.cmd is 'rdconfig':
 858         testrun.read(testrun.logger, options)
 859     elif options.cmd is 'wrconfig':
 860         find_tests(testrun, options)
 861         testrun.write(options)
 862         exit(0)
 863     else:
 864         fail('Unknown command specified')
 865
 866     testrun.complete_outputdirs(options)
 867     testrun.run(options)
 868     testrun.summary()
 869     exit(0)
 870
 871
 872 if __name__ == '__main__':
 873     main(argv[1:])