tests/test-runner/cmd/test-runner.py

   1 #!/usr/bin/python
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2013 by Delphix. All rights reserved.
  16 #
  17
  18 import ConfigParser
  19 import os
  20 import logging
  21 from datetime import datetime
  22 from optparse import OptionParser
  23 from pwd import getpwnam
  24 from pwd import getpwuid
  25 from select import select
  26 from subprocess import PIPE
  27 from subprocess import Popen
  28 from sys import argv
  29 from sys import exit
  30 from threading import Timer
  31 from time import time
  32
  33 BASEDIR = '/var/tmp/test_results'
  34 TESTDIR = '/usr/share/zfs/'
  35 KILL = 'kill'
  36 TRUE = 'true'
  37 SUDO = 'sudo'
  38
  39
  40 class Result(object):
  41     total = 0
  42     runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0}
  43
  44     def __init__(self):
  45         self.starttime = None
  46         self.returncode = None
  47         self.runtime = ''
  48         self.stdout = []
  49         self.stderr = []
  50         self.result = ''
  51
  52     def done(self, proc, killed):
  53         """
  54         Finalize the results of this Cmd.
  55         """
  56         Result.total += 1
  57         m, s = divmod(time() - self.starttime, 60)
  58         self.runtime = '%02d:%02d' % (m, s)
  59         self.returncode = proc.returncode
  60         if killed:
  61             self.result = 'KILLED'
  62             Result.runresults['KILLED'] += 1
  63         elif self.returncode is 0:
  64             self.result = 'PASS'
  65             Result.runresults['PASS'] += 1
  66         elif self.returncode is 4:
  67             self.result = 'SKIP'
  68             Result.runresults['SKIP'] += 1
  69         elif self.returncode is not 0:
  70             self.result = 'FAIL'
  71             Result.runresults['FAIL'] += 1
  72
  73
  74 class Output(object):
  75     """
  76     This class is a slightly modified version of the 'Stream' class found
  77     here: http://goo.gl/aSGfv
  78     """
  79     def __init__(self, stream):
  80         self.stream = stream
  81         self._buf = ''
  82         self.lines = []
  83
  84     def fileno(self):
  85         return self.stream.fileno()
  86
  87     def read(self, drain=0):
  88         """
  89         Read from the file descriptor. If 'drain' set, read until EOF.
  90         """
  91         while self._read() is not None:
  92             if not drain:
  93                 break
  94
  95     def _read(self):
  96         """
  97         Read up to 4k of data from this output stream. Collect the output
  98         up to the last newline, and append it to any leftover data from a
  99         previous call. The lines are stored as a (timestamp, data) tuple
 100         for easy sorting/merging later.
 101         """
 102         fd = self.fileno()
 103         buf = os.read(fd, 4096)
 104         if not buf:
 105             return None
 106         if '\n' not in buf:
 107             self._buf += buf
 108             return []
 109
 110         buf = self._buf + buf
 111         tmp, rest = buf.rsplit('\n', 1)
 112         self._buf = rest
 113         now = datetime.now()
 114         rows = tmp.split('\n')
 115         self.lines += [(now, r) for r in rows]
 116
 117
 118 class Cmd(object):
 119     verified_users = []
 120
 121     def __init__(self, pathname, outputdir=None, timeout=None, user=None):
 122         self.pathname = pathname
 123         self.outputdir = outputdir or 'BASEDIR'
 124         self.timeout = timeout or 60
 125         self.user = user or ''
 126         self.killed = False
 127         self.result = Result()
 128
 129     def __str__(self):
 130         return "Pathname: %s\nOutputdir: %s\nTimeout: %s\nUser: %s\n" % (
 131                 self.pathname, self.outputdir, self.timeout, self.user)
 132
 133     def kill_cmd(self, proc):
 134         """
 135         Kill a running command due to timeout, or ^C from the keyboard. If
 136         sudo is required, this user was verified previously.
 137         """
 138         self.killed = True
 139         do_sudo = len(self.user) != 0
 140         signal = '-TERM'
 141
 142         cmd = [SUDO, KILL, signal, str(proc.pid)]
 143         if not do_sudo:
 144             del cmd[0]
 145
 146         try:
 147             kp = Popen(cmd)
 148             kp.wait()
 149         except:
 150             pass
 151
 152     def update_cmd_privs(self, cmd, user):
 153         """
 154         If a user has been specified to run this Cmd and we're not already
 155         running as that user, prepend the appropriate sudo command to run
 156         as that user.
 157         """
 158         me = getpwuid(os.getuid())
 159
 160         if not user or user is me:
 161             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 162                 cmd += '.ksh'
 163             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 164                 cmd += '.sh'
 165             return cmd
 166
 167         if not os.path.isfile(cmd):
 168             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 169                 cmd += '.ksh'
 170             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 171                 cmd += '.sh'
 172
 173         ret = '%s -E -u %s %s' % (SUDO, user, cmd)
 174         return ret.split(' ')
 175
 176     def collect_output(self, proc):
 177         """
 178         Read from stdout/stderr as data becomes available, until the
 179         process is no longer running. Return the lines from the stdout and
 180         stderr Output objects.
 181         """
 182         out = Output(proc.stdout)
 183         err = Output(proc.stderr)
 184         res = []
 185         while proc.returncode is None:
 186             proc.poll()
 187             res = select([out, err], [], [], .1)
 188             for fd in res[0]:
 189                 fd.read()
 190         for fd in res[0]:
 191             fd.read(drain=1)
 192
 193         return out.lines, err.lines
 194
 195     def run(self, options):
 196         """
 197         This is the main function that runs each individual test.
 198         Determine whether or not the command requires sudo, and modify it
 199         if needed. Run the command, and update the result object.
 200         """
 201         if options.dryrun is True:
 202             print self
 203             return
 204
 205         privcmd = self.update_cmd_privs(self.pathname, self.user)
 206         try:
 207             old = os.umask(0)
 208             if not os.path.isdir(self.outputdir):
 209                 os.makedirs(self.outputdir, mode=0777)
 210             os.umask(old)
 211         except OSError, e:
 212             fail('%s' % e)
 213
 214         self.result.starttime = time()
 215         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
 216         t = Timer(int(self.timeout), self.kill_cmd, [proc])
 217
 218         try:
 219             t.start()
 220             self.result.stdout, self.result.stderr = self.collect_output(proc)
 221         except KeyboardInterrupt:
 222             self.kill_cmd(proc)
 223             fail('\nRun terminated at user request.')
 224         finally:
 225             t.cancel()
 226
 227         self.result.done(proc, self.killed)
 228
 229     def skip(self):
 230         """
 231         Initialize enough of the test result that we can log a skipped
 232         command.
 233         """
 234         Result.total += 1
 235         Result.runresults['SKIP'] += 1
 236         self.result.stdout = self.result.stderr = []
 237         self.result.starttime = time()
 238         m, s = divmod(time() - self.result.starttime, 60)
 239         self.result.runtime = '%02d:%02d' % (m, s)
 240         self.result.result = 'SKIP'
 241
 242     def log(self, logger, options):
 243         """
 244         This function is responsible for writing all output. This includes
 245         the console output, the logfile of all results (with timestamped
 246         merged stdout and stderr), and for each test, the unmodified
 247         stdout/stderr/merged in it's own file.
 248         """
 249         if logger is None:
 250             return
 251
 252         logname = getpwuid(os.getuid()).pw_name
 253         user = ' (run as %s)' % (self.user if len(self.user) else logname)
 254         msga = 'Test: %s%s ' % (self.pathname, user)
 255         msgb = '[%s] [%s]' % (self.result.runtime, self.result.result)
 256         pad = ' ' * (80 - (len(msga) + len(msgb)))
 257
 258         # If -q is specified, only print a line for tests that didn't pass.
 259         # This means passing tests need to be logged as DEBUG, or the one
 260         # line summary will only be printed in the logfile for failures.
 261         if not options.quiet:
 262             logger.info('%s%s%s' % (msga, pad, msgb))
 263         elif self.result.result is not 'PASS':
 264             logger.info('%s%s%s' % (msga, pad, msgb))
 265         else:
 266             logger.debug('%s%s%s' % (msga, pad, msgb))
 267
 268         lines = self.result.stdout + self.result.stderr
 269         for dt, line in sorted(lines):
 270             logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line))
 271
 272         if len(self.result.stdout):
 273             with open(os.path.join(self.outputdir, 'stdout'), 'w') as out:
 274                 for _, line in self.result.stdout:
 275                     os.write(out.fileno(), '%s\n' % line)
 276         if len(self.result.stderr):
 277             with open(os.path.join(self.outputdir, 'stderr'), 'w') as err:
 278                 for _, line in self.result.stderr:
 279                     os.write(err.fileno(), '%s\n' % line)
 280         if len(self.result.stdout) and len(self.result.stderr):
 281             with open(os.path.join(self.outputdir, 'merged'), 'w') as merged:
 282                 for _, line in sorted(lines):
 283                     os.write(merged.fileno(), '%s\n' % line)
 284
 285
 286 class Test(Cmd):
 287     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
 288              'post_user']
 289
 290     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 291                  pre=None, pre_user=None, post=None, post_user=None):
 292         super(Test, self).__init__(pathname, outputdir, timeout, user)
 293         self.pre = pre or ''
 294         self.pre_user = pre_user or ''
 295         self.post = post or ''
 296         self.post_user = post_user or ''
 297
 298     def __str__(self):
 299         post_user = pre_user = ''
 300         if len(self.pre_user):
 301             pre_user = ' (as %s)' % (self.pre_user)
 302         if len(self.post_user):
 303             post_user = ' (as %s)' % (self.post_user)
 304         return "Pathname: %s\nOutputdir: %s\nTimeout: %s\nPre: %s%s\nPost: " \
 305                "%s%s\nUser: %s\n" % (self.pathname, self.outputdir,
 306                 self.timeout, self.pre, pre_user, self.post, post_user,
 307                 self.user)
 308
 309     def verify(self, logger):
 310         """
 311         Check the pre/post scripts, user and Test. Omit the Test from this
 312         run if there are any problems.
 313         """
 314         files = [self.pre, self.pathname, self.post]
 315         users = [self.pre_user, self.user, self.post_user]
 316
 317         for f in [f for f in files if len(f)]:
 318             if not verify_file(f):
 319                 logger.info("Warning: Test '%s' not added to this run because"
 320                             " it failed verification." % f)
 321                 return False
 322
 323         for user in [user for user in users if len(user)]:
 324             if not verify_user(user, logger):
 325                 logger.info("Not adding Test '%s' to this run." %
 326                             self.pathname)
 327                 return False
 328
 329         return True
 330
 331     def run(self, logger, options):
 332         """
 333         Create Cmd instances for the pre/post scripts. If the pre script
 334         doesn't pass, skip this Test. Run the post script regardless.
 335         """
 336         pretest = Cmd(self.pre, outputdir=os.path.join(self.outputdir,
 337                       os.path.basename(self.pre)), timeout=self.timeout,
 338                       user=self.pre_user)
 339         test = Cmd(self.pathname, outputdir=self.outputdir,
 340                    timeout=self.timeout, user=self.user)
 341         posttest = Cmd(self.post, outputdir=os.path.join(self.outputdir,
 342                        os.path.basename(self.post)), timeout=self.timeout,
 343                        user=self.post_user)
 344
 345         cont = True
 346         if len(pretest.pathname):
 347             pretest.run(options)
 348             cont = pretest.result.result is 'PASS'
 349             pretest.log(logger, options)
 350
 351         if cont:
 352             test.run(options)
 353         else:
 354             test.skip()
 355
 356         test.log(logger, options)
 357
 358         if len(posttest.pathname):
 359             posttest.run(options)
 360             posttest.log(logger, options)
 361
 362
 363 class TestGroup(Test):
 364     props = Test.props + ['tests']
 365
 366     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 367                  pre=None, pre_user=None, post=None, post_user=None,
 368                  tests=None):
 369         super(TestGroup, self).__init__(pathname, outputdir, timeout, user,
 370                                         pre, pre_user, post, post_user)
 371         self.tests = tests or []
 372
 373     def __str__(self):
 374         post_user = pre_user = ''
 375         if len(self.pre_user):
 376             pre_user = ' (as %s)' % (self.pre_user)
 377         if len(self.post_user):
 378             post_user = ' (as %s)' % (self.post_user)
 379         return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %s\n" \
 380                "Pre: %s%s\nPost: %s%s\nUser: %s\n" % (self.pathname,
 381                 self.outputdir, self.tests, self.timeout, self.pre, pre_user,
 382                 self.post, post_user, self.user)
 383
 384     def verify(self, logger):
 385         """
 386         Check the pre/post scripts, user and tests in this TestGroup. Omit
 387         the TestGroup entirely, or simply delete the relevant tests in the
 388         group, if that's all that's required.
 389         """
 390         # If the pre or post scripts are relative pathnames, convert to
 391         # absolute, so they stand a chance of passing verification.
 392         if len(self.pre) and not os.path.isabs(self.pre):
 393             self.pre = os.path.join(self.pathname, self.pre)
 394         if len(self.post) and not os.path.isabs(self.post):
 395             self.post = os.path.join(self.pathname, self.post)
 396
 397         auxfiles = [self.pre, self.post]
 398         users = [self.pre_user, self.user, self.post_user]
 399
 400         for f in [f for f in auxfiles if len(f)]:
 401             if self.pathname != os.path.dirname(f):
 402                 logger.info("Warning: TestGroup '%s' not added to this run. "
 403                             "Auxiliary script '%s' exists in a different "
 404                             "directory." % (self.pathname, f))
 405                 return False
 406
 407             if not verify_file(f):
 408                 logger.info("Warning: TestGroup '%s' not added to this run. "
 409                             "Auxiliary script '%s' failed verification." %
 410                             (self.pathname, f))
 411                 return False
 412
 413         for user in [user for user in users if len(user)]:
 414             if not verify_user(user, logger):
 415                 logger.info("Not adding TestGroup '%s' to this run." %
 416                             self.pathname)
 417                 return False
 418
 419         # If one of the tests is invalid, delete it, log it, and drive on.
 420         for test in self.tests:
 421             if not verify_file(os.path.join(self.pathname, test)):
 422                 del self.tests[self.tests.index(test)]
 423                 logger.info("Warning: Test '%s' removed from TestGroup '%s' "
 424                             "because it failed verification." % (test,
 425                             self.pathname))
 426
 427         return len(self.tests) is not 0
 428
 429     def run(self, logger, options):
 430         """
 431         Create Cmd instances for the pre/post scripts. If the pre script
 432         doesn't pass, skip all the tests in this TestGroup. Run the post
 433         script regardless.
 434         """
 435         pretest = Cmd(self.pre, outputdir=os.path.join(self.outputdir,
 436                       os.path.basename(self.pre)), timeout=self.timeout,
 437                       user=self.pre_user)
 438         posttest = Cmd(self.post, outputdir=os.path.join(self.outputdir,
 439                        os.path.basename(self.post)), timeout=self.timeout,
 440                        user=self.post_user)
 441
 442         cont = True
 443         if len(pretest.pathname):
 444             pretest.run(options)
 445             cont = pretest.result.result is 'PASS'
 446             pretest.log(logger, options)
 447
 448         for fname in self.tests:
 449             test = Cmd(os.path.join(self.pathname, fname),
 450                        outputdir=os.path.join(self.outputdir, fname),
 451                        timeout=self.timeout, user=self.user)
 452             if cont:
 453                 test.run(options)
 454             else:
 455                 test.skip()
 456
 457             test.log(logger, options)
 458
 459         if len(posttest.pathname):
 460             posttest.run(options)
 461             posttest.log(logger, options)
 462
 463
 464 class TestRun(object):
 465     props = ['quiet', 'outputdir']
 466
 467     def __init__(self, options):
 468         self.tests = {}
 469         self.testgroups = {}
 470         self.starttime = time()
 471         self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
 472         self.outputdir = os.path.join(options.outputdir, self.timestamp)
 473         self.logger = self.setup_logging(options)
 474         self.defaults = [
 475             ('outputdir', BASEDIR),
 476             ('quiet', False),
 477             ('timeout', 60),
 478             ('user', ''),
 479             ('pre', ''),
 480             ('pre_user', ''),
 481             ('post', ''),
 482             ('post_user', '')
 483         ]
 484
 485     def __str__(self):
 486         s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
 487         s += 'TESTS:\n'
 488         for key in sorted(self.tests.keys()):
 489             s += '%s%s' % (self.tests[key].__str__(), '\n')
 490         s += 'TESTGROUPS:\n'
 491         for key in sorted(self.testgroups.keys()):
 492             s += '%s%s' % (self.testgroups[key].__str__(), '\n')
 493         return s
 494
 495     def addtest(self, pathname, options):
 496         """
 497         Create a new Test, and apply any properties that were passed in
 498         from the command line. If it passes verification, add it to the
 499         TestRun.
 500         """
 501         test = Test(pathname)
 502         for prop in Test.props:
 503             setattr(test, prop, getattr(options, prop))
 504
 505         if test.verify(self.logger):
 506             self.tests[pathname] = test
 507
 508     def addtestgroup(self, dirname, filenames, options):
 509         """
 510         Create a new TestGroup, and apply any properties that were passed
 511         in from the command line. If it passes verification, add it to the
 512         TestRun.
 513         """
 514         if dirname not in self.testgroups:
 515             testgroup = TestGroup(dirname)
 516             for prop in Test.props:
 517                 setattr(testgroup, prop, getattr(options, prop))
 518
 519             # Prevent pre/post scripts from running as regular tests
 520             for f in [testgroup.pre, testgroup.post]:
 521                 if f in filenames:
 522                     del filenames[filenames.index(f)]
 523
 524             self.testgroups[dirname] = testgroup
 525             self.testgroups[dirname].tests = sorted(filenames)
 526
 527             testgroup.verify(self.logger)
 528
 529     def read(self, logger, options):
 530         """
 531         Read in the specified runfile, and apply the TestRun properties
 532         listed in the 'DEFAULT' section to our TestRun. Then read each
 533         section, and apply the appropriate properties to the Test or
 534         TestGroup. Properties from individual sections override those set
 535         in the 'DEFAULT' section. If the Test or TestGroup passes
 536         verification, add it to the TestRun.
 537         """
 538         config = ConfigParser.RawConfigParser()
 539         if not len(config.read(options.runfile)):
 540             fail("Coulnd't read config file %s" % options.runfile)
 541
 542         for opt in TestRun.props:
 543             if config.has_option('DEFAULT', opt):
 544                 setattr(self, opt, config.get('DEFAULT', opt))
 545         self.outputdir = os.path.join(self.outputdir, self.timestamp)
 546
 547         for section in config.sections():
 548             if 'tests' in config.options(section):
 549                 if os.path.isdir(section):
 550                     pathname = section
 551                 elif os.path.isdir(os.path.join(options.testdir, section)):
 552                     pathname = os.path.join(options.testdir, section)
 553                 else:
 554                     pathname = section
 555
 556                 testgroup = TestGroup(os.path.abspath(pathname))
 557                 for prop in TestGroup.props:
 558                     try:
 559                         setattr(testgroup, prop, config.get('DEFAULT', prop))
 560                         setattr(testgroup, prop, config.get(section, prop))
 561                     except ConfigParser.NoOptionError:
 562                         pass
 563
 564                 # Repopulate tests using eval to convert the string to a list
 565                 testgroup.tests = eval(config.get(section, 'tests'))
 566
 567                 if testgroup.verify(logger):
 568                     self.testgroups[section] = testgroup
 569             else:
 570                 test = Test(section)
 571                 for prop in Test.props:
 572                     try:
 573                         setattr(test, prop, config.get('DEFAULT', prop))
 574                         setattr(test, prop, config.get(section, prop))
 575                     except ConfigParser.NoOptionError:
 576                         pass
 577                 if test.verify(logger):
 578                     self.tests[section] = test
 579
 580     def write(self, options):
 581         """
 582         Create a configuration file for editing and later use. The
 583         'DEFAULT' section of the config file is created from the
 584         properties that were specified on the command line. Tests are
 585         simply added as sections that inherit everything from the
 586         'DEFAULT' section. TestGroups are the same, except they get an
 587         option including all the tests to run in that directory.
 588         """
 589
 590         defaults = dict([(prop, getattr(options, prop)) for prop, _ in
 591                         self.defaults])
 592         config = ConfigParser.RawConfigParser(defaults)
 593
 594         for test in sorted(self.tests.keys()):
 595             config.add_section(test)
 596
 597         for testgroup in sorted(self.testgroups.keys()):
 598             config.add_section(testgroup)
 599             config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
 600
 601         try:
 602             with open(options.template, 'w') as f:
 603                 return config.write(f)
 604         except IOError:
 605             fail('Could not open \'%s\' for writing.' % options.template)
 606
 607     def complete_outputdirs(self, options):
 608         """
 609         Collect all the pathnames for Tests, and TestGroups. Work
 610         backwards one pathname component at a time, to create a unique
 611         directory name in which to deposit test output. Tests will be able
 612         to write output files directly in the newly modified outputdir.
 613         TestGroups will be able to create one subdirectory per test in the
 614         outputdir, and are guaranteed uniqueness because a group can only
 615         contain files in one directory. Pre and post tests will create a
 616         directory rooted at the outputdir of the Test or TestGroup in
 617         question for their output.
 618         """
 619         done = False
 620         components = 0
 621         tmp_dict = dict(self.tests.items() + self.testgroups.items())
 622         total = len(tmp_dict)
 623         base = self.outputdir
 624
 625         while not done:
 626             l = []
 627             components -= 1
 628             for testfile in tmp_dict.keys():
 629                 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
 630                 if not uniq in l:
 631                     l.append(uniq)
 632                     tmp_dict[testfile].outputdir = os.path.join(base, uniq)
 633                 else:
 634                     break
 635             done = total == len(l)
 636
 637     def setup_logging(self, options):
 638         """
 639         Two loggers are set up here. The first is for the logfile which
 640         will contain one line summarizing the test, including the test
 641         name, result, and running time. This logger will also capture the
 642         timestamped combined stdout and stderr of each run. The second
 643         logger is optional console output, which will contain only the one
 644         line summary. The loggers are initialized at two different levels
 645         to facilitate segregating the output.
 646         """
 647         if options.dryrun is True:
 648             return
 649
 650         testlogger = logging.getLogger(__name__)
 651         testlogger.setLevel(logging.DEBUG)
 652
 653         if options.cmd is not 'wrconfig':
 654             try:
 655                 old = os.umask(0)
 656                 os.makedirs(self.outputdir, mode=0777)
 657                 os.umask(old)
 658             except OSError, e:
 659                 fail('%s' % e)
 660             filename = os.path.join(self.outputdir, 'log')
 661
 662             logfile = logging.FileHandler(filename)
 663             logfile.setLevel(logging.DEBUG)
 664             logfilefmt = logging.Formatter('%(message)s')
 665             logfile.setFormatter(logfilefmt)
 666             testlogger.addHandler(logfile)
 667
 668         cons = logging.StreamHandler()
 669         cons.setLevel(logging.INFO)
 670         consfmt = logging.Formatter('%(message)s')
 671         cons.setFormatter(consfmt)
 672         testlogger.addHandler(cons)
 673
 674         return testlogger
 675
 676     def run(self, options):
 677         """
 678         Walk through all the Tests and TestGroups, calling run().
 679         """
 680         try:
 681             os.chdir(self.outputdir)
 682         except OSError:
 683             fail('Could not change to directory %s' % self.outputdir)
 684         for test in sorted(self.tests.keys()):
 685             self.tests[test].run(self.logger, options)
 686         for testgroup in sorted(self.testgroups.keys()):
 687             self.testgroups[testgroup].run(self.logger, options)
 688
 689     def summary(self):
 690         if Result.total is 0:
 691             return
 692
 693         print '\nResults Summary'
 694         for key in Result.runresults.keys():
 695             if Result.runresults[key] is not 0:
 696                 print '%s\t% 4d' % (key, Result.runresults[key])
 697
 698         m, s = divmod(time() - self.starttime, 60)
 699         h, m = divmod(m, 60)
 700         print '\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)
 701         print 'Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
 702                float(Result.total)) * 100)
 703         print 'Log directory:\t%s' % self.outputdir
 704
 705
 706 def verify_file(pathname):
 707     """
 708     Verify that the supplied pathname is an executable regular file.
 709     """
 710     if os.path.isdir(pathname) or os.path.islink(pathname):
 711         return False
 712
 713     if (os.path.isfile(pathname) and os.access(pathname, os.X_OK)) or \
 714             (os.path.isfile(pathname+'.ksh') and os.access(pathname+'.ksh', os.X_OK)) or \
 715             (os.path.isfile(pathname+'.sh') and os.access(pathname+'.sh', os.X_OK)):
 716         return True
 717
 718     return False
 719
 720
 721 def verify_user(user, logger):
 722     """
 723     Verify that the specified user exists on this system, and can execute
 724     sudo without being prompted for a password.
 725     """
 726     testcmd = [SUDO, '-n', '-u', user, TRUE]
 727     can_sudo = exists = True
 728
 729     if user in Cmd.verified_users:
 730         return True
 731
 732     try:
 733         _ = getpwnam(user)
 734     except KeyError:
 735         exists = False
 736         logger.info("Warning: user '%s' does not exist.", user)
 737         return False
 738
 739     p = Popen(testcmd)
 740     p.wait()
 741     if p.returncode is not 0:
 742         logger.info("Warning: user '%s' cannot use passwordless sudo.", user)
 743         return False
 744     else:
 745         Cmd.verified_users.append(user)
 746
 747     return True
 748
 749
 750 def find_tests(testrun, options):
 751     """
 752     For the given list of pathnames, add files as Tests. For directories,
 753     if do_groups is True, add the directory as a TestGroup. If False,
 754     recursively search for executable files.
 755     """
 756
 757     for p in sorted(options.pathnames):
 758         if os.path.isdir(p):
 759             for dirname, _, filenames in os.walk(p):
 760                 if options.do_groups:
 761                     testrun.addtestgroup(dirname, filenames, options)
 762                 else:
 763                     for f in sorted(filenames):
 764                         testrun.addtest(os.path.join(dirname, f), options)
 765         else:
 766             testrun.addtest(p, options)
 767
 768
 769 def fail(retstr, ret=1):
 770     print '%s: %s' % (argv[0], retstr)
 771     exit(ret)
 772
 773
 774 def options_cb(option, opt_str, value, parser):
 775     path_options = ['runfile', 'outputdir', 'template', 'testdir']
 776
 777     if option.dest is 'runfile' and '-w' in parser.rargs or \
 778         option.dest is 'template' and '-c' in parser.rargs:
 779         fail('-c and -w are mutually exclusive.')
 780
 781     if opt_str in parser.rargs:
 782         fail('%s may only be specified once.' % opt_str)
 783
 784     if option.dest is 'runfile':
 785         parser.values.cmd = 'rdconfig'
 786     if option.dest is 'template':
 787         parser.values.cmd = 'wrconfig'
 788
 789     setattr(parser.values, option.dest, value)
 790     if option.dest in path_options:
 791         setattr(parser.values, option.dest, os.path.abspath(value))
 792
 793
 794 def parse_args():
 795     parser = OptionParser()
 796     parser.add_option('-c', action='callback', callback=options_cb,
 797                       type='string', dest='runfile', metavar='runfile',
 798                       help='Specify tests to run via config file.')
 799     parser.add_option('-d', action='store_true', default=False, dest='dryrun',
 800                       help='Dry run. Print tests, but take no other action.')
 801     parser.add_option('-g', action='store_true', default=False,
 802                       dest='do_groups', help='Make directories TestGroups.')
 803     parser.add_option('-o', action='callback', callback=options_cb,
 804                       default=BASEDIR, dest='outputdir', type='string',
 805                       metavar='outputdir', help='Specify an output directory.')
 806     parser.add_option('-i', action='callback', callback=options_cb,
 807                       default=TESTDIR, dest='testdir', type='string',
 808                       metavar='testdir', help='Specify a test directory.')
 809     parser.add_option('-p', action='callback', callback=options_cb,
 810                       default='', dest='pre', metavar='script',
 811                       type='string', help='Specify a pre script.')
 812     parser.add_option('-P', action='callback', callback=options_cb,
 813                       default='', dest='post', metavar='script',
 814                       type='string', help='Specify a post script.')
 815     parser.add_option('-q', action='store_true', default=False, dest='quiet',
 816                       help='Silence on the console during a test run.')
 817     parser.add_option('-t', action='callback', callback=options_cb, default=60,
 818                       dest='timeout', metavar='seconds', type='int',
 819                       help='Timeout (in seconds) for an individual test.')
 820     parser.add_option('-u', action='callback', callback=options_cb,
 821                       default='', dest='user', metavar='user', type='string',
 822                       help='Specify a different user name to run as.')
 823     parser.add_option('-w', action='callback', callback=options_cb,
 824                       default=None, dest='template', metavar='template',
 825                       type='string', help='Create a new config file.')
 826     parser.add_option('-x', action='callback', callback=options_cb, default='',
 827                       dest='pre_user', metavar='pre_user', type='string',
 828                       help='Specify a user to execute the pre script.')
 829     parser.add_option('-X', action='callback', callback=options_cb, default='',
 830                       dest='post_user', metavar='post_user', type='string',
 831                       help='Specify a user to execute the post script.')
 832     (options, pathnames) = parser.parse_args()
 833
 834     if not options.runfile and not options.template:
 835         options.cmd = 'runtests'
 836
 837     if options.runfile and len(pathnames):
 838         fail('Extraneous arguments.')
 839
 840     options.pathnames = [os.path.abspath(path) for path in pathnames]
 841
 842     return options
 843
 844
 845 def main(args):
 846     options = parse_args()
 847     testrun = TestRun(options)
 848
 849     if options.cmd is 'runtests':
 850         find_tests(testrun, options)
 851     elif options.cmd is 'rdconfig':
 852         testrun.read(testrun.logger, options)
 853     elif options.cmd is 'wrconfig':
 854         find_tests(testrun, options)
 855         testrun.write(options)
 856         exit(0)
 857     else:
 858         fail('Unknown command specified')
 859
 860     testrun.complete_outputdirs(options)
 861     testrun.run(options)
 862     testrun.summary()
 863     exit(0)
 864
 865
 866 if __name__ == '__main__':
 867     main(argv[1:])