tests/test-runner/bin/test-runner.py

   1 #!/usr/bin/python
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  16 # Copyright (c) 2017 Datto Inc.
  17 #
  18 # This script must remain compatible with Python 2.6+ and Python 3.4+.
  19 #
  20
  21 # some python 2.7 system don't have a configparser shim
  22 try:
  23     import configparser
  24 except ImportError:
  25     import ConfigParser as configparser
  26
  27 import os
  28 import logging
  29 from datetime import datetime
  30 from optparse import OptionParser
  31 from pwd import getpwnam
  32 from pwd import getpwuid
  33 from select import select
  34 from subprocess import PIPE
  35 from subprocess import Popen
  36 from sys import argv
  37 from sys import maxsize
  38 from threading import Timer
  39 from time import time
  40
  41 BASEDIR = '/var/tmp/test_results'
  42 TESTDIR = '/usr/share/zfs/'
  43 KILL = 'kill'
  44 TRUE = 'true'
  45 SUDO = 'sudo'
  46
  47
  48 class Result(object):
  49     total = 0
  50     runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0}
  51
  52     def __init__(self):
  53         self.starttime = None
  54         self.returncode = None
  55         self.runtime = ''
  56         self.stdout = []
  57         self.stderr = []
  58         self.result = ''
  59
  60     def done(self, proc, killed):
  61         """
  62         Finalize the results of this Cmd.
  63         """
  64         Result.total += 1
  65         m, s = divmod(time() - self.starttime, 60)
  66         self.runtime = '%02d:%02d' % (m, s)
  67         self.returncode = proc.returncode
  68         if killed:
  69             self.result = 'KILLED'
  70             Result.runresults['KILLED'] += 1
  71         elif self.returncode is 0:
  72             self.result = 'PASS'
  73             Result.runresults['PASS'] += 1
  74         elif self.returncode is 4:
  75             self.result = 'SKIP'
  76             Result.runresults['SKIP'] += 1
  77         elif self.returncode is not 0:
  78             self.result = 'FAIL'
  79             Result.runresults['FAIL'] += 1
  80
  81
  82 class Output(object):
  83     """
  84     This class is a slightly modified version of the 'Stream' class found
  85     here: http://goo.gl/aSGfv
  86     """
  87     def __init__(self, stream):
  88         self.stream = stream
  89         self._buf = ''
  90         self.lines = []
  91
  92     def fileno(self):
  93         return self.stream.fileno()
  94
  95     def read(self, drain=0):
  96         """
  97         Read from the file descriptor. If 'drain' set, read until EOF.
  98         """
  99         while self._read() is not None:
 100             if not drain:
 101                 break
 102
 103     def _read(self):
 104         """
 105         Read up to 4k of data from this output stream. Collect the output
 106         up to the last newline, and append it to any leftover data from a
 107         previous call. The lines are stored as a (timestamp, data) tuple
 108         for easy sorting/merging later.
 109         """
 110         fd = self.fileno()
 111         buf = os.read(fd, 4096)
 112         if not buf:
 113             return None
 114         if '\n' not in buf:
 115             self._buf += buf
 116             return []
 117
 118         buf = self._buf + buf
 119         tmp, rest = buf.rsplit('\n', 1)
 120         self._buf = rest
 121         now = datetime.now()
 122         rows = tmp.split('\n')
 123         self.lines += [(now, r) for r in rows]
 124
 125
 126 class Cmd(object):
 127     verified_users = []
 128
 129     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 130                  tags=None):
 131         self.pathname = pathname
 132         self.outputdir = outputdir or 'BASEDIR'
 133         self.timeout = timeout
 134         self.user = user or ''
 135         self.killed = False
 136         self.result = Result()
 137
 138         if self.timeout is None:
 139             self.timeout = 60
 140
 141     def __str__(self):
 142         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % \
 143             (self.pathname, self.outputdir, self.timeout, self.user)
 144
 145     def kill_cmd(self, proc):
 146         """
 147         Kill a running command due to timeout, or ^C from the keyboard. If
 148         sudo is required, this user was verified previously.
 149         """
 150         self.killed = True
 151         do_sudo = len(self.user) != 0
 152         signal = '-TERM'
 153
 154         cmd = [SUDO, KILL, signal, str(proc.pid)]
 155         if not do_sudo:
 156             del cmd[0]
 157
 158         try:
 159             kp = Popen(cmd)
 160             kp.wait()
 161         except Exception:
 162             pass
 163
 164     def update_cmd_privs(self, cmd, user):
 165         """
 166         If a user has been specified to run this Cmd and we're not already
 167         running as that user, prepend the appropriate sudo command to run
 168         as that user.
 169         """
 170         me = getpwuid(os.getuid())
 171
 172         if not user or user is me:
 173             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 174                 cmd += '.ksh'
 175             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 176                 cmd += '.sh'
 177             return cmd
 178
 179         if not os.path.isfile(cmd):
 180             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 181                 cmd += '.ksh'
 182             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 183                 cmd += '.sh'
 184
 185         ret = '%s -E -u %s %s' % (SUDO, user, cmd)
 186         return ret.split(' ')
 187
 188     def collect_output(self, proc):
 189         """
 190         Read from stdout/stderr as data becomes available, until the
 191         process is no longer running. Return the lines from the stdout and
 192         stderr Output objects.
 193         """
 194         out = Output(proc.stdout)
 195         err = Output(proc.stderr)
 196         res = []
 197         while proc.returncode is None:
 198             proc.poll()
 199             res = select([out, err], [], [], .1)
 200             for fd in res[0]:
 201                 fd.read()
 202         for fd in res[0]:
 203             fd.read(drain=1)
 204
 205         return out.lines, err.lines
 206
 207     def run(self, options):
 208         """
 209         This is the main function that runs each individual test.
 210         Determine whether or not the command requires sudo, and modify it
 211         if needed. Run the command, and update the result object.
 212         """
 213         if options.dryrun is True:
 214             print(self)
 215             return
 216
 217         privcmd = self.update_cmd_privs(self.pathname, self.user)
 218         try:
 219             old = os.umask(0)
 220             if not os.path.isdir(self.outputdir):
 221                 os.makedirs(self.outputdir, mode=0o777)
 222             os.umask(old)
 223         except OSError as e:
 224             fail('%s' % e)
 225
 226         self.result.starttime = time()
 227         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
 228         # Allow a special timeout value of 0 to mean infinity
 229         if int(self.timeout) == 0:
 230             self.timeout = maxsize
 231         t = Timer(int(self.timeout), self.kill_cmd, [proc])
 232
 233         try:
 234             t.start()
 235             self.result.stdout, self.result.stderr = self.collect_output(proc)
 236         except KeyboardInterrupt:
 237             self.kill_cmd(proc)
 238             fail('\nRun terminated at user request.')
 239         finally:
 240             t.cancel()
 241
 242         self.result.done(proc, self.killed)
 243
 244     def skip(self):
 245         """
 246         Initialize enough of the test result that we can log a skipped
 247         command.
 248         """
 249         Result.total += 1
 250         Result.runresults['SKIP'] += 1
 251         self.result.stdout = self.result.stderr = []
 252         self.result.starttime = time()
 253         m, s = divmod(time() - self.result.starttime, 60)
 254         self.result.runtime = '%02d:%02d' % (m, s)
 255         self.result.result = 'SKIP'
 256
 257     def log(self, logger, options):
 258         """
 259         This function is responsible for writing all output. This includes
 260         the console output, the logfile of all results (with timestamped
 261         merged stdout and stderr), and for each test, the unmodified
 262         stdout/stderr/merged in it's own file.
 263         """
 264         if logger is None:
 265             return
 266
 267         logname = getpwuid(os.getuid()).pw_name
 268         user = ' (run as %s)' % (self.user if len(self.user) else logname)
 269         msga = 'Test: %s%s ' % (self.pathname, user)
 270         msgb = '[%s] [%s]' % (self.result.runtime, self.result.result)
 271         pad = ' ' * (80 - (len(msga) + len(msgb)))
 272
 273         # If -q is specified, only print a line for tests that didn't pass.
 274         # This means passing tests need to be logged as DEBUG, or the one
 275         # line summary will only be printed in the logfile for failures.
 276         if not options.quiet:
 277             logger.info('%s%s%s' % (msga, pad, msgb))
 278         elif self.result.result is not 'PASS':
 279             logger.info('%s%s%s' % (msga, pad, msgb))
 280         else:
 281             logger.debug('%s%s%s' % (msga, pad, msgb))
 282
 283         lines = sorted(self.result.stdout + self.result.stderr,
 284                        key=lambda x: x[0])
 285
 286         for dt, line in lines:
 287             logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line))
 288
 289         if len(self.result.stdout):
 290             with open(os.path.join(self.outputdir, 'stdout'), 'w') as out:
 291                 for _, line in self.result.stdout:
 292                     os.write(out.fileno(), '%s\n' % line)
 293         if len(self.result.stderr):
 294             with open(os.path.join(self.outputdir, 'stderr'), 'w') as err:
 295                 for _, line in self.result.stderr:
 296                     os.write(err.fileno(), '%s\n' % line)
 297         if len(self.result.stdout) and len(self.result.stderr):
 298             with open(os.path.join(self.outputdir, 'merged'), 'w') as merged:
 299                 for _, line in lines:
 300                     os.write(merged.fileno(), '%s\n' % line)
 301
 302
 303 class Test(Cmd):
 304     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
 305              'post_user', 'tags']
 306
 307     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 308                  pre=None, pre_user=None, post=None, post_user=None,
 309                  tags=None):
 310         super(Test, self).__init__(pathname, outputdir, timeout, user)
 311         self.pre = pre or ''
 312         self.pre_user = pre_user or ''
 313         self.post = post or ''
 314         self.post_user = post_user or ''
 315         self.tags = tags or []
 316
 317     def __str__(self):
 318         post_user = pre_user = ''
 319         if len(self.pre_user):
 320             pre_user = ' (as %s)' % (self.pre_user)
 321         if len(self.post_user):
 322             post_user = ' (as %s)' % (self.post_user)
 323         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \
 324                "%s%s\nUser: %s\nTags: %s\n" % \
 325                (self.pathname, self.outputdir, self.timeout, self.pre,
 326                 pre_user, self.post, post_user, self.user, self.tags)
 327
 328     def verify(self, logger):
 329         """
 330         Check the pre/post scripts, user and Test. Omit the Test from this
 331         run if there are any problems.
 332         """
 333         files = [self.pre, self.pathname, self.post]
 334         users = [self.pre_user, self.user, self.post_user]
 335
 336         for f in [f for f in files if len(f)]:
 337             if not verify_file(f):
 338                 logger.info("Warning: Test '%s' not added to this run because"
 339                             " it failed verification." % f)
 340                 return False
 341
 342         for user in [user for user in users if len(user)]:
 343             if not verify_user(user, logger):
 344                 logger.info("Not adding Test '%s' to this run." %
 345                             self.pathname)
 346                 return False
 347
 348         return True
 349
 350     def run(self, logger, options):
 351         """
 352         Create Cmd instances for the pre/post scripts. If the pre script
 353         doesn't pass, skip this Test. Run the post script regardless.
 354         """
 355         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 356         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 357                       user=self.pre_user)
 358         test = Cmd(self.pathname, outputdir=self.outputdir,
 359                    timeout=self.timeout, user=self.user)
 360         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 361         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 362                        user=self.post_user)
 363
 364         cont = True
 365         if len(pretest.pathname):
 366             pretest.run(options)
 367             cont = pretest.result.result is 'PASS'
 368             pretest.log(logger, options)
 369
 370         if cont:
 371             test.run(options)
 372         else:
 373             test.skip()
 374
 375         test.log(logger, options)
 376
 377         if len(posttest.pathname):
 378             posttest.run(options)
 379             posttest.log(logger, options)
 380
 381
 382 class TestGroup(Test):
 383     props = Test.props + ['tests']
 384
 385     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 386                  pre=None, pre_user=None, post=None, post_user=None,
 387                  tests=None, tags=None):
 388         super(TestGroup, self).__init__(pathname, outputdir, timeout, user,
 389                                         pre, pre_user, post, post_user, tags)
 390         self.tests = tests or []
 391
 392     def __str__(self):
 393         post_user = pre_user = ''
 394         if len(self.pre_user):
 395             pre_user = ' (as %s)' % (self.pre_user)
 396         if len(self.post_user):
 397             post_user = ' (as %s)' % (self.post_user)
 398         return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %s\n" \
 399                "Pre: %s%s\nPost: %s%s\nUser: %s\nTags: %s\n" % \
 400                (self.pathname, self.outputdir, self.tests, self.timeout,
 401                 self.pre, pre_user, self.post, post_user, self.user, self.tags)
 402
 403     def verify(self, logger):
 404         """
 405         Check the pre/post scripts, user and tests in this TestGroup. Omit
 406         the TestGroup entirely, or simply delete the relevant tests in the
 407         group, if that's all that's required.
 408         """
 409         # If the pre or post scripts are relative pathnames, convert to
 410         # absolute, so they stand a chance of passing verification.
 411         if len(self.pre) and not os.path.isabs(self.pre):
 412             self.pre = os.path.join(self.pathname, self.pre)
 413         if len(self.post) and not os.path.isabs(self.post):
 414             self.post = os.path.join(self.pathname, self.post)
 415
 416         auxfiles = [self.pre, self.post]
 417         users = [self.pre_user, self.user, self.post_user]
 418
 419         for f in [f for f in auxfiles if len(f)]:
 420             if self.pathname != os.path.dirname(f):
 421                 logger.info("Warning: TestGroup '%s' not added to this run. "
 422                             "Auxiliary script '%s' exists in a different "
 423                             "directory." % (self.pathname, f))
 424                 return False
 425
 426             if not verify_file(f):
 427                 logger.info("Warning: TestGroup '%s' not added to this run. "
 428                             "Auxiliary script '%s' failed verification." %
 429                             (self.pathname, f))
 430                 return False
 431
 432         for user in [user for user in users if len(user)]:
 433             if not verify_user(user, logger):
 434                 logger.info("Not adding TestGroup '%s' to this run." %
 435                             self.pathname)
 436                 return False
 437
 438         # If one of the tests is invalid, delete it, log it, and drive on.
 439         for test in self.tests:
 440             if not verify_file(os.path.join(self.pathname, test)):
 441                 del self.tests[self.tests.index(test)]
 442                 logger.info("Warning: Test '%s' removed from TestGroup '%s' "
 443                             "because it failed verification." %
 444                             (test, self.pathname))
 445
 446         return len(self.tests) is not 0
 447
 448     def run(self, logger, options):
 449         """
 450         Create Cmd instances for the pre/post scripts. If the pre script
 451         doesn't pass, skip all the tests in this TestGroup. Run the post
 452         script regardless.
 453         """
 454         # tags assigned to this test group also include the test names
 455         if options.tags and not set(self.tags).intersection(set(options.tags)):
 456             return
 457
 458         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 459         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 460                       user=self.pre_user)
 461         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 462         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 463                        user=self.post_user)
 464
 465         cont = True
 466         if len(pretest.pathname):
 467             pretest.run(options)
 468             cont = pretest.result.result is 'PASS'
 469             pretest.log(logger, options)
 470
 471         for fname in self.tests:
 472             test = Cmd(os.path.join(self.pathname, fname),
 473                        outputdir=os.path.join(self.outputdir, fname),
 474                        timeout=self.timeout, user=self.user)
 475             if cont:
 476                 test.run(options)
 477             else:
 478                 test.skip()
 479
 480             test.log(logger, options)
 481
 482         if len(posttest.pathname):
 483             posttest.run(options)
 484             posttest.log(logger, options)
 485
 486
 487 class TestRun(object):
 488     props = ['quiet', 'outputdir']
 489
 490     def __init__(self, options):
 491         self.tests = {}
 492         self.testgroups = {}
 493         self.starttime = time()
 494         self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
 495         self.outputdir = os.path.join(options.outputdir, self.timestamp)
 496         self.logger = self.setup_logging(options)
 497         self.defaults = [
 498             ('outputdir', BASEDIR),
 499             ('quiet', False),
 500             ('timeout', 60),
 501             ('user', ''),
 502             ('pre', ''),
 503             ('pre_user', ''),
 504             ('post', ''),
 505             ('post_user', ''),
 506             ('tags', [])
 507         ]
 508
 509     def __str__(self):
 510         s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
 511         s += 'TESTS:\n'
 512         for key in sorted(self.tests.keys()):
 513             s += '%s%s' % (self.tests[key].__str__(), '\n')
 514         s += 'TESTGROUPS:\n'
 515         for key in sorted(self.testgroups.keys()):
 516             s += '%s%s' % (self.testgroups[key].__str__(), '\n')
 517         return s
 518
 519     def addtest(self, pathname, options):
 520         """
 521         Create a new Test, and apply any properties that were passed in
 522         from the command line. If it passes verification, add it to the
 523         TestRun.
 524         """
 525         test = Test(pathname)
 526         for prop in Test.props:
 527             setattr(test, prop, getattr(options, prop))
 528
 529         if test.verify(self.logger):
 530             self.tests[pathname] = test
 531
 532     def addtestgroup(self, dirname, filenames, options):
 533         """
 534         Create a new TestGroup, and apply any properties that were passed
 535         in from the command line. If it passes verification, add it to the
 536         TestRun.
 537         """
 538         if dirname not in self.testgroups:
 539             testgroup = TestGroup(dirname)
 540             for prop in Test.props:
 541                 setattr(testgroup, prop, getattr(options, prop))
 542
 543             # Prevent pre/post scripts from running as regular tests
 544             for f in [testgroup.pre, testgroup.post]:
 545                 if f in filenames:
 546                     del filenames[filenames.index(f)]
 547
 548             self.testgroups[dirname] = testgroup
 549             self.testgroups[dirname].tests = sorted(filenames)
 550
 551             testgroup.verify(self.logger)
 552
 553     def read(self, logger, options):
 554         """
 555         Read in the specified runfile, and apply the TestRun properties
 556         listed in the 'DEFAULT' section to our TestRun. Then read each
 557         section, and apply the appropriate properties to the Test or
 558         TestGroup. Properties from individual sections override those set
 559         in the 'DEFAULT' section. If the Test or TestGroup passes
 560         verification, add it to the TestRun.
 561         """
 562         config = configparser.RawConfigParser()
 563         if not len(config.read(options.runfile)):
 564             fail("Coulnd't read config file %s" % options.runfile)
 565
 566         for opt in TestRun.props:
 567             if config.has_option('DEFAULT', opt):
 568                 setattr(self, opt, config.get('DEFAULT', opt))
 569         self.outputdir = os.path.join(self.outputdir, self.timestamp)
 570
 571         for section in config.sections():
 572             if 'tests' in config.options(section):
 573                 if os.path.isdir(section):
 574                     pathname = section
 575                 elif os.path.isdir(os.path.join(options.testdir, section)):
 576                     pathname = os.path.join(options.testdir, section)
 577                 else:
 578                     pathname = section
 579
 580                 testgroup = TestGroup(os.path.abspath(pathname))
 581                 for prop in TestGroup.props:
 582                     for sect in ['DEFAULT', section]:
 583                         if config.has_option(sect, prop):
 584                             if prop is "tags":
 585                                 setattr(testgroup, prop,
 586                                         eval(config.get(sect, prop)))
 587                             else:
 588                                 setattr(testgroup, prop,
 589                                         config.get(sect, prop))
 590
 591                 # Repopulate tests using eval to convert the string to a list
 592                 testgroup.tests = eval(config.get(section, 'tests'))
 593
 594                 if testgroup.verify(logger):
 595                     self.testgroups[section] = testgroup
 596             else:
 597                 test = Test(section)
 598                 for prop in Test.props:
 599                     for sect in ['DEFAULT', section]:
 600                         if config.has_option(sect, prop):
 601                             setattr(test, prop, config.get(sect, prop))
 602
 603                 if test.verify(logger):
 604                     self.tests[section] = test
 605
 606     def write(self, options):
 607         """
 608         Create a configuration file for editing and later use. The
 609         'DEFAULT' section of the config file is created from the
 610         properties that were specified on the command line. Tests are
 611         simply added as sections that inherit everything from the
 612         'DEFAULT' section. TestGroups are the same, except they get an
 613         option including all the tests to run in that directory.
 614         """
 615
 616         defaults = dict([(prop, getattr(options, prop)) for prop, _ in
 617                          self.defaults])
 618         config = configparser.RawConfigParser(defaults)
 619
 620         for test in sorted(self.tests.keys()):
 621             config.add_section(test)
 622
 623         for testgroup in sorted(self.testgroups.keys()):
 624             config.add_section(testgroup)
 625             config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
 626
 627         try:
 628             with open(options.template, 'w') as f:
 629                 return config.write(f)
 630         except IOError:
 631             fail('Could not open \'%s\' for writing.' % options.template)
 632
 633     def complete_outputdirs(self):
 634         """
 635         Collect all the pathnames for Tests, and TestGroups. Work
 636         backwards one pathname component at a time, to create a unique
 637         directory name in which to deposit test output. Tests will be able
 638         to write output files directly in the newly modified outputdir.
 639         TestGroups will be able to create one subdirectory per test in the
 640         outputdir, and are guaranteed uniqueness because a group can only
 641         contain files in one directory. Pre and post tests will create a
 642         directory rooted at the outputdir of the Test or TestGroup in
 643         question for their output.
 644         """
 645         done = False
 646         components = 0
 647         tmp_dict = dict(list(self.tests.items()) +
 648                         list(self.testgroups.items()))
 649         total = len(tmp_dict)
 650         base = self.outputdir
 651
 652         while not done:
 653             paths = []
 654             components -= 1
 655             for testfile in list(tmp_dict.keys()):
 656                 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
 657                 if uniq not in paths:
 658                     paths.append(uniq)
 659                     tmp_dict[testfile].outputdir = os.path.join(base, uniq)
 660                 else:
 661                     break
 662             done = total == len(paths)
 663
 664     def setup_logging(self, options):
 665         """
 666         Two loggers are set up here. The first is for the logfile which
 667         will contain one line summarizing the test, including the test
 668         name, result, and running time. This logger will also capture the
 669         timestamped combined stdout and stderr of each run. The second
 670         logger is optional console output, which will contain only the one
 671         line summary. The loggers are initialized at two different levels
 672         to facilitate segregating the output.
 673         """
 674         if options.dryrun is True:
 675             return
 676
 677         testlogger = logging.getLogger(__name__)
 678         testlogger.setLevel(logging.DEBUG)
 679
 680         if options.cmd is not 'wrconfig':
 681             try:
 682                 old = os.umask(0)
 683                 os.makedirs(self.outputdir, mode=0o777)
 684                 os.umask(old)
 685             except OSError as e:
 686                 fail('%s' % e)
 687             filename = os.path.join(self.outputdir, 'log')
 688
 689             logfile = logging.FileHandler(filename)
 690             logfile.setLevel(logging.DEBUG)
 691             logfilefmt = logging.Formatter('%(message)s')
 692             logfile.setFormatter(logfilefmt)
 693             testlogger.addHandler(logfile)
 694
 695         cons = logging.StreamHandler()
 696         cons.setLevel(logging.INFO)
 697         consfmt = logging.Formatter('%(message)s')
 698         cons.setFormatter(consfmt)
 699         testlogger.addHandler(cons)
 700
 701         return testlogger
 702
 703     def run(self, options):
 704         """
 705         Walk through all the Tests and TestGroups, calling run().
 706         """
 707         try:
 708             os.chdir(self.outputdir)
 709         except OSError:
 710             fail('Could not change to directory %s' % self.outputdir)
 711         # make a symlink to the output for the currently running test
 712         logsymlink = os.path.join(self.outputdir, '../current')
 713         if os.path.islink(logsymlink):
 714             os.unlink(logsymlink)
 715         if not os.path.exists(logsymlink):
 716             os.symlink(self.outputdir, logsymlink)
 717         else:
 718             print('Could not make a symlink to directory %s' % (
 719                 self.outputdir))
 720         iteration = 0
 721         while iteration < options.iterations:
 722             for test in sorted(self.tests.keys()):
 723                 self.tests[test].run(self.logger, options)
 724             for testgroup in sorted(self.testgroups.keys()):
 725                 self.testgroups[testgroup].run(self.logger, options)
 726             iteration += 1
 727
 728     def summary(self):
 729         if Result.total is 0:
 730             return 2
 731
 732         print('\nResults Summary')
 733         for key in list(Result.runresults.keys()):
 734             if Result.runresults[key] is not 0:
 735                 print('%s\t% 4d' % (key, Result.runresults[key]))
 736
 737         m, s = divmod(time() - self.starttime, 60)
 738         h, m = divmod(m, 60)
 739         print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
 740         print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
 741                                             float(Result.total)) * 100))
 742         print('Log directory:\t%s' % self.outputdir)
 743
 744         if Result.runresults['FAIL'] > 0:
 745             return 1
 746
 747         if Result.runresults['KILLED'] > 0:
 748             return 1
 749
 750         return 0
 751
 752
 753 def verify_file(pathname):
 754     """
 755     Verify that the supplied pathname is an executable regular file.
 756     """
 757     if os.path.isdir(pathname) or os.path.islink(pathname):
 758         return False
 759
 760     for ext in '', '.ksh', '.sh':
 761         script_path = pathname + ext
 762         if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
 763             return True
 764
 765     return False
 766
 767
 768 def verify_user(user, logger):
 769     """
 770     Verify that the specified user exists on this system, and can execute
 771     sudo without being prompted for a password.
 772     """
 773     testcmd = [SUDO, '-n', '-u', user, TRUE]
 774
 775     if user in Cmd.verified_users:
 776         return True
 777
 778     try:
 779         getpwnam(user)
 780     except KeyError:
 781         logger.info("Warning: user '%s' does not exist.", user)
 782         return False
 783
 784     p = Popen(testcmd)
 785     p.wait()
 786     if p.returncode is not 0:
 787         logger.info("Warning: user '%s' cannot use passwordless sudo.", user)
 788         return False
 789     else:
 790         Cmd.verified_users.append(user)
 791
 792     return True
 793
 794
 795 def find_tests(testrun, options):
 796     """
 797     For the given list of pathnames, add files as Tests. For directories,
 798     if do_groups is True, add the directory as a TestGroup. If False,
 799     recursively search for executable files.
 800     """
 801
 802     for p in sorted(options.pathnames):
 803         if os.path.isdir(p):
 804             for dirname, _, filenames in os.walk(p):
 805                 if options.do_groups:
 806                     testrun.addtestgroup(dirname, filenames, options)
 807                 else:
 808                     for f in sorted(filenames):
 809                         testrun.addtest(os.path.join(dirname, f), options)
 810         else:
 811             testrun.addtest(p, options)
 812
 813
 814 def fail(retstr, ret=1):
 815     print('%s: %s' % (argv[0], retstr))
 816     exit(ret)
 817
 818
 819 def options_cb(option, opt_str, value, parser):
 820     path_options = ['runfile', 'outputdir', 'template', 'testdir']
 821
 822     if option.dest is 'runfile' and '-w' in parser.rargs or \
 823             option.dest is 'template' and '-c' in parser.rargs:
 824         fail('-c and -w are mutually exclusive.')
 825
 826     if opt_str in parser.rargs:
 827         fail('%s may only be specified once.' % opt_str)
 828
 829     if option.dest is 'runfile':
 830         parser.values.cmd = 'rdconfig'
 831     if option.dest is 'template':
 832         parser.values.cmd = 'wrconfig'
 833     if option.dest is 'tags':
 834         value = [x.strip() for x in value.split(',')]
 835
 836     setattr(parser.values, option.dest, value)
 837     if option.dest in path_options:
 838         setattr(parser.values, option.dest, os.path.abspath(value))
 839
 840
 841 def parse_args():
 842     parser = OptionParser()
 843     parser.add_option('-c', action='callback', callback=options_cb,
 844                       type='string', dest='runfile', metavar='runfile',
 845                       help='Specify tests to run via config file.')
 846     parser.add_option('-d', action='store_true', default=False, dest='dryrun',
 847                       help='Dry run. Print tests, but take no other action.')
 848     parser.add_option('-g', action='store_true', default=False,
 849                       dest='do_groups', help='Make directories TestGroups.')
 850     parser.add_option('-o', action='callback', callback=options_cb,
 851                       default=BASEDIR, dest='outputdir', type='string',
 852                       metavar='outputdir', help='Specify an output directory.')
 853     parser.add_option('-i', action='callback', callback=options_cb,
 854                       default=TESTDIR, dest='testdir', type='string',
 855                       metavar='testdir', help='Specify a test directory.')
 856     parser.add_option('-p', action='callback', callback=options_cb,
 857                       default='', dest='pre', metavar='script',
 858                       type='string', help='Specify a pre script.')
 859     parser.add_option('-P', action='callback', callback=options_cb,
 860                       default='', dest='post', metavar='script',
 861                       type='string', help='Specify a post script.')
 862     parser.add_option('-q', action='store_true', default=False, dest='quiet',
 863                       help='Silence on the console during a test run.')
 864     parser.add_option('-t', action='callback', callback=options_cb, default=60,
 865                       dest='timeout', metavar='seconds', type='int',
 866                       help='Timeout (in seconds) for an individual test.')
 867     parser.add_option('-u', action='callback', callback=options_cb,
 868                       default='', dest='user', metavar='user', type='string',
 869                       help='Specify a different user name to run as.')
 870     parser.add_option('-w', action='callback', callback=options_cb,
 871                       default=None, dest='template', metavar='template',
 872                       type='string', help='Create a new config file.')
 873     parser.add_option('-x', action='callback', callback=options_cb, default='',
 874                       dest='pre_user', metavar='pre_user', type='string',
 875                       help='Specify a user to execute the pre script.')
 876     parser.add_option('-X', action='callback', callback=options_cb, default='',
 877                       dest='post_user', metavar='post_user', type='string',
 878                       help='Specify a user to execute the post script.')
 879     parser.add_option('-T', action='callback', callback=options_cb, default='',
 880                       dest='tags', metavar='tags', type='string',
 881                       help='Specify tags to execute specific test groups.')
 882     parser.add_option('-I', action='callback', callback=options_cb, default=1,
 883                       dest='iterations', metavar='iterations', type='int',
 884                       help='Number of times to run the test run.')
 885     (options, pathnames) = parser.parse_args()
 886
 887     if not options.runfile and not options.template:
 888         options.cmd = 'runtests'
 889
 890     if options.runfile and len(pathnames):
 891         fail('Extraneous arguments.')
 892
 893     options.pathnames = [os.path.abspath(path) for path in pathnames]
 894
 895     return options
 896
 897
 898 def main():
 899     options = parse_args()
 900     testrun = TestRun(options)
 901
 902     if options.cmd is 'runtests':
 903         find_tests(testrun, options)
 904     elif options.cmd is 'rdconfig':
 905         testrun.read(testrun.logger, options)
 906     elif options.cmd is 'wrconfig':
 907         find_tests(testrun, options)
 908         testrun.write(options)
 909         exit(0)
 910     else:
 911         fail('Unknown command specified')
 912
 913     testrun.complete_outputdirs()
 914     testrun.run(options)
 915     exit(testrun.summary())
 916
 917
 918 if __name__ == '__main__':
 919     main()