tests/test-runner/bin/test-runner.py

   1 #!/usr/bin/python
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2012, 2015 by Delphix. All rights reserved.
  16 # Copyright (c) 2017 Datto Inc.
  17 #
  18
  19 # some python 2.7 system don't have a configparser shim
  20 try:
  21     import configparser
  22 except ImportError:
  23     import ConfigParser as configparser
  24
  25 import os
  26 import logging
  27 from datetime import datetime
  28 from optparse import OptionParser
  29 from pwd import getpwnam
  30 from pwd import getpwuid
  31 from select import select
  32 from subprocess import PIPE
  33 from subprocess import Popen
  34 from sys import argv
  35 from sys import maxsize
  36 from threading import Timer
  37 from time import time
  38
  39 BASEDIR = '/var/tmp/test_results'
  40 TESTDIR = '/usr/share/zfs/'
  41 KILL = 'kill'
  42 TRUE = 'true'
  43 SUDO = 'sudo'
  44
  45
  46 class Result(object):
  47     total = 0
  48     runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0}
  49
  50     def __init__(self):
  51         self.starttime = None
  52         self.returncode = None
  53         self.runtime = ''
  54         self.stdout = []
  55         self.stderr = []
  56         self.result = ''
  57
  58     def done(self, proc, killed):
  59         """
  60         Finalize the results of this Cmd.
  61         """
  62         Result.total += 1
  63         m, s = divmod(time() - self.starttime, 60)
  64         self.runtime = '%02d:%02d' % (m, s)
  65         self.returncode = proc.returncode
  66         if killed:
  67             self.result = 'KILLED'
  68             Result.runresults['KILLED'] += 1
  69         elif self.returncode is 0:
  70             self.result = 'PASS'
  71             Result.runresults['PASS'] += 1
  72         elif self.returncode is 4:
  73             self.result = 'SKIP'
  74             Result.runresults['SKIP'] += 1
  75         elif self.returncode is not 0:
  76             self.result = 'FAIL'
  77             Result.runresults['FAIL'] += 1
  78
  79
  80 class Output(object):
  81     """
  82     This class is a slightly modified version of the 'Stream' class found
  83     here: http://goo.gl/aSGfv
  84     """
  85     def __init__(self, stream):
  86         self.stream = stream
  87         self._buf = ''
  88         self.lines = []
  89
  90     def fileno(self):
  91         return self.stream.fileno()
  92
  93     def read(self, drain=0):
  94         """
  95         Read from the file descriptor. If 'drain' set, read until EOF.
  96         """
  97         while self._read() is not None:
  98             if not drain:
  99                 break
 100
 101     def _read(self):
 102         """
 103         Read up to 4k of data from this output stream. Collect the output
 104         up to the last newline, and append it to any leftover data from a
 105         previous call. The lines are stored as a (timestamp, data) tuple
 106         for easy sorting/merging later.
 107         """
 108         fd = self.fileno()
 109         buf = os.read(fd, 4096)
 110         if not buf:
 111             return None
 112         if '\n' not in buf:
 113             self._buf += buf
 114             return []
 115
 116         buf = self._buf + buf
 117         tmp, rest = buf.rsplit('\n', 1)
 118         self._buf = rest
 119         now = datetime.now()
 120         rows = tmp.split('\n')
 121         self.lines += [(now, r) for r in rows]
 122
 123
 124 class Cmd(object):
 125     verified_users = []
 126
 127     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 128                  tags=None):
 129         self.pathname = pathname
 130         self.outputdir = outputdir or 'BASEDIR'
 131         self.timeout = timeout
 132         self.user = user or ''
 133         self.killed = False
 134         self.result = Result()
 135
 136         if self.timeout is None:
 137             self.timeout = 60
 138
 139     def __str__(self):
 140         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % \
 141             (self.pathname, self.outputdir, self.timeout, self.user)
 142
 143     def kill_cmd(self, proc):
 144         """
 145         Kill a running command due to timeout, or ^C from the keyboard. If
 146         sudo is required, this user was verified previously.
 147         """
 148         self.killed = True
 149         do_sudo = len(self.user) != 0
 150         signal = '-TERM'
 151
 152         cmd = [SUDO, KILL, signal, str(proc.pid)]
 153         if not do_sudo:
 154             del cmd[0]
 155
 156         try:
 157             kp = Popen(cmd)
 158             kp.wait()
 159         except Exception:
 160             pass
 161
 162     def update_cmd_privs(self, cmd, user):
 163         """
 164         If a user has been specified to run this Cmd and we're not already
 165         running as that user, prepend the appropriate sudo command to run
 166         as that user.
 167         """
 168         me = getpwuid(os.getuid())
 169
 170         if not user or user is me:
 171             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 172                 cmd += '.ksh'
 173             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 174                 cmd += '.sh'
 175             return cmd
 176
 177         if not os.path.isfile(cmd):
 178             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 179                 cmd += '.ksh'
 180             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 181                 cmd += '.sh'
 182
 183         ret = '%s -E -u %s %s' % (SUDO, user, cmd)
 184         return ret.split(' ')
 185
 186     def collect_output(self, proc):
 187         """
 188         Read from stdout/stderr as data becomes available, until the
 189         process is no longer running. Return the lines from the stdout and
 190         stderr Output objects.
 191         """
 192         out = Output(proc.stdout)
 193         err = Output(proc.stderr)
 194         res = []
 195         while proc.returncode is None:
 196             proc.poll()
 197             res = select([out, err], [], [], .1)
 198             for fd in res[0]:
 199                 fd.read()
 200         for fd in res[0]:
 201             fd.read(drain=1)
 202
 203         return out.lines, err.lines
 204
 205     def run(self, options):
 206         """
 207         This is the main function that runs each individual test.
 208         Determine whether or not the command requires sudo, and modify it
 209         if needed. Run the command, and update the result object.
 210         """
 211         if options.dryrun is True:
 212             print(self)
 213             return
 214
 215         privcmd = self.update_cmd_privs(self.pathname, self.user)
 216         try:
 217             old = os.umask(0)
 218             if not os.path.isdir(self.outputdir):
 219                 os.makedirs(self.outputdir, mode=0o777)
 220             os.umask(old)
 221         except OSError as e:
 222             fail('%s' % e)
 223
 224         self.result.starttime = time()
 225         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
 226         # Allow a special timeout value of 0 to mean infinity
 227         if int(self.timeout) == 0:
 228             self.timeout = maxsize
 229         t = Timer(int(self.timeout), self.kill_cmd, [proc])
 230
 231         try:
 232             t.start()
 233             self.result.stdout, self.result.stderr = self.collect_output(proc)
 234         except KeyboardInterrupt:
 235             self.kill_cmd(proc)
 236             fail('\nRun terminated at user request.')
 237         finally:
 238             t.cancel()
 239
 240         self.result.done(proc, self.killed)
 241
 242     def skip(self):
 243         """
 244         Initialize enough of the test result that we can log a skipped
 245         command.
 246         """
 247         Result.total += 1
 248         Result.runresults['SKIP'] += 1
 249         self.result.stdout = self.result.stderr = []
 250         self.result.starttime = time()
 251         m, s = divmod(time() - self.result.starttime, 60)
 252         self.result.runtime = '%02d:%02d' % (m, s)
 253         self.result.result = 'SKIP'
 254
 255     def log(self, logger, options):
 256         """
 257         This function is responsible for writing all output. This includes
 258         the console output, the logfile of all results (with timestamped
 259         merged stdout and stderr), and for each test, the unmodified
 260         stdout/stderr/merged in it's own file.
 261         """
 262         if logger is None:
 263             return
 264
 265         logname = getpwuid(os.getuid()).pw_name
 266         user = ' (run as %s)' % (self.user if len(self.user) else logname)
 267         msga = 'Test: %s%s ' % (self.pathname, user)
 268         msgb = '[%s] [%s]' % (self.result.runtime, self.result.result)
 269         pad = ' ' * (80 - (len(msga) + len(msgb)))
 270
 271         # If -q is specified, only print a line for tests that didn't pass.
 272         # This means passing tests need to be logged as DEBUG, or the one
 273         # line summary will only be printed in the logfile for failures.
 274         if not options.quiet:
 275             logger.info('%s%s%s' % (msga, pad, msgb))
 276         elif self.result.result is not 'PASS':
 277             logger.info('%s%s%s' % (msga, pad, msgb))
 278         else:
 279             logger.debug('%s%s%s' % (msga, pad, msgb))
 280
 281         lines = sorted(self.result.stdout + self.result.stderr,
 282                        key=lambda x: x[0])
 283
 284         for dt, line in lines:
 285             logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line))
 286
 287         if len(self.result.stdout):
 288             with open(os.path.join(self.outputdir, 'stdout'), 'w') as out:
 289                 for _, line in self.result.stdout:
 290                     os.write(out.fileno(), '%s\n' % line)
 291         if len(self.result.stderr):
 292             with open(os.path.join(self.outputdir, 'stderr'), 'w') as err:
 293                 for _, line in self.result.stderr:
 294                     os.write(err.fileno(), '%s\n' % line)
 295         if len(self.result.stdout) and len(self.result.stderr):
 296             with open(os.path.join(self.outputdir, 'merged'), 'w') as merged:
 297                 for _, line in lines:
 298                     os.write(merged.fileno(), '%s\n' % line)
 299
 300
 301 class Test(Cmd):
 302     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
 303              'post_user', 'tags']
 304
 305     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 306                  pre=None, pre_user=None, post=None, post_user=None,
 307                  tags=None):
 308         super(Test, self).__init__(pathname, outputdir, timeout, user)
 309         self.pre = pre or ''
 310         self.pre_user = pre_user or ''
 311         self.post = post or ''
 312         self.post_user = post_user or ''
 313         self.tags = tags or []
 314
 315     def __str__(self):
 316         post_user = pre_user = ''
 317         if len(self.pre_user):
 318             pre_user = ' (as %s)' % (self.pre_user)
 319         if len(self.post_user):
 320             post_user = ' (as %s)' % (self.post_user)
 321         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \
 322                "%s%s\nUser: %s\nTags: %s\n" % \
 323                (self.pathname, self.outputdir, self.timeout, self.pre,
 324                 pre_user, self.post, post_user, self.user, self.tags)
 325
 326     def verify(self, logger):
 327         """
 328         Check the pre/post scripts, user and Test. Omit the Test from this
 329         run if there are any problems.
 330         """
 331         files = [self.pre, self.pathname, self.post]
 332         users = [self.pre_user, self.user, self.post_user]
 333
 334         for f in [f for f in files if len(f)]:
 335             if not verify_file(f):
 336                 logger.info("Warning: Test '%s' not added to this run because"
 337                             " it failed verification." % f)
 338                 return False
 339
 340         for user in [user for user in users if len(user)]:
 341             if not verify_user(user, logger):
 342                 logger.info("Not adding Test '%s' to this run." %
 343                             self.pathname)
 344                 return False
 345
 346         return True
 347
 348     def run(self, logger, options):
 349         """
 350         Create Cmd instances for the pre/post scripts. If the pre script
 351         doesn't pass, skip this Test. Run the post script regardless.
 352         """
 353         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 354         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 355                       user=self.pre_user)
 356         test = Cmd(self.pathname, outputdir=self.outputdir,
 357                    timeout=self.timeout, user=self.user)
 358         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 359         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 360                        user=self.post_user)
 361
 362         cont = True
 363         if len(pretest.pathname):
 364             pretest.run(options)
 365             cont = pretest.result.result is 'PASS'
 366             pretest.log(logger, options)
 367
 368         if cont:
 369             test.run(options)
 370         else:
 371             test.skip()
 372
 373         test.log(logger, options)
 374
 375         if len(posttest.pathname):
 376             posttest.run(options)
 377             posttest.log(logger, options)
 378
 379
 380 class TestGroup(Test):
 381     props = Test.props + ['tests']
 382
 383     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 384                  pre=None, pre_user=None, post=None, post_user=None,
 385                  tests=None, tags=None):
 386         super(TestGroup, self).__init__(pathname, outputdir, timeout, user,
 387                                         pre, pre_user, post, post_user, tags)
 388         self.tests = tests or []
 389
 390     def __str__(self):
 391         post_user = pre_user = ''
 392         if len(self.pre_user):
 393             pre_user = ' (as %s)' % (self.pre_user)
 394         if len(self.post_user):
 395             post_user = ' (as %s)' % (self.post_user)
 396         return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %s\n" \
 397                "Pre: %s%s\nPost: %s%s\nUser: %s\nTags: %s\n" % \
 398                (self.pathname, self.outputdir, self.tests, self.timeout,
 399                 self.pre, pre_user, self.post, post_user, self.user, self.tags)
 400
 401     def verify(self, logger):
 402         """
 403         Check the pre/post scripts, user and tests in this TestGroup. Omit
 404         the TestGroup entirely, or simply delete the relevant tests in the
 405         group, if that's all that's required.
 406         """
 407         # If the pre or post scripts are relative pathnames, convert to
 408         # absolute, so they stand a chance of passing verification.
 409         if len(self.pre) and not os.path.isabs(self.pre):
 410             self.pre = os.path.join(self.pathname, self.pre)
 411         if len(self.post) and not os.path.isabs(self.post):
 412             self.post = os.path.join(self.pathname, self.post)
 413
 414         auxfiles = [self.pre, self.post]
 415         users = [self.pre_user, self.user, self.post_user]
 416
 417         for f in [f for f in auxfiles if len(f)]:
 418             if self.pathname != os.path.dirname(f):
 419                 logger.info("Warning: TestGroup '%s' not added to this run. "
 420                             "Auxiliary script '%s' exists in a different "
 421                             "directory." % (self.pathname, f))
 422                 return False
 423
 424             if not verify_file(f):
 425                 logger.info("Warning: TestGroup '%s' not added to this run. "
 426                             "Auxiliary script '%s' failed verification." %
 427                             (self.pathname, f))
 428                 return False
 429
 430         for user in [user for user in users if len(user)]:
 431             if not verify_user(user, logger):
 432                 logger.info("Not adding TestGroup '%s' to this run." %
 433                             self.pathname)
 434                 return False
 435
 436         # If one of the tests is invalid, delete it, log it, and drive on.
 437         for test in self.tests:
 438             if not verify_file(os.path.join(self.pathname, test)):
 439                 del self.tests[self.tests.index(test)]
 440                 logger.info("Warning: Test '%s' removed from TestGroup '%s' "
 441                             "because it failed verification." %
 442                             (test, self.pathname))
 443
 444         return len(self.tests) is not 0
 445
 446     def run(self, logger, options):
 447         """
 448         Create Cmd instances for the pre/post scripts. If the pre script
 449         doesn't pass, skip all the tests in this TestGroup. Run the post
 450         script regardless.
 451         """
 452         # tags assigned to this test group also include the test names
 453         if options.tags and not set(self.tags).intersection(set(options.tags)):
 454             return
 455
 456         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 457         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 458                       user=self.pre_user)
 459         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 460         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 461                        user=self.post_user)
 462
 463         cont = True
 464         if len(pretest.pathname):
 465             pretest.run(options)
 466             cont = pretest.result.result is 'PASS'
 467             pretest.log(logger, options)
 468
 469         for fname in self.tests:
 470             test = Cmd(os.path.join(self.pathname, fname),
 471                        outputdir=os.path.join(self.outputdir, fname),
 472                        timeout=self.timeout, user=self.user)
 473             if cont:
 474                 test.run(options)
 475             else:
 476                 test.skip()
 477
 478             test.log(logger, options)
 479
 480         if len(posttest.pathname):
 481             posttest.run(options)
 482             posttest.log(logger, options)
 483
 484
 485 class TestRun(object):
 486     props = ['quiet', 'outputdir']
 487
 488     def __init__(self, options):
 489         self.tests = {}
 490         self.testgroups = {}
 491         self.starttime = time()
 492         self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
 493         self.outputdir = os.path.join(options.outputdir, self.timestamp)
 494         self.logger = self.setup_logging(options)
 495         self.defaults = [
 496             ('outputdir', BASEDIR),
 497             ('quiet', False),
 498             ('timeout', 60),
 499             ('user', ''),
 500             ('pre', ''),
 501             ('pre_user', ''),
 502             ('post', ''),
 503             ('post_user', ''),
 504             ('tags', [])
 505         ]
 506
 507     def __str__(self):
 508         s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
 509         s += 'TESTS:\n'
 510         for key in sorted(self.tests.keys()):
 511             s += '%s%s' % (self.tests[key].__str__(), '\n')
 512         s += 'TESTGROUPS:\n'
 513         for key in sorted(self.testgroups.keys()):
 514             s += '%s%s' % (self.testgroups[key].__str__(), '\n')
 515         return s
 516
 517     def addtest(self, pathname, options):
 518         """
 519         Create a new Test, and apply any properties that were passed in
 520         from the command line. If it passes verification, add it to the
 521         TestRun.
 522         """
 523         test = Test(pathname)
 524         for prop in Test.props:
 525             setattr(test, prop, getattr(options, prop))
 526
 527         if test.verify(self.logger):
 528             self.tests[pathname] = test
 529
 530     def addtestgroup(self, dirname, filenames, options):
 531         """
 532         Create a new TestGroup, and apply any properties that were passed
 533         in from the command line. If it passes verification, add it to the
 534         TestRun.
 535         """
 536         if dirname not in self.testgroups:
 537             testgroup = TestGroup(dirname)
 538             for prop in Test.props:
 539                 setattr(testgroup, prop, getattr(options, prop))
 540
 541             # Prevent pre/post scripts from running as regular tests
 542             for f in [testgroup.pre, testgroup.post]:
 543                 if f in filenames:
 544                     del filenames[filenames.index(f)]
 545
 546             self.testgroups[dirname] = testgroup
 547             self.testgroups[dirname].tests = sorted(filenames)
 548
 549             testgroup.verify(self.logger)
 550
 551     def read(self, logger, options):
 552         """
 553         Read in the specified runfile, and apply the TestRun properties
 554         listed in the 'DEFAULT' section to our TestRun. Then read each
 555         section, and apply the appropriate properties to the Test or
 556         TestGroup. Properties from individual sections override those set
 557         in the 'DEFAULT' section. If the Test or TestGroup passes
 558         verification, add it to the TestRun.
 559         """
 560         config = configparser.RawConfigParser()
 561         if not len(config.read(options.runfile)):
 562             fail("Coulnd't read config file %s" % options.runfile)
 563
 564         for opt in TestRun.props:
 565             if config.has_option('DEFAULT', opt):
 566                 setattr(self, opt, config.get('DEFAULT', opt))
 567         self.outputdir = os.path.join(self.outputdir, self.timestamp)
 568
 569         for section in config.sections():
 570             if 'tests' in config.options(section):
 571                 if os.path.isdir(section):
 572                     pathname = section
 573                 elif os.path.isdir(os.path.join(options.testdir, section)):
 574                     pathname = os.path.join(options.testdir, section)
 575                 else:
 576                     pathname = section
 577
 578                 testgroup = TestGroup(os.path.abspath(pathname))
 579                 for prop in TestGroup.props:
 580                     for sect in ['DEFAULT', section]:
 581                         if config.has_option(sect, prop):
 582                             if prop is "tags":
 583                                 setattr(testgroup, prop,
 584                                         eval(config.get(sect, prop)))
 585                             else:
 586                                 setattr(testgroup, prop,
 587                                         config.get(sect, prop))
 588
 589                 # Repopulate tests using eval to convert the string to a list
 590                 testgroup.tests = eval(config.get(section, 'tests'))
 591
 592                 if testgroup.verify(logger):
 593                     self.testgroups[section] = testgroup
 594             else:
 595                 test = Test(section)
 596                 for prop in Test.props:
 597                     for sect in ['DEFAULT', section]:
 598                         if config.has_option(sect, prop):
 599                             setattr(test, prop, config.get(sect, prop))
 600
 601                 if test.verify(logger):
 602                     self.tests[section] = test
 603
 604     def write(self, options):
 605         """
 606         Create a configuration file for editing and later use. The
 607         'DEFAULT' section of the config file is created from the
 608         properties that were specified on the command line. Tests are
 609         simply added as sections that inherit everything from the
 610         'DEFAULT' section. TestGroups are the same, except they get an
 611         option including all the tests to run in that directory.
 612         """
 613
 614         defaults = dict([(prop, getattr(options, prop)) for prop, _ in
 615                          self.defaults])
 616         config = configparser.RawConfigParser(defaults)
 617
 618         for test in sorted(self.tests.keys()):
 619             config.add_section(test)
 620
 621         for testgroup in sorted(self.testgroups.keys()):
 622             config.add_section(testgroup)
 623             config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
 624
 625         try:
 626             with open(options.template, 'w') as f:
 627                 return config.write(f)
 628         except IOError:
 629             fail('Could not open \'%s\' for writing.' % options.template)
 630
 631     def complete_outputdirs(self):
 632         """
 633         Collect all the pathnames for Tests, and TestGroups. Work
 634         backwards one pathname component at a time, to create a unique
 635         directory name in which to deposit test output. Tests will be able
 636         to write output files directly in the newly modified outputdir.
 637         TestGroups will be able to create one subdirectory per test in the
 638         outputdir, and are guaranteed uniqueness because a group can only
 639         contain files in one directory. Pre and post tests will create a
 640         directory rooted at the outputdir of the Test or TestGroup in
 641         question for their output.
 642         """
 643         done = False
 644         components = 0
 645         tmp_dict = dict(list(self.tests.items()) +
 646                         list(self.testgroups.items()))
 647         total = len(tmp_dict)
 648         base = self.outputdir
 649
 650         while not done:
 651             paths = []
 652             components -= 1
 653             for testfile in list(tmp_dict.keys()):
 654                 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
 655                 if uniq not in paths:
 656                     paths.append(uniq)
 657                     tmp_dict[testfile].outputdir = os.path.join(base, uniq)
 658                 else:
 659                     break
 660             done = total == len(paths)
 661
 662     def setup_logging(self, options):
 663         """
 664         Two loggers are set up here. The first is for the logfile which
 665         will contain one line summarizing the test, including the test
 666         name, result, and running time. This logger will also capture the
 667         timestamped combined stdout and stderr of each run. The second
 668         logger is optional console output, which will contain only the one
 669         line summary. The loggers are initialized at two different levels
 670         to facilitate segregating the output.
 671         """
 672         if options.dryrun is True:
 673             return
 674
 675         testlogger = logging.getLogger(__name__)
 676         testlogger.setLevel(logging.DEBUG)
 677
 678         if options.cmd is not 'wrconfig':
 679             try:
 680                 old = os.umask(0)
 681                 os.makedirs(self.outputdir, mode=0o777)
 682                 os.umask(old)
 683             except OSError as e:
 684                 fail('%s' % e)
 685             filename = os.path.join(self.outputdir, 'log')
 686
 687             logfile = logging.FileHandler(filename)
 688             logfile.setLevel(logging.DEBUG)
 689             logfilefmt = logging.Formatter('%(message)s')
 690             logfile.setFormatter(logfilefmt)
 691             testlogger.addHandler(logfile)
 692
 693         cons = logging.StreamHandler()
 694         cons.setLevel(logging.INFO)
 695         consfmt = logging.Formatter('%(message)s')
 696         cons.setFormatter(consfmt)
 697         testlogger.addHandler(cons)
 698
 699         return testlogger
 700
 701     def run(self, options):
 702         """
 703         Walk through all the Tests and TestGroups, calling run().
 704         """
 705         try:
 706             os.chdir(self.outputdir)
 707         except OSError:
 708             fail('Could not change to directory %s' % self.outputdir)
 709         # make a symlink to the output for the currently running test
 710         logsymlink = os.path.join(self.outputdir, '../current')
 711         if os.path.islink(logsymlink):
 712             os.unlink(logsymlink)
 713         if not os.path.exists(logsymlink):
 714             os.symlink(self.outputdir, logsymlink)
 715         else:
 716             print('Could not make a symlink to directory %s' % (
 717                 self.outputdir))
 718         iteration = 0
 719         while iteration < options.iterations:
 720             for test in sorted(self.tests.keys()):
 721                 self.tests[test].run(self.logger, options)
 722             for testgroup in sorted(self.testgroups.keys()):
 723                 self.testgroups[testgroup].run(self.logger, options)
 724             iteration += 1
 725
 726     def summary(self):
 727         if Result.total is 0:
 728             return 2
 729
 730         print('\nResults Summary')
 731         for key in list(Result.runresults.keys()):
 732             if Result.runresults[key] is not 0:
 733                 print('%s\t% 4d' % (key, Result.runresults[key]))
 734
 735         m, s = divmod(time() - self.starttime, 60)
 736         h, m = divmod(m, 60)
 737         print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
 738         print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
 739                                             float(Result.total)) * 100))
 740         print('Log directory:\t%s' % self.outputdir)
 741
 742         if Result.runresults['FAIL'] > 0:
 743             return 1
 744
 745         if Result.runresults['KILLED'] > 0:
 746             return 1
 747
 748         return 0
 749
 750
 751 def verify_file(pathname):
 752     """
 753     Verify that the supplied pathname is an executable regular file.
 754     """
 755     if os.path.isdir(pathname) or os.path.islink(pathname):
 756         return False
 757
 758     for ext in '', '.ksh', '.sh':
 759         script_path = pathname + ext
 760         if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
 761             return True
 762
 763     return False
 764
 765
 766 def verify_user(user, logger):
 767     """
 768     Verify that the specified user exists on this system, and can execute
 769     sudo without being prompted for a password.
 770     """
 771     testcmd = [SUDO, '-n', '-u', user, TRUE]
 772
 773     if user in Cmd.verified_users:
 774         return True
 775
 776     try:
 777         getpwnam(user)
 778     except KeyError:
 779         logger.info("Warning: user '%s' does not exist.", user)
 780         return False
 781
 782     p = Popen(testcmd)
 783     p.wait()
 784     if p.returncode is not 0:
 785         logger.info("Warning: user '%s' cannot use passwordless sudo.", user)
 786         return False
 787     else:
 788         Cmd.verified_users.append(user)
 789
 790     return True
 791
 792
 793 def find_tests(testrun, options):
 794     """
 795     For the given list of pathnames, add files as Tests. For directories,
 796     if do_groups is True, add the directory as a TestGroup. If False,
 797     recursively search for executable files.
 798     """
 799
 800     for p in sorted(options.pathnames):
 801         if os.path.isdir(p):
 802             for dirname, _, filenames in os.walk(p):
 803                 if options.do_groups:
 804                     testrun.addtestgroup(dirname, filenames, options)
 805                 else:
 806                     for f in sorted(filenames):
 807                         testrun.addtest(os.path.join(dirname, f), options)
 808         else:
 809             testrun.addtest(p, options)
 810
 811
 812 def fail(retstr, ret=1):
 813     print('%s: %s' % (argv[0], retstr))
 814     exit(ret)
 815
 816
 817 def options_cb(option, opt_str, value, parser):
 818     path_options = ['runfile', 'outputdir', 'template', 'testdir']
 819
 820     if option.dest is 'runfile' and '-w' in parser.rargs or \
 821             option.dest is 'template' and '-c' in parser.rargs:
 822         fail('-c and -w are mutually exclusive.')
 823
 824     if opt_str in parser.rargs:
 825         fail('%s may only be specified once.' % opt_str)
 826
 827     if option.dest is 'runfile':
 828         parser.values.cmd = 'rdconfig'
 829     if option.dest is 'template':
 830         parser.values.cmd = 'wrconfig'
 831     if option.dest is 'tags':
 832         value = [x.strip() for x in value.split(',')]
 833
 834     setattr(parser.values, option.dest, value)
 835     if option.dest in path_options:
 836         setattr(parser.values, option.dest, os.path.abspath(value))
 837
 838
 839 def parse_args():
 840     parser = OptionParser()
 841     parser.add_option('-c', action='callback', callback=options_cb,
 842                       type='string', dest='runfile', metavar='runfile',
 843                       help='Specify tests to run via config file.')
 844     parser.add_option('-d', action='store_true', default=False, dest='dryrun',
 845                       help='Dry run. Print tests, but take no other action.')
 846     parser.add_option('-g', action='store_true', default=False,
 847                       dest='do_groups', help='Make directories TestGroups.')
 848     parser.add_option('-o', action='callback', callback=options_cb,
 849                       default=BASEDIR, dest='outputdir', type='string',
 850                       metavar='outputdir', help='Specify an output directory.')
 851     parser.add_option('-i', action='callback', callback=options_cb,
 852                       default=TESTDIR, dest='testdir', type='string',
 853                       metavar='testdir', help='Specify a test directory.')
 854     parser.add_option('-p', action='callback', callback=options_cb,
 855                       default='', dest='pre', metavar='script',
 856                       type='string', help='Specify a pre script.')
 857     parser.add_option('-P', action='callback', callback=options_cb,
 858                       default='', dest='post', metavar='script',
 859                       type='string', help='Specify a post script.')
 860     parser.add_option('-q', action='store_true', default=False, dest='quiet',
 861                       help='Silence on the console during a test run.')
 862     parser.add_option('-t', action='callback', callback=options_cb, default=60,
 863                       dest='timeout', metavar='seconds', type='int',
 864                       help='Timeout (in seconds) for an individual test.')
 865     parser.add_option('-u', action='callback', callback=options_cb,
 866                       default='', dest='user', metavar='user', type='string',
 867                       help='Specify a different user name to run as.')
 868     parser.add_option('-w', action='callback', callback=options_cb,
 869                       default=None, dest='template', metavar='template',
 870                       type='string', help='Create a new config file.')
 871     parser.add_option('-x', action='callback', callback=options_cb, default='',
 872                       dest='pre_user', metavar='pre_user', type='string',
 873                       help='Specify a user to execute the pre script.')
 874     parser.add_option('-X', action='callback', callback=options_cb, default='',
 875                       dest='post_user', metavar='post_user', type='string',
 876                       help='Specify a user to execute the post script.')
 877     parser.add_option('-T', action='callback', callback=options_cb, default='',
 878                       dest='tags', metavar='tags', type='string',
 879                       help='Specify tags to execute specific test groups.')
 880     parser.add_option('-I', action='callback', callback=options_cb, default=1,
 881                       dest='iterations', metavar='iterations', type='int',
 882                       help='Number of times to run the test run.')
 883     (options, pathnames) = parser.parse_args()
 884
 885     if not options.runfile and not options.template:
 886         options.cmd = 'runtests'
 887
 888     if options.runfile and len(pathnames):
 889         fail('Extraneous arguments.')
 890
 891     options.pathnames = [os.path.abspath(path) for path in pathnames]
 892
 893     return options
 894
 895
 896 def main():
 897     options = parse_args()
 898     testrun = TestRun(options)
 899
 900     if options.cmd is 'runtests':
 901         find_tests(testrun, options)
 902     elif options.cmd is 'rdconfig':
 903         testrun.read(testrun.logger, options)
 904     elif options.cmd is 'wrconfig':
 905         find_tests(testrun, options)
 906         testrun.write(options)
 907         exit(0)
 908     else:
 909         fail('Unknown command specified')
 910
 911     testrun.complete_outputdirs()
 912     testrun.run(options)
 913     exit(testrun.summary())
 914
 915
 916 if __name__ == '__main__':
 917     main()