tests/test-runner/cmd/test-runner.py

   1 #!/usr/bin/python
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  16 # Copyright (c) 2017 Datto Inc.
  17 #
  18
  19 # some python 2.7 system don't have a configparser shim
  20 try:
  21     import configparser
  22 except ImportError:
  23     import ConfigParser as configparser
  24
  25 import os
  26 import sys
  27
  28 from datetime import datetime
  29 from optparse import OptionParser
  30 from pwd import getpwnam
  31 from pwd import getpwuid
  32 from select import select
  33 from subprocess import PIPE
  34 from subprocess import Popen
  35 from threading import Timer
  36 from time import time
  37
  38 BASEDIR = '/var/tmp/test_results'
  39 TESTDIR = '/usr/share/zfs/'
  40 KILL = 'kill'
  41 TRUE = 'true'
  42 SUDO = 'sudo'
  43 LOG_FILE = 'LOG_FILE'
  44 LOG_OUT = 'LOG_OUT'
  45 LOG_ERR = 'LOG_ERR'
  46 LOG_FILE_OBJ = None
  47
  48
  49 class Result(object):
  50     total = 0
  51     runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0}
  52
  53     def __init__(self):
  54         self.starttime = None
  55         self.returncode = None
  56         self.runtime = ''
  57         self.stdout = []
  58         self.stderr = []
  59         self.result = ''
  60
  61     def done(self, proc, killed):
  62         """
  63         Finalize the results of this Cmd.
  64         """
  65         Result.total += 1
  66         m, s = divmod(time() - self.starttime, 60)
  67         self.runtime = '%02d:%02d' % (m, s)
  68         self.returncode = proc.returncode
  69         if killed:
  70             self.result = 'KILLED'
  71             Result.runresults['KILLED'] += 1
  72         elif self.returncode is 0:
  73             self.result = 'PASS'
  74             Result.runresults['PASS'] += 1
  75         elif self.returncode is 4:
  76             self.result = 'SKIP'
  77             Result.runresults['SKIP'] += 1
  78         elif self.returncode is not 0:
  79             self.result = 'FAIL'
  80             Result.runresults['FAIL'] += 1
  81
  82
  83 class Output(object):
  84     """
  85     This class is a slightly modified version of the 'Stream' class found
  86     here: http://goo.gl/aSGfv
  87     """
  88     def __init__(self, stream):
  89         self.stream = stream
  90         self._buf = b''
  91         self.lines = []
  92
  93     def fileno(self):
  94         return self.stream.fileno()
  95
  96     def read(self, drain=0):
  97         """
  98         Read from the file descriptor. If 'drain' set, read until EOF.
  99         """
 100         while self._read() is not None:
 101             if not drain:
 102                 break
 103
 104     def _read(self):
 105         """
 106         Read up to 4k of data from this output stream. Collect the output
 107         up to the last newline, and append it to any leftover data from a
 108         previous call. The lines are stored as a (timestamp, data) tuple
 109         for easy sorting/merging later.
 110         """
 111         fd = self.fileno()
 112         buf = os.read(fd, 4096)
 113         if not buf:
 114             return None
 115         if b'\n' not in buf:
 116             self._buf += buf
 117             return []
 118
 119         buf = self._buf + buf
 120         tmp, rest = buf.rsplit(b'\n', 1)
 121         self._buf = rest
 122         now = datetime.now()
 123         rows = tmp.split(b'\n')
 124         self.lines += [(now, r) for r in rows]
 125
 126
 127 class Cmd(object):
 128     verified_users = []
 129
 130     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 131                  tags=None):
 132         self.pathname = pathname
 133         self.outputdir = outputdir or 'BASEDIR'
 134         self.timeout = timeout
 135         self.user = user or ''
 136         self.killed = False
 137         self.result = Result()
 138
 139         if self.timeout is None:
 140             self.timeout = 60
 141
 142     def __str__(self):
 143         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % \
 144             (self.pathname, self.outputdir, self.timeout, self.user)
 145
 146     def kill_cmd(self, proc):
 147         """
 148         Kill a running command due to timeout, or ^C from the keyboard. If
 149         sudo is required, this user was verified previously.
 150         """
 151         self.killed = True
 152         do_sudo = len(self.user) != 0
 153         signal = '-TERM'
 154
 155         cmd = [SUDO, KILL, signal, str(proc.pid)]
 156         if not do_sudo:
 157             del cmd[0]
 158
 159         try:
 160             kp = Popen(cmd)
 161             kp.wait()
 162         except Exception:
 163             pass
 164
 165     def update_cmd_privs(self, cmd, user):
 166         """
 167         If a user has been specified to run this Cmd and we're not already
 168         running as that user, prepend the appropriate sudo command to run
 169         as that user.
 170         """
 171         me = getpwuid(os.getuid())
 172
 173         if not user or user is me:
 174             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 175                 cmd += '.ksh'
 176             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 177                 cmd += '.sh'
 178             return cmd
 179
 180         if not os.path.isfile(cmd):
 181             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 182                 cmd += '.ksh'
 183             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 184                 cmd += '.sh'
 185
 186         ret = '%s -E -u %s %s' % (SUDO, user, cmd)
 187         return ret.split(' ')
 188
 189     def collect_output(self, proc):
 190         """
 191         Read from stdout/stderr as data becomes available, until the
 192         process is no longer running. Return the lines from the stdout and
 193         stderr Output objects.
 194         """
 195         out = Output(proc.stdout)
 196         err = Output(proc.stderr)
 197         res = []
 198         while proc.returncode is None:
 199             proc.poll()
 200             res = select([out, err], [], [], .1)
 201             for fd in res[0]:
 202                 fd.read()
 203         for fd in res[0]:
 204             fd.read(drain=1)
 205
 206         return out.lines, err.lines
 207
 208     def run(self, options):
 209         """
 210         This is the main function that runs each individual test.
 211         Determine whether or not the command requires sudo, and modify it
 212         if needed. Run the command, and update the result object.
 213         """
 214         if options.dryrun is True:
 215             print(self)
 216             return
 217
 218         privcmd = self.update_cmd_privs(self.pathname, self.user)
 219         try:
 220             old = os.umask(0)
 221             if not os.path.isdir(self.outputdir):
 222                 os.makedirs(self.outputdir, mode=0o777)
 223             os.umask(old)
 224         except OSError as e:
 225             fail('%s' % e)
 226
 227         self.result.starttime = time()
 228         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
 229         # Allow a special timeout value of 0 to mean infinity
 230         if int(self.timeout) == 0:
 231             self.timeout = sys.maxsize
 232         t = Timer(int(self.timeout), self.kill_cmd, [proc])
 233
 234         try:
 235             t.start()
 236             self.result.stdout, self.result.stderr = self.collect_output(proc)
 237         except KeyboardInterrupt:
 238             self.kill_cmd(proc)
 239             fail('\nRun terminated at user request.')
 240         finally:
 241             t.cancel()
 242
 243         self.result.done(proc, self.killed)
 244
 245     def skip(self):
 246         """
 247         Initialize enough of the test result that we can log a skipped
 248         command.
 249         """
 250         Result.total += 1
 251         Result.runresults['SKIP'] += 1
 252         self.result.stdout = self.result.stderr = []
 253         self.result.starttime = time()
 254         m, s = divmod(time() - self.result.starttime, 60)
 255         self.result.runtime = '%02d:%02d' % (m, s)
 256         self.result.result = 'SKIP'
 257
 258     def log(self, options):
 259         """
 260         This function is responsible for writing all output. This includes
 261         the console output, the logfile of all results (with timestamped
 262         merged stdout and stderr), and for each test, the unmodified
 263         stdout/stderr/merged in it's own file.
 264         """
 265
 266         logname = getpwuid(os.getuid()).pw_name
 267         user = ' (run as %s)' % (self.user if len(self.user) else logname)
 268         msga = 'Test: %s%s ' % (self.pathname, user)
 269         msgb = '[%s] [%s]\n' % (self.result.runtime, self.result.result)
 270         pad = ' ' * (80 - (len(msga) + len(msgb)))
 271         result_line = msga + pad + msgb
 272
 273         # The result line is always written to the log file. If -q was
 274         # specified only failures are written to the console, otherwise
 275         # the result line is written to the console.
 276         write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE)
 277         if not options.quiet:
 278             write_log(result_line, LOG_OUT)
 279         elif options.quiet and self.result.result is not 'PASS':
 280             write_log(result_line, LOG_OUT)
 281
 282         lines = sorted(self.result.stdout + self.result.stderr,
 283                        key=lambda x: x[0])
 284
 285         # Write timestamped output (stdout and stderr) to the logfile
 286         for dt, line in lines:
 287             timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11],
 288                                   encoding='utf-8')
 289             write_log(b'%s %s\n' % (timestamp, line), LOG_FILE)
 290
 291         # Write the separate stdout/stderr/merged files, if the data exists
 292         if len(self.result.stdout):
 293             with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out:
 294                 for _, line in self.result.stdout:
 295                     os.write(out.fileno(), b'%s\n' % line)
 296         if len(self.result.stderr):
 297             with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err:
 298                 for _, line in self.result.stderr:
 299                     os.write(err.fileno(), b'%s\n' % line)
 300         if len(self.result.stdout) and len(self.result.stderr):
 301             with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
 302                 for _, line in lines:
 303                     os.write(merged.fileno(), b'%s\n' % line)
 304
 305
 306 class Test(Cmd):
 307     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
 308              'post_user', 'tags']
 309
 310     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 311                  pre=None, pre_user=None, post=None, post_user=None,
 312                  tags=None):
 313         super(Test, self).__init__(pathname, outputdir, timeout, user)
 314         self.pre = pre or ''
 315         self.pre_user = pre_user or ''
 316         self.post = post or ''
 317         self.post_user = post_user or ''
 318         self.tags = tags or []
 319
 320     def __str__(self):
 321         post_user = pre_user = ''
 322         if len(self.pre_user):
 323             pre_user = ' (as %s)' % (self.pre_user)
 324         if len(self.post_user):
 325             post_user = ' (as %s)' % (self.post_user)
 326         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \
 327                "%s%s\nUser: %s\nTags: %s\n" % \
 328                (self.pathname, self.outputdir, self.timeout, self.pre,
 329                 pre_user, self.post, post_user, self.user, self.tags)
 330
 331     def verify(self):
 332         """
 333         Check the pre/post scripts, user and Test. Omit the Test from this
 334         run if there are any problems.
 335         """
 336         files = [self.pre, self.pathname, self.post]
 337         users = [self.pre_user, self.user, self.post_user]
 338
 339         for f in [f for f in files if len(f)]:
 340             if not verify_file(f):
 341                 write_log("Warning: Test '%s' not added to this run because"
 342                           " it failed verification.\n" % f, LOG_ERR)
 343                 return False
 344
 345         for user in [user for user in users if len(user)]:
 346             if not verify_user(user):
 347                 write_log("Not adding Test '%s' to this run.\n" %
 348                           self.pathname, LOG_ERR)
 349                 return False
 350
 351         return True
 352
 353     def run(self, options):
 354         """
 355         Create Cmd instances for the pre/post scripts. If the pre script
 356         doesn't pass, skip this Test. Run the post script regardless.
 357         """
 358         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 359         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 360                       user=self.pre_user)
 361         test = Cmd(self.pathname, outputdir=self.outputdir,
 362                    timeout=self.timeout, user=self.user)
 363         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 364         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 365                        user=self.post_user)
 366
 367         cont = True
 368         if len(pretest.pathname):
 369             pretest.run(options)
 370             cont = pretest.result.result is 'PASS'
 371             pretest.log(options)
 372
 373         if cont:
 374             test.run(options)
 375         else:
 376             test.skip()
 377
 378         test.log(options)
 379
 380         if len(posttest.pathname):
 381             posttest.run(options)
 382             posttest.log(options)
 383
 384
 385 class TestGroup(Test):
 386     props = Test.props + ['tests']
 387
 388     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 389                  pre=None, pre_user=None, post=None, post_user=None,
 390                  tests=None, tags=None):
 391         super(TestGroup, self).__init__(pathname, outputdir, timeout, user,
 392                                         pre, pre_user, post, post_user, tags)
 393         self.tests = tests or []
 394
 395     def __str__(self):
 396         post_user = pre_user = ''
 397         if len(self.pre_user):
 398             pre_user = ' (as %s)' % (self.pre_user)
 399         if len(self.post_user):
 400             post_user = ' (as %s)' % (self.post_user)
 401         return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %s\n" \
 402                "Pre: %s%s\nPost: %s%s\nUser: %s\nTags: %s\n" % \
 403                (self.pathname, self.outputdir, self.tests, self.timeout,
 404                 self.pre, pre_user, self.post, post_user, self.user, self.tags)
 405
 406     def verify(self):
 407         """
 408         Check the pre/post scripts, user and tests in this TestGroup. Omit
 409         the TestGroup entirely, or simply delete the relevant tests in the
 410         group, if that's all that's required.
 411         """
 412         # If the pre or post scripts are relative pathnames, convert to
 413         # absolute, so they stand a chance of passing verification.
 414         if len(self.pre) and not os.path.isabs(self.pre):
 415             self.pre = os.path.join(self.pathname, self.pre)
 416         if len(self.post) and not os.path.isabs(self.post):
 417             self.post = os.path.join(self.pathname, self.post)
 418
 419         auxfiles = [self.pre, self.post]
 420         users = [self.pre_user, self.user, self.post_user]
 421
 422         for f in [f for f in auxfiles if len(f)]:
 423             if self.pathname != os.path.dirname(f):
 424                 write_log("Warning: TestGroup '%s' not added to this run. "
 425                           "Auxiliary script '%s' exists in a different "
 426                           "directory.\n" % (self.pathname, f), LOG_ERR)
 427                 return False
 428
 429             if not verify_file(f):
 430                 write_log("Warning: TestGroup '%s' not added to this run. "
 431                           "Auxiliary script '%s' failed verification.\n" %
 432                           (self.pathname, f), LOG_ERR)
 433                 return False
 434
 435         for user in [user for user in users if len(user)]:
 436             if not verify_user(user):
 437                 write_log("Not adding TestGroup '%s' to this run.\n" %
 438                           self.pathname, LOG_ERR)
 439                 return False
 440
 441         # If one of the tests is invalid, delete it, log it, and drive on.
 442         for test in self.tests:
 443             if not verify_file(os.path.join(self.pathname, test)):
 444                 del self.tests[self.tests.index(test)]
 445                 write_log("Warning: Test '%s' removed from TestGroup '%s' "
 446                           "because it failed verification.\n" %
 447                           (test, self.pathname), LOG_ERR)
 448
 449         return len(self.tests) is not 0
 450
 451     def run(self, options):
 452         """
 453         Create Cmd instances for the pre/post scripts. If the pre script
 454         doesn't pass, skip all the tests in this TestGroup. Run the post
 455         script regardless.
 456         """
 457         # tags assigned to this test group also include the test names
 458         if options.tags and not set(self.tags).intersection(set(options.tags)):
 459             return
 460
 461         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 462         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 463                       user=self.pre_user)
 464         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 465         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 466                        user=self.post_user)
 467
 468         cont = True
 469         if len(pretest.pathname):
 470             pretest.run(options)
 471             cont = pretest.result.result is 'PASS'
 472             pretest.log(options)
 473
 474         for fname in self.tests:
 475             test = Cmd(os.path.join(self.pathname, fname),
 476                        outputdir=os.path.join(self.outputdir, fname),
 477                        timeout=self.timeout, user=self.user)
 478             if cont:
 479                 test.run(options)
 480             else:
 481                 test.skip()
 482
 483             test.log(options)
 484
 485         if len(posttest.pathname):
 486             posttest.run(options)
 487             posttest.log(options)
 488
 489
 490 class TestRun(object):
 491     props = ['quiet', 'outputdir']
 492
 493     def __init__(self, options):
 494         self.tests = {}
 495         self.testgroups = {}
 496         self.starttime = time()
 497         self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
 498         self.outputdir = os.path.join(options.outputdir, self.timestamp)
 499         self.setup_logging(options)
 500         self.defaults = [
 501             ('outputdir', BASEDIR),
 502             ('quiet', False),
 503             ('timeout', 60),
 504             ('user', ''),
 505             ('pre', ''),
 506             ('pre_user', ''),
 507             ('post', ''),
 508             ('post_user', ''),
 509             ('tags', [])
 510         ]
 511
 512     def __str__(self):
 513         s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
 514         s += 'TESTS:\n'
 515         for key in sorted(self.tests.keys()):
 516             s += '%s%s' % (self.tests[key].__str__(), '\n')
 517         s += 'TESTGROUPS:\n'
 518         for key in sorted(self.testgroups.keys()):
 519             s += '%s%s' % (self.testgroups[key].__str__(), '\n')
 520         return s
 521
 522     def addtest(self, pathname, options):
 523         """
 524         Create a new Test, and apply any properties that were passed in
 525         from the command line. If it passes verification, add it to the
 526         TestRun.
 527         """
 528         test = Test(pathname)
 529         for prop in Test.props:
 530             setattr(test, prop, getattr(options, prop))
 531
 532         if test.verify():
 533             self.tests[pathname] = test
 534
 535     def addtestgroup(self, dirname, filenames, options):
 536         """
 537         Create a new TestGroup, and apply any properties that were passed
 538         in from the command line. If it passes verification, add it to the
 539         TestRun.
 540         """
 541         if dirname not in self.testgroups:
 542             testgroup = TestGroup(dirname)
 543             for prop in Test.props:
 544                 setattr(testgroup, prop, getattr(options, prop))
 545
 546             # Prevent pre/post scripts from running as regular tests
 547             for f in [testgroup.pre, testgroup.post]:
 548                 if f in filenames:
 549                     del filenames[filenames.index(f)]
 550
 551             self.testgroups[dirname] = testgroup
 552             self.testgroups[dirname].tests = sorted(filenames)
 553
 554             testgroup.verify()
 555
 556     def read(self, options):
 557         """
 558         Read in the specified runfile, and apply the TestRun properties
 559         listed in the 'DEFAULT' section to our TestRun. Then read each
 560         section, and apply the appropriate properties to the Test or
 561         TestGroup. Properties from individual sections override those set
 562         in the 'DEFAULT' section. If the Test or TestGroup passes
 563         verification, add it to the TestRun.
 564         """
 565         config = configparser.RawConfigParser()
 566         if not len(config.read(options.runfile)):
 567             fail("Coulnd't read config file %s" % options.runfile)
 568
 569         for opt in TestRun.props:
 570             if config.has_option('DEFAULT', opt):
 571                 setattr(self, opt, config.get('DEFAULT', opt))
 572         self.outputdir = os.path.join(self.outputdir, self.timestamp)
 573
 574         for section in config.sections():
 575             if 'tests' in config.options(section):
 576                 if os.path.isdir(section):
 577                     pathname = section
 578                 elif os.path.isdir(os.path.join(options.testdir, section)):
 579                     pathname = os.path.join(options.testdir, section)
 580                 else:
 581                     pathname = section
 582
 583                 testgroup = TestGroup(os.path.abspath(pathname))
 584                 for prop in TestGroup.props:
 585                     for sect in ['DEFAULT', section]:
 586                         if config.has_option(sect, prop):
 587                             if prop is "tags":
 588                                 setattr(testgroup, prop,
 589                                         eval(config.get(sect, prop)))
 590                             else:
 591                                 setattr(testgroup, prop,
 592                                         config.get(sect, prop))
 593
 594                 # Repopulate tests using eval to convert the string to a list
 595                 testgroup.tests = eval(config.get(section, 'tests'))
 596
 597                 if testgroup.verify():
 598                     self.testgroups[section] = testgroup
 599             else:
 600                 test = Test(section)
 601                 for prop in Test.props:
 602                     for sect in ['DEFAULT', section]:
 603                         if config.has_option(sect, prop):
 604                             setattr(test, prop, config.get(sect, prop))
 605
 606                 if test.verify():
 607                     self.tests[section] = test
 608
 609     def write(self, options):
 610         """
 611         Create a configuration file for editing and later use. The
 612         'DEFAULT' section of the config file is created from the
 613         properties that were specified on the command line. Tests are
 614         simply added as sections that inherit everything from the
 615         'DEFAULT' section. TestGroups are the same, except they get an
 616         option including all the tests to run in that directory.
 617         """
 618
 619         defaults = dict([(prop, getattr(options, prop)) for prop, _ in
 620                          self.defaults])
 621         config = configparser.RawConfigParser(defaults)
 622
 623         for test in sorted(self.tests.keys()):
 624             config.add_section(test)
 625
 626         for testgroup in sorted(self.testgroups.keys()):
 627             config.add_section(testgroup)
 628             config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
 629
 630         try:
 631             with open(options.template, 'w') as f:
 632                 return config.write(f)
 633         except IOError:
 634             fail('Could not open \'%s\' for writing.' % options.template)
 635
 636     def complete_outputdirs(self):
 637         """
 638         Collect all the pathnames for Tests, and TestGroups. Work
 639         backwards one pathname component at a time, to create a unique
 640         directory name in which to deposit test output. Tests will be able
 641         to write output files directly in the newly modified outputdir.
 642         TestGroups will be able to create one subdirectory per test in the
 643         outputdir, and are guaranteed uniqueness because a group can only
 644         contain files in one directory. Pre and post tests will create a
 645         directory rooted at the outputdir of the Test or TestGroup in
 646         question for their output.
 647         """
 648         done = False
 649         components = 0
 650         tmp_dict = dict(list(self.tests.items()) +
 651                         list(self.testgroups.items()))
 652         total = len(tmp_dict)
 653         base = self.outputdir
 654
 655         while not done:
 656             paths = []
 657             components -= 1
 658             for testfile in list(tmp_dict.keys()):
 659                 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
 660                 if uniq not in paths:
 661                     paths.append(uniq)
 662                     tmp_dict[testfile].outputdir = os.path.join(base, uniq)
 663                 else:
 664                     break
 665             done = total == len(paths)
 666
 667     def setup_logging(self, options):
 668         """
 669         This funtion creates the output directory and gets a file object
 670         for the logfile. This function must be called before write_log()
 671         can be used.
 672         """
 673         if options.dryrun is True:
 674             return
 675
 676         global LOG_FILE_OBJ
 677         if options.cmd is not 'wrconfig':
 678             try:
 679                 old = os.umask(0)
 680                 os.makedirs(self.outputdir, mode=0o777)
 681                 os.umask(old)
 682                 filename = os.path.join(self.outputdir, 'log')
 683                 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb')
 684             except OSError as e:
 685                 fail('%s' % e)
 686
 687     def run(self, options):
 688         """
 689         Walk through all the Tests and TestGroups, calling run().
 690         """
 691         try:
 692             os.chdir(self.outputdir)
 693         except OSError:
 694             fail('Could not change to directory %s' % self.outputdir)
 695         # make a symlink to the output for the currently running test
 696         logsymlink = os.path.join(self.outputdir, '../current')
 697         if os.path.islink(logsymlink):
 698             os.unlink(logsymlink)
 699         if not os.path.exists(logsymlink):
 700             os.symlink(self.outputdir, logsymlink)
 701         else:
 702             write_log('Could not make a symlink to directory %s\n' %
 703                       self.outputdir, LOG_ERR)
 704         iteration = 0
 705         while iteration < options.iterations:
 706             for test in sorted(self.tests.keys()):
 707                 self.tests[test].run(options)
 708             for testgroup in sorted(self.testgroups.keys()):
 709                 self.testgroups[testgroup].run(options)
 710             iteration += 1
 711
 712     def summary(self):
 713         if Result.total is 0:
 714             return 2
 715
 716         print('\nResults Summary')
 717         for key in list(Result.runresults.keys()):
 718             if Result.runresults[key] is not 0:
 719                 print('%s\t% 4d' % (key, Result.runresults[key]))
 720
 721         m, s = divmod(time() - self.starttime, 60)
 722         h, m = divmod(m, 60)
 723         print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
 724         print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
 725                                             float(Result.total)) * 100))
 726         print('Log directory:\t%s' % self.outputdir)
 727
 728         if Result.runresults['FAIL'] > 0:
 729             return 1
 730
 731         if Result.runresults['KILLED'] > 0:
 732             return 1
 733
 734         return 0
 735
 736
 737 def write_log(msg, target):
 738     """
 739     Write the provided message to standard out, standard error or
 740     the logfile. If specifying LOG_FILE, then `msg` must be a bytes
 741     like object. This way we can still handle output from tests that
 742     may be in unexpected encodings.
 743     """
 744     if target == LOG_OUT:
 745         os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8'))
 746     elif target == LOG_ERR:
 747         os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8'))
 748     elif target == LOG_FILE:
 749         os.write(LOG_FILE_OBJ.fileno(), msg)
 750     else:
 751         fail('log_msg called with unknown target "%s"' % target)
 752
 753
 754 def verify_file(pathname):
 755     """
 756     Verify that the supplied pathname is an executable regular file.
 757     """
 758     if os.path.isdir(pathname) or os.path.islink(pathname):
 759         return False
 760
 761     for ext in '', '.ksh', '.sh':
 762         script_path = pathname + ext
 763         if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
 764             return True
 765
 766     return False
 767
 768
 769 def verify_user(user):
 770     """
 771     Verify that the specified user exists on this system, and can execute
 772     sudo without being prompted for a password.
 773     """
 774     testcmd = [SUDO, '-n', '-u', user, TRUE]
 775
 776     if user in Cmd.verified_users:
 777         return True
 778
 779     try:
 780         getpwnam(user)
 781     except KeyError:
 782         write_log("Warning: user '%s' does not exist.\n" % user,
 783                   LOG_ERR)
 784         return False
 785
 786     p = Popen(testcmd)
 787     p.wait()
 788     if p.returncode is not 0:
 789         write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user,
 790                   LOG_ERR)
 791         return False
 792     else:
 793         Cmd.verified_users.append(user)
 794
 795     return True
 796
 797
 798 def find_tests(testrun, options):
 799     """
 800     For the given list of pathnames, add files as Tests. For directories,
 801     if do_groups is True, add the directory as a TestGroup. If False,
 802     recursively search for executable files.
 803     """
 804
 805     for p in sorted(options.pathnames):
 806         if os.path.isdir(p):
 807             for dirname, _, filenames in os.walk(p):
 808                 if options.do_groups:
 809                     testrun.addtestgroup(dirname, filenames, options)
 810                 else:
 811                     for f in sorted(filenames):
 812                         testrun.addtest(os.path.join(dirname, f), options)
 813         else:
 814             testrun.addtest(p, options)
 815
 816
 817 def fail(retstr, ret=1):
 818     print('%s: %s' % (sys.argv[0], retstr))
 819     exit(ret)
 820
 821
 822 def options_cb(option, opt_str, value, parser):
 823     path_options = ['runfile', 'outputdir', 'template', 'testdir']
 824
 825     if option.dest is 'runfile' and '-w' in parser.rargs or \
 826             option.dest is 'template' and '-c' in parser.rargs:
 827         fail('-c and -w are mutually exclusive.')
 828
 829     if opt_str in parser.rargs:
 830         fail('%s may only be specified once.' % opt_str)
 831
 832     if option.dest is 'runfile':
 833         parser.values.cmd = 'rdconfig'
 834     if option.dest is 'template':
 835         parser.values.cmd = 'wrconfig'
 836     if option.dest is 'tags':
 837         value = [x.strip() for x in value.split(',')]
 838
 839     setattr(parser.values, option.dest, value)
 840     if option.dest in path_options:
 841         setattr(parser.values, option.dest, os.path.abspath(value))
 842
 843
 844 def parse_args():
 845     parser = OptionParser()
 846     parser.add_option('-c', action='callback', callback=options_cb,
 847                       type='string', dest='runfile', metavar='runfile',
 848                       help='Specify tests to run via config file.')
 849     parser.add_option('-d', action='store_true', default=False, dest='dryrun',
 850                       help='Dry run. Print tests, but take no other action.')
 851     parser.add_option('-g', action='store_true', default=False,
 852                       dest='do_groups', help='Make directories TestGroups.')
 853     parser.add_option('-o', action='callback', callback=options_cb,
 854                       default=BASEDIR, dest='outputdir', type='string',
 855                       metavar='outputdir', help='Specify an output directory.')
 856     parser.add_option('-i', action='callback', callback=options_cb,
 857                       default=TESTDIR, dest='testdir', type='string',
 858                       metavar='testdir', help='Specify a test directory.')
 859     parser.add_option('-p', action='callback', callback=options_cb,
 860                       default='', dest='pre', metavar='script',
 861                       type='string', help='Specify a pre script.')
 862     parser.add_option('-P', action='callback', callback=options_cb,
 863                       default='', dest='post', metavar='script',
 864                       type='string', help='Specify a post script.')
 865     parser.add_option('-q', action='store_true', default=False, dest='quiet',
 866                       help='Silence on the console during a test run.')
 867     parser.add_option('-t', action='callback', callback=options_cb, default=60,
 868                       dest='timeout', metavar='seconds', type='int',
 869                       help='Timeout (in seconds) for an individual test.')
 870     parser.add_option('-u', action='callback', callback=options_cb,
 871                       default='', dest='user', metavar='user', type='string',
 872                       help='Specify a different user name to run as.')
 873     parser.add_option('-w', action='callback', callback=options_cb,
 874                       default=None, dest='template', metavar='template',
 875                       type='string', help='Create a new config file.')
 876     parser.add_option('-x', action='callback', callback=options_cb, default='',
 877                       dest='pre_user', metavar='pre_user', type='string',
 878                       help='Specify a user to execute the pre script.')
 879     parser.add_option('-X', action='callback', callback=options_cb, default='',
 880                       dest='post_user', metavar='post_user', type='string',
 881                       help='Specify a user to execute the post script.')
 882     parser.add_option('-T', action='callback', callback=options_cb, default='',
 883                       dest='tags', metavar='tags', type='string',
 884                       help='Specify tags to execute specific test groups.')
 885     parser.add_option('-I', action='callback', callback=options_cb, default=1,
 886                       dest='iterations', metavar='iterations', type='int',
 887                       help='Number of times to run the test run.')
 888     (options, pathnames) = parser.parse_args()
 889
 890     if not options.runfile and not options.template:
 891         options.cmd = 'runtests'
 892
 893     if options.runfile and len(pathnames):
 894         fail('Extraneous arguments.')
 895
 896     options.pathnames = [os.path.abspath(path) for path in pathnames]
 897
 898     return options
 899
 900
 901 def main():
 902     options = parse_args()
 903     testrun = TestRun(options)
 904
 905     if options.cmd is 'runtests':
 906         find_tests(testrun, options)
 907     elif options.cmd is 'rdconfig':
 908         testrun.read(options)
 909     elif options.cmd is 'wrconfig':
 910         find_tests(testrun, options)
 911         testrun.write(options)
 912         exit(0)
 913     else:
 914         fail('Unknown command specified')
 915
 916     testrun.complete_outputdirs()
 917     testrun.run(options)
 918     exit(testrun.summary())
 919
 920
 921 if __name__ == '__main__':
 922     main()