tests/test-runner/bin/test-runner.py

   1 #!/usr/bin/python
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  16 # Copyright (c) 2017 Datto Inc.
  17 #
  18 # This script must remain compatible with Python 2.6+ and Python 3.4+.
  19 #
  20
  21 # some python 2.7 system don't have a configparser shim
  22 try:
  23     import configparser
  24 except ImportError:
  25     import ConfigParser as configparser
  26
  27 import os
  28 import sys
  29
  30 from datetime import datetime
  31 from optparse import OptionParser
  32 from pwd import getpwnam
  33 from pwd import getpwuid
  34 from select import select
  35 from subprocess import PIPE
  36 from subprocess import Popen
  37 from threading import Timer
  38 from time import time
  39
  40 BASEDIR = '/var/tmp/test_results'
  41 TESTDIR = '/usr/share/zfs/'
  42 KILL = 'kill'
  43 TRUE = 'true'
  44 SUDO = 'sudo'
  45 LOG_FILE = 'LOG_FILE'
  46 LOG_OUT = 'LOG_OUT'
  47 LOG_ERR = 'LOG_ERR'
  48 LOG_FILE_OBJ = None
  49
  50
  51 class Result(object):
  52     total = 0
  53     runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0}
  54
  55     def __init__(self):
  56         self.starttime = None
  57         self.returncode = None
  58         self.runtime = ''
  59         self.stdout = []
  60         self.stderr = []
  61         self.result = ''
  62
  63     def done(self, proc, killed):
  64         """
  65         Finalize the results of this Cmd.
  66         """
  67         Result.total += 1
  68         m, s = divmod(time() - self.starttime, 60)
  69         self.runtime = '%02d:%02d' % (m, s)
  70         self.returncode = proc.returncode
  71         if killed:
  72             self.result = 'KILLED'
  73             Result.runresults['KILLED'] += 1
  74         elif self.returncode == 0:
  75             self.result = 'PASS'
  76             Result.runresults['PASS'] += 1
  77         elif self.returncode == 4:
  78             self.result = 'SKIP'
  79             Result.runresults['SKIP'] += 1
  80         elif self.returncode != 0:
  81             self.result = 'FAIL'
  82             Result.runresults['FAIL'] += 1
  83
  84
  85 class Output(object):
  86     """
  87     This class is a slightly modified version of the 'Stream' class found
  88     here: http://goo.gl/aSGfv
  89     """
  90     def __init__(self, stream):
  91         self.stream = stream
  92         self._buf = b''
  93         self.lines = []
  94
  95     def fileno(self):
  96         return self.stream.fileno()
  97
  98     def read(self, drain=0):
  99         """
 100         Read from the file descriptor. If 'drain' set, read until EOF.
 101         """
 102         while self._read() is not None:
 103             if not drain:
 104                 break
 105
 106     def _read(self):
 107         """
 108         Read up to 4k of data from this output stream. Collect the output
 109         up to the last newline, and append it to any leftover data from a
 110         previous call. The lines are stored as a (timestamp, data) tuple
 111         for easy sorting/merging later.
 112         """
 113         fd = self.fileno()
 114         buf = os.read(fd, 4096)
 115         if not buf:
 116             return None
 117         if b'\n' not in buf:
 118             self._buf += buf
 119             return []
 120
 121         buf = self._buf + buf
 122         tmp, rest = buf.rsplit(b'\n', 1)
 123         self._buf = rest
 124         now = datetime.now()
 125         rows = tmp.split(b'\n')
 126         self.lines += [(now, r) for r in rows]
 127
 128
 129 class Cmd(object):
 130     verified_users = []
 131
 132     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 133                  tags=None):
 134         self.pathname = pathname
 135         self.outputdir = outputdir or 'BASEDIR'
 136         self.timeout = timeout
 137         self.user = user or ''
 138         self.killed = False
 139         self.result = Result()
 140
 141         if self.timeout is None:
 142             self.timeout = 60
 143
 144     def __str__(self):
 145         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % \
 146             (self.pathname, self.outputdir, self.timeout, self.user)
 147
 148     def kill_cmd(self, proc):
 149         """
 150         Kill a running command due to timeout, or ^C from the keyboard. If
 151         sudo is required, this user was verified previously.
 152         """
 153         self.killed = True
 154         do_sudo = len(self.user) != 0
 155         signal = '-TERM'
 156
 157         cmd = [SUDO, KILL, signal, str(proc.pid)]
 158         if not do_sudo:
 159             del cmd[0]
 160
 161         try:
 162             kp = Popen(cmd)
 163             kp.wait()
 164         except Exception:
 165             pass
 166
 167     def update_cmd_privs(self, cmd, user):
 168         """
 169         If a user has been specified to run this Cmd and we're not already
 170         running as that user, prepend the appropriate sudo command to run
 171         as that user.
 172         """
 173         me = getpwuid(os.getuid())
 174
 175         if not user or user is me:
 176             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 177                 cmd += '.ksh'
 178             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 179                 cmd += '.sh'
 180             return cmd
 181
 182         if not os.path.isfile(cmd):
 183             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 184                 cmd += '.ksh'
 185             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 186                 cmd += '.sh'
 187
 188         ret = '%s -E -u %s %s' % (SUDO, user, cmd)
 189         return ret.split(' ')
 190
 191     def collect_output(self, proc):
 192         """
 193         Read from stdout/stderr as data becomes available, until the
 194         process is no longer running. Return the lines from the stdout and
 195         stderr Output objects.
 196         """
 197         out = Output(proc.stdout)
 198         err = Output(proc.stderr)
 199         res = []
 200         while proc.returncode is None:
 201             proc.poll()
 202             res = select([out, err], [], [], .1)
 203             for fd in res[0]:
 204                 fd.read()
 205         for fd in res[0]:
 206             fd.read(drain=1)
 207
 208         return out.lines, err.lines
 209
 210     def run(self, options):
 211         """
 212         This is the main function that runs each individual test.
 213         Determine whether or not the command requires sudo, and modify it
 214         if needed. Run the command, and update the result object.
 215         """
 216         if options.dryrun is True:
 217             print(self)
 218             return
 219
 220         privcmd = self.update_cmd_privs(self.pathname, self.user)
 221         try:
 222             old = os.umask(0)
 223             if not os.path.isdir(self.outputdir):
 224                 os.makedirs(self.outputdir, mode=0o777)
 225             os.umask(old)
 226         except OSError as e:
 227             fail('%s' % e)
 228
 229         self.result.starttime = time()
 230         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
 231         # Allow a special timeout value of 0 to mean infinity
 232         if int(self.timeout) == 0:
 233             self.timeout = sys.maxsize
 234         t = Timer(int(self.timeout), self.kill_cmd, [proc])
 235
 236         try:
 237             t.start()
 238             self.result.stdout, self.result.stderr = self.collect_output(proc)
 239         except KeyboardInterrupt:
 240             self.kill_cmd(proc)
 241             fail('\nRun terminated at user request.')
 242         finally:
 243             t.cancel()
 244
 245         self.result.done(proc, self.killed)
 246
 247     def skip(self):
 248         """
 249         Initialize enough of the test result that we can log a skipped
 250         command.
 251         """
 252         Result.total += 1
 253         Result.runresults['SKIP'] += 1
 254         self.result.stdout = self.result.stderr = []
 255         self.result.starttime = time()
 256         m, s = divmod(time() - self.result.starttime, 60)
 257         self.result.runtime = '%02d:%02d' % (m, s)
 258         self.result.result = 'SKIP'
 259
 260     def log(self, options):
 261         """
 262         This function is responsible for writing all output. This includes
 263         the console output, the logfile of all results (with timestamped
 264         merged stdout and stderr), and for each test, the unmodified
 265         stdout/stderr/merged in it's own file.
 266         """
 267
 268         logname = getpwuid(os.getuid()).pw_name
 269         user = ' (run as %s)' % (self.user if len(self.user) else logname)
 270         msga = 'Test: %s%s ' % (self.pathname, user)
 271         msgb = '[%s] [%s]\n' % (self.result.runtime, self.result.result)
 272         pad = ' ' * (80 - (len(msga) + len(msgb)))
 273         result_line = msga + pad + msgb
 274
 275         # The result line is always written to the log file. If -q was
 276         # specified only failures are written to the console, otherwise
 277         # the result line is written to the console.
 278         write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE)
 279         if not options.quiet:
 280             write_log(result_line, LOG_OUT)
 281         elif options.quiet and self.result.result != 'PASS':
 282             write_log(result_line, LOG_OUT)
 283
 284         lines = sorted(self.result.stdout + self.result.stderr,
 285                        key=lambda x: x[0])
 286
 287         # Write timestamped output (stdout and stderr) to the logfile
 288         for dt, line in lines:
 289             timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11],
 290                                   encoding='utf-8')
 291             write_log(b'%s %s\n' % (timestamp, line), LOG_FILE)
 292
 293         # Write the separate stdout/stderr/merged files, if the data exists
 294         if len(self.result.stdout):
 295             with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out:
 296                 for _, line in self.result.stdout:
 297                     os.write(out.fileno(), b'%s\n' % line)
 298         if len(self.result.stderr):
 299             with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err:
 300                 for _, line in self.result.stderr:
 301                     os.write(err.fileno(), b'%s\n' % line)
 302         if len(self.result.stdout) and len(self.result.stderr):
 303             with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
 304                 for _, line in lines:
 305                     os.write(merged.fileno(), b'%s\n' % line)
 306
 307
 308 class Test(Cmd):
 309     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
 310              'post_user', 'tags']
 311
 312     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 313                  pre=None, pre_user=None, post=None, post_user=None,
 314                  tags=None):
 315         super(Test, self).__init__(pathname, outputdir, timeout, user)
 316         self.pre = pre or ''
 317         self.pre_user = pre_user or ''
 318         self.post = post or ''
 319         self.post_user = post_user or ''
 320         self.tags = tags or []
 321
 322     def __str__(self):
 323         post_user = pre_user = ''
 324         if len(self.pre_user):
 325             pre_user = ' (as %s)' % (self.pre_user)
 326         if len(self.post_user):
 327             post_user = ' (as %s)' % (self.post_user)
 328         return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \
 329                "%s%s\nUser: %s\nTags: %s\n" % \
 330                (self.pathname, self.outputdir, self.timeout, self.pre,
 331                 pre_user, self.post, post_user, self.user, self.tags)
 332
 333     def verify(self):
 334         """
 335         Check the pre/post scripts, user and Test. Omit the Test from this
 336         run if there are any problems.
 337         """
 338         files = [self.pre, self.pathname, self.post]
 339         users = [self.pre_user, self.user, self.post_user]
 340
 341         for f in [f for f in files if len(f)]:
 342             if not verify_file(f):
 343                 write_log("Warning: Test '%s' not added to this run because"
 344                           " it failed verification.\n" % f, LOG_ERR)
 345                 return False
 346
 347         for user in [user for user in users if len(user)]:
 348             if not verify_user(user):
 349                 write_log("Not adding Test '%s' to this run.\n" %
 350                           self.pathname, LOG_ERR)
 351                 return False
 352
 353         return True
 354
 355     def run(self, options):
 356         """
 357         Create Cmd instances for the pre/post scripts. If the pre script
 358         doesn't pass, skip this Test. Run the post script regardless.
 359         """
 360         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 361         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 362                       user=self.pre_user)
 363         test = Cmd(self.pathname, outputdir=self.outputdir,
 364                    timeout=self.timeout, user=self.user)
 365         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 366         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 367                        user=self.post_user)
 368
 369         cont = True
 370         if len(pretest.pathname):
 371             pretest.run(options)
 372             cont = pretest.result.result == 'PASS'
 373             pretest.log(options)
 374
 375         if cont:
 376             test.run(options)
 377         else:
 378             test.skip()
 379
 380         test.log(options)
 381
 382         if len(posttest.pathname):
 383             posttest.run(options)
 384             posttest.log(options)
 385
 386
 387 class TestGroup(Test):
 388     props = Test.props + ['tests']
 389
 390     def __init__(self, pathname, outputdir=None, timeout=None, user=None,
 391                  pre=None, pre_user=None, post=None, post_user=None,
 392                  tests=None, tags=None):
 393         super(TestGroup, self).__init__(pathname, outputdir, timeout, user,
 394                                         pre, pre_user, post, post_user, tags)
 395         self.tests = tests or []
 396
 397     def __str__(self):
 398         post_user = pre_user = ''
 399         if len(self.pre_user):
 400             pre_user = ' (as %s)' % (self.pre_user)
 401         if len(self.post_user):
 402             post_user = ' (as %s)' % (self.post_user)
 403         return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %s\n" \
 404                "Pre: %s%s\nPost: %s%s\nUser: %s\nTags: %s\n" % \
 405                (self.pathname, self.outputdir, self.tests, self.timeout,
 406                 self.pre, pre_user, self.post, post_user, self.user, self.tags)
 407
 408     def verify(self):
 409         """
 410         Check the pre/post scripts, user and tests in this TestGroup. Omit
 411         the TestGroup entirely, or simply delete the relevant tests in the
 412         group, if that's all that's required.
 413         """
 414         # If the pre or post scripts are relative pathnames, convert to
 415         # absolute, so they stand a chance of passing verification.
 416         if len(self.pre) and not os.path.isabs(self.pre):
 417             self.pre = os.path.join(self.pathname, self.pre)
 418         if len(self.post) and not os.path.isabs(self.post):
 419             self.post = os.path.join(self.pathname, self.post)
 420
 421         auxfiles = [self.pre, self.post]
 422         users = [self.pre_user, self.user, self.post_user]
 423
 424         for f in [f for f in auxfiles if len(f)]:
 425             if self.pathname != os.path.dirname(f):
 426                 write_log("Warning: TestGroup '%s' not added to this run. "
 427                           "Auxiliary script '%s' exists in a different "
 428                           "directory.\n" % (self.pathname, f), LOG_ERR)
 429                 return False
 430
 431             if not verify_file(f):
 432                 write_log("Warning: TestGroup '%s' not added to this run. "
 433                           "Auxiliary script '%s' failed verification.\n" %
 434                           (self.pathname, f), LOG_ERR)
 435                 return False
 436
 437         for user in [user for user in users if len(user)]:
 438             if not verify_user(user):
 439                 write_log("Not adding TestGroup '%s' to this run.\n" %
 440                           self.pathname, LOG_ERR)
 441                 return False
 442
 443         # If one of the tests is invalid, delete it, log it, and drive on.
 444         for test in self.tests:
 445             if not verify_file(os.path.join(self.pathname, test)):
 446                 del self.tests[self.tests.index(test)]
 447                 write_log("Warning: Test '%s' removed from TestGroup '%s' "
 448                           "because it failed verification.\n" %
 449                           (test, self.pathname), LOG_ERR)
 450
 451         return len(self.tests) != 0
 452
 453     def run(self, options):
 454         """
 455         Create Cmd instances for the pre/post scripts. If the pre script
 456         doesn't pass, skip all the tests in this TestGroup. Run the post
 457         script regardless.
 458         """
 459         # tags assigned to this test group also include the test names
 460         if options.tags and not set(self.tags).intersection(set(options.tags)):
 461             return
 462
 463         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 464         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 465                       user=self.pre_user)
 466         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 467         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 468                        user=self.post_user)
 469
 470         cont = True
 471         if len(pretest.pathname):
 472             pretest.run(options)
 473             cont = pretest.result.result == 'PASS'
 474             pretest.log(options)
 475
 476         for fname in self.tests:
 477             test = Cmd(os.path.join(self.pathname, fname),
 478                        outputdir=os.path.join(self.outputdir, fname),
 479                        timeout=self.timeout, user=self.user)
 480             if cont:
 481                 test.run(options)
 482             else:
 483                 test.skip()
 484
 485             test.log(options)
 486
 487         if len(posttest.pathname):
 488             posttest.run(options)
 489             posttest.log(options)
 490
 491
 492 class TestRun(object):
 493     props = ['quiet', 'outputdir']
 494
 495     def __init__(self, options):
 496         self.tests = {}
 497         self.testgroups = {}
 498         self.starttime = time()
 499         self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
 500         self.outputdir = os.path.join(options.outputdir, self.timestamp)
 501         self.setup_logging(options)
 502         self.defaults = [
 503             ('outputdir', BASEDIR),
 504             ('quiet', False),
 505             ('timeout', 60),
 506             ('user', ''),
 507             ('pre', ''),
 508             ('pre_user', ''),
 509             ('post', ''),
 510             ('post_user', ''),
 511             ('tags', [])
 512         ]
 513
 514     def __str__(self):
 515         s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
 516         s += 'TESTS:\n'
 517         for key in sorted(self.tests.keys()):
 518             s += '%s%s' % (self.tests[key].__str__(), '\n')
 519         s += 'TESTGROUPS:\n'
 520         for key in sorted(self.testgroups.keys()):
 521             s += '%s%s' % (self.testgroups[key].__str__(), '\n')
 522         return s
 523
 524     def addtest(self, pathname, options):
 525         """
 526         Create a new Test, and apply any properties that were passed in
 527         from the command line. If it passes verification, add it to the
 528         TestRun.
 529         """
 530         test = Test(pathname)
 531         for prop in Test.props:
 532             setattr(test, prop, getattr(options, prop))
 533
 534         if test.verify():
 535             self.tests[pathname] = test
 536
 537     def addtestgroup(self, dirname, filenames, options):
 538         """
 539         Create a new TestGroup, and apply any properties that were passed
 540         in from the command line. If it passes verification, add it to the
 541         TestRun.
 542         """
 543         if dirname not in self.testgroups:
 544             testgroup = TestGroup(dirname)
 545             for prop in Test.props:
 546                 setattr(testgroup, prop, getattr(options, prop))
 547
 548             # Prevent pre/post scripts from running as regular tests
 549             for f in [testgroup.pre, testgroup.post]:
 550                 if f in filenames:
 551                     del filenames[filenames.index(f)]
 552
 553             self.testgroups[dirname] = testgroup
 554             self.testgroups[dirname].tests = sorted(filenames)
 555
 556             testgroup.verify()
 557
 558     def read(self, options):
 559         """
 560         Read in the specified runfile, and apply the TestRun properties
 561         listed in the 'DEFAULT' section to our TestRun. Then read each
 562         section, and apply the appropriate properties to the Test or
 563         TestGroup. Properties from individual sections override those set
 564         in the 'DEFAULT' section. If the Test or TestGroup passes
 565         verification, add it to the TestRun.
 566         """
 567         config = configparser.RawConfigParser()
 568         if not len(config.read(options.runfile)):
 569             fail("Coulnd't read config file %s" % options.runfile)
 570
 571         for opt in TestRun.props:
 572             if config.has_option('DEFAULT', opt):
 573                 setattr(self, opt, config.get('DEFAULT', opt))
 574         self.outputdir = os.path.join(self.outputdir, self.timestamp)
 575
 576         for section in config.sections():
 577             if 'tests' in config.options(section):
 578                 if os.path.isdir(section):
 579                     pathname = section
 580                 elif os.path.isdir(os.path.join(options.testdir, section)):
 581                     pathname = os.path.join(options.testdir, section)
 582                 else:
 583                     pathname = section
 584
 585                 testgroup = TestGroup(os.path.abspath(pathname))
 586                 for prop in TestGroup.props:
 587                     for sect in ['DEFAULT', section]:
 588                         if config.has_option(sect, prop):
 589                             if prop == "tags":
 590                                 setattr(testgroup, prop,
 591                                         eval(config.get(sect, prop)))
 592                             else:
 593                                 setattr(testgroup, prop,
 594                                         config.get(sect, prop))
 595
 596                 # Repopulate tests using eval to convert the string to a list
 597                 testgroup.tests = eval(config.get(section, 'tests'))
 598
 599                 if testgroup.verify():
 600                     self.testgroups[section] = testgroup
 601             else:
 602                 test = Test(section)
 603                 for prop in Test.props:
 604                     for sect in ['DEFAULT', section]:
 605                         if config.has_option(sect, prop):
 606                             setattr(test, prop, config.get(sect, prop))
 607
 608                 if test.verify():
 609                     self.tests[section] = test
 610
 611     def write(self, options):
 612         """
 613         Create a configuration file for editing and later use. The
 614         'DEFAULT' section of the config file is created from the
 615         properties that were specified on the command line. Tests are
 616         simply added as sections that inherit everything from the
 617         'DEFAULT' section. TestGroups are the same, except they get an
 618         option including all the tests to run in that directory.
 619         """
 620
 621         defaults = dict([(prop, getattr(options, prop)) for prop, _ in
 622                          self.defaults])
 623         config = configparser.RawConfigParser(defaults)
 624
 625         for test in sorted(self.tests.keys()):
 626             config.add_section(test)
 627
 628         for testgroup in sorted(self.testgroups.keys()):
 629             config.add_section(testgroup)
 630             config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
 631
 632         try:
 633             with open(options.template, 'w') as f:
 634                 return config.write(f)
 635         except IOError:
 636             fail('Could not open \'%s\' for writing.' % options.template)
 637
 638     def complete_outputdirs(self):
 639         """
 640         Collect all the pathnames for Tests, and TestGroups. Work
 641         backwards one pathname component at a time, to create a unique
 642         directory name in which to deposit test output. Tests will be able
 643         to write output files directly in the newly modified outputdir.
 644         TestGroups will be able to create one subdirectory per test in the
 645         outputdir, and are guaranteed uniqueness because a group can only
 646         contain files in one directory. Pre and post tests will create a
 647         directory rooted at the outputdir of the Test or TestGroup in
 648         question for their output.
 649         """
 650         done = False
 651         components = 0
 652         tmp_dict = dict(list(self.tests.items()) +
 653                         list(self.testgroups.items()))
 654         total = len(tmp_dict)
 655         base = self.outputdir
 656
 657         while not done:
 658             paths = []
 659             components -= 1
 660             for testfile in list(tmp_dict.keys()):
 661                 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
 662                 if uniq not in paths:
 663                     paths.append(uniq)
 664                     tmp_dict[testfile].outputdir = os.path.join(base, uniq)
 665                 else:
 666                     break
 667             done = total == len(paths)
 668
 669     def setup_logging(self, options):
 670         """
 671         This funtion creates the output directory and gets a file object
 672         for the logfile. This function must be called before write_log()
 673         can be used.
 674         """
 675         if options.dryrun is True:
 676             return
 677
 678         global LOG_FILE_OBJ
 679         if options.cmd != 'wrconfig':
 680             try:
 681                 old = os.umask(0)
 682                 os.makedirs(self.outputdir, mode=0o777)
 683                 os.umask(old)
 684                 filename = os.path.join(self.outputdir, 'log')
 685                 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb')
 686             except OSError as e:
 687                 fail('%s' % e)
 688
 689     def run(self, options):
 690         """
 691         Walk through all the Tests and TestGroups, calling run().
 692         """
 693         try:
 694             os.chdir(self.outputdir)
 695         except OSError:
 696             fail('Could not change to directory %s' % self.outputdir)
 697         # make a symlink to the output for the currently running test
 698         logsymlink = os.path.join(self.outputdir, '../current')
 699         if os.path.islink(logsymlink):
 700             os.unlink(logsymlink)
 701         if not os.path.exists(logsymlink):
 702             os.symlink(self.outputdir, logsymlink)
 703         else:
 704             write_log('Could not make a symlink to directory %s\n' %
 705                       self.outputdir, LOG_ERR)
 706         iteration = 0
 707         while iteration < options.iterations:
 708             for test in sorted(self.tests.keys()):
 709                 self.tests[test].run(options)
 710             for testgroup in sorted(self.testgroups.keys()):
 711                 self.testgroups[testgroup].run(options)
 712             iteration += 1
 713
 714     def summary(self):
 715         if Result.total == 0:
 716             return 2
 717
 718         print('\nResults Summary')
 719         for key in list(Result.runresults.keys()):
 720             if Result.runresults[key] != 0:
 721                 print('%s\t% 4d' % (key, Result.runresults[key]))
 722
 723         m, s = divmod(time() - self.starttime, 60)
 724         h, m = divmod(m, 60)
 725         print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
 726         print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
 727                                             float(Result.total)) * 100))
 728         print('Log directory:\t%s' % self.outputdir)
 729
 730         if Result.runresults['FAIL'] > 0:
 731             return 1
 732
 733         if Result.runresults['KILLED'] > 0:
 734             return 1
 735
 736         return 0
 737
 738
 739 def write_log(msg, target):
 740     """
 741     Write the provided message to standard out, standard error or
 742     the logfile. If specifying LOG_FILE, then `msg` must be a bytes
 743     like object. This way we can still handle output from tests that
 744     may be in unexpected encodings.
 745     """
 746     if target == LOG_OUT:
 747         os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8'))
 748     elif target == LOG_ERR:
 749         os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8'))
 750     elif target == LOG_FILE:
 751         os.write(LOG_FILE_OBJ.fileno(), msg)
 752     else:
 753         fail('log_msg called with unknown target "%s"' % target)
 754
 755
 756 def verify_file(pathname):
 757     """
 758     Verify that the supplied pathname is an executable regular file.
 759     """
 760     if os.path.isdir(pathname) or os.path.islink(pathname):
 761         return False
 762
 763     for ext in '', '.ksh', '.sh':
 764         script_path = pathname + ext
 765         if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
 766             return True
 767
 768     return False
 769
 770
 771 def verify_user(user):
 772     """
 773     Verify that the specified user exists on this system, and can execute
 774     sudo without being prompted for a password.
 775     """
 776     testcmd = [SUDO, '-n', '-u', user, TRUE]
 777
 778     if user in Cmd.verified_users:
 779         return True
 780
 781     try:
 782         getpwnam(user)
 783     except KeyError:
 784         write_log("Warning: user '%s' does not exist.\n" % user,
 785                   LOG_ERR)
 786         return False
 787
 788     p = Popen(testcmd)
 789     p.wait()
 790     if p.returncode != 0:
 791         write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user,
 792                   LOG_ERR)
 793         return False
 794     else:
 795         Cmd.verified_users.append(user)
 796
 797     return True
 798
 799
 800 def find_tests(testrun, options):
 801     """
 802     For the given list of pathnames, add files as Tests. For directories,
 803     if do_groups is True, add the directory as a TestGroup. If False,
 804     recursively search for executable files.
 805     """
 806
 807     for p in sorted(options.pathnames):
 808         if os.path.isdir(p):
 809             for dirname, _, filenames in os.walk(p):
 810                 if options.do_groups:
 811                     testrun.addtestgroup(dirname, filenames, options)
 812                 else:
 813                     for f in sorted(filenames):
 814                         testrun.addtest(os.path.join(dirname, f), options)
 815         else:
 816             testrun.addtest(p, options)
 817
 818
 819 def fail(retstr, ret=1):
 820     print('%s: %s' % (sys.argv[0], retstr))
 821     exit(ret)
 822
 823
 824 def options_cb(option, opt_str, value, parser):
 825     path_options = ['runfile', 'outputdir', 'template', 'testdir']
 826
 827     if option.dest == 'runfile' and '-w' in parser.rargs or \
 828             option.dest == 'template' and '-c' in parser.rargs:
 829         fail('-c and -w are mutually exclusive.')
 830
 831     if opt_str in parser.rargs:
 832         fail('%s may only be specified once.' % opt_str)
 833
 834     if option.dest == 'runfile':
 835         parser.values.cmd = 'rdconfig'
 836     if option.dest == 'template':
 837         parser.values.cmd = 'wrconfig'
 838     if option.dest == 'tags':
 839         value = [x.strip() for x in value.split(',')]
 840
 841     setattr(parser.values, option.dest, value)
 842     if option.dest in path_options:
 843         setattr(parser.values, option.dest, os.path.abspath(value))
 844
 845
 846 def parse_args():
 847     parser = OptionParser()
 848     parser.add_option('-c', action='callback', callback=options_cb,
 849                       type='string', dest='runfile', metavar='runfile',
 850                       help='Specify tests to run via config file.')
 851     parser.add_option('-d', action='store_true', default=False, dest='dryrun',
 852                       help='Dry run. Print tests, but take no other action.')
 853     parser.add_option('-g', action='store_true', default=False,
 854                       dest='do_groups', help='Make directories TestGroups.')
 855     parser.add_option('-o', action='callback', callback=options_cb,
 856                       default=BASEDIR, dest='outputdir', type='string',
 857                       metavar='outputdir', help='Specify an output directory.')
 858     parser.add_option('-i', action='callback', callback=options_cb,
 859                       default=TESTDIR, dest='testdir', type='string',
 860                       metavar='testdir', help='Specify a test directory.')
 861     parser.add_option('-p', action='callback', callback=options_cb,
 862                       default='', dest='pre', metavar='script',
 863                       type='string', help='Specify a pre script.')
 864     parser.add_option('-P', action='callback', callback=options_cb,
 865                       default='', dest='post', metavar='script',
 866                       type='string', help='Specify a post script.')
 867     parser.add_option('-q', action='store_true', default=False, dest='quiet',
 868                       help='Silence on the console during a test run.')
 869     parser.add_option('-t', action='callback', callback=options_cb, default=60,
 870                       dest='timeout', metavar='seconds', type='int',
 871                       help='Timeout (in seconds) for an individual test.')
 872     parser.add_option('-u', action='callback', callback=options_cb,
 873                       default='', dest='user', metavar='user', type='string',
 874                       help='Specify a different user name to run as.')
 875     parser.add_option('-w', action='callback', callback=options_cb,
 876                       default=None, dest='template', metavar='template',
 877                       type='string', help='Create a new config file.')
 878     parser.add_option('-x', action='callback', callback=options_cb, default='',
 879                       dest='pre_user', metavar='pre_user', type='string',
 880                       help='Specify a user to execute the pre script.')
 881     parser.add_option('-X', action='callback', callback=options_cb, default='',
 882                       dest='post_user', metavar='post_user', type='string',
 883                       help='Specify a user to execute the post script.')
 884     parser.add_option('-T', action='callback', callback=options_cb, default='',
 885                       dest='tags', metavar='tags', type='string',
 886                       help='Specify tags to execute specific test groups.')
 887     parser.add_option('-I', action='callback', callback=options_cb, default=1,
 888                       dest='iterations', metavar='iterations', type='int',
 889                       help='Number of times to run the test run.')
 890     (options, pathnames) = parser.parse_args()
 891
 892     if not options.runfile and not options.template:
 893         options.cmd = 'runtests'
 894
 895     if options.runfile and len(pathnames):
 896         fail('Extraneous arguments.')
 897
 898     options.pathnames = [os.path.abspath(path) for path in pathnames]
 899
 900     return options
 901
 902
 903 def main():
 904     options = parse_args()
 905     testrun = TestRun(options)
 906
 907     if options.cmd == 'runtests':
 908         find_tests(testrun, options)
 909     elif options.cmd == 'rdconfig':
 910         testrun.read(options)
 911     elif options.cmd == 'wrconfig':
 912         find_tests(testrun, options)
 913         testrun.write(options)
 914         exit(0)
 915     else:
 916         fail('Unknown command specified')
 917
 918     testrun.complete_outputdirs()
 919     testrun.run(options)
 920     exit(testrun.summary())
 921
 922
 923 if __name__ == '__main__':
 924     main()