tests/test-runner/bin/test-runner.py.in

   1 #!/usr/bin/env @PYTHON_SHEBANG@
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  16 # Copyright (c) 2019 Datto Inc.
  17 #
  18 # This script must remain compatible with Python 2.6+ and Python 3.4+.
  19 #
  20
  21 # some python 2.7 system don't have a configparser shim
  22 try:
  23     import configparser
  24 except ImportError:
  25     import ConfigParser as configparser
  26
  27 import os
  28 import sys
  29 import ctypes
  30 import re
  31
  32 from datetime import datetime
  33 from optparse import OptionParser
  34 from pwd import getpwnam
  35 from pwd import getpwuid
  36 from select import select
  37 from subprocess import PIPE
  38 from subprocess import Popen
  39 from threading import Timer
  40 from time import time
  41
  42 BASEDIR = '/var/tmp/test_results'
  43 TESTDIR = '/usr/share/zfs/'
  44 KILL = 'kill'
  45 TRUE = 'true'
  46 SUDO = 'sudo'
  47 LOG_FILE = 'LOG_FILE'
  48 LOG_OUT = 'LOG_OUT'
  49 LOG_ERR = 'LOG_ERR'
  50 LOG_FILE_OBJ = None
  51
  52 # some python 2.7 system don't have a concept of monotonic time
  53 CLOCK_MONOTONIC_RAW = 4  # see <linux/time.h>
  54
  55
  56 class timespec(ctypes.Structure):
  57     _fields_ = [
  58         ('tv_sec', ctypes.c_long),
  59         ('tv_nsec', ctypes.c_long)
  60     ]
  61
  62
  63 librt = ctypes.CDLL('librt.so.1', use_errno=True)
  64 clock_gettime = librt.clock_gettime
  65 clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)]
  66
  67
  68 def monotonic_time():
  69     t = timespec()
  70     if clock_gettime(CLOCK_MONOTONIC_RAW, ctypes.pointer(t)) != 0:
  71         errno_ = ctypes.get_errno()
  72         raise OSError(errno_, os.strerror(errno_))
  73     return t.tv_sec + t.tv_nsec * 1e-9
  74
  75
  76 class Result(object):
  77     total = 0
  78     runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0}
  79
  80     def __init__(self):
  81         self.starttime = None
  82         self.returncode = None
  83         self.runtime = ''
  84         self.stdout = []
  85         self.stderr = []
  86         self.result = ''
  87
  88     def done(self, proc, killed, reran):
  89         """
  90         Finalize the results of this Cmd.
  91         """
  92         Result.total += 1
  93         m, s = divmod(monotonic_time() - self.starttime, 60)
  94         self.runtime = '%02d:%02d' % (m, s)
  95         self.returncode = proc.returncode
  96         if reran is True:
  97             Result.runresults['RERAN'] += 1
  98         if killed:
  99             self.result = 'KILLED'
 100             Result.runresults['KILLED'] += 1
 101         elif self.returncode == 0:
 102             self.result = 'PASS'
 103             Result.runresults['PASS'] += 1
 104         elif self.returncode == 4:
 105             self.result = 'SKIP'
 106             Result.runresults['SKIP'] += 1
 107         elif self.returncode != 0:
 108             self.result = 'FAIL'
 109             Result.runresults['FAIL'] += 1
 110
 111
 112 class Output(object):
 113     """
 114     This class is a slightly modified version of the 'Stream' class found
 115     here: http://goo.gl/aSGfv
 116     """
 117     def __init__(self, stream):
 118         self.stream = stream
 119         self._buf = b''
 120         self.lines = []
 121
 122     def fileno(self):
 123         return self.stream.fileno()
 124
 125     def read(self, drain=0):
 126         """
 127         Read from the file descriptor. If 'drain' set, read until EOF.
 128         """
 129         while self._read() is not None:
 130             if not drain:
 131                 break
 132
 133     def _read(self):
 134         """
 135         Read up to 4k of data from this output stream. Collect the output
 136         up to the last newline, and append it to any leftover data from a
 137         previous call. The lines are stored as a (timestamp, data) tuple
 138         for easy sorting/merging later.
 139         """
 140         fd = self.fileno()
 141         buf = os.read(fd, 4096)
 142         if not buf:
 143             return None
 144         if b'\n' not in buf:
 145             self._buf += buf
 146             return []
 147
 148         buf = self._buf + buf
 149         tmp, rest = buf.rsplit(b'\n', 1)
 150         self._buf = rest
 151         now = datetime.now()
 152         rows = tmp.split(b'\n')
 153         self.lines += [(now, r) for r in rows]
 154
 155
 156 class Cmd(object):
 157     verified_users = []
 158
 159     def __init__(self, pathname, identifier=None, outputdir=None,
 160                  timeout=None, user=None, tags=None):
 161         self.pathname = pathname
 162         self.identifier = identifier
 163         self.outputdir = outputdir or 'BASEDIR'
 164         """
 165         The timeout for tests is measured in wall-clock time
 166         """
 167         self.timeout = timeout
 168         self.user = user or ''
 169         self.killed = False
 170         self.reran = None
 171         self.result = Result()
 172
 173         if self.timeout is None:
 174             self.timeout = 60
 175
 176     def __str__(self):
 177         return '''\
 178 Pathname: %s
 179 Identifier: %s
 180 Outputdir: %s
 181 Timeout: %d
 182 User: %s
 183 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user)
 184
 185     def kill_cmd(self, proc, keyboard_interrupt=False):
 186         """
 187         Kill a running command due to timeout, or ^C from the keyboard. If
 188         sudo is required, this user was verified previously.
 189         """
 190         self.killed = True
 191         do_sudo = len(self.user) != 0
 192         signal = '-TERM'
 193
 194         cmd = [SUDO, KILL, signal, str(proc.pid)]
 195         if not do_sudo:
 196             del cmd[0]
 197
 198         try:
 199             kp = Popen(cmd)
 200             kp.wait()
 201         except Exception:
 202             pass
 203
 204         """
 205         If this is not a user-initiated kill and the test has not been
 206         reran before we consider if the test needs to be reran:
 207         If the test has spent some time hibernating and didn't run the whole
 208         length of time before being timed out we will rerun the test.
 209         """
 210         if keyboard_interrupt is False and self.reran is None:
 211             runtime = monotonic_time() - self.result.starttime
 212             if int(self.timeout) > runtime:
 213                 self.killed = False
 214                 self.reran = False
 215                 self.run(False)
 216                 self.reran = True
 217
 218     def update_cmd_privs(self, cmd, user):
 219         """
 220         If a user has been specified to run this Cmd and we're not already
 221         running as that user, prepend the appropriate sudo command to run
 222         as that user.
 223         """
 224         me = getpwuid(os.getuid())
 225
 226         if not user or user is me:
 227             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 228                 cmd += '.ksh'
 229             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 230                 cmd += '.sh'
 231             return cmd
 232
 233         if not os.path.isfile(cmd):
 234             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 235                 cmd += '.ksh'
 236             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 237                 cmd += '.sh'
 238
 239         ret = '%s -E -u %s %s' % (SUDO, user, cmd)
 240         return ret.split(' ')
 241
 242     def collect_output(self, proc):
 243         """
 244         Read from stdout/stderr as data becomes available, until the
 245         process is no longer running. Return the lines from the stdout and
 246         stderr Output objects.
 247         """
 248         out = Output(proc.stdout)
 249         err = Output(proc.stderr)
 250         res = []
 251         while proc.returncode is None:
 252             proc.poll()
 253             res = select([out, err], [], [], .1)
 254             for fd in res[0]:
 255                 fd.read()
 256         for fd in res[0]:
 257             fd.read(drain=1)
 258
 259         return out.lines, err.lines
 260
 261     def run(self, dryrun):
 262         """
 263         This is the main function that runs each individual test.
 264         Determine whether or not the command requires sudo, and modify it
 265         if needed. Run the command, and update the result object.
 266         """
 267         if dryrun is True:
 268             print(self)
 269             return
 270
 271         privcmd = self.update_cmd_privs(self.pathname, self.user)
 272         try:
 273             old = os.umask(0)
 274             if not os.path.isdir(self.outputdir):
 275                 os.makedirs(self.outputdir, mode=0o777)
 276             os.umask(old)
 277         except OSError as e:
 278             fail('%s' % e)
 279
 280         self.result.starttime = monotonic_time()
 281         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
 282         # Allow a special timeout value of 0 to mean infinity
 283         if int(self.timeout) == 0:
 284             self.timeout = sys.maxsize
 285         t = Timer(int(self.timeout), self.kill_cmd, [proc])
 286
 287         try:
 288             t.start()
 289             self.result.stdout, self.result.stderr = self.collect_output(proc)
 290         except KeyboardInterrupt:
 291             self.kill_cmd(proc, True)
 292             fail('\nRun terminated at user request.')
 293         finally:
 294             t.cancel()
 295
 296         if self.reran is not False:
 297             self.result.done(proc, self.killed, self.reran)
 298
 299     def skip(self):
 300         """
 301         Initialize enough of the test result that we can log a skipped
 302         command.
 303         """
 304         Result.total += 1
 305         Result.runresults['SKIP'] += 1
 306         self.result.stdout = self.result.stderr = []
 307         self.result.starttime = monotonic_time()
 308         m, s = divmod(monotonic_time() - self.result.starttime, 60)
 309         self.result.runtime = '%02d:%02d' % (m, s)
 310         self.result.result = 'SKIP'
 311
 312     def log(self, options, suppress_console=False):
 313         """
 314         This function is responsible for writing all output. This includes
 315         the console output, the logfile of all results (with timestamped
 316         merged stdout and stderr), and for each test, the unmodified
 317         stdout/stderr/merged in its own file.
 318         """
 319
 320         logname = getpwuid(os.getuid()).pw_name
 321         rer = ''
 322         if self.reran is True:
 323             rer = ' (RERAN)'
 324         user = ' (run as %s)' % (self.user if len(self.user) else logname)
 325         if self.identifier:
 326             msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user)
 327         else:
 328             msga = 'Test: %s%s ' % (self.pathname, user)
 329         msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer)
 330         pad = ' ' * (80 - (len(msga) + len(msgb)))
 331         result_line = msga + pad + msgb
 332
 333         # The result line is always written to the log file. If -q was
 334         # specified only failures are written to the console, otherwise
 335         # the result line is written to the console. The console output
 336         # may be suppressed by calling log() with suppress_console=True.
 337         write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE)
 338         if not suppress_console:
 339             if not options.quiet:
 340                 write_log(result_line, LOG_OUT)
 341             elif options.quiet and self.result.result != 'PASS':
 342                 write_log(result_line, LOG_OUT)
 343
 344         lines = sorted(self.result.stdout + self.result.stderr,
 345                        key=lambda x: x[0])
 346
 347         # Write timestamped output (stdout and stderr) to the logfile
 348         for dt, line in lines:
 349             timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11],
 350                                   encoding='utf-8')
 351             write_log(b'%s %s\n' % (timestamp, line), LOG_FILE)
 352
 353         # Write the separate stdout/stderr/merged files, if the data exists
 354         if len(self.result.stdout):
 355             with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out:
 356                 for _, line in self.result.stdout:
 357                     os.write(out.fileno(), b'%s\n' % line)
 358         if len(self.result.stderr):
 359             with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err:
 360                 for _, line in self.result.stderr:
 361                     os.write(err.fileno(), b'%s\n' % line)
 362         if len(self.result.stdout) and len(self.result.stderr):
 363             with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
 364                 for _, line in lines:
 365                     os.write(merged.fileno(), b'%s\n' % line)
 366
 367
 368 class Test(Cmd):
 369     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
 370              'post_user', 'failsafe', 'failsafe_user', 'tags']
 371
 372     def __init__(self, pathname,
 373                  pre=None, pre_user=None, post=None, post_user=None,
 374                  failsafe=None, failsafe_user=None, tags=None, **kwargs):
 375         super(Test, self).__init__(pathname, **kwargs)
 376         self.pre = pre or ''
 377         self.pre_user = pre_user or ''
 378         self.post = post or ''
 379         self.post_user = post_user or ''
 380         self.failsafe = failsafe or ''
 381         self.failsafe_user = failsafe_user or ''
 382         self.tags = tags or []
 383
 384     def __str__(self):
 385         post_user = pre_user = failsafe_user = ''
 386         if len(self.pre_user):
 387             pre_user = ' (as %s)' % (self.pre_user)
 388         if len(self.post_user):
 389             post_user = ' (as %s)' % (self.post_user)
 390         if len(self.failsafe_user):
 391             failsafe_user = ' (as %s)' % (self.failsafe_user)
 392         return '''\
 393 Pathname: %s
 394 Identifier: %s
 395 Outputdir: %s
 396 Timeout: %d
 397 User: %s
 398 Pre: %s%s
 399 Post: %s%s
 400 Failsafe: %s%s
 401 Tags: %s
 402 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user,
 403             self.pre, pre_user, self.post, post_user, self.failsafe,
 404             failsafe_user, self.tags)
 405
 406     def verify(self):
 407         """
 408         Check the pre/post/failsafe scripts, user and Test. Omit the Test from
 409         this run if there are any problems.
 410         """
 411         files = [self.pre, self.pathname, self.post, self.failsafe]
 412         users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
 413
 414         for f in [f for f in files if len(f)]:
 415             if not verify_file(f):
 416                 write_log("Warning: Test '%s' not added to this run because"
 417                           " it failed verification.\n" % f, LOG_ERR)
 418                 return False
 419
 420         for user in [user for user in users if len(user)]:
 421             if not verify_user(user):
 422                 write_log("Not adding Test '%s' to this run.\n" %
 423                           self.pathname, LOG_ERR)
 424                 return False
 425
 426         return True
 427
 428     def run(self, options):
 429         """
 430         Create Cmd instances for the pre/post/failsafe scripts. If the pre
 431         script doesn't pass, skip this Test. Run the post script regardless.
 432         If the Test is killed, also run the failsafe script.
 433         """
 434         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 435         pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir,
 436                       timeout=self.timeout, user=self.pre_user)
 437         test = Cmd(self.pathname, identifier=self.identifier,
 438                    outputdir=self.outputdir, timeout=self.timeout,
 439                    user=self.user)
 440         odir = os.path.join(self.outputdir, os.path.basename(self.failsafe))
 441         failsafe = Cmd(self.failsafe, identifier=self.identifier,
 442                        outputdir=odir, timeout=self.timeout,
 443                        user=self.failsafe_user)
 444         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 445         posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir,
 446                        timeout=self.timeout, user=self.post_user)
 447
 448         cont = True
 449         if len(pretest.pathname):
 450             pretest.run(options.dryrun)
 451             cont = pretest.result.result == 'PASS'
 452             pretest.log(options)
 453
 454         if cont:
 455             test.run(options.dryrun)
 456             if test.result.result == 'KILLED' and len(failsafe.pathname):
 457                 failsafe.run(options.dryrun)
 458                 failsafe.log(options, suppress_console=True)
 459         else:
 460             test.skip()
 461
 462         test.log(options)
 463
 464         if len(posttest.pathname):
 465             posttest.run(options.dryrun)
 466             posttest.log(options)
 467
 468
 469 class TestGroup(Test):
 470     props = Test.props + ['tests']
 471
 472     def __init__(self, pathname, tests=None, **kwargs):
 473         super(TestGroup, self).__init__(pathname, **kwargs)
 474         self.tests = tests or []
 475
 476     def __str__(self):
 477         post_user = pre_user = failsafe_user = ''
 478         if len(self.pre_user):
 479             pre_user = ' (as %s)' % (self.pre_user)
 480         if len(self.post_user):
 481             post_user = ' (as %s)' % (self.post_user)
 482         if len(self.failsafe_user):
 483             failsafe_user = ' (as %s)' % (self.failsafe_user)
 484         return '''\
 485 Pathname: %s
 486 Identifier: %s
 487 Outputdir: %s
 488 Tests: %s
 489 Timeout: %s
 490 User: %s
 491 Pre: %s%s
 492 Post: %s%s
 493 Failsafe: %s%s
 494 Tags: %s
 495 ''' % (self.pathname, self.identifier, self.outputdir, self.tests,
 496             self.timeout, self.user, self.pre, pre_user, self.post, post_user,
 497             self.failsafe, failsafe_user, self.tags)
 498
 499     def filter(self, keeplist):
 500         self.tests = [x for x in self.tests if x in keeplist]
 501
 502     def verify(self):
 503         """
 504         Check the pre/post/failsafe scripts, user and tests in this TestGroup.
 505         Omit the TestGroup entirely, or simply delete the relevant tests in the
 506         group, if that's all that's required.
 507         """
 508         # If the pre/post/failsafe scripts are relative pathnames, convert to
 509         # absolute, so they stand a chance of passing verification.
 510         if len(self.pre) and not os.path.isabs(self.pre):
 511             self.pre = os.path.join(self.pathname, self.pre)
 512         if len(self.post) and not os.path.isabs(self.post):
 513             self.post = os.path.join(self.pathname, self.post)
 514         if len(self.failsafe) and not os.path.isabs(self.failsafe):
 515             self.post = os.path.join(self.pathname, self.post)
 516
 517         auxfiles = [self.pre, self.post, self.failsafe]
 518         users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
 519
 520         for f in [f for f in auxfiles if len(f)]:
 521             if f != self.failsafe and self.pathname != os.path.dirname(f):
 522                 write_log("Warning: TestGroup '%s' not added to this run. "
 523                           "Auxiliary script '%s' exists in a different "
 524                           "directory.\n" % (self.pathname, f), LOG_ERR)
 525                 return False
 526
 527             if not verify_file(f):
 528                 write_log("Warning: TestGroup '%s' not added to this run. "
 529                           "Auxiliary script '%s' failed verification.\n" %
 530                           (self.pathname, f), LOG_ERR)
 531                 return False
 532
 533         for user in [user for user in users if len(user)]:
 534             if not verify_user(user):
 535                 write_log("Not adding TestGroup '%s' to this run.\n" %
 536                           self.pathname, LOG_ERR)
 537                 return False
 538
 539         # If one of the tests is invalid, delete it, log it, and drive on.
 540         for test in self.tests:
 541             if not verify_file(os.path.join(self.pathname, test)):
 542                 del self.tests[self.tests.index(test)]
 543                 write_log("Warning: Test '%s' removed from TestGroup '%s' "
 544                           "because it failed verification.\n" %
 545                           (test, self.pathname), LOG_ERR)
 546
 547         return len(self.tests) != 0
 548
 549     def run(self, options):
 550         """
 551         Create Cmd instances for the pre/post/failsafe scripts. If the pre
 552         script doesn't pass, skip all the tests in this TestGroup. Run the
 553         post script regardless. Run the failsafe script when a test is killed.
 554         """
 555         # tags assigned to this test group also include the test names
 556         if options.tags and not set(self.tags).intersection(set(options.tags)):
 557             return
 558
 559         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 560         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 561                       user=self.pre_user, identifier=self.identifier)
 562         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 563         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 564                        user=self.post_user, identifier=self.identifier)
 565
 566         cont = True
 567         if len(pretest.pathname):
 568             pretest.run(options.dryrun)
 569             cont = pretest.result.result == 'PASS'
 570             pretest.log(options)
 571
 572         for fname in self.tests:
 573             odir = os.path.join(self.outputdir, fname)
 574             test = Cmd(os.path.join(self.pathname, fname), outputdir=odir,
 575                        timeout=self.timeout, user=self.user,
 576                        identifier=self.identifier)
 577             odir = os.path.join(odir, os.path.basename(self.failsafe))
 578             failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout,
 579                            user=self.failsafe_user, identifier=self.identifier)
 580             if cont:
 581                 test.run(options.dryrun)
 582                 if test.result.result == 'KILLED' and len(failsafe.pathname):
 583                     failsafe.run(options.dryrun)
 584                     failsafe.log(options, suppress_console=True)
 585             else:
 586                 test.skip()
 587
 588             test.log(options)
 589
 590         if len(posttest.pathname):
 591             posttest.run(options.dryrun)
 592             posttest.log(options)
 593
 594
 595 class TestRun(object):
 596     props = ['quiet', 'outputdir']
 597
 598     def __init__(self, options):
 599         self.tests = {}
 600         self.testgroups = {}
 601         self.starttime = time()
 602         self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
 603         self.outputdir = os.path.join(options.outputdir, self.timestamp)
 604         self.setup_logging(options)
 605         self.defaults = [
 606             ('outputdir', BASEDIR),
 607             ('quiet', False),
 608             ('timeout', 60),
 609             ('user', ''),
 610             ('pre', ''),
 611             ('pre_user', ''),
 612             ('post', ''),
 613             ('post_user', ''),
 614             ('failsafe', ''),
 615             ('failsafe_user', ''),
 616             ('tags', [])
 617         ]
 618
 619     def __str__(self):
 620         s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
 621         s += 'TESTS:\n'
 622         for key in sorted(self.tests.keys()):
 623             s += '%s%s' % (self.tests[key].__str__(), '\n')
 624         s += 'TESTGROUPS:\n'
 625         for key in sorted(self.testgroups.keys()):
 626             s += '%s%s' % (self.testgroups[key].__str__(), '\n')
 627         return s
 628
 629     def addtest(self, pathname, options):
 630         """
 631         Create a new Test, and apply any properties that were passed in
 632         from the command line. If it passes verification, add it to the
 633         TestRun.
 634         """
 635         test = Test(pathname)
 636         for prop in Test.props:
 637             setattr(test, prop, getattr(options, prop))
 638
 639         if test.verify():
 640             self.tests[pathname] = test
 641
 642     def addtestgroup(self, dirname, filenames, options):
 643         """
 644         Create a new TestGroup, and apply any properties that were passed
 645         in from the command line. If it passes verification, add it to the
 646         TestRun.
 647         """
 648         if dirname not in self.testgroups:
 649             testgroup = TestGroup(dirname)
 650             for prop in Test.props:
 651                 setattr(testgroup, prop, getattr(options, prop))
 652
 653             # Prevent pre/post/failsafe scripts from running as regular tests
 654             for f in [testgroup.pre, testgroup.post, testgroup.failsafe]:
 655                 if f in filenames:
 656                     del filenames[filenames.index(f)]
 657
 658             self.testgroups[dirname] = testgroup
 659             self.testgroups[dirname].tests = sorted(filenames)
 660
 661             testgroup.verify()
 662
 663     def filter(self, keeplist):
 664         for group in list(self.testgroups.keys()):
 665             if group not in keeplist:
 666                 del self.testgroups[group]
 667                 continue
 668
 669             g = self.testgroups[group]
 670
 671             if g.pre and os.path.basename(g.pre) in keeplist[group]:
 672                 continue
 673
 674             g.filter(keeplist[group])
 675
 676         for test in list(self.tests.keys()):
 677             directory, base = os.path.split(test)
 678             if directory not in keeplist or base not in keeplist[directory]:
 679                 del self.tests[test]
 680
 681     def read(self, options):
 682         """
 683         Read in the specified runfiles, and apply the TestRun properties
 684         listed in the 'DEFAULT' section to our TestRun. Then read each
 685         section, and apply the appropriate properties to the Test or
 686         TestGroup. Properties from individual sections override those set
 687         in the 'DEFAULT' section. If the Test or TestGroup passes
 688         verification, add it to the TestRun.
 689         """
 690         config = configparser.RawConfigParser()
 691         parsed = config.read(options.runfiles)
 692         failed = options.runfiles - set(parsed)
 693         if len(failed):
 694             files = ' '.join(sorted(failed))
 695             fail("Couldn't read config files: %s" % files)
 696
 697         for opt in TestRun.props:
 698             if config.has_option('DEFAULT', opt):
 699                 setattr(self, opt, config.get('DEFAULT', opt))
 700         self.outputdir = os.path.join(self.outputdir, self.timestamp)
 701
 702         testdir = options.testdir
 703
 704         for section in config.sections():
 705             if 'tests' in config.options(section):
 706                 parts = section.split(':', 1)
 707                 sectiondir = parts[0]
 708                 identifier = parts[1] if len(parts) == 2 else None
 709                 if os.path.isdir(sectiondir):
 710                     pathname = sectiondir
 711                 elif os.path.isdir(os.path.join(testdir, sectiondir)):
 712                     pathname = os.path.join(testdir, sectiondir)
 713                 else:
 714                     pathname = sectiondir
 715
 716                 testgroup = TestGroup(os.path.abspath(pathname),
 717                                       identifier=identifier)
 718                 for prop in TestGroup.props:
 719                     for sect in ['DEFAULT', section]:
 720                         if config.has_option(sect, prop):
 721                             if prop == 'tags':
 722                                 setattr(testgroup, prop,
 723                                         eval(config.get(sect, prop)))
 724                             elif prop == 'failsafe':
 725                                 failsafe = config.get(sect, prop)
 726                                 setattr(testgroup, prop,
 727                                         os.path.join(testdir, failsafe))
 728                             else:
 729                                 setattr(testgroup, prop,
 730                                         config.get(sect, prop))
 731
 732                 # Repopulate tests using eval to convert the string to a list
 733                 testgroup.tests = eval(config.get(section, 'tests'))
 734
 735                 if testgroup.verify():
 736                     self.testgroups[section] = testgroup
 737             else:
 738                 test = Test(section)
 739                 for prop in Test.props:
 740                     for sect in ['DEFAULT', section]:
 741                         if config.has_option(sect, prop):
 742                             if prop == 'failsafe':
 743                                 failsafe = config.get(sect, prop)
 744                                 setattr(test, prop,
 745                                         os.path.join(testdir, failsafe))
 746                             else:
 747                                 setattr(test, prop, config.get(sect, prop))
 748
 749                 if test.verify():
 750                     self.tests[section] = test
 751
 752     def write(self, options):
 753         """
 754         Create a configuration file for editing and later use. The
 755         'DEFAULT' section of the config file is created from the
 756         properties that were specified on the command line. Tests are
 757         simply added as sections that inherit everything from the
 758         'DEFAULT' section. TestGroups are the same, except they get an
 759         option including all the tests to run in that directory.
 760         """
 761
 762         defaults = dict([(prop, getattr(options, prop)) for prop, _ in
 763                          self.defaults])
 764         config = configparser.RawConfigParser(defaults)
 765
 766         for test in sorted(self.tests.keys()):
 767             config.add_section(test)
 768             for prop in Test.props:
 769                 if prop not in self.props:
 770                     config.set(test, prop,
 771                                getattr(self.tests[test], prop))
 772
 773         for testgroup in sorted(self.testgroups.keys()):
 774             config.add_section(testgroup)
 775             config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
 776             for prop in TestGroup.props:
 777                 if prop not in self.props:
 778                     config.set(testgroup, prop,
 779                                getattr(self.testgroups[testgroup], prop))
 780
 781         try:
 782             with open(options.template, 'w') as f:
 783                 return config.write(f)
 784         except IOError:
 785             fail('Could not open \'%s\' for writing.' % options.template)
 786
 787     def complete_outputdirs(self):
 788         """
 789         Collect all the pathnames for Tests, and TestGroups. Work
 790         backwards one pathname component at a time, to create a unique
 791         directory name in which to deposit test output. Tests will be able
 792         to write output files directly in the newly modified outputdir.
 793         TestGroups will be able to create one subdirectory per test in the
 794         outputdir, and are guaranteed uniqueness because a group can only
 795         contain files in one directory. Pre and post tests will create a
 796         directory rooted at the outputdir of the Test or TestGroup in
 797         question for their output. Failsafe scripts will create a directory
 798         rooted at the outputdir of each Test for their output.
 799         """
 800         done = False
 801         components = 0
 802         tmp_dict = dict(list(self.tests.items()) +
 803                         list(self.testgroups.items()))
 804         total = len(tmp_dict)
 805         base = self.outputdir
 806
 807         while not done:
 808             paths = []
 809             components -= 1
 810             for testfile in list(tmp_dict.keys()):
 811                 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
 812                 if uniq not in paths:
 813                     paths.append(uniq)
 814                     tmp_dict[testfile].outputdir = os.path.join(base, uniq)
 815                 else:
 816                     break
 817             done = total == len(paths)
 818
 819     def setup_logging(self, options):
 820         """
 821         This function creates the output directory and gets a file object
 822         for the logfile. This function must be called before write_log()
 823         can be used.
 824         """
 825         if options.dryrun is True:
 826             return
 827
 828         global LOG_FILE_OBJ
 829         if not options.template:
 830             try:
 831                 old = os.umask(0)
 832                 os.makedirs(self.outputdir, mode=0o777)
 833                 os.umask(old)
 834                 filename = os.path.join(self.outputdir, 'log')
 835                 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb')
 836             except OSError as e:
 837                 fail('%s' % e)
 838
 839     def run(self, options):
 840         """
 841         Walk through all the Tests and TestGroups, calling run().
 842         """
 843         try:
 844             os.chdir(self.outputdir)
 845         except OSError:
 846             fail('Could not change to directory %s' % self.outputdir)
 847         # make a symlink to the output for the currently running test
 848         logsymlink = os.path.join(self.outputdir, '../current')
 849         if os.path.islink(logsymlink):
 850             os.unlink(logsymlink)
 851         if not os.path.exists(logsymlink):
 852             os.symlink(self.outputdir, logsymlink)
 853         else:
 854             write_log('Could not make a symlink to directory %s\n' %
 855                       self.outputdir, LOG_ERR)
 856         iteration = 0
 857         while iteration < options.iterations:
 858             for test in sorted(self.tests.keys()):
 859                 self.tests[test].run(options)
 860             for testgroup in sorted(self.testgroups.keys()):
 861                 self.testgroups[testgroup].run(options)
 862             iteration += 1
 863
 864     def summary(self):
 865         if Result.total == 0:
 866             return 2
 867
 868         print('\nResults Summary')
 869         for key in list(Result.runresults.keys()):
 870             if Result.runresults[key] != 0:
 871                 print('%s\t% 4d' % (key, Result.runresults[key]))
 872
 873         m, s = divmod(time() - self.starttime, 60)
 874         h, m = divmod(m, 60)
 875         print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
 876         print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
 877                                             float(Result.total)) * 100))
 878         print('Log directory:\t%s' % self.outputdir)
 879
 880         if Result.runresults['FAIL'] > 0:
 881             return 1
 882
 883         if Result.runresults['KILLED'] > 0:
 884             return 1
 885
 886         if Result.runresults['RERAN'] > 0:
 887             return 3
 888
 889         return 0
 890
 891
 892 def write_log(msg, target):
 893     """
 894     Write the provided message to standard out, standard error or
 895     the logfile. If specifying LOG_FILE, then `msg` must be a bytes
 896     like object. This way we can still handle output from tests that
 897     may be in unexpected encodings.
 898     """
 899     if target == LOG_OUT:
 900         os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8'))
 901     elif target == LOG_ERR:
 902         os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8'))
 903     elif target == LOG_FILE:
 904         os.write(LOG_FILE_OBJ.fileno(), msg)
 905     else:
 906         fail('log_msg called with unknown target "%s"' % target)
 907
 908
 909 def verify_file(pathname):
 910     """
 911     Verify that the supplied pathname is an executable regular file.
 912     """
 913     if os.path.isdir(pathname) or os.path.islink(pathname):
 914         return False
 915
 916     for ext in '', '.ksh', '.sh':
 917         script_path = pathname + ext
 918         if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
 919             return True
 920
 921     return False
 922
 923
 924 def verify_user(user):
 925     """
 926     Verify that the specified user exists on this system, and can execute
 927     sudo without being prompted for a password.
 928     """
 929     testcmd = [SUDO, '-n', '-u', user, TRUE]
 930
 931     if user in Cmd.verified_users:
 932         return True
 933
 934     try:
 935         getpwnam(user)
 936     except KeyError:
 937         write_log("Warning: user '%s' does not exist.\n" % user,
 938                   LOG_ERR)
 939         return False
 940
 941     p = Popen(testcmd)
 942     p.wait()
 943     if p.returncode != 0:
 944         write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user,
 945                   LOG_ERR)
 946         return False
 947     else:
 948         Cmd.verified_users.append(user)
 949
 950     return True
 951
 952
 953 def find_tests(testrun, options):
 954     """
 955     For the given list of pathnames, add files as Tests. For directories,
 956     if do_groups is True, add the directory as a TestGroup. If False,
 957     recursively search for executable files.
 958     """
 959
 960     for p in sorted(options.pathnames):
 961         if os.path.isdir(p):
 962             for dirname, _, filenames in os.walk(p):
 963                 if options.do_groups:
 964                     testrun.addtestgroup(dirname, filenames, options)
 965                 else:
 966                     for f in sorted(filenames):
 967                         testrun.addtest(os.path.join(dirname, f), options)
 968         else:
 969             testrun.addtest(p, options)
 970
 971
 972 def filter_tests(testrun, options):
 973     try:
 974         fh = open(options.logfile, "r")
 975     except Exception as e:
 976         fail('%s' % e)
 977
 978     failed = {}
 979     while True:
 980         line = fh.readline()
 981         if not line:
 982             break
 983         m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line)
 984         if not m:
 985             continue
 986         group, test = m.group(1, 2)
 987         try:
 988             failed[group].append(test)
 989         except KeyError:
 990             failed[group] = [test]
 991     fh.close()
 992
 993     testrun.filter(failed)
 994
 995
 996 def fail(retstr, ret=1):
 997     print('%s: %s' % (sys.argv[0], retstr))
 998     exit(ret)
 999
1000
1001 def options_cb(option, opt_str, value, parser):
1002     path_options = ['outputdir', 'template', 'testdir', 'logfile']
1003
1004     if opt_str in parser.rargs:
1005         fail('%s may only be specified once.' % opt_str)
1006
1007     if option.dest == 'runfiles':
1008         parser.values.cmd = 'rdconfig'
1009         value = set(os.path.abspath(p) for p in value.split(','))
1010     if option.dest == 'tags':
1011         value = [x.strip() for x in value.split(',')]
1012
1013     if option.dest in path_options:
1014         setattr(parser.values, option.dest, os.path.abspath(value))
1015     else:
1016         setattr(parser.values, option.dest, value)
1017
1018
1019 def parse_args():
1020     parser = OptionParser()
1021     parser.add_option('-c', action='callback', callback=options_cb,
1022                       type='string', dest='runfiles', metavar='runfiles',
1023                       help='Specify tests to run via config files.')
1024     parser.add_option('-d', action='store_true', default=False, dest='dryrun',
1025                       help='Dry run. Print tests, but take no other action.')
1026     parser.add_option('-l', action='callback', callback=options_cb,
1027                       default=None, dest='logfile', metavar='logfile',
1028                       type='string',
1029                       help='Read logfile and re-run tests which failed.')
1030     parser.add_option('-g', action='store_true', default=False,
1031                       dest='do_groups', help='Make directories TestGroups.')
1032     parser.add_option('-o', action='callback', callback=options_cb,
1033                       default=BASEDIR, dest='outputdir', type='string',
1034                       metavar='outputdir', help='Specify an output directory.')
1035     parser.add_option('-i', action='callback', callback=options_cb,
1036                       default=TESTDIR, dest='testdir', type='string',
1037                       metavar='testdir', help='Specify a test directory.')
1038     parser.add_option('-p', action='callback', callback=options_cb,
1039                       default='', dest='pre', metavar='script',
1040                       type='string', help='Specify a pre script.')
1041     parser.add_option('-P', action='callback', callback=options_cb,
1042                       default='', dest='post', metavar='script',
1043                       type='string', help='Specify a post script.')
1044     parser.add_option('-q', action='store_true', default=False, dest='quiet',
1045                       help='Silence on the console during a test run.')
1046     parser.add_option('-s', action='callback', callback=options_cb,
1047                       default='', dest='failsafe', metavar='script',
1048                       type='string', help='Specify a failsafe script.')
1049     parser.add_option('-S', action='callback', callback=options_cb,
1050                       default='', dest='failsafe_user',
1051                       metavar='failsafe_user', type='string',
1052                       help='Specify a user to execute the failsafe script.')
1053     parser.add_option('-t', action='callback', callback=options_cb, default=60,
1054                       dest='timeout', metavar='seconds', type='int',
1055                       help='Timeout (in seconds) for an individual test.')
1056     parser.add_option('-u', action='callback', callback=options_cb,
1057                       default='', dest='user', metavar='user', type='string',
1058                       help='Specify a different user name to run as.')
1059     parser.add_option('-w', action='callback', callback=options_cb,
1060                       default=None, dest='template', metavar='template',
1061                       type='string', help='Create a new config file.')
1062     parser.add_option('-x', action='callback', callback=options_cb, default='',
1063                       dest='pre_user', metavar='pre_user', type='string',
1064                       help='Specify a user to execute the pre script.')
1065     parser.add_option('-X', action='callback', callback=options_cb, default='',
1066                       dest='post_user', metavar='post_user', type='string',
1067                       help='Specify a user to execute the post script.')
1068     parser.add_option('-T', action='callback', callback=options_cb, default='',
1069                       dest='tags', metavar='tags', type='string',
1070                       help='Specify tags to execute specific test groups.')
1071     parser.add_option('-I', action='callback', callback=options_cb, default=1,
1072                       dest='iterations', metavar='iterations', type='int',
1073                       help='Number of times to run the test run.')
1074     (options, pathnames) = parser.parse_args()
1075
1076     if options.runfiles and len(pathnames):
1077         fail('Extraneous arguments.')
1078
1079     options.pathnames = [os.path.abspath(path) for path in pathnames]
1080
1081     return options
1082
1083
1084 def main():
1085     options = parse_args()
1086
1087     testrun = TestRun(options)
1088
1089     if options.runfiles:
1090         testrun.read(options)
1091     else:
1092         find_tests(testrun, options)
1093
1094     if options.logfile:
1095         filter_tests(testrun, options)
1096
1097     if options.template:
1098         testrun.write(options)
1099         exit(0)
1100
1101     testrun.complete_outputdirs()
1102     testrun.run(options)
1103     exit(testrun.summary())
1104
1105
1106 if __name__ == '__main__':
1107     main()