tests/test-runner/bin/test-runner.py.in

   1 #!/usr/bin/env @PYTHON_SHEBANG@
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  16 # Copyright (c) 2019 Datto Inc.
  17 #
  18 # This script must remain compatible with Python 2.6+ and Python 3.4+.
  19 #
  20
  21 # some python 2.7 system don't have a configparser shim
  22 try:
  23     import configparser
  24 except ImportError:
  25     import ConfigParser as configparser
  26
  27 import os
  28 import sys
  29 import ctypes
  30 import re
  31
  32 from datetime import datetime
  33 from optparse import OptionParser
  34 from pwd import getpwnam
  35 from pwd import getpwuid
  36 from select import select
  37 from subprocess import PIPE
  38 from subprocess import Popen
  39 from subprocess import check_output
  40 from threading import Timer
  41 from time import time
  42
  43 BASEDIR = '/var/tmp/test_results'
  44 TESTDIR = '/usr/share/zfs/'
  45 KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak'
  46 KILL = 'kill'
  47 TRUE = 'true'
  48 SUDO = 'sudo'
  49 LOG_FILE = 'LOG_FILE'
  50 LOG_OUT = 'LOG_OUT'
  51 LOG_ERR = 'LOG_ERR'
  52 LOG_FILE_OBJ = None
  53
  54 # some python 2.7 system don't have a concept of monotonic time
  55 CLOCK_MONOTONIC_RAW = 4  # see <linux/time.h>
  56
  57
  58 class timespec(ctypes.Structure):
  59     _fields_ = [
  60         ('tv_sec', ctypes.c_long),
  61         ('tv_nsec', ctypes.c_long)
  62     ]
  63
  64
  65 librt = ctypes.CDLL('librt.so.1', use_errno=True)
  66 clock_gettime = librt.clock_gettime
  67 clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)]
  68
  69
  70 def monotonic_time():
  71     t = timespec()
  72     if clock_gettime(CLOCK_MONOTONIC_RAW, ctypes.pointer(t)) != 0:
  73         errno_ = ctypes.get_errno()
  74         raise OSError(errno_, os.strerror(errno_))
  75     return t.tv_sec + t.tv_nsec * 1e-9
  76
  77
  78 class Result(object):
  79     total = 0
  80     runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0}
  81
  82     def __init__(self):
  83         self.starttime = None
  84         self.returncode = None
  85         self.runtime = ''
  86         self.stdout = []
  87         self.stderr = []
  88         self.kmemleak = ''
  89         self.result = ''
  90
  91     def done(self, proc, killed, reran):
  92         """
  93         Finalize the results of this Cmd.
  94         """
  95         Result.total += 1
  96         m, s = divmod(monotonic_time() - self.starttime, 60)
  97         self.runtime = '%02d:%02d' % (m, s)
  98         self.returncode = proc.returncode
  99         if reran is True:
 100             Result.runresults['RERAN'] += 1
 101         if killed:
 102             self.result = 'KILLED'
 103             Result.runresults['KILLED'] += 1
 104         elif len(self.kmemleak) > 0:
 105             self.result = 'FAIL'
 106             Result.runresults['FAIL'] += 1
 107         elif self.returncode == 0:
 108             self.result = 'PASS'
 109             Result.runresults['PASS'] += 1
 110         elif self.returncode == 4:
 111             self.result = 'SKIP'
 112             Result.runresults['SKIP'] += 1
 113         elif self.returncode != 0:
 114             self.result = 'FAIL'
 115             Result.runresults['FAIL'] += 1
 116
 117
 118 class Output(object):
 119     """
 120     This class is a slightly modified version of the 'Stream' class found
 121     here: http://goo.gl/aSGfv
 122     """
 123     def __init__(self, stream):
 124         self.stream = stream
 125         self._buf = b''
 126         self.lines = []
 127
 128     def fileno(self):
 129         return self.stream.fileno()
 130
 131     def read(self, drain=0):
 132         """
 133         Read from the file descriptor. If 'drain' set, read until EOF.
 134         """
 135         while self._read() is not None:
 136             if not drain:
 137                 break
 138
 139     def _read(self):
 140         """
 141         Read up to 4k of data from this output stream. Collect the output
 142         up to the last newline, and append it to any leftover data from a
 143         previous call. The lines are stored as a (timestamp, data) tuple
 144         for easy sorting/merging later.
 145         """
 146         fd = self.fileno()
 147         buf = os.read(fd, 4096)
 148         if not buf:
 149             return None
 150         if b'\n' not in buf:
 151             self._buf += buf
 152             return []
 153
 154         buf = self._buf + buf
 155         tmp, rest = buf.rsplit(b'\n', 1)
 156         self._buf = rest
 157         now = datetime.now()
 158         rows = tmp.split(b'\n')
 159         self.lines += [(now, r) for r in rows]
 160
 161
 162 class Cmd(object):
 163     verified_users = []
 164
 165     def __init__(self, pathname, identifier=None, outputdir=None,
 166                  timeout=None, user=None, tags=None):
 167         self.pathname = pathname
 168         self.identifier = identifier
 169         self.outputdir = outputdir or 'BASEDIR'
 170         """
 171         The timeout for tests is measured in wall-clock time
 172         """
 173         self.timeout = timeout
 174         self.user = user or ''
 175         self.killed = False
 176         self.reran = None
 177         self.result = Result()
 178
 179         if self.timeout is None:
 180             self.timeout = 60
 181
 182     def __str__(self):
 183         return '''\
 184 Pathname: %s
 185 Identifier: %s
 186 Outputdir: %s
 187 Timeout: %d
 188 User: %s
 189 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user)
 190
 191     def kill_cmd(self, proc, keyboard_interrupt=False):
 192         """
 193         Kill a running command due to timeout, or ^C from the keyboard. If
 194         sudo is required, this user was verified previously.
 195         """
 196         self.killed = True
 197         do_sudo = len(self.user) != 0
 198         signal = '-TERM'
 199
 200         cmd = [SUDO, KILL, signal, str(proc.pid)]
 201         if not do_sudo:
 202             del cmd[0]
 203
 204         try:
 205             kp = Popen(cmd)
 206             kp.wait()
 207         except Exception:
 208             pass
 209
 210         """
 211         If this is not a user-initiated kill and the test has not been
 212         reran before we consider if the test needs to be reran:
 213         If the test has spent some time hibernating and didn't run the whole
 214         length of time before being timed out we will rerun the test.
 215         """
 216         if keyboard_interrupt is False and self.reran is None:
 217             runtime = monotonic_time() - self.result.starttime
 218             if int(self.timeout) > runtime:
 219                 self.killed = False
 220                 self.reran = False
 221                 self.run(False)
 222                 self.reran = True
 223
 224     def update_cmd_privs(self, cmd, user):
 225         """
 226         If a user has been specified to run this Cmd and we're not already
 227         running as that user, prepend the appropriate sudo command to run
 228         as that user.
 229         """
 230         me = getpwuid(os.getuid())
 231
 232         if not user or user is me:
 233             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 234                 cmd += '.ksh'
 235             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 236                 cmd += '.sh'
 237             return cmd
 238
 239         if not os.path.isfile(cmd):
 240             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 241                 cmd += '.ksh'
 242             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 243                 cmd += '.sh'
 244
 245         ret = '%s -E -u %s %s' % (SUDO, user, cmd)
 246         return ret.split(' ')
 247
 248     def collect_output(self, proc):
 249         """
 250         Read from stdout/stderr as data becomes available, until the
 251         process is no longer running. Return the lines from the stdout and
 252         stderr Output objects.
 253         """
 254         out = Output(proc.stdout)
 255         err = Output(proc.stderr)
 256         res = []
 257         while proc.returncode is None:
 258             proc.poll()
 259             res = select([out, err], [], [], .1)
 260             for fd in res[0]:
 261                 fd.read()
 262         for fd in res[0]:
 263             fd.read(drain=1)
 264
 265         return out.lines, err.lines
 266
 267     def run(self, dryrun, kmemleak):
 268         """
 269         This is the main function that runs each individual test.
 270         Determine whether or not the command requires sudo, and modify it
 271         if needed. Run the command, and update the result object.
 272         """
 273         if dryrun is True:
 274             print(self)
 275             return
 276
 277         privcmd = self.update_cmd_privs(self.pathname, self.user)
 278         try:
 279             old = os.umask(0)
 280             if not os.path.isdir(self.outputdir):
 281                 os.makedirs(self.outputdir, mode=0o777)
 282             os.umask(old)
 283         except OSError as e:
 284             fail('%s' % e)
 285
 286         self.result.starttime = monotonic_time()
 287
 288         if kmemleak:
 289             cmd = f'echo clear | {SUDO} tee {KMEMLEAK_FILE}'
 290             check_output(cmd, shell=True)
 291
 292         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
 293         # Allow a special timeout value of 0 to mean infinity
 294         if int(self.timeout) == 0:
 295             self.timeout = sys.maxsize
 296         t = Timer(int(self.timeout), self.kill_cmd, [proc])
 297
 298         try:
 299             t.start()
 300             self.result.stdout, self.result.stderr = self.collect_output(proc)
 301
 302             if kmemleak:
 303                 cmd = f'echo scan | {SUDO} tee {KMEMLEAK_FILE}'
 304                 check_output(cmd, shell=True)
 305                 cmd = f'{SUDO} cat {KMEMLEAK_FILE}'
 306                 self.result.kmemleak = check_output(cmd, shell=True)
 307         except KeyboardInterrupt:
 308             self.kill_cmd(proc, True)
 309             fail('\nRun terminated at user request.')
 310         finally:
 311             t.cancel()
 312
 313         if self.reran is not False:
 314             self.result.done(proc, self.killed, self.reran)
 315
 316     def skip(self):
 317         """
 318         Initialize enough of the test result that we can log a skipped
 319         command.
 320         """
 321         Result.total += 1
 322         Result.runresults['SKIP'] += 1
 323         self.result.stdout = self.result.stderr = []
 324         self.result.starttime = monotonic_time()
 325         m, s = divmod(monotonic_time() - self.result.starttime, 60)
 326         self.result.runtime = '%02d:%02d' % (m, s)
 327         self.result.result = 'SKIP'
 328
 329     def log(self, options, suppress_console=False):
 330         """
 331         This function is responsible for writing all output. This includes
 332         the console output, the logfile of all results (with timestamped
 333         merged stdout and stderr), and for each test, the unmodified
 334         stdout/stderr/merged in its own file.
 335         """
 336
 337         logname = getpwuid(os.getuid()).pw_name
 338         rer = ''
 339         if self.reran is True:
 340             rer = ' (RERAN)'
 341         user = ' (run as %s)' % (self.user if len(self.user) else logname)
 342         if self.identifier:
 343             msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user)
 344         else:
 345             msga = 'Test: %s%s ' % (self.pathname, user)
 346         msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer)
 347         pad = ' ' * (80 - (len(msga) + len(msgb)))
 348         result_line = msga + pad + msgb
 349
 350         # The result line is always written to the log file. If -q was
 351         # specified only failures are written to the console, otherwise
 352         # the result line is written to the console. The console output
 353         # may be suppressed by calling log() with suppress_console=True.
 354         write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE)
 355         if not suppress_console:
 356             if not options.quiet:
 357                 write_log(result_line, LOG_OUT)
 358             elif options.quiet and self.result.result != 'PASS':
 359                 write_log(result_line, LOG_OUT)
 360
 361         lines = sorted(self.result.stdout + self.result.stderr,
 362                        key=lambda x: x[0])
 363
 364         # Write timestamped output (stdout and stderr) to the logfile
 365         for dt, line in lines:
 366             timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11],
 367                                   encoding='utf-8')
 368             write_log(b'%s %s\n' % (timestamp, line), LOG_FILE)
 369
 370         # Write the separate stdout/stderr/merged files, if the data exists
 371         if len(self.result.stdout):
 372             with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out:
 373                 for _, line in self.result.stdout:
 374                     os.write(out.fileno(), b'%s\n' % line)
 375         if len(self.result.stderr):
 376             with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err:
 377                 for _, line in self.result.stderr:
 378                     os.write(err.fileno(), b'%s\n' % line)
 379         if len(self.result.stdout) and len(self.result.stderr):
 380             with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
 381                 for _, line in lines:
 382                     os.write(merged.fileno(), b'%s\n' % line)
 383         if len(self.result.kmemleak):
 384             with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem:
 385                 kmem.write(self.result.kmemleak)
 386
 387
 388 class Test(Cmd):
 389     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
 390              'post_user', 'failsafe', 'failsafe_user', 'tags']
 391
 392     def __init__(self, pathname,
 393                  pre=None, pre_user=None, post=None, post_user=None,
 394                  failsafe=None, failsafe_user=None, tags=None, **kwargs):
 395         super(Test, self).__init__(pathname, **kwargs)
 396         self.pre = pre or ''
 397         self.pre_user = pre_user or ''
 398         self.post = post or ''
 399         self.post_user = post_user or ''
 400         self.failsafe = failsafe or ''
 401         self.failsafe_user = failsafe_user or ''
 402         self.tags = tags or []
 403
 404     def __str__(self):
 405         post_user = pre_user = failsafe_user = ''
 406         if len(self.pre_user):
 407             pre_user = ' (as %s)' % (self.pre_user)
 408         if len(self.post_user):
 409             post_user = ' (as %s)' % (self.post_user)
 410         if len(self.failsafe_user):
 411             failsafe_user = ' (as %s)' % (self.failsafe_user)
 412         return '''\
 413 Pathname: %s
 414 Identifier: %s
 415 Outputdir: %s
 416 Timeout: %d
 417 User: %s
 418 Pre: %s%s
 419 Post: %s%s
 420 Failsafe: %s%s
 421 Tags: %s
 422 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user,
 423             self.pre, pre_user, self.post, post_user, self.failsafe,
 424             failsafe_user, self.tags)
 425
 426     def verify(self):
 427         """
 428         Check the pre/post/failsafe scripts, user and Test. Omit the Test from
 429         this run if there are any problems.
 430         """
 431         files = [self.pre, self.pathname, self.post, self.failsafe]
 432         users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
 433
 434         for f in [f for f in files if len(f)]:
 435             if not verify_file(f):
 436                 write_log("Warning: Test '%s' not added to this run because"
 437                           " it failed verification.\n" % f, LOG_ERR)
 438                 return False
 439
 440         for user in [user for user in users if len(user)]:
 441             if not verify_user(user):
 442                 write_log("Not adding Test '%s' to this run.\n" %
 443                           self.pathname, LOG_ERR)
 444                 return False
 445
 446         return True
 447
 448     def run(self, options):
 449         """
 450         Create Cmd instances for the pre/post/failsafe scripts. If the pre
 451         script doesn't pass, skip this Test. Run the post script regardless.
 452         If the Test is killed, also run the failsafe script.
 453         """
 454         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 455         pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir,
 456                       timeout=self.timeout, user=self.pre_user)
 457         test = Cmd(self.pathname, identifier=self.identifier,
 458                    outputdir=self.outputdir, timeout=self.timeout,
 459                    user=self.user)
 460         odir = os.path.join(self.outputdir, os.path.basename(self.failsafe))
 461         failsafe = Cmd(self.failsafe, identifier=self.identifier,
 462                        outputdir=odir, timeout=self.timeout,
 463                        user=self.failsafe_user)
 464         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 465         posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir,
 466                        timeout=self.timeout, user=self.post_user)
 467
 468         cont = True
 469         if len(pretest.pathname):
 470             pretest.run(options.dryrun, False)
 471             cont = pretest.result.result == 'PASS'
 472             pretest.log(options)
 473
 474         if cont:
 475             test.run(options.dryrun, options.kmemleak)
 476             if test.result.result == 'KILLED' and len(failsafe.pathname):
 477                 failsafe.run(options.dryrun, False)
 478                 failsafe.log(options, suppress_console=True)
 479         else:
 480             test.skip()
 481
 482         test.log(options)
 483
 484         if len(posttest.pathname):
 485             posttest.run(options.dryrun, False)
 486             posttest.log(options)
 487
 488
 489 class TestGroup(Test):
 490     props = Test.props + ['tests']
 491
 492     def __init__(self, pathname, tests=None, **kwargs):
 493         super(TestGroup, self).__init__(pathname, **kwargs)
 494         self.tests = tests or []
 495
 496     def __str__(self):
 497         post_user = pre_user = failsafe_user = ''
 498         if len(self.pre_user):
 499             pre_user = ' (as %s)' % (self.pre_user)
 500         if len(self.post_user):
 501             post_user = ' (as %s)' % (self.post_user)
 502         if len(self.failsafe_user):
 503             failsafe_user = ' (as %s)' % (self.failsafe_user)
 504         return '''\
 505 Pathname: %s
 506 Identifier: %s
 507 Outputdir: %s
 508 Tests: %s
 509 Timeout: %s
 510 User: %s
 511 Pre: %s%s
 512 Post: %s%s
 513 Failsafe: %s%s
 514 Tags: %s
 515 ''' % (self.pathname, self.identifier, self.outputdir, self.tests,
 516             self.timeout, self.user, self.pre, pre_user, self.post, post_user,
 517             self.failsafe, failsafe_user, self.tags)
 518
 519     def filter(self, keeplist):
 520         self.tests = [x for x in self.tests if x in keeplist]
 521
 522     def verify(self):
 523         """
 524         Check the pre/post/failsafe scripts, user and tests in this TestGroup.
 525         Omit the TestGroup entirely, or simply delete the relevant tests in the
 526         group, if that's all that's required.
 527         """
 528         # If the pre/post/failsafe scripts are relative pathnames, convert to
 529         # absolute, so they stand a chance of passing verification.
 530         if len(self.pre) and not os.path.isabs(self.pre):
 531             self.pre = os.path.join(self.pathname, self.pre)
 532         if len(self.post) and not os.path.isabs(self.post):
 533             self.post = os.path.join(self.pathname, self.post)
 534         if len(self.failsafe) and not os.path.isabs(self.failsafe):
 535             self.post = os.path.join(self.pathname, self.post)
 536
 537         auxfiles = [self.pre, self.post, self.failsafe]
 538         users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
 539
 540         for f in [f for f in auxfiles if len(f)]:
 541             if f != self.failsafe and self.pathname != os.path.dirname(f):
 542                 write_log("Warning: TestGroup '%s' not added to this run. "
 543                           "Auxiliary script '%s' exists in a different "
 544                           "directory.\n" % (self.pathname, f), LOG_ERR)
 545                 return False
 546
 547             if not verify_file(f):
 548                 write_log("Warning: TestGroup '%s' not added to this run. "
 549                           "Auxiliary script '%s' failed verification.\n" %
 550                           (self.pathname, f), LOG_ERR)
 551                 return False
 552
 553         for user in [user for user in users if len(user)]:
 554             if not verify_user(user):
 555                 write_log("Not adding TestGroup '%s' to this run.\n" %
 556                           self.pathname, LOG_ERR)
 557                 return False
 558
 559         # If one of the tests is invalid, delete it, log it, and drive on.
 560         for test in self.tests:
 561             if not verify_file(os.path.join(self.pathname, test)):
 562                 del self.tests[self.tests.index(test)]
 563                 write_log("Warning: Test '%s' removed from TestGroup '%s' "
 564                           "because it failed verification.\n" %
 565                           (test, self.pathname), LOG_ERR)
 566
 567         return len(self.tests) != 0
 568
 569     def run(self, options):
 570         """
 571         Create Cmd instances for the pre/post/failsafe scripts. If the pre
 572         script doesn't pass, skip all the tests in this TestGroup. Run the
 573         post script regardless. Run the failsafe script when a test is killed.
 574         """
 575         # tags assigned to this test group also include the test names
 576         if options.tags and not set(self.tags).intersection(set(options.tags)):
 577             return
 578
 579         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 580         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 581                       user=self.pre_user, identifier=self.identifier)
 582         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 583         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 584                        user=self.post_user, identifier=self.identifier)
 585
 586         cont = True
 587         if len(pretest.pathname):
 588             pretest.run(options.dryrun, False)
 589             cont = pretest.result.result == 'PASS'
 590             pretest.log(options)
 591
 592         for fname in self.tests:
 593             odir = os.path.join(self.outputdir, fname)
 594             test = Cmd(os.path.join(self.pathname, fname), outputdir=odir,
 595                        timeout=self.timeout, user=self.user,
 596                        identifier=self.identifier)
 597             odir = os.path.join(odir, os.path.basename(self.failsafe))
 598             failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout,
 599                            user=self.failsafe_user, identifier=self.identifier)
 600             if cont:
 601                 test.run(options.dryrun, options.kmemleak)
 602                 if test.result.result == 'KILLED' and len(failsafe.pathname):
 603                     failsafe.run(options.dryrun, False)
 604                     failsafe.log(options, suppress_console=True)
 605             else:
 606                 test.skip()
 607
 608             test.log(options)
 609
 610         if len(posttest.pathname):
 611             posttest.run(options.dryrun, False)
 612             posttest.log(options)
 613
 614
 615 class TestRun(object):
 616     props = ['quiet', 'outputdir']
 617
 618     def __init__(self, options):
 619         self.tests = {}
 620         self.testgroups = {}
 621         self.starttime = time()
 622         self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
 623         self.outputdir = os.path.join(options.outputdir, self.timestamp)
 624         self.setup_logging(options)
 625         self.defaults = [
 626             ('outputdir', BASEDIR),
 627             ('quiet', False),
 628             ('timeout', 60),
 629             ('user', ''),
 630             ('pre', ''),
 631             ('pre_user', ''),
 632             ('post', ''),
 633             ('post_user', ''),
 634             ('failsafe', ''),
 635             ('failsafe_user', ''),
 636             ('tags', [])
 637         ]
 638
 639     def __str__(self):
 640         s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
 641         s += 'TESTS:\n'
 642         for key in sorted(self.tests.keys()):
 643             s += '%s%s' % (self.tests[key].__str__(), '\n')
 644         s += 'TESTGROUPS:\n'
 645         for key in sorted(self.testgroups.keys()):
 646             s += '%s%s' % (self.testgroups[key].__str__(), '\n')
 647         return s
 648
 649     def addtest(self, pathname, options):
 650         """
 651         Create a new Test, and apply any properties that were passed in
 652         from the command line. If it passes verification, add it to the
 653         TestRun.
 654         """
 655         test = Test(pathname)
 656         for prop in Test.props:
 657             setattr(test, prop, getattr(options, prop))
 658
 659         if test.verify():
 660             self.tests[pathname] = test
 661
 662     def addtestgroup(self, dirname, filenames, options):
 663         """
 664         Create a new TestGroup, and apply any properties that were passed
 665         in from the command line. If it passes verification, add it to the
 666         TestRun.
 667         """
 668         if dirname not in self.testgroups:
 669             testgroup = TestGroup(dirname)
 670             for prop in Test.props:
 671                 setattr(testgroup, prop, getattr(options, prop))
 672
 673             # Prevent pre/post/failsafe scripts from running as regular tests
 674             for f in [testgroup.pre, testgroup.post, testgroup.failsafe]:
 675                 if f in filenames:
 676                     del filenames[filenames.index(f)]
 677
 678             self.testgroups[dirname] = testgroup
 679             self.testgroups[dirname].tests = sorted(filenames)
 680
 681             testgroup.verify()
 682
 683     def filter(self, keeplist):
 684         for group in list(self.testgroups.keys()):
 685             if group not in keeplist:
 686                 del self.testgroups[group]
 687                 continue
 688
 689             g = self.testgroups[group]
 690
 691             if g.pre and os.path.basename(g.pre) in keeplist[group]:
 692                 continue
 693
 694             g.filter(keeplist[group])
 695
 696         for test in list(self.tests.keys()):
 697             directory, base = os.path.split(test)
 698             if directory not in keeplist or base not in keeplist[directory]:
 699                 del self.tests[test]
 700
 701     def read(self, options):
 702         """
 703         Read in the specified runfiles, and apply the TestRun properties
 704         listed in the 'DEFAULT' section to our TestRun. Then read each
 705         section, and apply the appropriate properties to the Test or
 706         TestGroup. Properties from individual sections override those set
 707         in the 'DEFAULT' section. If the Test or TestGroup passes
 708         verification, add it to the TestRun.
 709         """
 710         config = configparser.RawConfigParser()
 711         parsed = config.read(options.runfiles)
 712         failed = options.runfiles - set(parsed)
 713         if len(failed):
 714             files = ' '.join(sorted(failed))
 715             fail("Couldn't read config files: %s" % files)
 716
 717         for opt in TestRun.props:
 718             if config.has_option('DEFAULT', opt):
 719                 setattr(self, opt, config.get('DEFAULT', opt))
 720         self.outputdir = os.path.join(self.outputdir, self.timestamp)
 721
 722         testdir = options.testdir
 723
 724         for section in config.sections():
 725             if 'tests' in config.options(section):
 726                 parts = section.split(':', 1)
 727                 sectiondir = parts[0]
 728                 identifier = parts[1] if len(parts) == 2 else None
 729                 if os.path.isdir(sectiondir):
 730                     pathname = sectiondir
 731                 elif os.path.isdir(os.path.join(testdir, sectiondir)):
 732                     pathname = os.path.join(testdir, sectiondir)
 733                 else:
 734                     pathname = sectiondir
 735
 736                 testgroup = TestGroup(os.path.abspath(pathname),
 737                                       identifier=identifier)
 738                 for prop in TestGroup.props:
 739                     for sect in ['DEFAULT', section]:
 740                         if config.has_option(sect, prop):
 741                             if prop == 'tags':
 742                                 setattr(testgroup, prop,
 743                                         eval(config.get(sect, prop)))
 744                             elif prop == 'failsafe':
 745                                 failsafe = config.get(sect, prop)
 746                                 setattr(testgroup, prop,
 747                                         os.path.join(testdir, failsafe))
 748                             else:
 749                                 setattr(testgroup, prop,
 750                                         config.get(sect, prop))
 751
 752                 # Repopulate tests using eval to convert the string to a list
 753                 testgroup.tests = eval(config.get(section, 'tests'))
 754
 755                 if testgroup.verify():
 756                     self.testgroups[section] = testgroup
 757             else:
 758                 test = Test(section)
 759                 for prop in Test.props:
 760                     for sect in ['DEFAULT', section]:
 761                         if config.has_option(sect, prop):
 762                             if prop == 'failsafe':
 763                                 failsafe = config.get(sect, prop)
 764                                 setattr(test, prop,
 765                                         os.path.join(testdir, failsafe))
 766                             else:
 767                                 setattr(test, prop, config.get(sect, prop))
 768
 769                 if test.verify():
 770                     self.tests[section] = test
 771
 772     def write(self, options):
 773         """
 774         Create a configuration file for editing and later use. The
 775         'DEFAULT' section of the config file is created from the
 776         properties that were specified on the command line. Tests are
 777         simply added as sections that inherit everything from the
 778         'DEFAULT' section. TestGroups are the same, except they get an
 779         option including all the tests to run in that directory.
 780         """
 781
 782         defaults = dict([(prop, getattr(options, prop)) for prop, _ in
 783                          self.defaults])
 784         config = configparser.RawConfigParser(defaults)
 785
 786         for test in sorted(self.tests.keys()):
 787             config.add_section(test)
 788             for prop in Test.props:
 789                 if prop not in self.props:
 790                     config.set(test, prop,
 791                                getattr(self.tests[test], prop))
 792
 793         for testgroup in sorted(self.testgroups.keys()):
 794             config.add_section(testgroup)
 795             config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
 796             for prop in TestGroup.props:
 797                 if prop not in self.props:
 798                     config.set(testgroup, prop,
 799                                getattr(self.testgroups[testgroup], prop))
 800
 801         try:
 802             with open(options.template, 'w') as f:
 803                 return config.write(f)
 804         except IOError:
 805             fail('Could not open \'%s\' for writing.' % options.template)
 806
 807     def complete_outputdirs(self):
 808         """
 809         Collect all the pathnames for Tests, and TestGroups. Work
 810         backwards one pathname component at a time, to create a unique
 811         directory name in which to deposit test output. Tests will be able
 812         to write output files directly in the newly modified outputdir.
 813         TestGroups will be able to create one subdirectory per test in the
 814         outputdir, and are guaranteed uniqueness because a group can only
 815         contain files in one directory. Pre and post tests will create a
 816         directory rooted at the outputdir of the Test or TestGroup in
 817         question for their output. Failsafe scripts will create a directory
 818         rooted at the outputdir of each Test for their output.
 819         """
 820         done = False
 821         components = 0
 822         tmp_dict = dict(list(self.tests.items()) +
 823                         list(self.testgroups.items()))
 824         total = len(tmp_dict)
 825         base = self.outputdir
 826
 827         while not done:
 828             paths = []
 829             components -= 1
 830             for testfile in list(tmp_dict.keys()):
 831                 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
 832                 if uniq not in paths:
 833                     paths.append(uniq)
 834                     tmp_dict[testfile].outputdir = os.path.join(base, uniq)
 835                 else:
 836                     break
 837             done = total == len(paths)
 838
 839     def setup_logging(self, options):
 840         """
 841         This function creates the output directory and gets a file object
 842         for the logfile. This function must be called before write_log()
 843         can be used.
 844         """
 845         if options.dryrun is True:
 846             return
 847
 848         global LOG_FILE_OBJ
 849         if not options.template:
 850             try:
 851                 old = os.umask(0)
 852                 os.makedirs(self.outputdir, mode=0o777)
 853                 os.umask(old)
 854                 filename = os.path.join(self.outputdir, 'log')
 855                 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb')
 856             except OSError as e:
 857                 fail('%s' % e)
 858
 859     def run(self, options):
 860         """
 861         Walk through all the Tests and TestGroups, calling run().
 862         """
 863         try:
 864             os.chdir(self.outputdir)
 865         except OSError:
 866             fail('Could not change to directory %s' % self.outputdir)
 867         # make a symlink to the output for the currently running test
 868         logsymlink = os.path.join(self.outputdir, '../current')
 869         if os.path.islink(logsymlink):
 870             os.unlink(logsymlink)
 871         if not os.path.exists(logsymlink):
 872             os.symlink(self.outputdir, logsymlink)
 873         else:
 874             write_log('Could not make a symlink to directory %s\n' %
 875                       self.outputdir, LOG_ERR)
 876
 877         if options.kmemleak:
 878             cmd = f'echo scan=0 | {SUDO} tee {KMEMLEAK_FILE}'
 879             check_output(cmd, shell=True)
 880
 881         iteration = 0
 882         while iteration < options.iterations:
 883             for test in sorted(self.tests.keys()):
 884                 self.tests[test].run(options)
 885             for testgroup in sorted(self.testgroups.keys()):
 886                 self.testgroups[testgroup].run(options)
 887             iteration += 1
 888
 889     def summary(self):
 890         if Result.total == 0:
 891             return 2
 892
 893         print('\nResults Summary')
 894         for key in list(Result.runresults.keys()):
 895             if Result.runresults[key] != 0:
 896                 print('%s\t% 4d' % (key, Result.runresults[key]))
 897
 898         m, s = divmod(time() - self.starttime, 60)
 899         h, m = divmod(m, 60)
 900         print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
 901         print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
 902                                             float(Result.total)) * 100))
 903         print('Log directory:\t%s' % self.outputdir)
 904
 905         if Result.runresults['FAIL'] > 0:
 906             return 1
 907
 908         if Result.runresults['KILLED'] > 0:
 909             return 1
 910
 911         if Result.runresults['RERAN'] > 0:
 912             return 3
 913
 914         return 0
 915
 916
 917 def write_log(msg, target):
 918     """
 919     Write the provided message to standard out, standard error or
 920     the logfile. If specifying LOG_FILE, then `msg` must be a bytes
 921     like object. This way we can still handle output from tests that
 922     may be in unexpected encodings.
 923     """
 924     if target == LOG_OUT:
 925         os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8'))
 926     elif target == LOG_ERR:
 927         os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8'))
 928     elif target == LOG_FILE:
 929         os.write(LOG_FILE_OBJ.fileno(), msg)
 930     else:
 931         fail('log_msg called with unknown target "%s"' % target)
 932
 933
 934 def verify_file(pathname):
 935     """
 936     Verify that the supplied pathname is an executable regular file.
 937     """
 938     if os.path.isdir(pathname) or os.path.islink(pathname):
 939         return False
 940
 941     for ext in '', '.ksh', '.sh':
 942         script_path = pathname + ext
 943         if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
 944             return True
 945
 946     return False
 947
 948
 949 def verify_user(user):
 950     """
 951     Verify that the specified user exists on this system, and can execute
 952     sudo without being prompted for a password.
 953     """
 954     testcmd = [SUDO, '-n', '-u', user, TRUE]
 955
 956     if user in Cmd.verified_users:
 957         return True
 958
 959     try:
 960         getpwnam(user)
 961     except KeyError:
 962         write_log("Warning: user '%s' does not exist.\n" % user,
 963                   LOG_ERR)
 964         return False
 965
 966     p = Popen(testcmd)
 967     p.wait()
 968     if p.returncode != 0:
 969         write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user,
 970                   LOG_ERR)
 971         return False
 972     else:
 973         Cmd.verified_users.append(user)
 974
 975     return True
 976
 977
 978 def find_tests(testrun, options):
 979     """
 980     For the given list of pathnames, add files as Tests. For directories,
 981     if do_groups is True, add the directory as a TestGroup. If False,
 982     recursively search for executable files.
 983     """
 984
 985     for p in sorted(options.pathnames):
 986         if os.path.isdir(p):
 987             for dirname, _, filenames in os.walk(p):
 988                 if options.do_groups:
 989                     testrun.addtestgroup(dirname, filenames, options)
 990                 else:
 991                     for f in sorted(filenames):
 992                         testrun.addtest(os.path.join(dirname, f), options)
 993         else:
 994             testrun.addtest(p, options)
 995
 996
 997 def filter_tests(testrun, options):
 998     try:
 999         fh = open(options.logfile, "r")
1000     except Exception as e:
1001         fail('%s' % e)
1002
1003     failed = {}
1004     while True:
1005         line = fh.readline()
1006         if not line:
1007             break
1008         m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line)
1009         if not m:
1010             continue
1011         group, test = m.group(1, 2)
1012         try:
1013             failed[group].append(test)
1014         except KeyError:
1015             failed[group] = [test]
1016     fh.close()
1017
1018     testrun.filter(failed)
1019
1020
1021 def fail(retstr, ret=1):
1022     print('%s: %s' % (sys.argv[0], retstr))
1023     exit(ret)
1024
1025
1026 def kmemleak_cb(option, opt_str, value, parser):
1027     if not os.path.exists(KMEMLEAK_FILE):
1028         fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " +
1029              "Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.")
1030
1031     setattr(parser.values, option.dest, True)
1032
1033
1034 def options_cb(option, opt_str, value, parser):
1035     path_options = ['outputdir', 'template', 'testdir', 'logfile']
1036
1037     if opt_str in parser.rargs:
1038         fail('%s may only be specified once.' % opt_str)
1039
1040     if option.dest == 'runfiles':
1041         parser.values.cmd = 'rdconfig'
1042         value = set(os.path.abspath(p) for p in value.split(','))
1043     if option.dest == 'tags':
1044         value = [x.strip() for x in value.split(',')]
1045
1046     if option.dest in path_options:
1047         setattr(parser.values, option.dest, os.path.abspath(value))
1048     else:
1049         setattr(parser.values, option.dest, value)
1050
1051
1052 def parse_args():
1053     parser = OptionParser()
1054     parser.add_option('-c', action='callback', callback=options_cb,
1055                       type='string', dest='runfiles', metavar='runfiles',
1056                       help='Specify tests to run via config files.')
1057     parser.add_option('-d', action='store_true', default=False, dest='dryrun',
1058                       help='Dry run. Print tests, but take no other action.')
1059     parser.add_option('-l', action='callback', callback=options_cb,
1060                       default=None, dest='logfile', metavar='logfile',
1061                       type='string',
1062                       help='Read logfile and re-run tests which failed.')
1063     parser.add_option('-g', action='store_true', default=False,
1064                       dest='do_groups', help='Make directories TestGroups.')
1065     parser.add_option('-o', action='callback', callback=options_cb,
1066                       default=BASEDIR, dest='outputdir', type='string',
1067                       metavar='outputdir', help='Specify an output directory.')
1068     parser.add_option('-i', action='callback', callback=options_cb,
1069                       default=TESTDIR, dest='testdir', type='string',
1070                       metavar='testdir', help='Specify a test directory.')
1071     parser.add_option('-m', action='callback', callback=kmemleak_cb,
1072                       default=False, dest='kmemleak',
1073                       help='Enable kmemleak reporting (Linux only)')
1074     parser.add_option('-p', action='callback', callback=options_cb,
1075                       default='', dest='pre', metavar='script',
1076                       type='string', help='Specify a pre script.')
1077     parser.add_option('-P', action='callback', callback=options_cb,
1078                       default='', dest='post', metavar='script',
1079                       type='string', help='Specify a post script.')
1080     parser.add_option('-q', action='store_true', default=False, dest='quiet',
1081                       help='Silence on the console during a test run.')
1082     parser.add_option('-s', action='callback', callback=options_cb,
1083                       default='', dest='failsafe', metavar='script',
1084                       type='string', help='Specify a failsafe script.')
1085     parser.add_option('-S', action='callback', callback=options_cb,
1086                       default='', dest='failsafe_user',
1087                       metavar='failsafe_user', type='string',
1088                       help='Specify a user to execute the failsafe script.')
1089     parser.add_option('-t', action='callback', callback=options_cb, default=60,
1090                       dest='timeout', metavar='seconds', type='int',
1091                       help='Timeout (in seconds) for an individual test.')
1092     parser.add_option('-u', action='callback', callback=options_cb,
1093                       default='', dest='user', metavar='user', type='string',
1094                       help='Specify a different user name to run as.')
1095     parser.add_option('-w', action='callback', callback=options_cb,
1096                       default=None, dest='template', metavar='template',
1097                       type='string', help='Create a new config file.')
1098     parser.add_option('-x', action='callback', callback=options_cb, default='',
1099                       dest='pre_user', metavar='pre_user', type='string',
1100                       help='Specify a user to execute the pre script.')
1101     parser.add_option('-X', action='callback', callback=options_cb, default='',
1102                       dest='post_user', metavar='post_user', type='string',
1103                       help='Specify a user to execute the post script.')
1104     parser.add_option('-T', action='callback', callback=options_cb, default='',
1105                       dest='tags', metavar='tags', type='string',
1106                       help='Specify tags to execute specific test groups.')
1107     parser.add_option('-I', action='callback', callback=options_cb, default=1,
1108                       dest='iterations', metavar='iterations', type='int',
1109                       help='Number of times to run the test run.')
1110     (options, pathnames) = parser.parse_args()
1111
1112     if options.runfiles and len(pathnames):
1113         fail('Extraneous arguments.')
1114
1115     options.pathnames = [os.path.abspath(path) for path in pathnames]
1116
1117     return options
1118
1119
1120 def main():
1121     options = parse_args()
1122
1123     testrun = TestRun(options)
1124
1125     if options.runfiles:
1126         testrun.read(options)
1127     else:
1128         find_tests(testrun, options)
1129
1130     if options.logfile:
1131         filter_tests(testrun, options)
1132
1133     if options.template:
1134         testrun.write(options)
1135         exit(0)
1136
1137     testrun.complete_outputdirs()
1138     testrun.run(options)
1139     exit(testrun.summary())
1140
1141
1142 if __name__ == '__main__':
1143     main()