tests/test-runner/bin/test-runner.py.in

   1 #!/usr/bin/env @PYTHON_SHEBANG@
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  16 # Copyright (c) 2019 Datto Inc.
  17 #
  18 # This script must remain compatible with Python 3.6+.
  19 #
  20
  21 import os
  22 import sys
  23 import ctypes
  24 import re
  25 import configparser
  26
  27 from datetime import datetime
  28 from optparse import OptionParser
  29 from pwd import getpwnam
  30 from pwd import getpwuid
  31 from select import select
  32 from subprocess import PIPE
  33 from subprocess import Popen
  34 from subprocess import check_output
  35 from threading import Timer
  36 from time import time, CLOCK_MONOTONIC
  37 from os.path import exists
  38
  39 BASEDIR = '/var/tmp/test_results'
  40 TESTDIR = '/usr/share/zfs/'
  41 KMEMLEAK_FILE = '/sys/kernel/debug/kmemleak'
  42 KILL = 'kill'
  43 TRUE = 'true'
  44 SUDO = 'sudo'
  45 LOG_FILE = 'LOG_FILE'
  46 LOG_OUT = 'LOG_OUT'
  47 LOG_ERR = 'LOG_ERR'
  48 LOG_FILE_OBJ = None
  49
  50 try:
  51     from time import monotonic as monotonic_time
  52 except ImportError:
  53     class timespec(ctypes.Structure):
  54         _fields_ = [
  55             ('tv_sec', ctypes.c_long),
  56             ('tv_nsec', ctypes.c_long)
  57         ]
  58
  59     librt = ctypes.CDLL('librt.so.1', use_errno=True)
  60     clock_gettime = librt.clock_gettime
  61     clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)]
  62
  63     def monotonic_time():
  64         t = timespec()
  65         if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(t)) != 0:
  66             errno_ = ctypes.get_errno()
  67             raise OSError(errno_, os.strerror(errno_))
  68         return t.tv_sec + t.tv_nsec * 1e-9
  69
  70
  71 class Result(object):
  72     total = 0
  73     runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0}
  74
  75     def __init__(self):
  76         self.starttime = None
  77         self.returncode = None
  78         self.runtime = ''
  79         self.stdout = []
  80         self.stderr = []
  81         self.kmemleak = ''
  82         self.result = ''
  83
  84     def done(self, proc, killed, reran):
  85         """
  86         Finalize the results of this Cmd.
  87         """
  88         Result.total += 1
  89         m, s = divmod(monotonic_time() - self.starttime, 60)
  90         self.runtime = '%02d:%02d' % (m, s)
  91         self.returncode = proc.returncode
  92         if reran is True:
  93             Result.runresults['RERAN'] += 1
  94         if killed:
  95             self.result = 'KILLED'
  96             Result.runresults['KILLED'] += 1
  97         elif len(self.kmemleak) > 0:
  98             self.result = 'FAIL'
  99             Result.runresults['FAIL'] += 1
 100         elif self.returncode == 0:
 101             self.result = 'PASS'
 102             Result.runresults['PASS'] += 1
 103         elif self.returncode == 4:
 104             self.result = 'SKIP'
 105             Result.runresults['SKIP'] += 1
 106         elif self.returncode != 0:
 107             self.result = 'FAIL'
 108             Result.runresults['FAIL'] += 1
 109
 110
 111 class Output(object):
 112     """
 113     This class is a slightly modified version of the 'Stream' class found
 114     here: http://goo.gl/aSGfv
 115     """
 116     def __init__(self, stream, debug=False):
 117         self.stream = stream
 118         self.debug = debug
 119         self._buf = b''
 120         self.lines = []
 121
 122     def fileno(self):
 123         return self.stream.fileno()
 124
 125     def read(self, drain=0):
 126         """
 127         Read from the file descriptor. If 'drain' set, read until EOF.
 128         """
 129         while self._read() is not None:
 130             if not drain:
 131                 break
 132
 133     def _read(self):
 134         """
 135         Read up to 4k of data from this output stream. Collect the output
 136         up to the last newline, and append it to any leftover data from a
 137         previous call. The lines are stored as a (timestamp, data) tuple
 138         for easy sorting/merging later.
 139         """
 140         fd = self.fileno()
 141         buf = os.read(fd, 4096)
 142         if not buf:
 143             return None
 144         if self.debug:
 145             os.write(sys.stderr.fileno(), buf)
 146         if b'\n' not in buf:
 147             self._buf += buf
 148             return []
 149
 150         buf = self._buf + buf
 151         tmp, rest = buf.rsplit(b'\n', 1)
 152         self._buf = rest
 153         now = datetime.now()
 154         rows = tmp.split(b'\n')
 155         self.lines += [(now, r) for r in rows]
 156
 157
 158 class Cmd(object):
 159     verified_users = []
 160
 161     def __init__(self, pathname, identifier=None, outputdir=None,
 162                  timeout=None, user=None, tags=None):
 163         self.pathname = pathname
 164         self.identifier = identifier
 165         self.outputdir = outputdir or 'BASEDIR'
 166         """
 167         The timeout for tests is measured in wall-clock time
 168         """
 169         self.timeout = timeout
 170         self.user = user or ''
 171         self.killed = False
 172         self.reran = None
 173         self.result = Result()
 174
 175         if self.timeout is None:
 176             self.timeout = 60
 177
 178     def __str__(self):
 179         return '''\
 180 Pathname: %s
 181 Identifier: %s
 182 Outputdir: %s
 183 Timeout: %d
 184 User: %s
 185 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user)
 186
 187     def kill_cmd(self, proc, options, kmemleak, keyboard_interrupt=False):
 188         """
 189         Kill a running command due to timeout, or ^C from the keyboard. If
 190         sudo is required, this user was verified previously.
 191         """
 192         self.killed = True
 193         do_sudo = len(self.user) != 0
 194         signal = '-TERM'
 195
 196         cmd = [SUDO, KILL, signal, str(proc.pid)]
 197         if not do_sudo:
 198             del cmd[0]
 199
 200         try:
 201             kp = Popen(cmd)
 202             kp.wait()
 203         except Exception:
 204             pass
 205
 206         """
 207         If this is not a user-initiated kill and the test has not been
 208         reran before we consider if the test needs to be reran:
 209         If the test has spent some time hibernating and didn't run the whole
 210         length of time before being timed out we will rerun the test.
 211         """
 212         if keyboard_interrupt is False and self.reran is None:
 213             runtime = monotonic_time() - self.result.starttime
 214             if int(self.timeout) > runtime:
 215                 self.killed = False
 216                 self.reran = False
 217                 self.run(options, dryrun=False, kmemleak=kmemleak)
 218                 self.reran = True
 219
 220     def update_cmd_privs(self, cmd, user):
 221         """
 222         If a user has been specified to run this Cmd and we're not already
 223         running as that user, prepend the appropriate sudo command to run
 224         as that user.
 225         """
 226         me = getpwuid(os.getuid())
 227
 228         if not user or user is me:
 229             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 230                 cmd += '.ksh'
 231             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 232                 cmd += '.sh'
 233             return cmd
 234
 235         if not os.path.isfile(cmd):
 236             if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK):
 237                 cmd += '.ksh'
 238             if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK):
 239                 cmd += '.sh'
 240
 241         ret = '%s -E -u %s %s' % (SUDO, user, cmd)
 242         return ret.split(' ')
 243
 244     def collect_output(self, proc, debug=False):
 245         """
 246         Read from stdout/stderr as data becomes available, until the
 247         process is no longer running. Return the lines from the stdout and
 248         stderr Output objects.
 249         """
 250         out = Output(proc.stdout, debug)
 251         err = Output(proc.stderr, debug)
 252         res = []
 253         while proc.returncode is None:
 254             proc.poll()
 255             res = select([out, err], [], [], .1)
 256             for fd in res[0]:
 257                 fd.read()
 258         for fd in res[0]:
 259             fd.read(drain=1)
 260
 261         return out.lines, err.lines
 262
 263     def run(self, options, dryrun=None, kmemleak=None):
 264         """
 265         This is the main function that runs each individual test.
 266         Determine whether or not the command requires sudo, and modify it
 267         if needed. Run the command, and update the result object.
 268         """
 269         if dryrun is None:
 270             dryrun = options.dryrun
 271         if dryrun is True:
 272             print(self)
 273             return
 274         if kmemleak is None:
 275             kmemleak = options.kmemleak
 276
 277         privcmd = self.update_cmd_privs(self.pathname, self.user)
 278         try:
 279             old = os.umask(0)
 280             if not os.path.isdir(self.outputdir):
 281                 os.makedirs(self.outputdir, mode=0o777)
 282             os.umask(old)
 283         except OSError as e:
 284             fail('%s' % e)
 285
 286         """
 287         Log each test we run to /dev/kmsg (on Linux), so if there's a kernel
 288         warning we'll be able to match it up to a particular test.
 289         """
 290         if options.kmsg is True and exists("/dev/kmsg"):
 291             try:
 292                 kp = Popen([SUDO, "sh", "-c",
 293                             f"echo ZTS run {self.pathname} > /dev/kmsg"])
 294                 kp.wait()
 295             except Exception:
 296                 pass
 297
 298         self.result.starttime = monotonic_time()
 299
 300         if kmemleak:
 301             cmd = f'{SUDO} sh -c "echo clear > {KMEMLEAK_FILE}"'
 302             check_output(cmd, shell=True)
 303
 304         proc = Popen(privcmd, stdout=PIPE, stderr=PIPE)
 305         # Allow a special timeout value of 0 to mean infinity
 306         if int(self.timeout) == 0:
 307             self.timeout = sys.maxsize / (10 ** 9)
 308         t = Timer(
 309             int(self.timeout), self.kill_cmd, [proc, options, kmemleak]
 310         )
 311
 312         try:
 313             t.start()
 314
 315             out, err = self.collect_output(proc, options.debug)
 316             self.result.stdout = out
 317             self.result.stderr = err
 318
 319             if kmemleak:
 320                 cmd = f'{SUDO} sh -c "echo scan > {KMEMLEAK_FILE}"'
 321                 check_output(cmd, shell=True)
 322                 cmd = f'{SUDO} cat {KMEMLEAK_FILE}'
 323                 self.result.kmemleak = check_output(cmd, shell=True)
 324         except KeyboardInterrupt:
 325             self.kill_cmd(proc, options, kmemleak, True)
 326             fail('\nRun terminated at user request.')
 327         finally:
 328             t.cancel()
 329
 330         if self.reran is not False:
 331             self.result.done(proc, self.killed, self.reran)
 332
 333     def skip(self):
 334         """
 335         Initialize enough of the test result that we can log a skipped
 336         command.
 337         """
 338         Result.total += 1
 339         Result.runresults['SKIP'] += 1
 340         self.result.stdout = self.result.stderr = []
 341         self.result.starttime = monotonic_time()
 342         m, s = divmod(monotonic_time() - self.result.starttime, 60)
 343         self.result.runtime = '%02d:%02d' % (m, s)
 344         self.result.result = 'SKIP'
 345
 346     def log(self, options, suppress_console=False):
 347         """
 348         This function is responsible for writing all output. This includes
 349         the console output, the logfile of all results (with timestamped
 350         merged stdout and stderr), and for each test, the unmodified
 351         stdout/stderr/merged in its own file.
 352         """
 353
 354         logname = getpwuid(os.getuid()).pw_name
 355         rer = ''
 356         if self.reran is True:
 357             rer = ' (RERAN)'
 358         user = ' (run as %s)' % (self.user if len(self.user) else logname)
 359         if self.identifier:
 360             msga = 'Test (%s): %s%s ' % (self.identifier, self.pathname, user)
 361         else:
 362             msga = 'Test: %s%s ' % (self.pathname, user)
 363         msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer)
 364         pad = ' ' * (80 - (len(msga) + len(msgb)))
 365         result_line = msga + pad + msgb
 366
 367         # The result line is always written to the log file. If -q was
 368         # specified only failures are written to the console, otherwise
 369         # the result line is written to the console. The console output
 370         # may be suppressed by calling log() with suppress_console=True.
 371         write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE)
 372         if not suppress_console:
 373             if not options.quiet:
 374                 write_log(result_line, LOG_OUT)
 375             elif options.quiet and self.result.result != 'PASS':
 376                 write_log(result_line, LOG_OUT)
 377
 378         lines = sorted(self.result.stdout + self.result.stderr,
 379                        key=lambda x: x[0])
 380
 381         # Write timestamped output (stdout and stderr) to the logfile
 382         for dt, line in lines:
 383             timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11],
 384                                   encoding='utf-8')
 385             write_log(b'%s %s\n' % (timestamp, line), LOG_FILE)
 386
 387         # Write the separate stdout/stderr/merged files, if the data exists
 388         if len(self.result.stdout):
 389             with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out:
 390                 for _, line in self.result.stdout:
 391                     os.write(out.fileno(), b'%s\n' % line)
 392         if len(self.result.stderr):
 393             with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err:
 394                 for _, line in self.result.stderr:
 395                     os.write(err.fileno(), b'%s\n' % line)
 396         if len(self.result.stdout) and len(self.result.stderr):
 397             with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged:
 398                 for _, line in lines:
 399                     os.write(merged.fileno(), b'%s\n' % line)
 400         if len(self.result.kmemleak):
 401             with open(os.path.join(self.outputdir, 'kmemleak'), 'wb') as kmem:
 402                 kmem.write(self.result.kmemleak)
 403
 404
 405 class Test(Cmd):
 406     props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
 407              'post_user', 'failsafe', 'failsafe_user', 'tags']
 408
 409     def __init__(self, pathname,
 410                  pre=None, pre_user=None, post=None, post_user=None,
 411                  failsafe=None, failsafe_user=None, tags=None, **kwargs):
 412         super(Test, self).__init__(pathname, **kwargs)
 413         self.pre = pre or ''
 414         self.pre_user = pre_user or ''
 415         self.post = post or ''
 416         self.post_user = post_user or ''
 417         self.failsafe = failsafe or ''
 418         self.failsafe_user = failsafe_user or ''
 419         self.tags = tags or []
 420
 421     def __str__(self):
 422         post_user = pre_user = failsafe_user = ''
 423         if len(self.pre_user):
 424             pre_user = ' (as %s)' % (self.pre_user)
 425         if len(self.post_user):
 426             post_user = ' (as %s)' % (self.post_user)
 427         if len(self.failsafe_user):
 428             failsafe_user = ' (as %s)' % (self.failsafe_user)
 429         return '''\
 430 Pathname: %s
 431 Identifier: %s
 432 Outputdir: %s
 433 Timeout: %d
 434 User: %s
 435 Pre: %s%s
 436 Post: %s%s
 437 Failsafe: %s%s
 438 Tags: %s
 439 ''' % (self.pathname, self.identifier, self.outputdir, self.timeout, self.user,
 440             self.pre, pre_user, self.post, post_user, self.failsafe,
 441             failsafe_user, self.tags)
 442
 443     def verify(self):
 444         """
 445         Check the pre/post/failsafe scripts, user and Test. Omit the Test from
 446         this run if there are any problems.
 447         """
 448         files = [self.pre, self.pathname, self.post, self.failsafe]
 449         users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
 450
 451         for f in [f for f in files if len(f)]:
 452             if not verify_file(f):
 453                 write_log("Warning: Test '%s' not added to this run because"
 454                           " it failed verification.\n" % f, LOG_ERR)
 455                 return False
 456
 457         for user in [user for user in users if len(user)]:
 458             if not verify_user(user):
 459                 write_log("Not adding Test '%s' to this run.\n" %
 460                           self.pathname, LOG_ERR)
 461                 return False
 462
 463         return True
 464
 465     def run(self, options, dryrun=None, kmemleak=None):
 466         """
 467         Create Cmd instances for the pre/post/failsafe scripts. If the pre
 468         script doesn't pass, skip this Test. Run the post script regardless.
 469         If the Test is killed, also run the failsafe script.
 470         """
 471         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 472         pretest = Cmd(self.pre, identifier=self.identifier, outputdir=odir,
 473                       timeout=self.timeout, user=self.pre_user)
 474         test = Cmd(self.pathname, identifier=self.identifier,
 475                    outputdir=self.outputdir, timeout=self.timeout,
 476                    user=self.user)
 477         odir = os.path.join(self.outputdir, os.path.basename(self.failsafe))
 478         failsafe = Cmd(self.failsafe, identifier=self.identifier,
 479                        outputdir=odir, timeout=self.timeout,
 480                        user=self.failsafe_user)
 481         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 482         posttest = Cmd(self.post, identifier=self.identifier, outputdir=odir,
 483                        timeout=self.timeout, user=self.post_user)
 484
 485         cont = True
 486         if len(pretest.pathname):
 487             pretest.run(options, kmemleak=False)
 488             cont = pretest.result.result == 'PASS'
 489             pretest.log(options)
 490
 491         if cont:
 492             test.run(options, kmemleak=kmemleak)
 493             if test.result.result == 'KILLED' and len(failsafe.pathname):
 494                 failsafe.run(options, kmemleak=False)
 495                 failsafe.log(options, suppress_console=True)
 496         else:
 497             test.skip()
 498
 499         test.log(options)
 500
 501         if len(posttest.pathname):
 502             posttest.run(options, kmemleak=False)
 503             posttest.log(options)
 504
 505
 506 class TestGroup(Test):
 507     props = Test.props + ['tests']
 508
 509     def __init__(self, pathname, tests=None, **kwargs):
 510         super(TestGroup, self).__init__(pathname, **kwargs)
 511         self.tests = tests or []
 512
 513     def __str__(self):
 514         post_user = pre_user = failsafe_user = ''
 515         if len(self.pre_user):
 516             pre_user = ' (as %s)' % (self.pre_user)
 517         if len(self.post_user):
 518             post_user = ' (as %s)' % (self.post_user)
 519         if len(self.failsafe_user):
 520             failsafe_user = ' (as %s)' % (self.failsafe_user)
 521         return '''\
 522 Pathname: %s
 523 Identifier: %s
 524 Outputdir: %s
 525 Tests: %s
 526 Timeout: %s
 527 User: %s
 528 Pre: %s%s
 529 Post: %s%s
 530 Failsafe: %s%s
 531 Tags: %s
 532 ''' % (self.pathname, self.identifier, self.outputdir, self.tests,
 533             self.timeout, self.user, self.pre, pre_user, self.post, post_user,
 534             self.failsafe, failsafe_user, self.tags)
 535
 536     def filter(self, keeplist):
 537         self.tests = [x for x in self.tests if x in keeplist]
 538
 539     def verify(self):
 540         """
 541         Check the pre/post/failsafe scripts, user and tests in this TestGroup.
 542         Omit the TestGroup entirely, or simply delete the relevant tests in the
 543         group, if that's all that's required.
 544         """
 545         # If the pre/post/failsafe scripts are relative pathnames, convert to
 546         # absolute, so they stand a chance of passing verification.
 547         if len(self.pre) and not os.path.isabs(self.pre):
 548             self.pre = os.path.join(self.pathname, self.pre)
 549         if len(self.post) and not os.path.isabs(self.post):
 550             self.post = os.path.join(self.pathname, self.post)
 551         if len(self.failsafe) and not os.path.isabs(self.failsafe):
 552             self.post = os.path.join(self.pathname, self.post)
 553
 554         auxfiles = [self.pre, self.post, self.failsafe]
 555         users = [self.pre_user, self.user, self.post_user, self.failsafe_user]
 556
 557         for f in [f for f in auxfiles if len(f)]:
 558             if f != self.failsafe and self.pathname != os.path.dirname(f):
 559                 write_log("Warning: TestGroup '%s' not added to this run. "
 560                           "Auxiliary script '%s' exists in a different "
 561                           "directory.\n" % (self.pathname, f), LOG_ERR)
 562                 return False
 563
 564             if not verify_file(f):
 565                 write_log("Warning: TestGroup '%s' not added to this run. "
 566                           "Auxiliary script '%s' failed verification.\n" %
 567                           (self.pathname, f), LOG_ERR)
 568                 return False
 569
 570         for user in [user for user in users if len(user)]:
 571             if not verify_user(user):
 572                 write_log("Not adding TestGroup '%s' to this run.\n" %
 573                           self.pathname, LOG_ERR)
 574                 return False
 575
 576         # If one of the tests is invalid, delete it, log it, and drive on.
 577         for test in self.tests:
 578             if not verify_file(os.path.join(self.pathname, test)):
 579                 del self.tests[self.tests.index(test)]
 580                 write_log("Warning: Test '%s' removed from TestGroup '%s' "
 581                           "because it failed verification.\n" %
 582                           (test, self.pathname), LOG_ERR)
 583
 584         return len(self.tests) != 0
 585
 586     def run(self, options, dryrun=None, kmemleak=None):
 587         """
 588         Create Cmd instances for the pre/post/failsafe scripts. If the pre
 589         script doesn't pass, skip all the tests in this TestGroup. Run the
 590         post script regardless. Run the failsafe script when a test is killed.
 591         """
 592         # tags assigned to this test group also include the test names
 593         if options.tags and not set(self.tags).intersection(set(options.tags)):
 594             return
 595
 596         odir = os.path.join(self.outputdir, os.path.basename(self.pre))
 597         pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
 598                       user=self.pre_user, identifier=self.identifier)
 599         odir = os.path.join(self.outputdir, os.path.basename(self.post))
 600         posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
 601                        user=self.post_user, identifier=self.identifier)
 602
 603         cont = True
 604         if len(pretest.pathname):
 605             pretest.run(options, dryrun=dryrun, kmemleak=False)
 606             cont = pretest.result.result == 'PASS'
 607             pretest.log(options)
 608
 609         for fname in self.tests:
 610             odir = os.path.join(self.outputdir, fname)
 611             test = Cmd(os.path.join(self.pathname, fname), outputdir=odir,
 612                        timeout=self.timeout, user=self.user,
 613                        identifier=self.identifier)
 614             odir = os.path.join(odir, os.path.basename(self.failsafe))
 615             failsafe = Cmd(self.failsafe, outputdir=odir, timeout=self.timeout,
 616                            user=self.failsafe_user, identifier=self.identifier)
 617             if cont:
 618                 test.run(options, dryrun=dryrun, kmemleak=kmemleak)
 619                 if test.result.result == 'KILLED' and len(failsafe.pathname):
 620                     failsafe.run(options, dryrun=dryrun, kmemleak=False)
 621                     failsafe.log(options, suppress_console=True)
 622             else:
 623                 test.skip()
 624
 625             test.log(options)
 626
 627         if len(posttest.pathname):
 628             posttest.run(options, dryrun=dryrun, kmemleak=False)
 629             posttest.log(options)
 630
 631
 632 class TestRun(object):
 633     props = ['quiet', 'outputdir', 'debug']
 634
 635     def __init__(self, options):
 636         self.tests = {}
 637         self.testgroups = {}
 638         self.starttime = time()
 639         self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
 640         self.outputdir = os.path.join(options.outputdir, self.timestamp)
 641         self.setup_logging(options)
 642         self.defaults = [
 643             ('outputdir', BASEDIR),
 644             ('quiet', False),
 645             ('timeout', 60),
 646             ('user', ''),
 647             ('pre', ''),
 648             ('pre_user', ''),
 649             ('post', ''),
 650             ('post_user', ''),
 651             ('failsafe', ''),
 652             ('failsafe_user', ''),
 653             ('tags', []),
 654             ('debug', False)
 655         ]
 656
 657     def __str__(self):
 658         s = 'TestRun:\n    outputdir: %s\n' % self.outputdir
 659         s += 'TESTS:\n'
 660         for key in sorted(self.tests.keys()):
 661             s += '%s%s' % (self.tests[key].__str__(), '\n')
 662         s += 'TESTGROUPS:\n'
 663         for key in sorted(self.testgroups.keys()):
 664             s += '%s%s' % (self.testgroups[key].__str__(), '\n')
 665         return s
 666
 667     def addtest(self, pathname, options):
 668         """
 669         Create a new Test, and apply any properties that were passed in
 670         from the command line. If it passes verification, add it to the
 671         TestRun.
 672         """
 673         test = Test(pathname)
 674         for prop in Test.props:
 675             setattr(test, prop, getattr(options, prop))
 676
 677         if test.verify():
 678             self.tests[pathname] = test
 679
 680     def addtestgroup(self, dirname, filenames, options):
 681         """
 682         Create a new TestGroup, and apply any properties that were passed
 683         in from the command line. If it passes verification, add it to the
 684         TestRun.
 685         """
 686         if dirname not in self.testgroups:
 687             testgroup = TestGroup(dirname)
 688             for prop in Test.props:
 689                 setattr(testgroup, prop, getattr(options, prop))
 690
 691             # Prevent pre/post/failsafe scripts from running as regular tests
 692             for f in [testgroup.pre, testgroup.post, testgroup.failsafe]:
 693                 if f in filenames:
 694                     del filenames[filenames.index(f)]
 695
 696             self.testgroups[dirname] = testgroup
 697             self.testgroups[dirname].tests = sorted(filenames)
 698
 699             testgroup.verify()
 700
 701     def filter(self, keeplist):
 702         for group in list(self.testgroups.keys()):
 703             if group not in keeplist:
 704                 del self.testgroups[group]
 705                 continue
 706
 707             g = self.testgroups[group]
 708
 709             if g.pre and os.path.basename(g.pre) in keeplist[group]:
 710                 continue
 711
 712             g.filter(keeplist[group])
 713
 714         for test in list(self.tests.keys()):
 715             directory, base = os.path.split(test)
 716             if directory not in keeplist or base not in keeplist[directory]:
 717                 del self.tests[test]
 718
 719     def read(self, options):
 720         """
 721         Read in the specified runfiles, and apply the TestRun properties
 722         listed in the 'DEFAULT' section to our TestRun. Then read each
 723         section, and apply the appropriate properties to the Test or
 724         TestGroup. Properties from individual sections override those set
 725         in the 'DEFAULT' section. If the Test or TestGroup passes
 726         verification, add it to the TestRun.
 727         """
 728         config = configparser.RawConfigParser()
 729         parsed = config.read(options.runfiles)
 730         failed = options.runfiles - set(parsed)
 731         if len(failed):
 732             files = ' '.join(sorted(failed))
 733             fail("Couldn't read config files: %s" % files)
 734
 735         for opt in TestRun.props:
 736             if config.has_option('DEFAULT', opt):
 737                 setattr(self, opt, config.get('DEFAULT', opt))
 738         self.outputdir = os.path.join(self.outputdir, self.timestamp)
 739
 740         testdir = options.testdir
 741
 742         for section in config.sections():
 743             if 'tests' in config.options(section):
 744                 parts = section.split(':', 1)
 745                 sectiondir = parts[0]
 746                 identifier = parts[1] if len(parts) == 2 else None
 747                 if os.path.isdir(sectiondir):
 748                     pathname = sectiondir
 749                 elif os.path.isdir(os.path.join(testdir, sectiondir)):
 750                     pathname = os.path.join(testdir, sectiondir)
 751                 else:
 752                     pathname = sectiondir
 753
 754                 testgroup = TestGroup(os.path.abspath(pathname),
 755                                       identifier=identifier)
 756                 for prop in TestGroup.props:
 757                     for sect in ['DEFAULT', section]:
 758                         if config.has_option(sect, prop):
 759                             if prop == 'tags':
 760                                 setattr(testgroup, prop,
 761                                         eval(config.get(sect, prop)))
 762                             elif prop == 'failsafe':
 763                                 failsafe = config.get(sect, prop)
 764                                 setattr(testgroup, prop,
 765                                         os.path.join(testdir, failsafe))
 766                             else:
 767                                 setattr(testgroup, prop,
 768                                         config.get(sect, prop))
 769
 770                 # Repopulate tests using eval to convert the string to a list
 771                 testgroup.tests = eval(config.get(section, 'tests'))
 772
 773                 if testgroup.verify():
 774                     self.testgroups[section] = testgroup
 775             else:
 776                 test = Test(section)
 777                 for prop in Test.props:
 778                     for sect in ['DEFAULT', section]:
 779                         if config.has_option(sect, prop):
 780                             if prop == 'failsafe':
 781                                 failsafe = config.get(sect, prop)
 782                                 setattr(test, prop,
 783                                         os.path.join(testdir, failsafe))
 784                             else:
 785                                 setattr(test, prop, config.get(sect, prop))
 786
 787                 if test.verify():
 788                     self.tests[section] = test
 789
 790     def write(self, options):
 791         """
 792         Create a configuration file for editing and later use. The
 793         'DEFAULT' section of the config file is created from the
 794         properties that were specified on the command line. Tests are
 795         simply added as sections that inherit everything from the
 796         'DEFAULT' section. TestGroups are the same, except they get an
 797         option including all the tests to run in that directory.
 798         """
 799
 800         defaults = dict([(prop, getattr(options, prop)) for prop, _ in
 801                          self.defaults])
 802         config = configparser.RawConfigParser(defaults)
 803
 804         for test in sorted(self.tests.keys()):
 805             config.add_section(test)
 806             for prop in Test.props:
 807                 if prop not in self.props:
 808                     config.set(test, prop,
 809                                getattr(self.tests[test], prop))
 810
 811         for testgroup in sorted(self.testgroups.keys()):
 812             config.add_section(testgroup)
 813             config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
 814             for prop in TestGroup.props:
 815                 if prop not in self.props:
 816                     config.set(testgroup, prop,
 817                                getattr(self.testgroups[testgroup], prop))
 818
 819         try:
 820             with open(options.template, 'w') as f:
 821                 return config.write(f)
 822         except IOError:
 823             fail('Could not open \'%s\' for writing.' % options.template)
 824
 825     def complete_outputdirs(self):
 826         """
 827         Collect all the pathnames for Tests, and TestGroups. Work
 828         backwards one pathname component at a time, to create a unique
 829         directory name in which to deposit test output. Tests will be able
 830         to write output files directly in the newly modified outputdir.
 831         TestGroups will be able to create one subdirectory per test in the
 832         outputdir, and are guaranteed uniqueness because a group can only
 833         contain files in one directory. Pre and post tests will create a
 834         directory rooted at the outputdir of the Test or TestGroup in
 835         question for their output. Failsafe scripts will create a directory
 836         rooted at the outputdir of each Test for their output.
 837         """
 838         done = False
 839         components = 0
 840         tmp_dict = dict(list(self.tests.items()) +
 841                         list(self.testgroups.items()))
 842         total = len(tmp_dict)
 843         base = self.outputdir
 844
 845         while not done:
 846             paths = []
 847             components -= 1
 848             for testfile in list(tmp_dict.keys()):
 849                 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
 850                 if uniq not in paths:
 851                     paths.append(uniq)
 852                     tmp_dict[testfile].outputdir = os.path.join(base, uniq)
 853                 else:
 854                     break
 855             done = total == len(paths)
 856
 857     def setup_logging(self, options):
 858         """
 859         This function creates the output directory and gets a file object
 860         for the logfile. This function must be called before write_log()
 861         can be used.
 862         """
 863         if options.dryrun is True:
 864             return
 865
 866         global LOG_FILE_OBJ
 867         if not options.template:
 868             try:
 869                 old = os.umask(0)
 870                 os.makedirs(self.outputdir, mode=0o777)
 871                 os.umask(old)
 872                 filename = os.path.join(self.outputdir, 'log')
 873                 LOG_FILE_OBJ = open(filename, buffering=0, mode='wb')
 874             except OSError as e:
 875                 fail('%s' % e)
 876
 877     def run(self, options):
 878         """
 879         Walk through all the Tests and TestGroups, calling run().
 880         """
 881         try:
 882             os.chdir(self.outputdir)
 883         except OSError:
 884             fail('Could not change to directory %s' % self.outputdir)
 885         # make a symlink to the output for the currently running test
 886         logsymlink = os.path.join(self.outputdir, '../current')
 887         if os.path.islink(logsymlink):
 888             os.unlink(logsymlink)
 889         if not os.path.exists(logsymlink):
 890             os.symlink(self.outputdir, logsymlink)
 891         else:
 892             write_log('Could not make a symlink to directory %s\n' %
 893                       self.outputdir, LOG_ERR)
 894
 895         if options.kmemleak:
 896             cmd = f'{SUDO} -c "echo scan=0 > {KMEMLEAK_FILE}"'
 897             check_output(cmd, shell=True)
 898
 899         iteration = 0
 900         while iteration < options.iterations:
 901             for test in sorted(self.tests.keys()):
 902                 self.tests[test].run(options)
 903             for testgroup in sorted(self.testgroups.keys()):
 904                 self.testgroups[testgroup].run(options)
 905             iteration += 1
 906
 907     def summary(self):
 908         if Result.total == 0:
 909             return 2
 910
 911         print('\nResults Summary')
 912         for key in list(Result.runresults.keys()):
 913             if Result.runresults[key] != 0:
 914                 print('%s\t% 4d' % (key, Result.runresults[key]))
 915
 916         m, s = divmod(time() - self.starttime, 60)
 917         h, m = divmod(m, 60)
 918         print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s))
 919         print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
 920                                             float(Result.total)) * 100))
 921         print('Log directory:\t%s' % self.outputdir)
 922
 923         if Result.runresults['FAIL'] > 0:
 924             return 1
 925
 926         if Result.runresults['KILLED'] > 0:
 927             return 1
 928
 929         if Result.runresults['RERAN'] > 0:
 930             return 3
 931
 932         return 0
 933
 934
 935 def write_log(msg, target):
 936     """
 937     Write the provided message to standard out, standard error or
 938     the logfile. If specifying LOG_FILE, then `msg` must be a bytes
 939     like object. This way we can still handle output from tests that
 940     may be in unexpected encodings.
 941     """
 942     if target == LOG_OUT:
 943         os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8'))
 944     elif target == LOG_ERR:
 945         os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8'))
 946     elif target == LOG_FILE:
 947         os.write(LOG_FILE_OBJ.fileno(), msg)
 948     else:
 949         fail('log_msg called with unknown target "%s"' % target)
 950
 951
 952 def verify_file(pathname):
 953     """
 954     Verify that the supplied pathname is an executable regular file.
 955     """
 956     if os.path.isdir(pathname) or os.path.islink(pathname):
 957         return False
 958
 959     for ext in '', '.ksh', '.sh':
 960         script_path = pathname + ext
 961         if os.path.isfile(script_path) and os.access(script_path, os.X_OK):
 962             return True
 963
 964     return False
 965
 966
 967 def verify_user(user):
 968     """
 969     Verify that the specified user exists on this system, and can execute
 970     sudo without being prompted for a password.
 971     """
 972     testcmd = [SUDO, '-n', '-u', user, TRUE]
 973
 974     if user in Cmd.verified_users:
 975         return True
 976
 977     try:
 978         getpwnam(user)
 979     except KeyError:
 980         write_log("Warning: user '%s' does not exist.\n" % user,
 981                   LOG_ERR)
 982         return False
 983
 984     p = Popen(testcmd)
 985     p.wait()
 986     if p.returncode != 0:
 987         write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user,
 988                   LOG_ERR)
 989         return False
 990     else:
 991         Cmd.verified_users.append(user)
 992
 993     return True
 994
 995
 996 def find_tests(testrun, options):
 997     """
 998     For the given list of pathnames, add files as Tests. For directories,
 999     if do_groups is True, add the directory as a TestGroup. If False,
1000     recursively search for executable files.
1001     """
1002
1003     for p in sorted(options.pathnames):
1004         if os.path.isdir(p):
1005             for dirname, _, filenames in os.walk(p):
1006                 if options.do_groups:
1007                     testrun.addtestgroup(dirname, filenames, options)
1008                 else:
1009                     for f in sorted(filenames):
1010                         testrun.addtest(os.path.join(dirname, f), options)
1011         else:
1012             testrun.addtest(p, options)
1013
1014
1015 def filter_tests(testrun, options):
1016     try:
1017         fh = open(options.logfile, "r")
1018     except Exception as e:
1019         fail('%s' % e)
1020
1021     failed = {}
1022     while True:
1023         line = fh.readline()
1024         if not line:
1025             break
1026         m = re.match(r'Test: .*(tests/.*)/(\S+).*\[FAIL\]', line)
1027         if not m:
1028             continue
1029         group, test = m.group(1, 2)
1030         try:
1031             failed[group].append(test)
1032         except KeyError:
1033             failed[group] = [test]
1034     fh.close()
1035
1036     testrun.filter(failed)
1037
1038
1039 def fail(retstr, ret=1):
1040     print('%s: %s' % (sys.argv[0], retstr))
1041     exit(ret)
1042
1043
1044 def kmemleak_cb(option, opt_str, value, parser):
1045     if not os.path.exists(KMEMLEAK_FILE):
1046         fail(f"File '{KMEMLEAK_FILE}' doesn't exist. " +
1047              "Enable CONFIG_DEBUG_KMEMLEAK in kernel configuration.")
1048
1049     setattr(parser.values, option.dest, True)
1050
1051
1052 def options_cb(option, opt_str, value, parser):
1053     path_options = ['outputdir', 'template', 'testdir', 'logfile']
1054
1055     if opt_str in parser.rargs:
1056         fail('%s may only be specified once.' % opt_str)
1057
1058     if option.dest == 'runfiles':
1059         parser.values.cmd = 'rdconfig'
1060         value = set(os.path.abspath(p) for p in value.split(','))
1061     if option.dest == 'tags':
1062         value = [x.strip() for x in value.split(',')]
1063
1064     if option.dest in path_options:
1065         setattr(parser.values, option.dest, os.path.abspath(value))
1066     else:
1067         setattr(parser.values, option.dest, value)
1068
1069
1070 def parse_args():
1071     parser = OptionParser()
1072     parser.add_option('-c', action='callback', callback=options_cb,
1073                       type='string', dest='runfiles', metavar='runfiles',
1074                       help='Specify tests to run via config files.')
1075     parser.add_option('-d', action='store_true', default=False, dest='dryrun',
1076                       help='Dry run. Print tests, but take no other action.')
1077     parser.add_option('-D', action='store_true', default=False, dest='debug',
1078                       help='Write all test output to stdout as it arrives.')
1079     parser.add_option('-l', action='callback', callback=options_cb,
1080                       default=None, dest='logfile', metavar='logfile',
1081                       type='string',
1082                       help='Read logfile and re-run tests which failed.')
1083     parser.add_option('-g', action='store_true', default=False,
1084                       dest='do_groups', help='Make directories TestGroups.')
1085     parser.add_option('-o', action='callback', callback=options_cb,
1086                       default=BASEDIR, dest='outputdir', type='string',
1087                       metavar='outputdir', help='Specify an output directory.')
1088     parser.add_option('-i', action='callback', callback=options_cb,
1089                       default=TESTDIR, dest='testdir', type='string',
1090                       metavar='testdir', help='Specify a test directory.')
1091     parser.add_option('-K', action='store_true', default=False, dest='kmsg',
1092                       help='Log tests names to /dev/kmsg')
1093     parser.add_option('-m', action='callback', callback=kmemleak_cb,
1094                       default=False, dest='kmemleak',
1095                       help='Enable kmemleak reporting (Linux only)')
1096     parser.add_option('-p', action='callback', callback=options_cb,
1097                       default='', dest='pre', metavar='script',
1098                       type='string', help='Specify a pre script.')
1099     parser.add_option('-P', action='callback', callback=options_cb,
1100                       default='', dest='post', metavar='script',
1101                       type='string', help='Specify a post script.')
1102     parser.add_option('-q', action='store_true', default=False, dest='quiet',
1103                       help='Silence on the console during a test run.')
1104     parser.add_option('-s', action='callback', callback=options_cb,
1105                       default='', dest='failsafe', metavar='script',
1106                       type='string', help='Specify a failsafe script.')
1107     parser.add_option('-S', action='callback', callback=options_cb,
1108                       default='', dest='failsafe_user',
1109                       metavar='failsafe_user', type='string',
1110                       help='Specify a user to execute the failsafe script.')
1111     parser.add_option('-t', action='callback', callback=options_cb, default=60,
1112                       dest='timeout', metavar='seconds', type='int',
1113                       help='Timeout (in seconds) for an individual test.')
1114     parser.add_option('-u', action='callback', callback=options_cb,
1115                       default='', dest='user', metavar='user', type='string',
1116                       help='Specify a different user name to run as.')
1117     parser.add_option('-w', action='callback', callback=options_cb,
1118                       default=None, dest='template', metavar='template',
1119                       type='string', help='Create a new config file.')
1120     parser.add_option('-x', action='callback', callback=options_cb, default='',
1121                       dest='pre_user', metavar='pre_user', type='string',
1122                       help='Specify a user to execute the pre script.')
1123     parser.add_option('-X', action='callback', callback=options_cb, default='',
1124                       dest='post_user', metavar='post_user', type='string',
1125                       help='Specify a user to execute the post script.')
1126     parser.add_option('-T', action='callback', callback=options_cb, default='',
1127                       dest='tags', metavar='tags', type='string',
1128                       help='Specify tags to execute specific test groups.')
1129     parser.add_option('-I', action='callback', callback=options_cb, default=1,
1130                       dest='iterations', metavar='iterations', type='int',
1131                       help='Number of times to run the test run.')
1132     (options, pathnames) = parser.parse_args()
1133
1134     if options.runfiles and len(pathnames):
1135         fail('Extraneous arguments.')
1136
1137     options.pathnames = [os.path.abspath(path) for path in pathnames]
1138
1139     return options
1140
1141
1142 def main():
1143     options = parse_args()
1144
1145     testrun = TestRun(options)
1146
1147     if options.runfiles:
1148         testrun.read(options)
1149     else:
1150         find_tests(testrun, options)
1151
1152     if options.logfile:
1153         filter_tests(testrun, options)
1154
1155     if options.template:
1156         testrun.write(options)
1157         exit(0)
1158
1159     testrun.complete_outputdirs()
1160     testrun.run(options)
1161     exit(testrun.summary())
1162
1163
1164 if __name__ == '__main__':
1165     main()