]>
Commit | Line | Data |
---|---|---|
6bb24f4d BB |
1 | #!/usr/bin/python |
2 | ||
3 | # | |
4 | # This file and its contents are supplied under the terms of the | |
5 | # Common Development and Distribution License ("CDDL"), version 1.0. | |
6 | # You may only use this file in accordance with the terms of version | |
7 | # 1.0 of the CDDL. | |
8 | # | |
9 | # A full copy of the text of the CDDL should have accompanied this | |
10 | # source. A copy of the CDDL is also available via the Internet at | |
11 | # http://www.illumos.org/license/CDDL. | |
12 | # | |
13 | ||
14 | # | |
dffce3c2 | 15 | # Copyright (c) 2012, 2018 by Delphix. All rights reserved. |
b31cf30a | 16 | # Copyright (c) 2019 Datto Inc. |
6bb24f4d | 17 | # |
6e72a5b9 BB |
18 | # This script must remain compatible with Python 2.6+ and Python 3.4+. |
19 | # | |
6bb24f4d | 20 | |
3ed2fbcc GK |
21 | # some python 2.7 system don't have a configparser shim |
22 | try: | |
23 | import configparser | |
24 | except ImportError: | |
25 | import ConfigParser as configparser | |
26 | ||
6bb24f4d | 27 | import os |
dffce3c2 | 28 | import sys |
b31cf30a | 29 | import ctypes |
dffce3c2 | 30 | |
6bb24f4d BB |
31 | from datetime import datetime |
32 | from optparse import OptionParser | |
33 | from pwd import getpwnam | |
34 | from pwd import getpwuid | |
35 | from select import select | |
36 | from subprocess import PIPE | |
37 | from subprocess import Popen | |
6bb24f4d BB |
38 | from threading import Timer |
39 | from time import time | |
40 | ||
41 | BASEDIR = '/var/tmp/test_results' | |
42 | TESTDIR = '/usr/share/zfs/' | |
43 | KILL = 'kill' | |
44 | TRUE = 'true' | |
45 | SUDO = 'sudo' | |
dffce3c2 JWK |
46 | LOG_FILE = 'LOG_FILE' |
47 | LOG_OUT = 'LOG_OUT' | |
48 | LOG_ERR = 'LOG_ERR' | |
49 | LOG_FILE_OBJ = None | |
6bb24f4d | 50 | |
b31cf30a AP |
51 | # some python 2.7 system don't have a concept of monotonic time |
52 | CLOCK_MONOTONIC_RAW = 4 # see <linux/time.h> | |
53 | ||
54 | ||
55 | class timespec(ctypes.Structure): | |
56 | _fields_ = [ | |
57 | ('tv_sec', ctypes.c_long), | |
58 | ('tv_nsec', ctypes.c_long) | |
59 | ] | |
60 | ||
61 | ||
62 | librt = ctypes.CDLL('librt.so.1', use_errno=True) | |
63 | clock_gettime = librt.clock_gettime | |
64 | clock_gettime.argtypes = [ctypes.c_int, ctypes.POINTER(timespec)] | |
65 | ||
66 | ||
67 | def monotonic_time(): | |
68 | t = timespec() | |
69 | if clock_gettime(CLOCK_MONOTONIC_RAW, ctypes.pointer(t)) != 0: | |
70 | errno_ = ctypes.get_errno() | |
71 | raise OSError(errno_, os.strerror(errno_)) | |
72 | return t.tv_sec + t.tv_nsec * 1e-9 | |
73 | ||
6bb24f4d BB |
74 | |
75 | class Result(object): | |
76 | total = 0 | |
b31cf30a | 77 | runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0, 'RERAN': 0} |
6bb24f4d BB |
78 | |
79 | def __init__(self): | |
80 | self.starttime = None | |
81 | self.returncode = None | |
82 | self.runtime = '' | |
83 | self.stdout = [] | |
84 | self.stderr = [] | |
85 | self.result = '' | |
86 | ||
b31cf30a | 87 | def done(self, proc, killed, reran): |
6bb24f4d BB |
88 | """ |
89 | Finalize the results of this Cmd. | |
90 | """ | |
91 | Result.total += 1 | |
b31cf30a | 92 | m, s = divmod(monotonic_time() - self.starttime, 60) |
6bb24f4d BB |
93 | self.runtime = '%02d:%02d' % (m, s) |
94 | self.returncode = proc.returncode | |
b31cf30a AP |
95 | if reran is True: |
96 | Result.runresults['RERAN'] += 1 | |
6bb24f4d BB |
97 | if killed: |
98 | self.result = 'KILLED' | |
99 | Result.runresults['KILLED'] += 1 | |
cca14128 | 100 | elif self.returncode == 0: |
6bb24f4d BB |
101 | self.result = 'PASS' |
102 | Result.runresults['PASS'] += 1 | |
cca14128 | 103 | elif self.returncode == 4: |
6bb24f4d BB |
104 | self.result = 'SKIP' |
105 | Result.runresults['SKIP'] += 1 | |
cca14128 | 106 | elif self.returncode != 0: |
6bb24f4d BB |
107 | self.result = 'FAIL' |
108 | Result.runresults['FAIL'] += 1 | |
109 | ||
110 | ||
111 | class Output(object): | |
112 | """ | |
113 | This class is a slightly modified version of the 'Stream' class found | |
114 | here: http://goo.gl/aSGfv | |
115 | """ | |
116 | def __init__(self, stream): | |
117 | self.stream = stream | |
dffce3c2 | 118 | self._buf = b'' |
6bb24f4d BB |
119 | self.lines = [] |
120 | ||
121 | def fileno(self): | |
122 | return self.stream.fileno() | |
123 | ||
124 | def read(self, drain=0): | |
125 | """ | |
126 | Read from the file descriptor. If 'drain' set, read until EOF. | |
127 | """ | |
128 | while self._read() is not None: | |
129 | if not drain: | |
130 | break | |
131 | ||
132 | def _read(self): | |
133 | """ | |
134 | Read up to 4k of data from this output stream. Collect the output | |
135 | up to the last newline, and append it to any leftover data from a | |
136 | previous call. The lines are stored as a (timestamp, data) tuple | |
137 | for easy sorting/merging later. | |
138 | """ | |
139 | fd = self.fileno() | |
140 | buf = os.read(fd, 4096) | |
141 | if not buf: | |
142 | return None | |
dffce3c2 | 143 | if b'\n' not in buf: |
6bb24f4d BB |
144 | self._buf += buf |
145 | return [] | |
146 | ||
147 | buf = self._buf + buf | |
dffce3c2 | 148 | tmp, rest = buf.rsplit(b'\n', 1) |
6bb24f4d BB |
149 | self._buf = rest |
150 | now = datetime.now() | |
dffce3c2 | 151 | rows = tmp.split(b'\n') |
6bb24f4d BB |
152 | self.lines += [(now, r) for r in rows] |
153 | ||
154 | ||
155 | class Cmd(object): | |
156 | verified_users = [] | |
157 | ||
9a810efb GDN |
158 | def __init__(self, pathname, outputdir=None, timeout=None, user=None, |
159 | tags=None): | |
6bb24f4d BB |
160 | self.pathname = pathname |
161 | self.outputdir = outputdir or 'BASEDIR' | |
b31cf30a AP |
162 | """ |
163 | The timeout for tests is measured in wall-clock time | |
164 | """ | |
679d73e9 | 165 | self.timeout = timeout |
6bb24f4d BB |
166 | self.user = user or '' |
167 | self.killed = False | |
b31cf30a | 168 | self.reran = None |
6bb24f4d BB |
169 | self.result = Result() |
170 | ||
9285493a GDN |
171 | if self.timeout is None: |
172 | self.timeout = 60 | |
679d73e9 | 173 | |
6bb24f4d | 174 | def __str__(self): |
90609171 JWK |
175 | return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nUser: %s\n" % \ |
176 | (self.pathname, self.outputdir, self.timeout, self.user) | |
6bb24f4d | 177 | |
b31cf30a | 178 | def kill_cmd(self, proc, keyboard_interrupt=False): |
6bb24f4d BB |
179 | """ |
180 | Kill a running command due to timeout, or ^C from the keyboard. If | |
181 | sudo is required, this user was verified previously. | |
182 | """ | |
183 | self.killed = True | |
184 | do_sudo = len(self.user) != 0 | |
185 | signal = '-TERM' | |
186 | ||
187 | cmd = [SUDO, KILL, signal, str(proc.pid)] | |
188 | if not do_sudo: | |
189 | del cmd[0] | |
190 | ||
191 | try: | |
192 | kp = Popen(cmd) | |
193 | kp.wait() | |
63e5e960 | 194 | except Exception: |
6bb24f4d BB |
195 | pass |
196 | ||
b31cf30a AP |
197 | """ |
198 | If this is not a user-initiated kill and the test has not been | |
199 | reran before we consider if the test needs to be reran: | |
200 | If the test has spent some time hibernating and didn't run the whole | |
201 | length of time before being timed out we will rerun the test. | |
202 | """ | |
203 | if keyboard_interrupt is False and self.reran is None: | |
204 | runtime = monotonic_time() - self.result.starttime | |
205 | if int(self.timeout) > runtime: | |
206 | self.killed = False | |
207 | self.reran = False | |
208 | self.run(False) | |
209 | self.reran = True | |
210 | ||
6bb24f4d BB |
211 | def update_cmd_privs(self, cmd, user): |
212 | """ | |
213 | If a user has been specified to run this Cmd and we're not already | |
214 | running as that user, prepend the appropriate sudo command to run | |
215 | as that user. | |
216 | """ | |
217 | me = getpwuid(os.getuid()) | |
218 | ||
219 | if not user or user is me: | |
f74b821a BB |
220 | if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): |
221 | cmd += '.ksh' | |
222 | if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): | |
223 | cmd += '.sh' | |
6bb24f4d BB |
224 | return cmd |
225 | ||
226 | if not os.path.isfile(cmd): | |
227 | if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): | |
228 | cmd += '.ksh' | |
229 | if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): | |
230 | cmd += '.sh' | |
231 | ||
232 | ret = '%s -E -u %s %s' % (SUDO, user, cmd) | |
233 | return ret.split(' ') | |
234 | ||
235 | def collect_output(self, proc): | |
236 | """ | |
237 | Read from stdout/stderr as data becomes available, until the | |
238 | process is no longer running. Return the lines from the stdout and | |
239 | stderr Output objects. | |
240 | """ | |
241 | out = Output(proc.stdout) | |
242 | err = Output(proc.stderr) | |
243 | res = [] | |
244 | while proc.returncode is None: | |
245 | proc.poll() | |
246 | res = select([out, err], [], [], .1) | |
247 | for fd in res[0]: | |
248 | fd.read() | |
249 | for fd in res[0]: | |
250 | fd.read(drain=1) | |
251 | ||
252 | return out.lines, err.lines | |
253 | ||
b31cf30a | 254 | def run(self, dryrun): |
6bb24f4d BB |
255 | """ |
256 | This is the main function that runs each individual test. | |
257 | Determine whether or not the command requires sudo, and modify it | |
258 | if needed. Run the command, and update the result object. | |
259 | """ | |
b31cf30a | 260 | if dryrun is True: |
3ed2fbcc | 261 | print(self) |
6bb24f4d BB |
262 | return |
263 | ||
264 | privcmd = self.update_cmd_privs(self.pathname, self.user) | |
265 | try: | |
266 | old = os.umask(0) | |
267 | if not os.path.isdir(self.outputdir): | |
3ed2fbcc | 268 | os.makedirs(self.outputdir, mode=0o777) |
6bb24f4d | 269 | os.umask(old) |
3ed2fbcc | 270 | except OSError as e: |
6bb24f4d BB |
271 | fail('%s' % e) |
272 | ||
b31cf30a | 273 | self.result.starttime = monotonic_time() |
f74b821a | 274 | proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) |
679d73e9 JWK |
275 | # Allow a special timeout value of 0 to mean infinity |
276 | if int(self.timeout) == 0: | |
dffce3c2 | 277 | self.timeout = sys.maxsize |
f74b821a BB |
278 | t = Timer(int(self.timeout), self.kill_cmd, [proc]) |
279 | ||
6bb24f4d | 280 | try: |
6bb24f4d BB |
281 | t.start() |
282 | self.result.stdout, self.result.stderr = self.collect_output(proc) | |
283 | except KeyboardInterrupt: | |
b31cf30a | 284 | self.kill_cmd(proc, True) |
6bb24f4d BB |
285 | fail('\nRun terminated at user request.') |
286 | finally: | |
287 | t.cancel() | |
288 | ||
b31cf30a AP |
289 | if self.reran is not False: |
290 | self.result.done(proc, self.killed, self.reran) | |
6bb24f4d BB |
291 | |
292 | def skip(self): | |
293 | """ | |
294 | Initialize enough of the test result that we can log a skipped | |
295 | command. | |
296 | """ | |
297 | Result.total += 1 | |
298 | Result.runresults['SKIP'] += 1 | |
299 | self.result.stdout = self.result.stderr = [] | |
b31cf30a AP |
300 | self.result.starttime = monotonic_time() |
301 | m, s = divmod(monotonic_time() - self.result.starttime, 60) | |
6bb24f4d BB |
302 | self.result.runtime = '%02d:%02d' % (m, s) |
303 | self.result.result = 'SKIP' | |
304 | ||
dffce3c2 | 305 | def log(self, options): |
6bb24f4d BB |
306 | """ |
307 | This function is responsible for writing all output. This includes | |
308 | the console output, the logfile of all results (with timestamped | |
309 | merged stdout and stderr), and for each test, the unmodified | |
310 | stdout/stderr/merged in it's own file. | |
311 | """ | |
6bb24f4d BB |
312 | |
313 | logname = getpwuid(os.getuid()).pw_name | |
b31cf30a AP |
314 | rer = '' |
315 | if self.reran is True: | |
316 | rer = ' (RERAN)' | |
6bb24f4d BB |
317 | user = ' (run as %s)' % (self.user if len(self.user) else logname) |
318 | msga = 'Test: %s%s ' % (self.pathname, user) | |
b31cf30a | 319 | msgb = '[%s] [%s]%s\n' % (self.result.runtime, self.result.result, rer) |
6bb24f4d | 320 | pad = ' ' * (80 - (len(msga) + len(msgb))) |
dffce3c2 | 321 | result_line = msga + pad + msgb |
6bb24f4d | 322 | |
dffce3c2 JWK |
323 | # The result line is always written to the log file. If -q was |
324 | # specified only failures are written to the console, otherwise | |
325 | # the result line is written to the console. | |
326 | write_log(bytearray(result_line, encoding='utf-8'), LOG_FILE) | |
6bb24f4d | 327 | if not options.quiet: |
dffce3c2 | 328 | write_log(result_line, LOG_OUT) |
cca14128 | 329 | elif options.quiet and self.result.result != 'PASS': |
dffce3c2 | 330 | write_log(result_line, LOG_OUT) |
6bb24f4d | 331 | |
717afc13 | 332 | lines = sorted(self.result.stdout + self.result.stderr, |
3ed2fbcc | 333 | key=lambda x: x[0]) |
717afc13 | 334 | |
dffce3c2 | 335 | # Write timestamped output (stdout and stderr) to the logfile |
717afc13 | 336 | for dt, line in lines: |
dffce3c2 JWK |
337 | timestamp = bytearray(dt.strftime("%H:%M:%S.%f ")[:11], |
338 | encoding='utf-8') | |
339 | write_log(b'%s %s\n' % (timestamp, line), LOG_FILE) | |
6bb24f4d | 340 | |
dffce3c2 | 341 | # Write the separate stdout/stderr/merged files, if the data exists |
6bb24f4d | 342 | if len(self.result.stdout): |
dffce3c2 | 343 | with open(os.path.join(self.outputdir, 'stdout'), 'wb') as out: |
6bb24f4d | 344 | for _, line in self.result.stdout: |
dffce3c2 | 345 | os.write(out.fileno(), b'%s\n' % line) |
6bb24f4d | 346 | if len(self.result.stderr): |
dffce3c2 | 347 | with open(os.path.join(self.outputdir, 'stderr'), 'wb') as err: |
6bb24f4d | 348 | for _, line in self.result.stderr: |
dffce3c2 | 349 | os.write(err.fileno(), b'%s\n' % line) |
6bb24f4d | 350 | if len(self.result.stdout) and len(self.result.stderr): |
dffce3c2 | 351 | with open(os.path.join(self.outputdir, 'merged'), 'wb') as merged: |
717afc13 | 352 | for _, line in lines: |
dffce3c2 | 353 | os.write(merged.fileno(), b'%s\n' % line) |
6bb24f4d BB |
354 | |
355 | ||
356 | class Test(Cmd): | |
357 | props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post', | |
9a810efb | 358 | 'post_user', 'tags'] |
6bb24f4d BB |
359 | |
360 | def __init__(self, pathname, outputdir=None, timeout=None, user=None, | |
9a810efb GDN |
361 | pre=None, pre_user=None, post=None, post_user=None, |
362 | tags=None): | |
6bb24f4d BB |
363 | super(Test, self).__init__(pathname, outputdir, timeout, user) |
364 | self.pre = pre or '' | |
365 | self.pre_user = pre_user or '' | |
366 | self.post = post or '' | |
367 | self.post_user = post_user or '' | |
9a810efb | 368 | self.tags = tags or [] |
6bb24f4d BB |
369 | |
370 | def __str__(self): | |
371 | post_user = pre_user = '' | |
372 | if len(self.pre_user): | |
373 | pre_user = ' (as %s)' % (self.pre_user) | |
374 | if len(self.post_user): | |
375 | post_user = ' (as %s)' % (self.post_user) | |
679d73e9 | 376 | return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \ |
9a810efb | 377 | "%s%s\nUser: %s\nTags: %s\n" % \ |
90609171 | 378 | (self.pathname, self.outputdir, self.timeout, self.pre, |
9a810efb | 379 | pre_user, self.post, post_user, self.user, self.tags) |
6bb24f4d | 380 | |
dffce3c2 | 381 | def verify(self): |
6bb24f4d BB |
382 | """ |
383 | Check the pre/post scripts, user and Test. Omit the Test from this | |
384 | run if there are any problems. | |
385 | """ | |
386 | files = [self.pre, self.pathname, self.post] | |
387 | users = [self.pre_user, self.user, self.post_user] | |
388 | ||
389 | for f in [f for f in files if len(f)]: | |
390 | if not verify_file(f): | |
dffce3c2 JWK |
391 | write_log("Warning: Test '%s' not added to this run because" |
392 | " it failed verification.\n" % f, LOG_ERR) | |
6bb24f4d BB |
393 | return False |
394 | ||
395 | for user in [user for user in users if len(user)]: | |
dffce3c2 JWK |
396 | if not verify_user(user): |
397 | write_log("Not adding Test '%s' to this run.\n" % | |
398 | self.pathname, LOG_ERR) | |
6bb24f4d BB |
399 | return False |
400 | ||
401 | return True | |
402 | ||
dffce3c2 | 403 | def run(self, options): |
6bb24f4d BB |
404 | """ |
405 | Create Cmd instances for the pre/post scripts. If the pre script | |
406 | doesn't pass, skip this Test. Run the post script regardless. | |
407 | """ | |
90609171 JWK |
408 | odir = os.path.join(self.outputdir, os.path.basename(self.pre)) |
409 | pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, | |
6bb24f4d BB |
410 | user=self.pre_user) |
411 | test = Cmd(self.pathname, outputdir=self.outputdir, | |
412 | timeout=self.timeout, user=self.user) | |
90609171 JWK |
413 | odir = os.path.join(self.outputdir, os.path.basename(self.post)) |
414 | posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, | |
6bb24f4d BB |
415 | user=self.post_user) |
416 | ||
417 | cont = True | |
418 | if len(pretest.pathname): | |
b31cf30a | 419 | pretest.run(options.dryrun) |
cca14128 | 420 | cont = pretest.result.result == 'PASS' |
dffce3c2 | 421 | pretest.log(options) |
6bb24f4d BB |
422 | |
423 | if cont: | |
b31cf30a | 424 | test.run(options.dryrun) |
6bb24f4d BB |
425 | else: |
426 | test.skip() | |
427 | ||
dffce3c2 | 428 | test.log(options) |
6bb24f4d BB |
429 | |
430 | if len(posttest.pathname): | |
b31cf30a | 431 | posttest.run(options.dryrun) |
dffce3c2 | 432 | posttest.log(options) |
6bb24f4d BB |
433 | |
434 | ||
435 | class TestGroup(Test): | |
436 | props = Test.props + ['tests'] | |
437 | ||
438 | def __init__(self, pathname, outputdir=None, timeout=None, user=None, | |
439 | pre=None, pre_user=None, post=None, post_user=None, | |
9a810efb | 440 | tests=None, tags=None): |
6bb24f4d | 441 | super(TestGroup, self).__init__(pathname, outputdir, timeout, user, |
9a810efb | 442 | pre, pre_user, post, post_user, tags) |
6bb24f4d BB |
443 | self.tests = tests or [] |
444 | ||
445 | def __str__(self): | |
446 | post_user = pre_user = '' | |
447 | if len(self.pre_user): | |
448 | pre_user = ' (as %s)' % (self.pre_user) | |
449 | if len(self.post_user): | |
450 | post_user = ' (as %s)' % (self.post_user) | |
9a810efb GDN |
451 | return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %s\n" \ |
452 | "Pre: %s%s\nPost: %s%s\nUser: %s\nTags: %s\n" % \ | |
90609171 | 453 | (self.pathname, self.outputdir, self.tests, self.timeout, |
9a810efb | 454 | self.pre, pre_user, self.post, post_user, self.user, self.tags) |
6bb24f4d | 455 | |
dffce3c2 | 456 | def verify(self): |
6bb24f4d BB |
457 | """ |
458 | Check the pre/post scripts, user and tests in this TestGroup. Omit | |
459 | the TestGroup entirely, or simply delete the relevant tests in the | |
460 | group, if that's all that's required. | |
461 | """ | |
462 | # If the pre or post scripts are relative pathnames, convert to | |
463 | # absolute, so they stand a chance of passing verification. | |
464 | if len(self.pre) and not os.path.isabs(self.pre): | |
465 | self.pre = os.path.join(self.pathname, self.pre) | |
466 | if len(self.post) and not os.path.isabs(self.post): | |
467 | self.post = os.path.join(self.pathname, self.post) | |
468 | ||
469 | auxfiles = [self.pre, self.post] | |
470 | users = [self.pre_user, self.user, self.post_user] | |
471 | ||
472 | for f in [f for f in auxfiles if len(f)]: | |
473 | if self.pathname != os.path.dirname(f): | |
dffce3c2 JWK |
474 | write_log("Warning: TestGroup '%s' not added to this run. " |
475 | "Auxiliary script '%s' exists in a different " | |
476 | "directory.\n" % (self.pathname, f), LOG_ERR) | |
6bb24f4d BB |
477 | return False |
478 | ||
479 | if not verify_file(f): | |
dffce3c2 JWK |
480 | write_log("Warning: TestGroup '%s' not added to this run. " |
481 | "Auxiliary script '%s' failed verification.\n" % | |
482 | (self.pathname, f), LOG_ERR) | |
6bb24f4d BB |
483 | return False |
484 | ||
485 | for user in [user for user in users if len(user)]: | |
dffce3c2 JWK |
486 | if not verify_user(user): |
487 | write_log("Not adding TestGroup '%s' to this run.\n" % | |
488 | self.pathname, LOG_ERR) | |
6bb24f4d BB |
489 | return False |
490 | ||
491 | # If one of the tests is invalid, delete it, log it, and drive on. | |
492 | for test in self.tests: | |
493 | if not verify_file(os.path.join(self.pathname, test)): | |
494 | del self.tests[self.tests.index(test)] | |
dffce3c2 JWK |
495 | write_log("Warning: Test '%s' removed from TestGroup '%s' " |
496 | "because it failed verification.\n" % | |
497 | (test, self.pathname), LOG_ERR) | |
6bb24f4d | 498 | |
cca14128 | 499 | return len(self.tests) != 0 |
6bb24f4d | 500 | |
dffce3c2 | 501 | def run(self, options): |
6bb24f4d BB |
502 | """ |
503 | Create Cmd instances for the pre/post scripts. If the pre script | |
504 | doesn't pass, skip all the tests in this TestGroup. Run the post | |
505 | script regardless. | |
506 | """ | |
9a810efb GDN |
507 | # tags assigned to this test group also include the test names |
508 | if options.tags and not set(self.tags).intersection(set(options.tags)): | |
509 | return | |
510 | ||
90609171 JWK |
511 | odir = os.path.join(self.outputdir, os.path.basename(self.pre)) |
512 | pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout, | |
6bb24f4d | 513 | user=self.pre_user) |
90609171 JWK |
514 | odir = os.path.join(self.outputdir, os.path.basename(self.post)) |
515 | posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout, | |
6bb24f4d BB |
516 | user=self.post_user) |
517 | ||
518 | cont = True | |
519 | if len(pretest.pathname): | |
b31cf30a | 520 | pretest.run(options.dryrun) |
cca14128 | 521 | cont = pretest.result.result == 'PASS' |
dffce3c2 | 522 | pretest.log(options) |
6bb24f4d BB |
523 | |
524 | for fname in self.tests: | |
525 | test = Cmd(os.path.join(self.pathname, fname), | |
526 | outputdir=os.path.join(self.outputdir, fname), | |
527 | timeout=self.timeout, user=self.user) | |
528 | if cont: | |
b31cf30a | 529 | test.run(options.dryrun) |
6bb24f4d BB |
530 | else: |
531 | test.skip() | |
532 | ||
dffce3c2 | 533 | test.log(options) |
6bb24f4d BB |
534 | |
535 | if len(posttest.pathname): | |
b31cf30a | 536 | posttest.run(options.dryrun) |
dffce3c2 | 537 | posttest.log(options) |
6bb24f4d BB |
538 | |
539 | ||
540 | class TestRun(object): | |
541 | props = ['quiet', 'outputdir'] | |
542 | ||
543 | def __init__(self, options): | |
544 | self.tests = {} | |
545 | self.testgroups = {} | |
546 | self.starttime = time() | |
547 | self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S') | |
548 | self.outputdir = os.path.join(options.outputdir, self.timestamp) | |
dffce3c2 | 549 | self.setup_logging(options) |
6bb24f4d BB |
550 | self.defaults = [ |
551 | ('outputdir', BASEDIR), | |
552 | ('quiet', False), | |
553 | ('timeout', 60), | |
554 | ('user', ''), | |
555 | ('pre', ''), | |
556 | ('pre_user', ''), | |
557 | ('post', ''), | |
9a810efb GDN |
558 | ('post_user', ''), |
559 | ('tags', []) | |
6bb24f4d BB |
560 | ] |
561 | ||
562 | def __str__(self): | |
563 | s = 'TestRun:\n outputdir: %s\n' % self.outputdir | |
564 | s += 'TESTS:\n' | |
565 | for key in sorted(self.tests.keys()): | |
566 | s += '%s%s' % (self.tests[key].__str__(), '\n') | |
567 | s += 'TESTGROUPS:\n' | |
568 | for key in sorted(self.testgroups.keys()): | |
569 | s += '%s%s' % (self.testgroups[key].__str__(), '\n') | |
570 | return s | |
571 | ||
572 | def addtest(self, pathname, options): | |
573 | """ | |
574 | Create a new Test, and apply any properties that were passed in | |
575 | from the command line. If it passes verification, add it to the | |
576 | TestRun. | |
577 | """ | |
578 | test = Test(pathname) | |
579 | for prop in Test.props: | |
580 | setattr(test, prop, getattr(options, prop)) | |
581 | ||
dffce3c2 | 582 | if test.verify(): |
6bb24f4d BB |
583 | self.tests[pathname] = test |
584 | ||
585 | def addtestgroup(self, dirname, filenames, options): | |
586 | """ | |
587 | Create a new TestGroup, and apply any properties that were passed | |
588 | in from the command line. If it passes verification, add it to the | |
589 | TestRun. | |
590 | """ | |
591 | if dirname not in self.testgroups: | |
592 | testgroup = TestGroup(dirname) | |
593 | for prop in Test.props: | |
594 | setattr(testgroup, prop, getattr(options, prop)) | |
595 | ||
596 | # Prevent pre/post scripts from running as regular tests | |
597 | for f in [testgroup.pre, testgroup.post]: | |
598 | if f in filenames: | |
599 | del filenames[filenames.index(f)] | |
600 | ||
601 | self.testgroups[dirname] = testgroup | |
602 | self.testgroups[dirname].tests = sorted(filenames) | |
603 | ||
dffce3c2 | 604 | testgroup.verify() |
6bb24f4d | 605 | |
dffce3c2 | 606 | def read(self, options): |
6bb24f4d BB |
607 | """ |
608 | Read in the specified runfile, and apply the TestRun properties | |
609 | listed in the 'DEFAULT' section to our TestRun. Then read each | |
610 | section, and apply the appropriate properties to the Test or | |
611 | TestGroup. Properties from individual sections override those set | |
612 | in the 'DEFAULT' section. If the Test or TestGroup passes | |
613 | verification, add it to the TestRun. | |
614 | """ | |
3ed2fbcc | 615 | config = configparser.RawConfigParser() |
6bb24f4d BB |
616 | if not len(config.read(options.runfile)): |
617 | fail("Coulnd't read config file %s" % options.runfile) | |
618 | ||
619 | for opt in TestRun.props: | |
620 | if config.has_option('DEFAULT', opt): | |
621 | setattr(self, opt, config.get('DEFAULT', opt)) | |
622 | self.outputdir = os.path.join(self.outputdir, self.timestamp) | |
623 | ||
624 | for section in config.sections(): | |
625 | if 'tests' in config.options(section): | |
626 | if os.path.isdir(section): | |
627 | pathname = section | |
628 | elif os.path.isdir(os.path.join(options.testdir, section)): | |
629 | pathname = os.path.join(options.testdir, section) | |
630 | else: | |
631 | pathname = section | |
632 | ||
633 | testgroup = TestGroup(os.path.abspath(pathname)) | |
634 | for prop in TestGroup.props: | |
90609171 JWK |
635 | for sect in ['DEFAULT', section]: |
636 | if config.has_option(sect, prop): | |
cca14128 | 637 | if prop == "tags": |
9a810efb GDN |
638 | setattr(testgroup, prop, |
639 | eval(config.get(sect, prop))) | |
640 | else: | |
641 | setattr(testgroup, prop, | |
642 | config.get(sect, prop)) | |
6bb24f4d BB |
643 | |
644 | # Repopulate tests using eval to convert the string to a list | |
645 | testgroup.tests = eval(config.get(section, 'tests')) | |
646 | ||
dffce3c2 | 647 | if testgroup.verify(): |
6bb24f4d BB |
648 | self.testgroups[section] = testgroup |
649 | else: | |
650 | test = Test(section) | |
651 | for prop in Test.props: | |
90609171 JWK |
652 | for sect in ['DEFAULT', section]: |
653 | if config.has_option(sect, prop): | |
654 | setattr(test, prop, config.get(sect, prop)) | |
655 | ||
dffce3c2 | 656 | if test.verify(): |
6bb24f4d BB |
657 | self.tests[section] = test |
658 | ||
659 | def write(self, options): | |
660 | """ | |
661 | Create a configuration file for editing and later use. The | |
662 | 'DEFAULT' section of the config file is created from the | |
663 | properties that were specified on the command line. Tests are | |
664 | simply added as sections that inherit everything from the | |
665 | 'DEFAULT' section. TestGroups are the same, except they get an | |
666 | option including all the tests to run in that directory. | |
667 | """ | |
668 | ||
669 | defaults = dict([(prop, getattr(options, prop)) for prop, _ in | |
90609171 | 670 | self.defaults]) |
3ed2fbcc | 671 | config = configparser.RawConfigParser(defaults) |
6bb24f4d BB |
672 | |
673 | for test in sorted(self.tests.keys()): | |
674 | config.add_section(test) | |
675 | ||
676 | for testgroup in sorted(self.testgroups.keys()): | |
677 | config.add_section(testgroup) | |
678 | config.set(testgroup, 'tests', self.testgroups[testgroup].tests) | |
679 | ||
680 | try: | |
681 | with open(options.template, 'w') as f: | |
682 | return config.write(f) | |
683 | except IOError: | |
684 | fail('Could not open \'%s\' for writing.' % options.template) | |
685 | ||
90609171 | 686 | def complete_outputdirs(self): |
6bb24f4d BB |
687 | """ |
688 | Collect all the pathnames for Tests, and TestGroups. Work | |
689 | backwards one pathname component at a time, to create a unique | |
690 | directory name in which to deposit test output. Tests will be able | |
691 | to write output files directly in the newly modified outputdir. | |
692 | TestGroups will be able to create one subdirectory per test in the | |
693 | outputdir, and are guaranteed uniqueness because a group can only | |
694 | contain files in one directory. Pre and post tests will create a | |
695 | directory rooted at the outputdir of the Test or TestGroup in | |
696 | question for their output. | |
697 | """ | |
698 | done = False | |
699 | components = 0 | |
3ed2fbcc GK |
700 | tmp_dict = dict(list(self.tests.items()) + |
701 | list(self.testgroups.items())) | |
6bb24f4d BB |
702 | total = len(tmp_dict) |
703 | base = self.outputdir | |
704 | ||
705 | while not done: | |
63e5e960 | 706 | paths = [] |
6bb24f4d | 707 | components -= 1 |
3ed2fbcc | 708 | for testfile in list(tmp_dict.keys()): |
6bb24f4d | 709 | uniq = '/'.join(testfile.split('/')[components:]).lstrip('/') |
63e5e960 GDN |
710 | if uniq not in paths: |
711 | paths.append(uniq) | |
6bb24f4d BB |
712 | tmp_dict[testfile].outputdir = os.path.join(base, uniq) |
713 | else: | |
714 | break | |
63e5e960 | 715 | done = total == len(paths) |
6bb24f4d BB |
716 | |
717 | def setup_logging(self, options): | |
718 | """ | |
dffce3c2 JWK |
719 | This funtion creates the output directory and gets a file object |
720 | for the logfile. This function must be called before write_log() | |
721 | can be used. | |
6bb24f4d BB |
722 | """ |
723 | if options.dryrun is True: | |
724 | return | |
725 | ||
dffce3c2 | 726 | global LOG_FILE_OBJ |
cca14128 | 727 | if options.cmd != 'wrconfig': |
6bb24f4d BB |
728 | try: |
729 | old = os.umask(0) | |
3ed2fbcc | 730 | os.makedirs(self.outputdir, mode=0o777) |
6bb24f4d | 731 | os.umask(old) |
dffce3c2 JWK |
732 | filename = os.path.join(self.outputdir, 'log') |
733 | LOG_FILE_OBJ = open(filename, buffering=0, mode='wb') | |
3ed2fbcc | 734 | except OSError as e: |
6bb24f4d | 735 | fail('%s' % e) |
6bb24f4d BB |
736 | |
737 | def run(self, options): | |
738 | """ | |
739 | Walk through all the Tests and TestGroups, calling run(). | |
740 | """ | |
741 | try: | |
742 | os.chdir(self.outputdir) | |
743 | except OSError: | |
744 | fail('Could not change to directory %s' % self.outputdir) | |
a85cefa3 GDN |
745 | # make a symlink to the output for the currently running test |
746 | logsymlink = os.path.join(self.outputdir, '../current') | |
df749224 GDN |
747 | if os.path.islink(logsymlink): |
748 | os.unlink(logsymlink) | |
749 | if not os.path.exists(logsymlink): | |
750 | os.symlink(self.outputdir, logsymlink) | |
751 | else: | |
dffce3c2 JWK |
752 | write_log('Could not make a symlink to directory %s\n' % |
753 | self.outputdir, LOG_ERR) | |
9a810efb GDN |
754 | iteration = 0 |
755 | while iteration < options.iterations: | |
756 | for test in sorted(self.tests.keys()): | |
dffce3c2 | 757 | self.tests[test].run(options) |
9a810efb | 758 | for testgroup in sorted(self.testgroups.keys()): |
dffce3c2 | 759 | self.testgroups[testgroup].run(options) |
9a810efb | 760 | iteration += 1 |
6bb24f4d BB |
761 | |
762 | def summary(self): | |
cca14128 | 763 | if Result.total == 0: |
fe46eebe | 764 | return 2 |
6bb24f4d | 765 | |
3ed2fbcc GK |
766 | print('\nResults Summary') |
767 | for key in list(Result.runresults.keys()): | |
cca14128 | 768 | if Result.runresults[key] != 0: |
3ed2fbcc | 769 | print('%s\t% 4d' % (key, Result.runresults[key])) |
6bb24f4d BB |
770 | |
771 | m, s = divmod(time() - self.starttime, 60) | |
772 | h, m = divmod(m, 60) | |
3ed2fbcc GK |
773 | print('\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)) |
774 | print('Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) / | |
775 | float(Result.total)) * 100)) | |
776 | print('Log directory:\t%s' % self.outputdir) | |
6bb24f4d | 777 | |
fe46eebe AP |
778 | if Result.runresults['FAIL'] > 0: |
779 | return 1 | |
b24827ac GDN |
780 | |
781 | if Result.runresults['KILLED'] > 0: | |
782 | return 1 | |
783 | ||
b31cf30a AP |
784 | if Result.runresults['RERAN'] > 0: |
785 | return 3 | |
786 | ||
fe46eebe AP |
787 | return 0 |
788 | ||
6bb24f4d | 789 | |
dffce3c2 JWK |
790 | def write_log(msg, target): |
791 | """ | |
792 | Write the provided message to standard out, standard error or | |
793 | the logfile. If specifying LOG_FILE, then `msg` must be a bytes | |
794 | like object. This way we can still handle output from tests that | |
795 | may be in unexpected encodings. | |
796 | """ | |
797 | if target == LOG_OUT: | |
798 | os.write(sys.stdout.fileno(), bytearray(msg, encoding='utf-8')) | |
799 | elif target == LOG_ERR: | |
800 | os.write(sys.stderr.fileno(), bytearray(msg, encoding='utf-8')) | |
801 | elif target == LOG_FILE: | |
802 | os.write(LOG_FILE_OBJ.fileno(), msg) | |
803 | else: | |
804 | fail('log_msg called with unknown target "%s"' % target) | |
805 | ||
806 | ||
6bb24f4d BB |
807 | def verify_file(pathname): |
808 | """ | |
809 | Verify that the supplied pathname is an executable regular file. | |
810 | """ | |
811 | if os.path.isdir(pathname) or os.path.islink(pathname): | |
812 | return False | |
813 | ||
9285493a GDN |
814 | for ext in '', '.ksh', '.sh': |
815 | script_path = pathname + ext | |
816 | if os.path.isfile(script_path) and os.access(script_path, os.X_OK): | |
817 | return True | |
6bb24f4d BB |
818 | |
819 | return False | |
820 | ||
821 | ||
dffce3c2 | 822 | def verify_user(user): |
6bb24f4d BB |
823 | """ |
824 | Verify that the specified user exists on this system, and can execute | |
825 | sudo without being prompted for a password. | |
826 | """ | |
827 | testcmd = [SUDO, '-n', '-u', user, TRUE] | |
6bb24f4d BB |
828 | |
829 | if user in Cmd.verified_users: | |
830 | return True | |
831 | ||
832 | try: | |
9285493a | 833 | getpwnam(user) |
6bb24f4d | 834 | except KeyError: |
dffce3c2 JWK |
835 | write_log("Warning: user '%s' does not exist.\n" % user, |
836 | LOG_ERR) | |
6bb24f4d BB |
837 | return False |
838 | ||
839 | p = Popen(testcmd) | |
840 | p.wait() | |
cca14128 | 841 | if p.returncode != 0: |
dffce3c2 JWK |
842 | write_log("Warning: user '%s' cannot use passwordless sudo.\n" % user, |
843 | LOG_ERR) | |
6bb24f4d BB |
844 | return False |
845 | else: | |
846 | Cmd.verified_users.append(user) | |
847 | ||
848 | return True | |
849 | ||
850 | ||
851 | def find_tests(testrun, options): | |
852 | """ | |
853 | For the given list of pathnames, add files as Tests. For directories, | |
854 | if do_groups is True, add the directory as a TestGroup. If False, | |
855 | recursively search for executable files. | |
856 | """ | |
857 | ||
858 | for p in sorted(options.pathnames): | |
859 | if os.path.isdir(p): | |
860 | for dirname, _, filenames in os.walk(p): | |
861 | if options.do_groups: | |
862 | testrun.addtestgroup(dirname, filenames, options) | |
863 | else: | |
864 | for f in sorted(filenames): | |
865 | testrun.addtest(os.path.join(dirname, f), options) | |
866 | else: | |
867 | testrun.addtest(p, options) | |
868 | ||
869 | ||
870 | def fail(retstr, ret=1): | |
dffce3c2 | 871 | print('%s: %s' % (sys.argv[0], retstr)) |
6bb24f4d BB |
872 | exit(ret) |
873 | ||
874 | ||
875 | def options_cb(option, opt_str, value, parser): | |
876 | path_options = ['runfile', 'outputdir', 'template', 'testdir'] | |
877 | ||
cca14128 | 878 | if option.dest == 'runfile' and '-w' in parser.rargs or \ |
879 | option.dest == 'template' and '-c' in parser.rargs: | |
6bb24f4d BB |
880 | fail('-c and -w are mutually exclusive.') |
881 | ||
882 | if opt_str in parser.rargs: | |
883 | fail('%s may only be specified once.' % opt_str) | |
884 | ||
cca14128 | 885 | if option.dest == 'runfile': |
6bb24f4d | 886 | parser.values.cmd = 'rdconfig' |
cca14128 | 887 | if option.dest == 'template': |
6bb24f4d | 888 | parser.values.cmd = 'wrconfig' |
cca14128 | 889 | if option.dest == 'tags': |
9a810efb | 890 | value = [x.strip() for x in value.split(',')] |
6bb24f4d BB |
891 | |
892 | setattr(parser.values, option.dest, value) | |
893 | if option.dest in path_options: | |
894 | setattr(parser.values, option.dest, os.path.abspath(value)) | |
895 | ||
896 | ||
897 | def parse_args(): | |
898 | parser = OptionParser() | |
899 | parser.add_option('-c', action='callback', callback=options_cb, | |
900 | type='string', dest='runfile', metavar='runfile', | |
901 | help='Specify tests to run via config file.') | |
902 | parser.add_option('-d', action='store_true', default=False, dest='dryrun', | |
903 | help='Dry run. Print tests, but take no other action.') | |
904 | parser.add_option('-g', action='store_true', default=False, | |
905 | dest='do_groups', help='Make directories TestGroups.') | |
906 | parser.add_option('-o', action='callback', callback=options_cb, | |
907 | default=BASEDIR, dest='outputdir', type='string', | |
908 | metavar='outputdir', help='Specify an output directory.') | |
909 | parser.add_option('-i', action='callback', callback=options_cb, | |
910 | default=TESTDIR, dest='testdir', type='string', | |
911 | metavar='testdir', help='Specify a test directory.') | |
912 | parser.add_option('-p', action='callback', callback=options_cb, | |
913 | default='', dest='pre', metavar='script', | |
914 | type='string', help='Specify a pre script.') | |
915 | parser.add_option('-P', action='callback', callback=options_cb, | |
916 | default='', dest='post', metavar='script', | |
917 | type='string', help='Specify a post script.') | |
918 | parser.add_option('-q', action='store_true', default=False, dest='quiet', | |
919 | help='Silence on the console during a test run.') | |
920 | parser.add_option('-t', action='callback', callback=options_cb, default=60, | |
921 | dest='timeout', metavar='seconds', type='int', | |
922 | help='Timeout (in seconds) for an individual test.') | |
923 | parser.add_option('-u', action='callback', callback=options_cb, | |
924 | default='', dest='user', metavar='user', type='string', | |
925 | help='Specify a different user name to run as.') | |
926 | parser.add_option('-w', action='callback', callback=options_cb, | |
927 | default=None, dest='template', metavar='template', | |
928 | type='string', help='Create a new config file.') | |
929 | parser.add_option('-x', action='callback', callback=options_cb, default='', | |
930 | dest='pre_user', metavar='pre_user', type='string', | |
931 | help='Specify a user to execute the pre script.') | |
932 | parser.add_option('-X', action='callback', callback=options_cb, default='', | |
933 | dest='post_user', metavar='post_user', type='string', | |
934 | help='Specify a user to execute the post script.') | |
9a810efb GDN |
935 | parser.add_option('-T', action='callback', callback=options_cb, default='', |
936 | dest='tags', metavar='tags', type='string', | |
937 | help='Specify tags to execute specific test groups.') | |
938 | parser.add_option('-I', action='callback', callback=options_cb, default=1, | |
939 | dest='iterations', metavar='iterations', type='int', | |
940 | help='Number of times to run the test run.') | |
6bb24f4d BB |
941 | (options, pathnames) = parser.parse_args() |
942 | ||
943 | if not options.runfile and not options.template: | |
944 | options.cmd = 'runtests' | |
945 | ||
946 | if options.runfile and len(pathnames): | |
947 | fail('Extraneous arguments.') | |
948 | ||
949 | options.pathnames = [os.path.abspath(path) for path in pathnames] | |
950 | ||
951 | return options | |
952 | ||
953 | ||
90609171 | 954 | def main(): |
6bb24f4d BB |
955 | options = parse_args() |
956 | testrun = TestRun(options) | |
957 | ||
cca14128 | 958 | if options.cmd == 'runtests': |
6bb24f4d | 959 | find_tests(testrun, options) |
cca14128 | 960 | elif options.cmd == 'rdconfig': |
dffce3c2 | 961 | testrun.read(options) |
cca14128 | 962 | elif options.cmd == 'wrconfig': |
6bb24f4d BB |
963 | find_tests(testrun, options) |
964 | testrun.write(options) | |
965 | exit(0) | |
966 | else: | |
967 | fail('Unknown command specified') | |
968 | ||
90609171 | 969 | testrun.complete_outputdirs() |
6bb24f4d | 970 | testrun.run(options) |
fe46eebe | 971 | exit(testrun.summary()) |
6bb24f4d BB |
972 | |
973 | ||
974 | if __name__ == '__main__': | |
90609171 | 975 | main() |