]>
Commit | Line | Data |
---|---|---|
3d004a37 | 1 | #!/usr/bin/env python3 |
23ea4f30 EH |
2 | # |
3 | # Copyright (c) 2017 Red Hat Inc | |
4 | # | |
5 | # Author: | |
6 | # Eduardo Habkost <ehabkost@redhat.com> | |
7 | # | |
8 | # This program is free software; you can redistribute it and/or modify | |
9 | # it under the terms of the GNU General Public License as published by | |
10 | # the Free Software Foundation; either version 2 of the License, or | |
11 | # (at your option) any later version. | |
12 | # | |
13 | # This program is distributed in the hope that it will be useful, | |
14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
16 | # GNU General Public License for more details. | |
17 | # | |
18 | # You should have received a copy of the GNU General Public License along | |
19 | # with this program; if not, write to the Free Software Foundation, Inc., | |
20 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | |
21 | ||
22 | """ | |
23 | Run QEMU with all combinations of -machine and -device types, | |
24 | check for crashes and unexpected errors. | |
25 | """ | |
26 | ||
8f8fd9ed | 27 | import os |
23ea4f30 | 28 | import sys |
23ea4f30 EH |
29 | import glob |
30 | import logging | |
31 | import traceback | |
32 | import re | |
33 | import random | |
34 | import argparse | |
35 | from itertools import chain | |
36 | ||
8f8fd9ed | 37 | sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) |
abf0bf99 | 38 | from qemu.machine import QEMUMachine |
c398a241 | 39 | from qemu.aqmp import ConnectError |
23ea4f30 EH |
40 | |
41 | logger = logging.getLogger('device-crash-test') | |
42 | dbg = logger.debug | |
43 | ||
44 | ||
1a14d4e1 | 45 | # Purposes of the following rule list: |
23ea4f30 EH |
46 | # * Avoiding verbose log messages when we find known non-fatal |
47 | # (exitcode=1) errors | |
48 | # * Avoiding fatal errors when we find known crashes | |
49 | # * Skipping machines/devices that are known not to work out of | |
50 | # the box, when running in --quick mode | |
51 | # | |
1a14d4e1 | 52 | # Keeping the rule list updated is desirable, but not required, |
23ea4f30 EH |
53 | # because unexpected cases where QEMU exits with exitcode=1 will |
54 | # just trigger a INFO message. | |
55 | ||
1a14d4e1 | 56 | # Valid error rule keys: |
23ea4f30 EH |
57 | # * accel: regexp, full match only |
58 | # * machine: regexp, full match only | |
59 | # * device: regexp, full match only | |
60 | # * log: regexp, partial match allowed | |
61 | # * exitcode: if not present, defaults to 1. If None, matches any exitcode | |
62 | # * warn: if True, matching failures will be logged as warnings | |
63 | # * expected: if True, QEMU is expected to always fail every time | |
64 | # when testing the corresponding test case | |
65 | # * loglevel: log level of log output when there's a match. | |
1a14d4e1 | 66 | ERROR_RULE_LIST = [ |
23ea4f30 EH |
67 | # Machines that won't work out of the box: |
68 | # MACHINE | ERROR MESSAGE | |
69 | {'machine':'niagara', 'expected':True}, # Unable to load a firmware for -M niagara | |
70 | {'machine':'boston', 'expected':True}, # Please provide either a -kernel or -bios argument | |
71 | {'machine':'leon3_generic', 'expected':True}, # Can't read bios image (null) | |
72 | ||
73 | # devices that don't work out of the box because they require extra options to "-device DEV": | |
74 | # DEVICE | ERROR MESSAGE | |
75 | {'device':'.*-(i386|x86_64)-cpu', 'expected':True}, # CPU socket-id is not set | |
23ea4f30 EH |
76 | {'device':'icp', 'expected':True}, # icp_realize: required link 'xics' not found: Property '.xics' not found |
77 | {'device':'ics', 'expected':True}, # ics_base_realize: required link 'xics' not found: Property '.xics' not found | |
78 | # "-device ide-cd" does work on more recent QEMU versions, so it doesn't have expected=True | |
79 | {'device':'ide-cd'}, # No drive specified | |
23ea4f30 EH |
80 | {'device':'ide-hd', 'expected':True}, # No drive specified |
81 | {'device':'ipmi-bmc-extern', 'expected':True}, # IPMI external bmc requires chardev attribute | |
82 | {'device':'isa-debugcon', 'expected':True}, # Can't create serial device, empty char device | |
83 | {'device':'isa-ipmi-bt', 'expected':True}, # IPMI device requires a bmc attribute to be set | |
84 | {'device':'isa-ipmi-kcs', 'expected':True}, # IPMI device requires a bmc attribute to be set | |
85 | {'device':'isa-parallel', 'expected':True}, # Can't create serial device, empty char device | |
23ea4f30 EH |
86 | {'device':'ivshmem-doorbell', 'expected':True}, # You must specify a 'chardev' |
87 | {'device':'ivshmem-plain', 'expected':True}, # You must specify a 'memdev' | |
23ea4f30 EH |
88 | {'device':'loader', 'expected':True}, # please include valid arguments |
89 | {'device':'nand', 'expected':True}, # Unsupported NAND block size 0x1 | |
90 | {'device':'nvdimm', 'expected':True}, # 'memdev' property is not set | |
91 | {'device':'nvme', 'expected':True}, # Device initialization failed | |
92 | {'device':'pc-dimm', 'expected':True}, # 'memdev' property is not set | |
93 | {'device':'pci-bridge', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. | |
94 | {'device':'pci-bridge-seat', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. | |
23ea4f30 EH |
95 | {'device':'pxb', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0. |
96 | {'device':'scsi-block', 'expected':True}, # drive property not set | |
23ea4f30 EH |
97 | {'device':'scsi-generic', 'expected':True}, # drive property not set |
98 | {'device':'scsi-hd', 'expected':True}, # drive property not set | |
99 | {'device':'spapr-pci-host-bridge', 'expected':True}, # BUID not specified for PHB | |
23ea4f30 EH |
100 | {'device':'spapr-rng', 'expected':True}, # spapr-rng needs an RNG backend! |
101 | {'device':'spapr-vty', 'expected':True}, # chardev property not set | |
102 | {'device':'tpm-tis', 'expected':True}, # tpm_tis: backend driver with id (null) could not be found | |
103 | {'device':'unimplemented-device', 'expected':True}, # property 'size' not specified or zero | |
104 | {'device':'usb-braille', 'expected':True}, # Property chardev is required | |
1ee53067 | 105 | {'device':'usb-mtp', 'expected':True}, # rootdir property must be configured |
23ea4f30 EH |
106 | {'device':'usb-redir', 'expected':True}, # Parameter 'chardev' is missing |
107 | {'device':'usb-serial', 'expected':True}, # Property chardev is required | |
108 | {'device':'usb-storage', 'expected':True}, # drive property not set | |
109 | {'device':'vfio-amd-xgbe', 'expected':True}, # -device vfio-amd-xgbe: vfio error: wrong host device name | |
110 | {'device':'vfio-calxeda-xgmac', 'expected':True}, # -device vfio-calxeda-xgmac: vfio error: wrong host device name | |
111 | {'device':'vfio-pci', 'expected':True}, # No provided host device | |
112 | {'device':'vfio-pci-igd-lpc-bridge', 'expected':True}, # VFIO dummy ISA/LPC bridge must have address 1f.0 | |
113 | {'device':'vhost-scsi.*', 'expected':True}, # vhost-scsi: missing wwpn | |
114 | {'device':'vhost-vsock-device', 'expected':True}, # guest-cid property must be greater than 2 | |
115 | {'device':'vhost-vsock-pci', 'expected':True}, # guest-cid property must be greater than 2 | |
116 | {'device':'virtio-9p-ccw', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL | |
117 | {'device':'virtio-9p-device', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL | |
118 | {'device':'virtio-9p-pci', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL | |
119 | {'device':'virtio-blk-ccw', 'expected':True}, # drive property not set | |
120 | {'device':'virtio-blk-device', 'expected':True}, # drive property not set | |
121 | {'device':'virtio-blk-device', 'expected':True}, # drive property not set | |
122 | {'device':'virtio-blk-pci', 'expected':True}, # drive property not set | |
123 | {'device':'virtio-crypto-ccw', 'expected':True}, # 'cryptodev' parameter expects a valid object | |
124 | {'device':'virtio-crypto-device', 'expected':True}, # 'cryptodev' parameter expects a valid object | |
125 | {'device':'virtio-crypto-pci', 'expected':True}, # 'cryptodev' parameter expects a valid object | |
126 | {'device':'virtio-input-host-device', 'expected':True}, # evdev property is required | |
127 | {'device':'virtio-input-host-pci', 'expected':True}, # evdev property is required | |
128 | {'device':'xen-pvdevice', 'expected':True}, # Device ID invalid, it must always be supplied | |
129 | {'device':'vhost-vsock-ccw', 'expected':True}, # guest-cid property must be greater than 2 | |
23ea4f30 EH |
130 | {'device':'zpci', 'expected':True}, # target must be defined |
131 | {'device':'pnv-(occ|icp|lpc)', 'expected':True}, # required link 'xics' not found: Property '.xics' not found | |
132 | {'device':'powernv-cpu-.*', 'expected':True}, # pnv_core_realize: required link 'xics' not found: Property '.xics' not found | |
133 | ||
134 | # ioapic devices are already created by pc and will fail: | |
135 | {'machine':'q35|pc.*', 'device':'kvm-ioapic', 'expected':True}, # Only 1 ioapics allowed | |
136 | {'machine':'q35|pc.*', 'device':'ioapic', 'expected':True}, # Only 1 ioapics allowed | |
137 | ||
2363d5ee TH |
138 | # "spapr-cpu-core needs a pseries machine" |
139 | {'machine':'(?!pseries).*', 'device':'.*-spapr-cpu-core', 'expected':True}, | |
140 | ||
23ea4f30 EH |
141 | # KVM-specific devices shouldn't be tried without accel=kvm: |
142 | {'accel':'(?!kvm).*', 'device':'kvmclock', 'expected':True}, | |
23ea4f30 EH |
143 | |
144 | # xen-specific machines and devices: | |
145 | {'accel':'(?!xen).*', 'machine':'xen.*', 'expected':True}, | |
146 | {'accel':'(?!xen).*', 'device':'xen-.*', 'expected':True}, | |
147 | ||
148 | # this fails on some machine-types, but not all, so they don't have expected=True: | |
149 | {'device':'vmgenid'}, # vmgenid requires DMA write support in fw_cfg, which this machine type does not provide | |
150 | ||
151 | # Silence INFO messages for errors that are common on multiple | |
152 | # devices/machines: | |
153 | {'log':r"No '[\w-]+' bus found for device '[\w-]+'"}, | |
154 | {'log':r"images* must be given with the 'pflash' parameter"}, | |
155 | {'log':r"(Guest|ROM|Flash|Kernel) image must be specified"}, | |
156 | {'log':r"[cC]ould not load [\w ]+ (BIOS|bios) '[\w-]+\.bin'"}, | |
157 | {'log':r"Couldn't find rom image '[\w-]+\.bin'"}, | |
158 | {'log':r"speed mismatch trying to attach usb device"}, | |
159 | {'log':r"Can't create a second ISA bus"}, | |
160 | {'log':r"duplicate fw_cfg file name"}, | |
161 | # sysbus-related error messages: most machines reject most dynamic sysbus devices: | |
162 | {'log':r"Option '-device [\w.,-]+' cannot be handled by this machine"}, | |
163 | {'log':r"Device [\w.,-]+ is not supported by this machine yet"}, | |
164 | {'log':r"Device [\w.,-]+ can not be dynamically instantiated"}, | |
165 | {'log':r"Platform Bus: Can not fit MMIO region of size "}, | |
166 | # other more specific errors we will ignore: | |
23ea4f30 EH |
167 | {'device':'.*-spapr-cpu-core', 'log':r"CPU core type should be"}, |
168 | {'log':r"MSI(-X)? is not supported by interrupt controller"}, | |
169 | {'log':r"pxb-pcie? devices cannot reside on a PCIe? bus"}, | |
170 | {'log':r"Ignoring smp_cpus value"}, | |
171 | {'log':r"sd_init failed: Drive 'sd0' is already in use because it has been automatically connected to another device"}, | |
172 | {'log':r"This CPU requires a smaller page size than the system is using"}, | |
173 | {'log':r"MSI-X support is mandatory in the S390 architecture"}, | |
174 | {'log':r"rom check and register reset failed"}, | |
175 | {'log':r"Unable to initialize GIC, CPUState for CPU#0 not valid"}, | |
176 | {'log':r"Multiple VT220 operator consoles are not supported"}, | |
177 | {'log':r"core 0 already populated"}, | |
178 | {'log':r"could not find stage1 bootloader"}, | |
179 | ||
180 | # other exitcode=1 failures not listed above will just generate INFO messages: | |
181 | {'exitcode':1, 'loglevel':logging.INFO}, | |
182 | ||
23ea4f30 EH |
183 | # everything else (including SIGABRT and SIGSEGV) will be a fatal error: |
184 | {'exitcode':None, 'fatal':True, 'loglevel':logging.FATAL}, | |
185 | ] | |
186 | ||
187 | ||
1a14d4e1 EH |
188 | def errorRuleTestCaseMatch(rule, t): |
189 | """Check if a test case specification can match a error rule | |
23ea4f30 | 190 | |
1a14d4e1 | 191 | This only checks if a error rule is a candidate match |
23ea4f30 | 192 | for a given test case, it won't check if the test case |
1a14d4e1 | 193 | results/output match the rule. See ruleListResultMatch(). |
23ea4f30 | 194 | """ |
1a14d4e1 | 195 | return (('machine' not in rule or |
23ea4f30 | 196 | 'machine' not in t or |
1a14d4e1 EH |
197 | re.match(rule['machine'] + '$', t['machine'])) and |
198 | ('accel' not in rule or | |
23ea4f30 | 199 | 'accel' not in t or |
1a14d4e1 EH |
200 | re.match(rule['accel'] + '$', t['accel'])) and |
201 | ('device' not in rule or | |
23ea4f30 | 202 | 'device' not in t or |
1a14d4e1 | 203 | re.match(rule['device'] + '$', t['device']))) |
23ea4f30 EH |
204 | |
205 | ||
1a14d4e1 | 206 | def ruleListCandidates(t): |
23ea4f30 | 207 | """Generate the list of candidates that can match a test case""" |
1a14d4e1 EH |
208 | for i, rule in enumerate(ERROR_RULE_LIST): |
209 | if errorRuleTestCaseMatch(rule, t): | |
210 | yield (i, rule) | |
23ea4f30 EH |
211 | |
212 | ||
213 | def findExpectedResult(t): | |
1a14d4e1 | 214 | """Check if there's an expected=True error rule for a test case |
23ea4f30 | 215 | |
1a14d4e1 EH |
216 | Returns (i, rule) tuple, where i is the index in |
217 | ERROR_RULE_LIST and rule is the error rule itself. | |
23ea4f30 | 218 | """ |
1a14d4e1 EH |
219 | for i, rule in ruleListCandidates(t): |
220 | if rule.get('expected'): | |
221 | return (i, rule) | |
23ea4f30 EH |
222 | |
223 | ||
1a14d4e1 EH |
224 | def ruleListResultMatch(rule, r): |
225 | """Check if test case results/output match a error rule | |
23ea4f30 EH |
226 | |
227 | It is valid to call this function only if | |
1a14d4e1 EH |
228 | errorRuleTestCaseMatch() is True for the rule (e.g. on |
229 | rules returned by ruleListCandidates()) | |
23ea4f30 | 230 | """ |
1a14d4e1 EH |
231 | assert errorRuleTestCaseMatch(rule, r['testcase']) |
232 | return ((rule.get('exitcode', 1) is None or | |
233 | r['exitcode'] == rule.get('exitcode', 1)) and | |
234 | ('log' not in rule or | |
235 | re.search(rule['log'], r['log'], re.MULTILINE))) | |
23ea4f30 EH |
236 | |
237 | ||
1a14d4e1 EH |
238 | def checkResultRuleList(r): |
239 | """Look up error rule for a given test case result | |
23ea4f30 | 240 | |
1a14d4e1 EH |
241 | Returns (i, rule) tuple, where i is the index in |
242 | ERROR_RULE_LIST and rule is the error rule itself. | |
23ea4f30 | 243 | """ |
1a14d4e1 EH |
244 | for i, rule in ruleListCandidates(r['testcase']): |
245 | if ruleListResultMatch(rule, r): | |
246 | return i, rule | |
23ea4f30 EH |
247 | |
248 | raise Exception("this should never happen") | |
249 | ||
250 | ||
251 | def qemuOptsEscape(s): | |
252 | """Escape option value QemuOpts""" | |
253 | return s.replace(",", ",,") | |
254 | ||
255 | ||
256 | def formatTestCase(t): | |
257 | """Format test case info as "key=value key=value" for prettier logging output""" | |
258 | return ' '.join('%s=%s' % (k, v) for k, v in t.items()) | |
259 | ||
260 | ||
261 | def qomListTypeNames(vm, **kwargs): | |
262 | """Run qom-list-types QMP command, return type names""" | |
263 | types = vm.command('qom-list-types', **kwargs) | |
264 | return [t['name'] for t in types] | |
265 | ||
266 | ||
267 | def infoQDM(vm): | |
268 | """Parse 'info qdm' output""" | |
269 | args = {'command-line': 'info qdm'} | |
270 | devhelp = vm.command('human-monitor-command', **args) | |
271 | for l in devhelp.split('\n'): | |
272 | l = l.strip() | |
273 | if l == '' or l.endswith(':'): | |
274 | continue | |
275 | d = {'name': re.search(r'name "([^"]+)"', l).group(1), | |
276 | 'no-user': (re.search(', no-user', l) is not None)} | |
277 | yield d | |
278 | ||
279 | ||
280 | class QemuBinaryInfo(object): | |
281 | def __init__(self, binary, devtype): | |
282 | if devtype is None: | |
283 | devtype = 'device' | |
284 | ||
285 | self.binary = binary | |
286 | self._machine_info = {} | |
287 | ||
288 | dbg("devtype: %r", devtype) | |
289 | args = ['-S', '-machine', 'none,accel=kvm:tcg'] | |
290 | dbg("querying info for QEMU binary: %s", binary) | |
291 | vm = QEMUMachine(binary=binary, args=args) | |
292 | vm.launch() | |
293 | try: | |
294 | self.alldevs = set(qomListTypeNames(vm, implements=devtype, abstract=False)) | |
295 | # there's no way to query DeviceClass::user_creatable using QMP, | |
296 | # so use 'info qdm': | |
297 | self.no_user_devs = set([d['name'] for d in infoQDM(vm, ) if d['no-user']]) | |
298 | self.machines = list(m['name'] for m in vm.command('query-machines')) | |
299 | self.user_devs = self.alldevs.difference(self.no_user_devs) | |
300 | self.kvm_available = vm.command('query-kvm')['enabled'] | |
301 | finally: | |
302 | vm.shutdown() | |
303 | ||
304 | def machineInfo(self, machine): | |
305 | """Query for information on a specific machine-type | |
306 | ||
307 | Results are cached internally, in case the same machine- | |
308 | type is queried multiple times. | |
309 | """ | |
310 | if machine in self._machine_info: | |
311 | return self._machine_info[machine] | |
312 | ||
313 | mi = {} | |
314 | args = ['-S', '-machine', '%s' % (machine)] | |
315 | dbg("querying machine info for binary=%s machine=%s", self.binary, machine) | |
316 | vm = QEMUMachine(binary=self.binary, args=args) | |
317 | try: | |
318 | vm.launch() | |
319 | mi['runnable'] = True | |
47b43acd | 320 | except Exception: |
23ea4f30 EH |
321 | dbg("exception trying to run binary=%s machine=%s", self.binary, machine, exc_info=sys.exc_info()) |
322 | dbg("log: %r", vm.get_log()) | |
323 | mi['runnable'] = False | |
324 | ||
325 | vm.shutdown() | |
326 | self._machine_info[machine] = mi | |
327 | return mi | |
328 | ||
329 | ||
330 | BINARY_INFO = {} | |
331 | ||
332 | ||
333 | def getBinaryInfo(args, binary): | |
334 | if binary not in BINARY_INFO: | |
335 | BINARY_INFO[binary] = QemuBinaryInfo(binary, args.devtype) | |
336 | return BINARY_INFO[binary] | |
337 | ||
338 | ||
339 | def checkOneCase(args, testcase): | |
340 | """Check one specific case | |
341 | ||
342 | Returns a dictionary containing failure information on error, | |
343 | or None on success | |
344 | """ | |
345 | binary = testcase['binary'] | |
346 | accel = testcase['accel'] | |
347 | machine = testcase['machine'] | |
348 | device = testcase['device'] | |
349 | ||
350 | dbg("will test: %r", testcase) | |
351 | ||
352 | args = ['-S', '-machine', '%s,accel=%s' % (machine, accel), | |
353 | '-device', qemuOptsEscape(device)] | |
354 | cmdline = ' '.join([binary] + args) | |
355 | dbg("will launch QEMU: %s", cmdline) | |
206439cd | 356 | vm = QEMUMachine(binary=binary, args=args, qmp_timer=15) |
23ea4f30 | 357 | |
c398a241 | 358 | exc = None |
23ea4f30 EH |
359 | exc_traceback = None |
360 | try: | |
361 | vm.launch() | |
c398a241 JS |
362 | except Exception as this_exc: |
363 | exc = this_exc | |
23ea4f30 EH |
364 | exc_traceback = traceback.format_exc() |
365 | dbg("Exception while running test case") | |
366 | finally: | |
367 | vm.shutdown() | |
368 | ec = vm.exitcode() | |
369 | log = vm.get_log() | |
370 | ||
c398a241 JS |
371 | if exc is not None or ec != 0: |
372 | return {'exc': exc, | |
373 | 'exc_traceback':exc_traceback, | |
23ea4f30 EH |
374 | 'exitcode':ec, |
375 | 'log':log, | |
376 | 'testcase':testcase, | |
377 | 'cmdline':cmdline} | |
378 | ||
379 | ||
380 | def binariesToTest(args, testcase): | |
381 | if args.qemu: | |
382 | r = args.qemu | |
383 | else: | |
8a478365 EH |
384 | r = [f.path for f in os.scandir('.') |
385 | if f.name.startswith('qemu-system-') and | |
386 | f.is_file() and os.access(f, os.X_OK)] | |
23ea4f30 EH |
387 | return r |
388 | ||
389 | ||
390 | def accelsToTest(args, testcase): | |
391 | if getBinaryInfo(args, testcase['binary']).kvm_available: | |
392 | yield 'kvm' | |
393 | yield 'tcg' | |
394 | ||
395 | ||
396 | def machinesToTest(args, testcase): | |
397 | return getBinaryInfo(args, testcase['binary']).machines | |
398 | ||
399 | ||
400 | def devicesToTest(args, testcase): | |
401 | return getBinaryInfo(args, testcase['binary']).user_devs | |
402 | ||
403 | ||
404 | TESTCASE_VARIABLES = [ | |
405 | ('binary', binariesToTest), | |
406 | ('accel', accelsToTest), | |
407 | ('machine', machinesToTest), | |
408 | ('device', devicesToTest), | |
409 | ] | |
410 | ||
411 | ||
412 | def genCases1(args, testcases, var, fn): | |
413 | """Generate new testcases for one variable | |
414 | ||
415 | If an existing item already has a variable set, don't | |
416 | generate new items and just return it directly. This | |
417 | allows the "-t" command-line option to be used to choose | |
418 | a specific test case. | |
419 | """ | |
420 | for testcase in testcases: | |
421 | if var in testcase: | |
422 | yield testcase.copy() | |
423 | else: | |
424 | for i in fn(args, testcase): | |
425 | t = testcase.copy() | |
426 | t[var] = i | |
427 | yield t | |
428 | ||
429 | ||
430 | def genCases(args, testcase): | |
431 | """Generate test cases for all variables | |
432 | """ | |
433 | cases = [testcase.copy()] | |
434 | for var, fn in TESTCASE_VARIABLES: | |
435 | dbg("var: %r, fn: %r", var, fn) | |
436 | cases = genCases1(args, cases, var, fn) | |
437 | return cases | |
438 | ||
439 | ||
440 | def casesToTest(args, testcase): | |
441 | cases = genCases(args, testcase) | |
442 | if args.random: | |
443 | cases = list(cases) | |
444 | cases = random.sample(cases, min(args.random, len(cases))) | |
445 | if args.debug: | |
446 | cases = list(cases) | |
447 | dbg("%d test cases to test", len(cases)) | |
448 | if args.shuffle: | |
449 | cases = list(cases) | |
450 | random.shuffle(cases) | |
451 | return cases | |
452 | ||
453 | ||
454 | def logFailure(f, level): | |
455 | t = f['testcase'] | |
456 | logger.log(level, "failed: %s", formatTestCase(t)) | |
457 | logger.log(level, "cmdline: %s", f['cmdline']) | |
458 | for l in f['log'].strip().split('\n'): | |
459 | logger.log(level, "log: %s", l) | |
460 | logger.log(level, "exit code: %r", f['exitcode']) | |
c398a241 JS |
461 | |
462 | # If the Exception is merely a QMP connect error, | |
463 | # reduce the logging level for its traceback to | |
464 | # improve visual clarity. | |
465 | if isinstance(f.get('exc'), ConnectError): | |
466 | logger.log(level, "%s.%s: %s", | |
467 | type(f['exc']).__module__, | |
468 | type(f['exc']).__qualname__, | |
469 | str(f['exc'])) | |
470 | level = logging.DEBUG | |
471 | ||
23ea4f30 EH |
472 | if f['exc_traceback']: |
473 | logger.log(level, "exception:") | |
474 | for l in f['exc_traceback'].split('\n'): | |
475 | logger.log(level, " %s", l.rstrip('\n')) | |
476 | ||
477 | ||
478 | def main(): | |
479 | parser = argparse.ArgumentParser(description="QEMU -device crash test") | |
480 | parser.add_argument('-t', metavar='KEY=VALUE', nargs='*', | |
481 | help="Limit test cases to KEY=VALUE", | |
482 | action='append', dest='testcases', default=[]) | |
483 | parser.add_argument('-d', '--debug', action='store_true', | |
484 | help='debug output') | |
485 | parser.add_argument('-v', '--verbose', action='store_true', default=True, | |
486 | help='verbose output') | |
487 | parser.add_argument('-q', '--quiet', dest='verbose', action='store_false', | |
488 | help='non-verbose output') | |
489 | parser.add_argument('-r', '--random', type=int, metavar='COUNT', | |
490 | help='run a random sample of COUNT test cases', | |
491 | default=0) | |
492 | parser.add_argument('--shuffle', action='store_true', | |
493 | help='Run test cases in random order') | |
494 | parser.add_argument('--dry-run', action='store_true', | |
495 | help="Don't run any tests, just generate list") | |
496 | parser.add_argument('-D', '--devtype', metavar='TYPE', | |
497 | help="Test only device types that implement TYPE") | |
498 | parser.add_argument('-Q', '--quick', action='store_true', default=True, | |
499 | help="Quick mode: skip test cases that are expected to fail") | |
500 | parser.add_argument('-F', '--full', action='store_false', dest='quick', | |
501 | help="Full mode: test cases that are expected to fail") | |
502 | parser.add_argument('--strict', action='store_true', dest='strict', | |
503 | help="Treat all warnings as fatal") | |
504 | parser.add_argument('qemu', nargs='*', metavar='QEMU', | |
505 | help='QEMU binary to run') | |
506 | args = parser.parse_args() | |
507 | ||
508 | if args.debug: | |
509 | lvl = logging.DEBUG | |
510 | elif args.verbose: | |
511 | lvl = logging.INFO | |
512 | else: | |
513 | lvl = logging.WARN | |
514 | logging.basicConfig(stream=sys.stdout, level=lvl, format='%(levelname)s: %(message)s') | |
515 | ||
76f86e78 JS |
516 | if not args.debug: |
517 | # Async QMP, when in use, is chatty about connection failures. | |
518 | # This script knowingly generates a ton of connection errors. | |
519 | # Silence this logger. | |
520 | logging.getLogger('qemu.aqmp.qmp_client').setLevel(logging.CRITICAL) | |
521 | ||
23ea4f30 EH |
522 | fatal_failures = [] |
523 | wl_stats = {} | |
524 | skipped = 0 | |
525 | total = 0 | |
526 | ||
527 | tc = {} | |
528 | dbg("testcases: %r", args.testcases) | |
529 | if args.testcases: | |
530 | for t in chain(*args.testcases): | |
531 | for kv in t.split(): | |
532 | k, v = kv.split('=', 1) | |
533 | tc[k] = v | |
534 | ||
535 | if len(binariesToTest(args, tc)) == 0: | |
f03868bd | 536 | print("No QEMU binary found", file=sys.stderr) |
23ea4f30 EH |
537 | parser.print_usage(sys.stderr) |
538 | return 1 | |
539 | ||
540 | for t in casesToTest(args, tc): | |
541 | logger.info("running test case: %s", formatTestCase(t)) | |
542 | total += 1 | |
543 | ||
544 | expected_match = findExpectedResult(t) | |
545 | if (args.quick and | |
546 | (expected_match or | |
547 | not getBinaryInfo(args, t['binary']).machineInfo(t['machine'])['runnable'])): | |
548 | dbg("skipped: %s", formatTestCase(t)) | |
549 | skipped += 1 | |
550 | continue | |
551 | ||
552 | if args.dry_run: | |
553 | continue | |
554 | ||
555 | try: | |
556 | f = checkOneCase(args, t) | |
557 | except KeyboardInterrupt: | |
558 | break | |
559 | ||
560 | if f: | |
1a14d4e1 EH |
561 | i, rule = checkResultRuleList(f) |
562 | dbg("testcase: %r, rule list match: %r", t, rule) | |
23ea4f30 | 563 | wl_stats.setdefault(i, []).append(f) |
1a14d4e1 | 564 | level = rule.get('loglevel', logging.DEBUG) |
23ea4f30 | 565 | logFailure(f, level) |
1a14d4e1 | 566 | if rule.get('fatal') or (args.strict and level >= logging.WARN): |
23ea4f30 EH |
567 | fatal_failures.append(f) |
568 | else: | |
569 | dbg("success: %s", formatTestCase(t)) | |
570 | if expected_match: | |
571 | logger.warn("Didn't fail as expected: %s", formatTestCase(t)) | |
572 | ||
573 | logger.info("Total: %d test cases", total) | |
574 | if skipped: | |
575 | logger.info("Skipped %d test cases", skipped) | |
576 | ||
577 | if args.debug: | |
1a14d4e1 EH |
578 | stats = sorted([(len(wl_stats.get(i, [])), rule) for i, rule in |
579 | enumerate(ERROR_RULE_LIST)], key=lambda x: x[0]) | |
580 | for count, rule in stats: | |
581 | dbg("error rule stats: %d: %r", count, rule) | |
23ea4f30 EH |
582 | |
583 | if fatal_failures: | |
584 | for f in fatal_failures: | |
585 | t = f['testcase'] | |
586 | logger.error("Fatal failure: %s", formatTestCase(t)) | |
587 | logger.error("Fatal failures on some machine/device combinations") | |
588 | return 1 | |
589 | ||
590 | if __name__ == '__main__': | |
591 | sys.exit(main()) |