]>
git.proxmox.com Git - mirror_qemu.git/blob - scripts/device-crash-test
3 # Copyright (c) 2017 Red Hat Inc
6 # Eduardo Habkost <ehabkost@redhat.com>
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License along
19 # with this program; if not, write to the Free Software Foundation, Inc.,
20 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23 Run QEMU with all combinations of -machine and -device types,
24 check for crashes and unexpected errors.
35 from itertools
import chain
37 sys
.path
.append(os
.path
.join(os
.path
.dirname(__file__
), '..', 'python'))
38 from qemu
.machine
import QEMUMachine
40 logger
= logging
.getLogger('device-crash-test')
44 # Purposes of the following rule list:
45 # * Avoiding verbose log messages when we find known non-fatal
47 # * Avoiding fatal errors when we find known crashes
48 # * Skipping machines/devices that are known not to work out of
49 # the box, when running in --quick mode
51 # Keeping the rule list updated is desirable, but not required,
52 # because unexpected cases where QEMU exits with exitcode=1 will
53 # just trigger a INFO message.
55 # Valid error rule keys:
56 # * accel: regexp, full match only
57 # * machine: regexp, full match only
58 # * device: regexp, full match only
59 # * log: regexp, partial match allowed
60 # * exitcode: if not present, defaults to 1. If None, matches any exitcode
61 # * warn: if True, matching failures will be logged as warnings
62 # * expected: if True, QEMU is expected to always fail every time
63 # when testing the corresponding test case
64 # * loglevel: log level of log output when there's a match.
66 # Machines that won't work out of the box:
67 # MACHINE | ERROR MESSAGE
68 {'machine':'niagara', 'expected':True}, # Unable to load a firmware for -M niagara
69 {'machine':'boston', 'expected':True}, # Please provide either a -kernel or -bios argument
70 {'machine':'leon3_generic', 'expected':True}, # Can't read bios image (null)
72 # devices that don't work out of the box because they require extra options to "-device DEV":
73 # DEVICE | ERROR MESSAGE
74 {'device':'.*-(i386|x86_64)-cpu', 'expected':True}, # CPU socket-id is not set
75 {'device':'icp', 'expected':True}, # icp_realize: required link 'xics' not found: Property '.xics' not found
76 {'device':'ics', 'expected':True}, # ics_base_realize: required link 'xics' not found: Property '.xics' not found
77 # "-device ide-cd" does work on more recent QEMU versions, so it doesn't have expected=True
78 {'device':'ide-cd'}, # No drive specified
79 {'device':'ide-hd', 'expected':True}, # No drive specified
80 {'device':'ipmi-bmc-extern', 'expected':True}, # IPMI external bmc requires chardev attribute
81 {'device':'isa-debugcon', 'expected':True}, # Can't create serial device, empty char device
82 {'device':'isa-ipmi-bt', 'expected':True}, # IPMI device requires a bmc attribute to be set
83 {'device':'isa-ipmi-kcs', 'expected':True}, # IPMI device requires a bmc attribute to be set
84 {'device':'isa-parallel', 'expected':True}, # Can't create serial device, empty char device
85 {'device':'ivshmem-doorbell', 'expected':True}, # You must specify a 'chardev'
86 {'device':'ivshmem-plain', 'expected':True}, # You must specify a 'memdev'
87 {'device':'loader', 'expected':True}, # please include valid arguments
88 {'device':'nand', 'expected':True}, # Unsupported NAND block size 0x1
89 {'device':'nvdimm', 'expected':True}, # 'memdev' property is not set
90 {'device':'nvme', 'expected':True}, # Device initialization failed
91 {'device':'pc-dimm', 'expected':True}, # 'memdev' property is not set
92 {'device':'pci-bridge', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
93 {'device':'pci-bridge-seat', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
94 {'device':'pxb', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
95 {'device':'scsi-block', 'expected':True}, # drive property not set
96 {'device':'scsi-generic', 'expected':True}, # drive property not set
97 {'device':'scsi-hd', 'expected':True}, # drive property not set
98 {'device':'spapr-pci-host-bridge', 'expected':True}, # BUID not specified for PHB
99 {'device':'spapr-rng', 'expected':True}, # spapr-rng needs an RNG backend!
100 {'device':'spapr-vty', 'expected':True}, # chardev property not set
101 {'device':'tpm-tis', 'expected':True}, # tpm_tis: backend driver with id (null) could not be found
102 {'device':'unimplemented-device', 'expected':True}, # property 'size' not specified or zero
103 {'device':'usb-braille', 'expected':True}, # Property chardev is required
104 {'device':'usb-mtp', 'expected':True}, # rootdir property must be configured
105 {'device':'usb-redir', 'expected':True}, # Parameter 'chardev' is missing
106 {'device':'usb-serial', 'expected':True}, # Property chardev is required
107 {'device':'usb-storage', 'expected':True}, # drive property not set
108 {'device':'vfio-amd-xgbe', 'expected':True}, # -device vfio-amd-xgbe: vfio error: wrong host device name
109 {'device':'vfio-calxeda-xgmac', 'expected':True}, # -device vfio-calxeda-xgmac: vfio error: wrong host device name
110 {'device':'vfio-pci', 'expected':True}, # No provided host device
111 {'device':'vfio-pci-igd-lpc-bridge', 'expected':True}, # VFIO dummy ISA/LPC bridge must have address 1f.0
112 {'device':'vhost-scsi.*', 'expected':True}, # vhost-scsi: missing wwpn
113 {'device':'vhost-vsock-device', 'expected':True}, # guest-cid property must be greater than 2
114 {'device':'vhost-vsock-pci', 'expected':True}, # guest-cid property must be greater than 2
115 {'device':'virtio-9p-ccw', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL
116 {'device':'virtio-9p-device', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL
117 {'device':'virtio-9p-pci', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL
118 {'device':'virtio-blk-ccw', 'expected':True}, # drive property not set
119 {'device':'virtio-blk-device', 'expected':True}, # drive property not set
120 {'device':'virtio-blk-device', 'expected':True}, # drive property not set
121 {'device':'virtio-blk-pci', 'expected':True}, # drive property not set
122 {'device':'virtio-crypto-ccw', 'expected':True}, # 'cryptodev' parameter expects a valid object
123 {'device':'virtio-crypto-device', 'expected':True}, # 'cryptodev' parameter expects a valid object
124 {'device':'virtio-crypto-pci', 'expected':True}, # 'cryptodev' parameter expects a valid object
125 {'device':'virtio-input-host-device', 'expected':True}, # evdev property is required
126 {'device':'virtio-input-host-pci', 'expected':True}, # evdev property is required
127 {'device':'xen-pvdevice', 'expected':True}, # Device ID invalid, it must always be supplied
128 {'device':'vhost-vsock-ccw', 'expected':True}, # guest-cid property must be greater than 2
129 {'device':'zpci', 'expected':True}, # target must be defined
130 {'device':'pnv-(occ|icp|lpc)', 'expected':True}, # required link 'xics' not found: Property '.xics' not found
131 {'device':'powernv-cpu-.*', 'expected':True}, # pnv_core_realize: required link 'xics' not found: Property '.xics' not found
133 # ioapic devices are already created by pc and will fail:
134 {'machine':'q35|pc.*', 'device':'kvm-ioapic', 'expected':True}, # Only 1 ioapics allowed
135 {'machine':'q35|pc.*', 'device':'ioapic', 'expected':True}, # Only 1 ioapics allowed
137 # "spapr-cpu-core needs a pseries machine"
138 {'machine':'(?!pseries).*', 'device':'.*-spapr-cpu-core', 'expected':True},
140 # KVM-specific devices shouldn't be tried without accel=kvm:
141 {'accel':'(?!kvm).*', 'device':'kvmclock', 'expected':True},
143 # xen-specific machines and devices:
144 {'accel':'(?!xen).*', 'machine':'xen.*', 'expected':True},
145 {'accel':'(?!xen).*', 'device':'xen-.*', 'expected':True},
147 # this fails on some machine-types, but not all, so they don't have expected=True:
148 {'device':'vmgenid'}, # vmgenid requires DMA write support in fw_cfg, which this machine type does not provide
150 # Silence INFO messages for errors that are common on multiple
152 {'log':r
"No '[\w-]+' bus found for device '[\w-]+'"},
153 {'log':r
"images* must be given with the 'pflash' parameter"},
154 {'log':r
"(Guest|ROM|Flash|Kernel) image must be specified"},
155 {'log':r
"[cC]ould not load [\w ]+ (BIOS|bios) '[\w-]+\.bin'"},
156 {'log':r
"Couldn't find rom image '[\w-]+\.bin'"},
157 {'log':r
"speed mismatch trying to attach usb device"},
158 {'log':r
"Can't create a second ISA bus"},
159 {'log':r
"duplicate fw_cfg file name"},
160 # sysbus-related error messages: most machines reject most dynamic sysbus devices:
161 {'log':r
"Option '-device [\w.,-]+' cannot be handled by this machine"},
162 {'log':r
"Device [\w.,-]+ is not supported by this machine yet"},
163 {'log':r
"Device [\w.,-]+ can not be dynamically instantiated"},
164 {'log':r
"Platform Bus: Can not fit MMIO region of size "},
165 # other more specific errors we will ignore:
166 {'device':'.*-spapr-cpu-core', 'log':r
"CPU core type should be"},
167 {'log':r
"MSI(-X)? is not supported by interrupt controller"},
168 {'log':r
"pxb-pcie? devices cannot reside on a PCIe? bus"},
169 {'log':r
"Ignoring smp_cpus value"},
170 {'log':r
"sd_init failed: Drive 'sd0' is already in use because it has been automatically connected to another device"},
171 {'log':r
"This CPU requires a smaller page size than the system is using"},
172 {'log':r
"MSI-X support is mandatory in the S390 architecture"},
173 {'log':r
"rom check and register reset failed"},
174 {'log':r
"Unable to initialize GIC, CPUState for CPU#0 not valid"},
175 {'log':r
"Multiple VT220 operator consoles are not supported"},
176 {'log':r
"core 0 already populated"},
177 {'log':r
"could not find stage1 bootloader"},
178 {'log':r
"No '.*' bus found for device"},
180 # other exitcode=1 failures not listed above will just generate INFO messages:
181 {'exitcode':1, 'loglevel':logging
.INFO
},
183 # everything else (including SIGABRT and SIGSEGV) will be a fatal error:
184 {'exitcode':None, 'fatal':True, 'loglevel':logging
.FATAL
},
188 def errorRuleTestCaseMatch(rule
, t
):
189 """Check if a test case specification can match a error rule
191 This only checks if a error rule is a candidate match
192 for a given test case, it won't check if the test case
193 results/output match the rule. See ruleListResultMatch().
195 return (('machine' not in rule
or
196 'machine' not in t
or
197 re
.match(rule
['machine'] + '$', t
['machine'])) and
198 ('accel' not in rule
or
200 re
.match(rule
['accel'] + '$', t
['accel'])) and
201 ('device' not in rule
or
203 re
.match(rule
['device'] + '$', t
['device'])))
206 def ruleListCandidates(t
):
207 """Generate the list of candidates that can match a test case"""
208 for i
, rule
in enumerate(ERROR_RULE_LIST
):
209 if errorRuleTestCaseMatch(rule
, t
):
213 def findExpectedResult(t
):
214 """Check if there's an expected=True error rule for a test case
216 Returns (i, rule) tuple, where i is the index in
217 ERROR_RULE_LIST and rule is the error rule itself.
219 for i
, rule
in ruleListCandidates(t
):
220 if rule
.get('expected'):
224 def ruleListResultMatch(rule
, r
):
225 """Check if test case results/output match a error rule
227 It is valid to call this function only if
228 errorRuleTestCaseMatch() is True for the rule (e.g. on
229 rules returned by ruleListCandidates())
231 assert errorRuleTestCaseMatch(rule
, r
['testcase'])
232 return ((rule
.get('exitcode', 1) is None or
233 r
['exitcode'] == rule
.get('exitcode', 1)) and
234 ('log' not in rule
or
235 re
.search(rule
['log'], r
['log'], re
.MULTILINE
)))
238 def checkResultRuleList(r
):
239 """Look up error rule for a given test case result
241 Returns (i, rule) tuple, where i is the index in
242 ERROR_RULE_LIST and rule is the error rule itself.
244 for i
, rule
in ruleListCandidates(r
['testcase']):
245 if ruleListResultMatch(rule
, r
):
248 raise Exception("this should never happen")
251 def qemuOptsEscape(s
):
252 """Escape option value QemuOpts"""
253 return s
.replace(",", ",,")
256 def formatTestCase(t
):
257 """Format test case info as "key=value key=value" for prettier logging output"""
258 return ' '.join('%s=%s' % (k
, v
) for k
, v
in t
.items())
261 def qomListTypeNames(vm
, **kwargs
):
262 """Run qom-list-types QMP command, return type names"""
263 types
= vm
.command('qom-list-types', **kwargs
)
264 return [t
['name'] for t
in types
]
268 """Parse 'info qdm' output"""
269 args
= {'command-line': 'info qdm'}
270 devhelp
= vm
.command('human-monitor-command', **args
)
271 for l
in devhelp
.split('\n'):
273 if l
== '' or l
.endswith(':'):
275 d
= {'name': re
.search(r
'name "([^"]+)"', l
).group(1),
276 'no-user': (re
.search(', no-user', l
) is not None)}
280 class QemuBinaryInfo(object):
281 def __init__(self
, binary
, devtype
):
286 self
._machine
_info
= {}
288 dbg("devtype: %r", devtype
)
289 args
= ['-S', '-machine', 'none,accel=kvm:tcg']
290 dbg("querying info for QEMU binary: %s", binary
)
291 vm
= QEMUMachine(binary
=binary
, args
=args
)
294 self
.alldevs
= set(qomListTypeNames(vm
, implements
=devtype
, abstract
=False))
295 # there's no way to query DeviceClass::user_creatable using QMP,
297 self
.no_user_devs
= set([d
['name'] for d
in infoQDM(vm
, ) if d
['no-user']])
298 self
.machines
= list(m
['name'] for m
in vm
.command('query-machines'))
299 self
.user_devs
= self
.alldevs
.difference(self
.no_user_devs
)
300 self
.kvm_available
= vm
.command('query-kvm')['enabled']
304 def machineInfo(self
, machine
):
305 """Query for information on a specific machine-type
307 Results are cached internally, in case the same machine-
308 type is queried multiple times.
310 if machine
in self
._machine
_info
:
311 return self
._machine
_info
[machine
]
314 args
= ['-S', '-machine', '%s' % (machine
)]
315 dbg("querying machine info for binary=%s machine=%s", self
.binary
, machine
)
316 vm
= QEMUMachine(binary
=self
.binary
, args
=args
)
319 mi
['runnable'] = True
320 except KeyboardInterrupt:
323 dbg("exception trying to run binary=%s machine=%s", self
.binary
, machine
, exc_info
=sys
.exc_info())
324 dbg("log: %r", vm
.get_log())
325 mi
['runnable'] = False
328 self
._machine
_info
[machine
] = mi
335 def getBinaryInfo(args
, binary
):
336 if binary
not in BINARY_INFO
:
337 BINARY_INFO
[binary
] = QemuBinaryInfo(binary
, args
.devtype
)
338 return BINARY_INFO
[binary
]
341 def checkOneCase(args
, testcase
):
342 """Check one specific case
344 Returns a dictionary containing failure information on error,
347 binary
= testcase
['binary']
348 accel
= testcase
['accel']
349 machine
= testcase
['machine']
350 device
= testcase
['device']
352 dbg("will test: %r", testcase
)
354 args
= ['-S', '-machine', '%s,accel=%s' % (machine
, accel
),
355 '-device', qemuOptsEscape(device
)]
356 cmdline
= ' '.join([binary
] + args
)
357 dbg("will launch QEMU: %s", cmdline
)
358 vm
= QEMUMachine(binary
=binary
, args
=args
)
363 except KeyboardInterrupt:
366 exc_traceback
= traceback
.format_exc()
367 dbg("Exception while running test case")
373 if exc_traceback
is not None or ec
!= 0:
374 return {'exc_traceback':exc_traceback
,
381 def binariesToTest(args
, testcase
):
385 r
= [f
.path
for f
in os
.scandir('.')
386 if f
.name
.startswith('qemu-system-') and
387 f
.is_file() and os
.access(f
, os
.X_OK
)]
391 def accelsToTest(args
, testcase
):
392 if getBinaryInfo(args
, testcase
['binary']).kvm_available
:
397 def machinesToTest(args
, testcase
):
398 return getBinaryInfo(args
, testcase
['binary']).machines
401 def devicesToTest(args
, testcase
):
402 return getBinaryInfo(args
, testcase
['binary']).user_devs
405 TESTCASE_VARIABLES
= [
406 ('binary', binariesToTest
),
407 ('accel', accelsToTest
),
408 ('machine', machinesToTest
),
409 ('device', devicesToTest
),
413 def genCases1(args
, testcases
, var
, fn
):
414 """Generate new testcases for one variable
416 If an existing item already has a variable set, don't
417 generate new items and just return it directly. This
418 allows the "-t" command-line option to be used to choose
419 a specific test case.
421 for testcase
in testcases
:
423 yield testcase
.copy()
425 for i
in fn(args
, testcase
):
431 def genCases(args
, testcase
):
432 """Generate test cases for all variables
434 cases
= [testcase
.copy()]
435 for var
, fn
in TESTCASE_VARIABLES
:
436 dbg("var: %r, fn: %r", var
, fn
)
437 cases
= genCases1(args
, cases
, var
, fn
)
441 def casesToTest(args
, testcase
):
442 cases
= genCases(args
, testcase
)
445 cases
= random
.sample(cases
, min(args
.random
, len(cases
)))
448 dbg("%d test cases to test", len(cases
))
451 random
.shuffle(cases
)
455 def logFailure(f
, level
):
457 logger
.log(level
, "failed: %s", formatTestCase(t
))
458 logger
.log(level
, "cmdline: %s", f
['cmdline'])
459 for l
in f
['log'].strip().split('\n'):
460 logger
.log(level
, "log: %s", l
)
461 logger
.log(level
, "exit code: %r", f
['exitcode'])
462 if f
['exc_traceback']:
463 logger
.log(level
, "exception:")
464 for l
in f
['exc_traceback'].split('\n'):
465 logger
.log(level
, " %s", l
.rstrip('\n'))
469 parser
= argparse
.ArgumentParser(description
="QEMU -device crash test")
470 parser
.add_argument('-t', metavar
='KEY=VALUE', nargs
='*',
471 help="Limit test cases to KEY=VALUE",
472 action
='append', dest
='testcases', default
=[])
473 parser
.add_argument('-d', '--debug', action
='store_true',
475 parser
.add_argument('-v', '--verbose', action
='store_true', default
=True,
476 help='verbose output')
477 parser
.add_argument('-q', '--quiet', dest
='verbose', action
='store_false',
478 help='non-verbose output')
479 parser
.add_argument('-r', '--random', type=int, metavar
='COUNT',
480 help='run a random sample of COUNT test cases',
482 parser
.add_argument('--shuffle', action
='store_true',
483 help='Run test cases in random order')
484 parser
.add_argument('--dry-run', action
='store_true',
485 help="Don't run any tests, just generate list")
486 parser
.add_argument('-D', '--devtype', metavar
='TYPE',
487 help="Test only device types that implement TYPE")
488 parser
.add_argument('-Q', '--quick', action
='store_true', default
=True,
489 help="Quick mode: skip test cases that are expected to fail")
490 parser
.add_argument('-F', '--full', action
='store_false', dest
='quick',
491 help="Full mode: test cases that are expected to fail")
492 parser
.add_argument('--strict', action
='store_true', dest
='strict',
493 help="Treat all warnings as fatal")
494 parser
.add_argument('qemu', nargs
='*', metavar
='QEMU',
495 help='QEMU binary to run')
496 args
= parser
.parse_args()
504 logging
.basicConfig(stream
=sys
.stdout
, level
=lvl
, format
='%(levelname)s: %(message)s')
512 dbg("testcases: %r", args
.testcases
)
514 for t
in chain(*args
.testcases
):
516 k
, v
= kv
.split('=', 1)
519 if len(binariesToTest(args
, tc
)) == 0:
520 print("No QEMU binary found", file=sys
.stderr
)
521 parser
.print_usage(sys
.stderr
)
524 for t
in casesToTest(args
, tc
):
525 logger
.info("running test case: %s", formatTestCase(t
))
528 expected_match
= findExpectedResult(t
)
531 not getBinaryInfo(args
, t
['binary']).machineInfo(t
['machine'])['runnable'])):
532 dbg("skipped: %s", formatTestCase(t
))
540 f
= checkOneCase(args
, t
)
541 except KeyboardInterrupt:
545 i
, rule
= checkResultRuleList(f
)
546 dbg("testcase: %r, rule list match: %r", t
, rule
)
547 wl_stats
.setdefault(i
, []).append(f
)
548 level
= rule
.get('loglevel', logging
.DEBUG
)
550 if rule
.get('fatal') or (args
.strict
and level
>= logging
.WARN
):
551 fatal_failures
.append(f
)
553 dbg("success: %s", formatTestCase(t
))
555 logger
.warn("Didn't fail as expected: %s", formatTestCase(t
))
557 logger
.info("Total: %d test cases", total
)
559 logger
.info("Skipped %d test cases", skipped
)
562 stats
= sorted([(len(wl_stats
.get(i
, [])), rule
) for i
, rule
in
563 enumerate(ERROR_RULE_LIST
)], key
=lambda x
: x
[0])
564 for count
, rule
in stats
:
565 dbg("error rule stats: %d: %r", count
, rule
)
568 for f
in fatal_failures
:
570 logger
.error("Fatal failure: %s", formatTestCase(t
))
571 logger
.error("Fatal failures on some machine/device combinations")
574 if __name__
== '__main__':