]>
git.proxmox.com Git - mirror_qemu.git/blob - scripts/device-crash-test
3 # Copyright (c) 2017 Red Hat Inc
6 # Eduardo Habkost <ehabkost@redhat.com>
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License along
19 # with this program; if not, write to the Free Software Foundation, Inc.,
20 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23 Run QEMU with all combinations of -machine and -device types,
24 check for crashes and unexpected errors.
35 from itertools
import chain
37 sys
.path
.append(os
.path
.join(os
.path
.dirname(__file__
), '..', 'python'))
38 from qemu
.machine
import QEMUMachine
40 logger
= logging
.getLogger('device-crash-test')
44 # Purposes of the following whitelist:
45 # * Avoiding verbose log messages when we find known non-fatal
47 # * Avoiding fatal errors when we find known crashes
48 # * Skipping machines/devices that are known not to work out of
49 # the box, when running in --quick mode
51 # Keeping the whitelist updated is desirable, but not required,
52 # because unexpected cases where QEMU exits with exitcode=1 will
53 # just trigger a INFO message.
55 # Valid whitelist entry keys:
56 # * accel: regexp, full match only
57 # * machine: regexp, full match only
58 # * device: regexp, full match only
59 # * log: regexp, partial match allowed
60 # * exitcode: if not present, defaults to 1. If None, matches any exitcode
61 # * warn: if True, matching failures will be logged as warnings
62 # * expected: if True, QEMU is expected to always fail every time
63 # when testing the corresponding test case
64 # * loglevel: log level of log output when there's a match.
66 # Machines that won't work out of the box:
67 # MACHINE | ERROR MESSAGE
68 {'machine':'niagara', 'expected':True}, # Unable to load a firmware for -M niagara
69 {'machine':'boston', 'expected':True}, # Please provide either a -kernel or -bios argument
70 {'machine':'leon3_generic', 'expected':True}, # Can't read bios image (null)
72 # devices that don't work out of the box because they require extra options to "-device DEV":
73 # DEVICE | ERROR MESSAGE
74 {'device':'.*-(i386|x86_64)-cpu', 'expected':True}, # CPU socket-id is not set
75 {'device':'icp', 'expected':True}, # icp_realize: required link 'xics' not found: Property '.xics' not found
76 {'device':'ics', 'expected':True}, # ics_base_realize: required link 'xics' not found: Property '.xics' not found
77 # "-device ide-cd" does work on more recent QEMU versions, so it doesn't have expected=True
78 {'device':'ide-cd'}, # No drive specified
79 {'device':'ide-drive', 'expected':True}, # No drive specified
80 {'device':'ide-hd', 'expected':True}, # No drive specified
81 {'device':'ipmi-bmc-extern', 'expected':True}, # IPMI external bmc requires chardev attribute
82 {'device':'isa-debugcon', 'expected':True}, # Can't create serial device, empty char device
83 {'device':'isa-ipmi-bt', 'expected':True}, # IPMI device requires a bmc attribute to be set
84 {'device':'isa-ipmi-kcs', 'expected':True}, # IPMI device requires a bmc attribute to be set
85 {'device':'isa-parallel', 'expected':True}, # Can't create serial device, empty char device
86 {'device':'ivshmem-doorbell', 'expected':True}, # You must specify a 'chardev'
87 {'device':'ivshmem-plain', 'expected':True}, # You must specify a 'memdev'
88 {'device':'loader', 'expected':True}, # please include valid arguments
89 {'device':'nand', 'expected':True}, # Unsupported NAND block size 0x1
90 {'device':'nvdimm', 'expected':True}, # 'memdev' property is not set
91 {'device':'nvme', 'expected':True}, # Device initialization failed
92 {'device':'pc-dimm', 'expected':True}, # 'memdev' property is not set
93 {'device':'pci-bridge', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
94 {'device':'pci-bridge-seat', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
95 {'device':'pxb', 'expected':True}, # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
96 {'device':'scsi-block', 'expected':True}, # drive property not set
97 {'device':'scsi-disk', 'expected':True}, # drive property not set
98 {'device':'scsi-generic', 'expected':True}, # drive property not set
99 {'device':'scsi-hd', 'expected':True}, # drive property not set
100 {'device':'spapr-pci-host-bridge', 'expected':True}, # BUID not specified for PHB
101 {'device':'spapr-rng', 'expected':True}, # spapr-rng needs an RNG backend!
102 {'device':'spapr-vty', 'expected':True}, # chardev property not set
103 {'device':'tpm-tis', 'expected':True}, # tpm_tis: backend driver with id (null) could not be found
104 {'device':'unimplemented-device', 'expected':True}, # property 'size' not specified or zero
105 {'device':'usb-braille', 'expected':True}, # Property chardev is required
106 {'device':'usb-mtp', 'expected':True}, # rootdir property must be configured
107 {'device':'usb-redir', 'expected':True}, # Parameter 'chardev' is missing
108 {'device':'usb-serial', 'expected':True}, # Property chardev is required
109 {'device':'usb-storage', 'expected':True}, # drive property not set
110 {'device':'vfio-amd-xgbe', 'expected':True}, # -device vfio-amd-xgbe: vfio error: wrong host device name
111 {'device':'vfio-calxeda-xgmac', 'expected':True}, # -device vfio-calxeda-xgmac: vfio error: wrong host device name
112 {'device':'vfio-pci', 'expected':True}, # No provided host device
113 {'device':'vfio-pci-igd-lpc-bridge', 'expected':True}, # VFIO dummy ISA/LPC bridge must have address 1f.0
114 {'device':'vhost-scsi.*', 'expected':True}, # vhost-scsi: missing wwpn
115 {'device':'vhost-vsock-device', 'expected':True}, # guest-cid property must be greater than 2
116 {'device':'vhost-vsock-pci', 'expected':True}, # guest-cid property must be greater than 2
117 {'device':'virtio-9p-ccw', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL
118 {'device':'virtio-9p-device', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL
119 {'device':'virtio-9p-pci', 'expected':True}, # 9pfs device couldn't find fsdev with the id = NULL
120 {'device':'virtio-blk-ccw', 'expected':True}, # drive property not set
121 {'device':'virtio-blk-device', 'expected':True}, # drive property not set
122 {'device':'virtio-blk-device', 'expected':True}, # drive property not set
123 {'device':'virtio-blk-pci', 'expected':True}, # drive property not set
124 {'device':'virtio-crypto-ccw', 'expected':True}, # 'cryptodev' parameter expects a valid object
125 {'device':'virtio-crypto-device', 'expected':True}, # 'cryptodev' parameter expects a valid object
126 {'device':'virtio-crypto-pci', 'expected':True}, # 'cryptodev' parameter expects a valid object
127 {'device':'virtio-input-host-device', 'expected':True}, # evdev property is required
128 {'device':'virtio-input-host-pci', 'expected':True}, # evdev property is required
129 {'device':'xen-pvdevice', 'expected':True}, # Device ID invalid, it must always be supplied
130 {'device':'vhost-vsock-ccw', 'expected':True}, # guest-cid property must be greater than 2
131 {'device':'zpci', 'expected':True}, # target must be defined
132 {'device':'pnv-(occ|icp|lpc)', 'expected':True}, # required link 'xics' not found: Property '.xics' not found
133 {'device':'powernv-cpu-.*', 'expected':True}, # pnv_core_realize: required link 'xics' not found: Property '.xics' not found
135 # ioapic devices are already created by pc and will fail:
136 {'machine':'q35|pc.*', 'device':'kvm-ioapic', 'expected':True}, # Only 1 ioapics allowed
137 {'machine':'q35|pc.*', 'device':'ioapic', 'expected':True}, # Only 1 ioapics allowed
139 # "spapr-cpu-core needs a pseries machine"
140 {'machine':'(?!pseries).*', 'device':'.*-spapr-cpu-core', 'expected':True},
142 # KVM-specific devices shouldn't be tried without accel=kvm:
143 {'accel':'(?!kvm).*', 'device':'kvmclock', 'expected':True},
145 # xen-specific machines and devices:
146 {'accel':'(?!xen).*', 'machine':'xen.*', 'expected':True},
147 {'accel':'(?!xen).*', 'device':'xen-.*', 'expected':True},
149 # this fails on some machine-types, but not all, so they don't have expected=True:
150 {'device':'vmgenid'}, # vmgenid requires DMA write support in fw_cfg, which this machine type does not provide
152 # Silence INFO messages for errors that are common on multiple
154 {'log':r
"No '[\w-]+' bus found for device '[\w-]+'"},
155 {'log':r
"images* must be given with the 'pflash' parameter"},
156 {'log':r
"(Guest|ROM|Flash|Kernel) image must be specified"},
157 {'log':r
"[cC]ould not load [\w ]+ (BIOS|bios) '[\w-]+\.bin'"},
158 {'log':r
"Couldn't find rom image '[\w-]+\.bin'"},
159 {'log':r
"speed mismatch trying to attach usb device"},
160 {'log':r
"Can't create a second ISA bus"},
161 {'log':r
"duplicate fw_cfg file name"},
162 # sysbus-related error messages: most machines reject most dynamic sysbus devices:
163 {'log':r
"Option '-device [\w.,-]+' cannot be handled by this machine"},
164 {'log':r
"Device [\w.,-]+ is not supported by this machine yet"},
165 {'log':r
"Device [\w.,-]+ can not be dynamically instantiated"},
166 {'log':r
"Platform Bus: Can not fit MMIO region of size "},
167 # other more specific errors we will ignore:
168 {'device':'.*-spapr-cpu-core', 'log':r
"CPU core type should be"},
169 {'log':r
"MSI(-X)? is not supported by interrupt controller"},
170 {'log':r
"pxb-pcie? devices cannot reside on a PCIe? bus"},
171 {'log':r
"Ignoring smp_cpus value"},
172 {'log':r
"sd_init failed: Drive 'sd0' is already in use because it has been automatically connected to another device"},
173 {'log':r
"This CPU requires a smaller page size than the system is using"},
174 {'log':r
"MSI-X support is mandatory in the S390 architecture"},
175 {'log':r
"rom check and register reset failed"},
176 {'log':r
"Unable to initialize GIC, CPUState for CPU#0 not valid"},
177 {'log':r
"Multiple VT220 operator consoles are not supported"},
178 {'log':r
"core 0 already populated"},
179 {'log':r
"could not find stage1 bootloader"},
181 # other exitcode=1 failures not listed above will just generate INFO messages:
182 {'exitcode':1, 'loglevel':logging
.INFO
},
184 # everything else (including SIGABRT and SIGSEGV) will be a fatal error:
185 {'exitcode':None, 'fatal':True, 'loglevel':logging
.FATAL
},
189 def whitelistTestCaseMatch(wl
, t
):
190 """Check if a test case specification can match a whitelist entry
192 This only checks if a whitelist entry is a candidate match
193 for a given test case, it won't check if the test case
194 results/output match the entry. See whitelistResultMatch().
196 return (('machine' not in wl
or
197 'machine' not in t
or
198 re
.match(wl
['machine'] + '$', t
['machine'])) and
199 ('accel' not in wl
or
201 re
.match(wl
['accel'] + '$', t
['accel'])) and
202 ('device' not in wl
or
204 re
.match(wl
['device'] + '$', t
['device'])))
207 def whitelistCandidates(t
):
208 """Generate the list of candidates that can match a test case"""
209 for i
, wl
in enumerate(ERROR_WHITELIST
):
210 if whitelistTestCaseMatch(wl
, t
):
214 def findExpectedResult(t
):
215 """Check if there's an expected=True whitelist entry for a test case
217 Returns (i, wl) tuple, where i is the index in
218 ERROR_WHITELIST and wl is the whitelist entry itself.
220 for i
, wl
in whitelistCandidates(t
):
221 if wl
.get('expected'):
225 def whitelistResultMatch(wl
, r
):
226 """Check if test case results/output match a whitelist entry
228 It is valid to call this function only if
229 whitelistTestCaseMatch() is True for the entry (e.g. on
230 entries returned by whitelistCandidates())
232 assert whitelistTestCaseMatch(wl
, r
['testcase'])
233 return ((wl
.get('exitcode', 1) is None or
234 r
['exitcode'] == wl
.get('exitcode', 1)) and
236 re
.search(wl
['log'], r
['log'], re
.MULTILINE
)))
239 def checkResultWhitelist(r
):
240 """Look up whitelist entry for a given test case result
242 Returns (i, wl) tuple, where i is the index in
243 ERROR_WHITELIST and wl is the whitelist entry itself.
245 for i
, wl
in whitelistCandidates(r
['testcase']):
246 if whitelistResultMatch(wl
, r
):
249 raise Exception("this should never happen")
252 def qemuOptsEscape(s
):
253 """Escape option value QemuOpts"""
254 return s
.replace(",", ",,")
257 def formatTestCase(t
):
258 """Format test case info as "key=value key=value" for prettier logging output"""
259 return ' '.join('%s=%s' % (k
, v
) for k
, v
in t
.items())
262 def qomListTypeNames(vm
, **kwargs
):
263 """Run qom-list-types QMP command, return type names"""
264 types
= vm
.command('qom-list-types', **kwargs
)
265 return [t
['name'] for t
in types
]
269 """Parse 'info qdm' output"""
270 args
= {'command-line': 'info qdm'}
271 devhelp
= vm
.command('human-monitor-command', **args
)
272 for l
in devhelp
.split('\n'):
274 if l
== '' or l
.endswith(':'):
276 d
= {'name': re
.search(r
'name "([^"]+)"', l
).group(1),
277 'no-user': (re
.search(', no-user', l
) is not None)}
281 class QemuBinaryInfo(object):
282 def __init__(self
, binary
, devtype
):
287 self
._machine
_info
= {}
289 dbg("devtype: %r", devtype
)
290 args
= ['-S', '-machine', 'none,accel=kvm:tcg']
291 dbg("querying info for QEMU binary: %s", binary
)
292 vm
= QEMUMachine(binary
=binary
, args
=args
)
295 self
.alldevs
= set(qomListTypeNames(vm
, implements
=devtype
, abstract
=False))
296 # there's no way to query DeviceClass::user_creatable using QMP,
298 self
.no_user_devs
= set([d
['name'] for d
in infoQDM(vm
, ) if d
['no-user']])
299 self
.machines
= list(m
['name'] for m
in vm
.command('query-machines'))
300 self
.user_devs
= self
.alldevs
.difference(self
.no_user_devs
)
301 self
.kvm_available
= vm
.command('query-kvm')['enabled']
305 def machineInfo(self
, machine
):
306 """Query for information on a specific machine-type
308 Results are cached internally, in case the same machine-
309 type is queried multiple times.
311 if machine
in self
._machine
_info
:
312 return self
._machine
_info
[machine
]
315 args
= ['-S', '-machine', '%s' % (machine
)]
316 dbg("querying machine info for binary=%s machine=%s", self
.binary
, machine
)
317 vm
= QEMUMachine(binary
=self
.binary
, args
=args
)
320 mi
['runnable'] = True
321 except KeyboardInterrupt:
324 dbg("exception trying to run binary=%s machine=%s", self
.binary
, machine
, exc_info
=sys
.exc_info())
325 dbg("log: %r", vm
.get_log())
326 mi
['runnable'] = False
329 self
._machine
_info
[machine
] = mi
336 def getBinaryInfo(args
, binary
):
337 if binary
not in BINARY_INFO
:
338 BINARY_INFO
[binary
] = QemuBinaryInfo(binary
, args
.devtype
)
339 return BINARY_INFO
[binary
]
342 def checkOneCase(args
, testcase
):
343 """Check one specific case
345 Returns a dictionary containing failure information on error,
348 binary
= testcase
['binary']
349 accel
= testcase
['accel']
350 machine
= testcase
['machine']
351 device
= testcase
['device']
353 dbg("will test: %r", testcase
)
355 args
= ['-S', '-machine', '%s,accel=%s' % (machine
, accel
),
356 '-device', qemuOptsEscape(device
)]
357 cmdline
= ' '.join([binary
] + args
)
358 dbg("will launch QEMU: %s", cmdline
)
359 vm
= QEMUMachine(binary
=binary
, args
=args
)
364 except KeyboardInterrupt:
367 exc_traceback
= traceback
.format_exc()
368 dbg("Exception while running test case")
374 if exc_traceback
is not None or ec
!= 0:
375 return {'exc_traceback':exc_traceback
,
382 def binariesToTest(args
, testcase
):
386 r
= [f
.path
for f
in os
.scandir('.')
387 if f
.name
.startswith('qemu-system-') and
388 f
.is_file() and os
.access(f
, os
.X_OK
)]
392 def accelsToTest(args
, testcase
):
393 if getBinaryInfo(args
, testcase
['binary']).kvm_available
:
398 def machinesToTest(args
, testcase
):
399 return getBinaryInfo(args
, testcase
['binary']).machines
402 def devicesToTest(args
, testcase
):
403 return getBinaryInfo(args
, testcase
['binary']).user_devs
406 TESTCASE_VARIABLES
= [
407 ('binary', binariesToTest
),
408 ('accel', accelsToTest
),
409 ('machine', machinesToTest
),
410 ('device', devicesToTest
),
414 def genCases1(args
, testcases
, var
, fn
):
415 """Generate new testcases for one variable
417 If an existing item already has a variable set, don't
418 generate new items and just return it directly. This
419 allows the "-t" command-line option to be used to choose
420 a specific test case.
422 for testcase
in testcases
:
424 yield testcase
.copy()
426 for i
in fn(args
, testcase
):
432 def genCases(args
, testcase
):
433 """Generate test cases for all variables
435 cases
= [testcase
.copy()]
436 for var
, fn
in TESTCASE_VARIABLES
:
437 dbg("var: %r, fn: %r", var
, fn
)
438 cases
= genCases1(args
, cases
, var
, fn
)
442 def casesToTest(args
, testcase
):
443 cases
= genCases(args
, testcase
)
446 cases
= random
.sample(cases
, min(args
.random
, len(cases
)))
449 dbg("%d test cases to test", len(cases
))
452 random
.shuffle(cases
)
456 def logFailure(f
, level
):
458 logger
.log(level
, "failed: %s", formatTestCase(t
))
459 logger
.log(level
, "cmdline: %s", f
['cmdline'])
460 for l
in f
['log'].strip().split('\n'):
461 logger
.log(level
, "log: %s", l
)
462 logger
.log(level
, "exit code: %r", f
['exitcode'])
463 if f
['exc_traceback']:
464 logger
.log(level
, "exception:")
465 for l
in f
['exc_traceback'].split('\n'):
466 logger
.log(level
, " %s", l
.rstrip('\n'))
470 parser
= argparse
.ArgumentParser(description
="QEMU -device crash test")
471 parser
.add_argument('-t', metavar
='KEY=VALUE', nargs
='*',
472 help="Limit test cases to KEY=VALUE",
473 action
='append', dest
='testcases', default
=[])
474 parser
.add_argument('-d', '--debug', action
='store_true',
476 parser
.add_argument('-v', '--verbose', action
='store_true', default
=True,
477 help='verbose output')
478 parser
.add_argument('-q', '--quiet', dest
='verbose', action
='store_false',
479 help='non-verbose output')
480 parser
.add_argument('-r', '--random', type=int, metavar
='COUNT',
481 help='run a random sample of COUNT test cases',
483 parser
.add_argument('--shuffle', action
='store_true',
484 help='Run test cases in random order')
485 parser
.add_argument('--dry-run', action
='store_true',
486 help="Don't run any tests, just generate list")
487 parser
.add_argument('-D', '--devtype', metavar
='TYPE',
488 help="Test only device types that implement TYPE")
489 parser
.add_argument('-Q', '--quick', action
='store_true', default
=True,
490 help="Quick mode: skip test cases that are expected to fail")
491 parser
.add_argument('-F', '--full', action
='store_false', dest
='quick',
492 help="Full mode: test cases that are expected to fail")
493 parser
.add_argument('--strict', action
='store_true', dest
='strict',
494 help="Treat all warnings as fatal")
495 parser
.add_argument('qemu', nargs
='*', metavar
='QEMU',
496 help='QEMU binary to run')
497 args
= parser
.parse_args()
505 logging
.basicConfig(stream
=sys
.stdout
, level
=lvl
, format
='%(levelname)s: %(message)s')
513 dbg("testcases: %r", args
.testcases
)
515 for t
in chain(*args
.testcases
):
517 k
, v
= kv
.split('=', 1)
520 if len(binariesToTest(args
, tc
)) == 0:
521 print("No QEMU binary found", file=sys
.stderr
)
522 parser
.print_usage(sys
.stderr
)
525 for t
in casesToTest(args
, tc
):
526 logger
.info("running test case: %s", formatTestCase(t
))
529 expected_match
= findExpectedResult(t
)
532 not getBinaryInfo(args
, t
['binary']).machineInfo(t
['machine'])['runnable'])):
533 dbg("skipped: %s", formatTestCase(t
))
541 f
= checkOneCase(args
, t
)
542 except KeyboardInterrupt:
546 i
, wl
= checkResultWhitelist(f
)
547 dbg("testcase: %r, whitelist match: %r", t
, wl
)
548 wl_stats
.setdefault(i
, []).append(f
)
549 level
= wl
.get('loglevel', logging
.DEBUG
)
551 if wl
.get('fatal') or (args
.strict
and level
>= logging
.WARN
):
552 fatal_failures
.append(f
)
554 dbg("success: %s", formatTestCase(t
))
556 logger
.warn("Didn't fail as expected: %s", formatTestCase(t
))
558 logger
.info("Total: %d test cases", total
)
560 logger
.info("Skipped %d test cases", skipped
)
563 stats
= sorted([(len(wl_stats
.get(i
, [])), wl
) for i
, wl
in
564 enumerate(ERROR_WHITELIST
)], key
=lambda x
: x
[0])
565 for count
, wl
in stats
:
566 dbg("whitelist entry stats: %d: %r", count
, wl
)
569 for f
in fatal_failures
:
571 logger
.error("Fatal failure: %s", formatTestCase(t
))
572 logger
.error("Fatal failures on some machine/device combinations")
575 if __name__
== '__main__':