scripts/device-crash-test

   1 #!/usr/bin/env python3
   2 #
   3 #  Copyright (c) 2017 Red Hat Inc
   4 #
   5 # Author:
   6 #  Eduardo Habkost <ehabkost@redhat.com>
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License along
  19 # with this program; if not, write to the Free Software Foundation, Inc.,
  20 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  21
  22 """
  23 Run QEMU with all combinations of -machine and -device types,
  24 check for crashes and unexpected errors.
  25 """
  26
  27 import os
  28 import sys
  29 import glob
  30 import logging
  31 import traceback
  32 import re
  33 import random
  34 import argparse
  35 from itertools import chain
  36 from pathlib import Path
  37
  38 try:
  39     from qemu.machine import QEMUMachine
  40     from qemu.qmp import ConnectError
  41 except ModuleNotFoundError as exc:
  42     path = Path(__file__).resolve()
  43     print(f"Module '{exc.name}' not found.")
  44     print("  Try 'make check-venv' from your build directory,")
  45     print("  and then one way to run this script is like so:")
  46     print(f'  > $builddir/pyvenv/bin/python3 "{path}"')
  47     sys.exit(1)
  48
  49 logger = logging.getLogger('device-crash-test')
  50 dbg = logger.debug
  51
  52
  53 # Purposes of the following rule list:
  54 # * Avoiding verbose log messages when we find known non-fatal
  55 #   (exitcode=1) errors
  56 # * Avoiding fatal errors when we find known crashes
  57 # * Skipping machines/devices that are known not to work out of
  58 #   the box, when running in --quick mode
  59 #
  60 # Keeping the rule list updated is desirable, but not required,
  61 # because unexpected cases where QEMU exits with exitcode=1 will
  62 # just trigger a INFO message.
  63
  64 # Valid error rule keys:
  65 # * accel: regexp, full match only
  66 # * machine: regexp, full match only
  67 # * device: regexp, full match only
  68 # * log: regexp, partial match allowed
  69 # * exitcode: if not present, defaults to 1. If None, matches any exitcode
  70 # * warn: if True, matching failures will be logged as warnings
  71 # * expected: if True, QEMU is expected to always fail every time
  72 #   when testing the corresponding test case
  73 # * loglevel: log level of log output when there's a match.
  74 ERROR_RULE_LIST = [
  75     # Machines that won't work out of the box:
  76     #             MACHINE                         | ERROR MESSAGE
  77     {'machine':'niagara', 'expected':True},       # Unable to load a firmware for -M niagara
  78     {'machine':'boston', 'expected':True},        # Please provide either a -kernel or -bios argument
  79     {'machine':'leon3_generic', 'expected':True}, # Can't read bios image (null)
  80
  81     # devices that don't work out of the box because they require extra options to "-device DEV":
  82     #            DEVICE                                    | ERROR MESSAGE
  83     {'device':'.*-(i386|x86_64)-cpu', 'expected':True},    # CPU socket-id is not set
  84     {'device':'icp', 'expected':True},                     # icp_realize: required link 'xics' not found: Property '.xics' not found
  85     {'device':'ics', 'expected':True},                     # ics_base_realize: required link 'xics' not found: Property '.xics' not found
  86     # "-device ide-cd" does work on more recent QEMU versions, so it doesn't have expected=True
  87     {'device':'ide-cd'},                                 # No drive specified
  88     {'device':'ide-hd', 'expected':True},                  # No drive specified
  89     {'device':'ipmi-bmc-extern', 'expected':True},         # IPMI external bmc requires chardev attribute
  90     {'device':'isa-debugcon', 'expected':True},            # Can't create serial device, empty char device
  91     {'device':'isa-ipmi-bt', 'expected':True},             # IPMI device requires a bmc attribute to be set
  92     {'device':'isa-ipmi-kcs', 'expected':True},            # IPMI device requires a bmc attribute to be set
  93     {'device':'isa-parallel', 'expected':True},            # Can't create serial device, empty char device
  94     {'device':'ivshmem-doorbell', 'expected':True},        # You must specify a 'chardev'
  95     {'device':'ivshmem-plain', 'expected':True},           # You must specify a 'memdev'
  96     {'device':'loader', 'expected':True},                  # please include valid arguments
  97     {'device':'nand', 'expected':True},                    # Unsupported NAND block size 0x1
  98     {'device':'nvdimm', 'expected':True},                  # 'memdev' property is not set
  99     {'device':'nvme', 'expected':True},                    # Device initialization failed
 100     {'device':'pc-dimm', 'expected':True},                 # 'memdev' property is not set
 101     {'device':'pci-bridge', 'expected':True},              # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
 102     {'device':'pci-bridge-seat', 'expected':True},         # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
 103     {'device':'pxb', 'expected':True},                     # Bridge chassis not specified. Each bridge is required to be assigned a unique chassis id > 0.
 104     {'device':'pxb-cxl', 'expected':True},                 # pxb-cxl devices cannot reside on a PCI bus.
 105     {'device':'scsi-block', 'expected':True},              # drive property not set
 106     {'device':'scsi-generic', 'expected':True},            # drive property not set
 107     {'device':'scsi-hd', 'expected':True},                 # drive property not set
 108     {'device':'spapr-pci-host-bridge', 'expected':True},   # BUID not specified for PHB
 109     {'device':'spapr-rng', 'expected':True},               # spapr-rng needs an RNG backend!
 110     {'device':'spapr-vty', 'expected':True},               # chardev property not set
 111     {'device':'tpm-tis', 'expected':True},                 # tpm_tis: backend driver with id (null) could not be found
 112     {'device':'unimplemented-device', 'expected':True},    # property 'size' not specified or zero
 113     {'device':'usb-braille', 'expected':True},             # Property chardev is required
 114     {'device':'usb-mtp', 'expected':True},                 # rootdir property must be configured
 115     {'device':'usb-redir', 'expected':True},               # Parameter 'chardev' is missing
 116     {'device':'usb-serial', 'expected':True},              # Property chardev is required
 117     {'device':'usb-storage', 'expected':True},             # drive property not set
 118     {'device':'vfio-amd-xgbe', 'expected':True},           # -device vfio-amd-xgbe: vfio error: wrong host device name
 119     {'device':'vfio-calxeda-xgmac', 'expected':True},      # -device vfio-calxeda-xgmac: vfio error: wrong host device name
 120     {'device':'vfio-pci', 'expected':True},                # No provided host device
 121     {'device':'vfio-pci-igd-lpc-bridge', 'expected':True}, # VFIO dummy ISA/LPC bridge must have address 1f.0
 122     {'device':'vhost-scsi.*', 'expected':True},            # vhost-scsi: missing wwpn
 123     {'device':'vhost-vsock-device', 'expected':True},      # guest-cid property must be greater than 2
 124     {'device':'vhost-vsock-pci', 'expected':True},         # guest-cid property must be greater than 2
 125     {'device':'virtio-9p-ccw', 'expected':True},           # 9pfs device couldn't find fsdev with the id = NULL
 126     {'device':'virtio-9p-device', 'expected':True},        # 9pfs device couldn't find fsdev with the id = NULL
 127     {'device':'virtio-9p-pci', 'expected':True},           # 9pfs device couldn't find fsdev with the id = NULL
 128     {'device':'virtio-blk-ccw', 'expected':True},          # drive property not set
 129     {'device':'virtio-blk-device', 'expected':True},       # drive property not set
 130     {'device':'virtio-blk-device', 'expected':True},       # drive property not set
 131     {'device':'virtio-blk-pci', 'expected':True},          # drive property not set
 132     {'device':'virtio-crypto-ccw', 'expected':True},       # 'cryptodev' parameter expects a valid object
 133     {'device':'virtio-crypto-device', 'expected':True},    # 'cryptodev' parameter expects a valid object
 134     {'device':'virtio-crypto-pci', 'expected':True},       # 'cryptodev' parameter expects a valid object
 135     {'device':'virtio-input-host-device', 'expected':True}, # evdev property is required
 136     {'device':'virtio-input-host-pci', 'expected':True},   # evdev property is required
 137     {'device':'xen-pvdevice', 'expected':True},            # Device ID invalid, it must always be supplied
 138     {'device':'vhost-vsock-ccw', 'expected':True},         # guest-cid property must be greater than 2
 139     {'device':'zpci', 'expected':True},                    # target must be defined
 140     {'device':'pnv-(occ|icp|lpc)', 'expected':True},       # required link 'xics' not found: Property '.xics' not found
 141     {'device':'powernv-cpu-.*', 'expected':True},          # pnv_core_realize: required link 'xics' not found: Property '.xics' not found
 142
 143     # ioapic devices are already created by pc and will fail:
 144     {'machine':'q35|pc.*', 'device':'kvm-ioapic', 'expected':True}, # Only 1 ioapics allowed
 145     {'machine':'q35|pc.*', 'device':'ioapic', 'expected':True},     # Only 1 ioapics allowed
 146
 147     # "spapr-cpu-core needs a pseries machine"
 148     {'machine':'(?!pseries).*', 'device':'.*-spapr-cpu-core', 'expected':True},
 149
 150     # KVM-specific devices shouldn't be tried without accel=kvm:
 151     {'accel':'(?!kvm).*', 'device':'kvmclock', 'expected':True},
 152
 153     # xen-specific machines and devices:
 154     {'accel':'(?!xen).*', 'machine':'xen.*', 'expected':True},
 155     {'accel':'(?!xen).*', 'device':'xen-.*', 'expected':True},
 156
 157     # this fails on some machine-types, but not all, so they don't have expected=True:
 158     {'device':'vmgenid'}, # vmgenid requires DMA write support in fw_cfg, which this machine type does not provide
 159
 160     # Silence INFO messages for errors that are common on multiple
 161     # devices/machines:
 162     {'log':r"No '[\w-]+' bus found for device '[\w-]+'"},
 163     {'log':r"images* must be given with the 'pflash' parameter"},
 164     {'log':r"(Guest|ROM|Flash|Kernel) image must be specified"},
 165     {'log':r"[cC]ould not load [\w ]+ (BIOS|bios) '[\w-]+\.bin'"},
 166     {'log':r"Couldn't find rom image '[\w-]+\.bin'"},
 167     {'log':r"speed mismatch trying to attach usb device"},
 168     {'log':r"Can't create a second ISA bus"},
 169     {'log':r"duplicate fw_cfg file name"},
 170     # sysbus-related error messages: most machines reject most dynamic sysbus devices:
 171     {'log':r"Option '-device [\w.,-]+' cannot be handled by this machine"},
 172     {'log':r"Device [\w.,-]+ is not supported by this machine yet"},
 173     {'log':r"Device [\w.,-]+ can not be dynamically instantiated"},
 174     {'log':r"Platform Bus: Can not fit MMIO region of size "},
 175     # other more specific errors we will ignore:
 176     {'device':'.*-spapr-cpu-core', 'log':r"CPU core type should be"},
 177     {'log':r"MSI(-X)? is not supported by interrupt controller"},
 178     {'log':r"pxb-pcie? devices cannot reside on a PCIe? bus"},
 179     {'log':r"Ignoring smp_cpus value"},
 180     {'log':r"sd_init failed: Drive 'sd0' is already in use because it has been automatically connected to another device"},
 181     {'log':r"This CPU requires a smaller page size than the system is using"},
 182     {'log':r"MSI-X support is mandatory in the S390 architecture"},
 183     {'log':r"rom check and register reset failed"},
 184     {'log':r"Unable to initialize GIC, CPUState for CPU#0 not valid"},
 185     {'log':r"Multiple VT220 operator consoles are not supported"},
 186     {'log':r"core 0 already populated"},
 187     {'log':r"could not find stage1 bootloader"},
 188
 189     # other exitcode=1 failures not listed above will just generate INFO messages:
 190     {'exitcode':1, 'loglevel':logging.INFO},
 191
 192     # everything else (including SIGABRT and SIGSEGV) will be a fatal error:
 193     {'exitcode':None, 'fatal':True, 'loglevel':logging.FATAL},
 194 ]
 195
 196
 197 def errorRuleTestCaseMatch(rule, t):
 198     """Check if a test case specification can match a error rule
 199
 200     This only checks if a error rule is a candidate match
 201     for a given test case, it won't check if the test case
 202     results/output match the rule.  See ruleListResultMatch().
 203     """
 204     return (('machine' not in rule or
 205              'machine' not in t or
 206              re.match(rule['machine'] + '$', t['machine'])) and
 207             ('accel' not in rule or
 208              'accel' not in t or
 209              re.match(rule['accel'] + '$', t['accel'])) and
 210             ('device' not in rule or
 211              'device' not in t or
 212              re.match(rule['device'] + '$', t['device'])))
 213
 214
 215 def ruleListCandidates(t):
 216     """Generate the list of candidates that can match a test case"""
 217     for i, rule in enumerate(ERROR_RULE_LIST):
 218         if errorRuleTestCaseMatch(rule, t):
 219             yield (i, rule)
 220
 221
 222 def findExpectedResult(t):
 223     """Check if there's an expected=True error rule for a test case
 224
 225     Returns (i, rule) tuple, where i is the index in
 226     ERROR_RULE_LIST and rule is the error rule itself.
 227     """
 228     for i, rule in ruleListCandidates(t):
 229         if rule.get('expected'):
 230             return (i, rule)
 231
 232
 233 def ruleListResultMatch(rule, r):
 234     """Check if test case results/output match a error rule
 235
 236     It is valid to call this function only if
 237     errorRuleTestCaseMatch() is True for the rule (e.g. on
 238     rules returned by ruleListCandidates())
 239     """
 240     assert errorRuleTestCaseMatch(rule, r['testcase'])
 241     return ((rule.get('exitcode', 1) is None or
 242              r['exitcode'] == rule.get('exitcode', 1)) and
 243             ('log' not in rule or
 244              re.search(rule['log'], r['log'], re.MULTILINE)))
 245
 246
 247 def checkResultRuleList(r):
 248     """Look up error rule for a given test case result
 249
 250     Returns (i, rule) tuple, where i is the index in
 251     ERROR_RULE_LIST and rule is the error rule itself.
 252     """
 253     for i, rule in ruleListCandidates(r['testcase']):
 254         if ruleListResultMatch(rule, r):
 255             return i, rule
 256
 257     raise Exception("this should never happen")
 258
 259
 260 def qemuOptsEscape(s):
 261     """Escape option value QemuOpts"""
 262     return s.replace(",", ",,")
 263
 264
 265 def formatTestCase(t):
 266     """Format test case info as "key=value key=value" for prettier logging output"""
 267     return ' '.join('%s=%s' % (k, v) for k, v in t.items())
 268
 269
 270 def qomListTypeNames(vm, **kwargs):
 271     """Run qom-list-types QMP command, return type names"""
 272     types = vm.command('qom-list-types', **kwargs)
 273     return [t['name'] for t in types]
 274
 275
 276 def infoQDM(vm):
 277     """Parse 'info qdm' output"""
 278     args = {'command-line': 'info qdm'}
 279     devhelp = vm.command('human-monitor-command', **args)
 280     for l in devhelp.split('\n'):
 281         l = l.strip()
 282         if l == '' or l.endswith(':'):
 283             continue
 284         d = {'name': re.search(r'name "([^"]+)"', l).group(1),
 285              'no-user': (re.search(', no-user', l) is not None)}
 286         yield d
 287
 288
 289 class QemuBinaryInfo(object):
 290     def __init__(self, binary, devtype):
 291         if devtype is None:
 292             devtype = 'device'
 293
 294         self.binary = binary
 295         self._machine_info = {}
 296
 297         dbg("devtype: %r", devtype)
 298         args = ['-S', '-machine', 'none,accel=kvm:tcg']
 299         dbg("querying info for QEMU binary: %s", binary)
 300         vm = QEMUMachine(binary=binary, args=args)
 301         vm.launch()
 302         try:
 303             self.alldevs = set(qomListTypeNames(vm, implements=devtype, abstract=False))
 304             # there's no way to query DeviceClass::user_creatable using QMP,
 305             # so use 'info qdm':
 306             self.no_user_devs = set([d['name'] for d in infoQDM(vm, ) if d['no-user']])
 307             self.machines = list(m['name'] for m in vm.command('query-machines'))
 308             self.user_devs = self.alldevs.difference(self.no_user_devs)
 309             self.kvm_available = vm.command('query-kvm')['enabled']
 310         finally:
 311             vm.shutdown()
 312
 313     def machineInfo(self, machine):
 314         """Query for information on a specific machine-type
 315
 316         Results are cached internally, in case the same machine-
 317         type is queried multiple times.
 318         """
 319         if machine in self._machine_info:
 320             return self._machine_info[machine]
 321
 322         mi = {}
 323         args = ['-S', '-machine', '%s' % (machine)]
 324         dbg("querying machine info for binary=%s machine=%s", self.binary, machine)
 325         vm = QEMUMachine(binary=self.binary, args=args)
 326         try:
 327             vm.launch()
 328             mi['runnable'] = True
 329         except Exception:
 330             dbg("exception trying to run binary=%s machine=%s", self.binary, machine, exc_info=sys.exc_info())
 331             dbg("log: %r", vm.get_log())
 332             mi['runnable'] = False
 333
 334         vm.shutdown()
 335         self._machine_info[machine] = mi
 336         return mi
 337
 338
 339 BINARY_INFO = {}
 340
 341
 342 def getBinaryInfo(args, binary):
 343     if binary not in BINARY_INFO:
 344         BINARY_INFO[binary] = QemuBinaryInfo(binary, args.devtype)
 345     return BINARY_INFO[binary]
 346
 347
 348 def checkOneCase(args, testcase):
 349     """Check one specific case
 350
 351     Returns a dictionary containing failure information on error,
 352     or None on success
 353     """
 354     binary = testcase['binary']
 355     accel = testcase['accel']
 356     machine = testcase['machine']
 357     device = testcase['device']
 358
 359     dbg("will test: %r", testcase)
 360
 361     args = ['-S', '-machine', '%s,accel=%s' % (machine, accel),
 362             '-device', qemuOptsEscape(device)]
 363     cmdline = ' '.join([binary] + args)
 364     dbg("will launch QEMU: %s", cmdline)
 365     vm = QEMUMachine(binary=binary, args=args, qmp_timer=15)
 366
 367     exc = None
 368     exc_traceback = None
 369     try:
 370         vm.launch()
 371     except Exception as this_exc:
 372         exc = this_exc
 373         exc_traceback = traceback.format_exc()
 374         dbg("Exception while running test case")
 375     finally:
 376         vm.shutdown()
 377         ec = vm.exitcode()
 378         log = vm.get_log()
 379
 380     if exc is not None or ec != 0:
 381         return {'exc': exc,
 382                 'exc_traceback':exc_traceback,
 383                 'exitcode':ec,
 384                 'log':log,
 385                 'testcase':testcase,
 386                 'cmdline':cmdline}
 387
 388
 389 def binariesToTest(args, testcase):
 390     if args.qemu:
 391         r = args.qemu
 392     else:
 393         r = [f.path for f in os.scandir('.')
 394              if f.name.startswith('qemu-system-') and
 395                 f.is_file() and os.access(f, os.X_OK)]
 396     return r
 397
 398
 399 def accelsToTest(args, testcase):
 400     if getBinaryInfo(args, testcase['binary']).kvm_available and not args.tcg_only:
 401         yield 'kvm'
 402     yield 'tcg'
 403
 404
 405 def machinesToTest(args, testcase):
 406     return getBinaryInfo(args, testcase['binary']).machines
 407
 408
 409 def devicesToTest(args, testcase):
 410     return getBinaryInfo(args, testcase['binary']).user_devs
 411
 412
 413 TESTCASE_VARIABLES = [
 414     ('binary', binariesToTest),
 415     ('accel', accelsToTest),
 416     ('machine', machinesToTest),
 417     ('device', devicesToTest),
 418 ]
 419
 420
 421 def genCases1(args, testcases, var, fn):
 422     """Generate new testcases for one variable
 423
 424     If an existing item already has a variable set, don't
 425     generate new items and just return it directly. This
 426     allows the "-t" command-line option to be used to choose
 427     a specific test case.
 428     """
 429     for testcase in testcases:
 430         if var in testcase:
 431             yield testcase.copy()
 432         else:
 433             for i in fn(args, testcase):
 434                 t = testcase.copy()
 435                 t[var] = i
 436                 yield t
 437
 438
 439 def genCases(args, testcase):
 440     """Generate test cases for all variables
 441     """
 442     cases = [testcase.copy()]
 443     for var, fn in TESTCASE_VARIABLES:
 444         dbg("var: %r, fn: %r", var, fn)
 445         cases = genCases1(args, cases, var, fn)
 446     return cases
 447
 448
 449 def casesToTest(args, testcase):
 450     cases = genCases(args, testcase)
 451     if args.random:
 452         cases = list(cases)
 453         cases = random.sample(cases, min(args.random, len(cases)))
 454     if args.debug:
 455         cases = list(cases)
 456         dbg("%d test cases to test", len(cases))
 457     if args.shuffle:
 458         cases = list(cases)
 459         random.shuffle(cases)
 460     return cases
 461
 462
 463 def logFailure(f, level):
 464     t = f['testcase']
 465     logger.log(level, "failed: %s", formatTestCase(t))
 466     logger.log(level, "cmdline: %s", f['cmdline'])
 467     for l in f['log'].strip().split('\n'):
 468         logger.log(level, "log: %s", l)
 469     logger.log(level, "exit code: %r", f['exitcode'])
 470
 471     # If the Exception is merely a QMP connect error,
 472     # reduce the logging level for its traceback to
 473     # improve visual clarity.
 474     if isinstance(f.get('exc'), ConnectError):
 475         logger.log(level, "%s.%s: %s",
 476                    type(f['exc']).__module__,
 477                    type(f['exc']).__qualname__,
 478                    str(f['exc']))
 479         level = logging.DEBUG
 480
 481     if f['exc_traceback']:
 482         logger.log(level, "exception:")
 483         for l in f['exc_traceback'].split('\n'):
 484             logger.log(level, "  %s", l.rstrip('\n'))
 485
 486
 487 def main():
 488     parser = argparse.ArgumentParser(description="QEMU -device crash test")
 489     parser.add_argument('-t', metavar='KEY=VALUE', nargs='*',
 490                         help="Limit test cases to KEY=VALUE",
 491                         action='append', dest='testcases', default=[])
 492     parser.add_argument('-d', '--debug', action='store_true',
 493                         help='debug output')
 494     parser.add_argument('-v', '--verbose', action='store_true', default=True,
 495                         help='verbose output')
 496     parser.add_argument('-q', '--quiet', dest='verbose', action='store_false',
 497                         help='non-verbose output')
 498     parser.add_argument('-r', '--random', type=int, metavar='COUNT',
 499                         help='run a random sample of COUNT test cases',
 500                         default=0)
 501     parser.add_argument('--shuffle', action='store_true',
 502                         help='Run test cases in random order')
 503     parser.add_argument('--dry-run', action='store_true',
 504                         help="Don't run any tests, just generate list")
 505     parser.add_argument('-D', '--devtype', metavar='TYPE',
 506                         help="Test only device types that implement TYPE")
 507     parser.add_argument('-Q', '--quick', action='store_true', default=True,
 508                         help="Quick mode: skip test cases that are expected to fail")
 509     parser.add_argument('-F', '--full', action='store_false', dest='quick',
 510                         help="Full mode: test cases that are expected to fail")
 511     parser.add_argument('--strict', action='store_true', dest='strict',
 512                         help="Treat all warnings as fatal")
 513     parser.add_argument('--tcg-only', action='store_true', dest='tcg_only',
 514                         help="Only test with TCG accelerator")
 515     parser.add_argument('qemu', nargs='*', metavar='QEMU',
 516                         help='QEMU binary to run')
 517     args = parser.parse_args()
 518
 519     if args.debug:
 520         lvl = logging.DEBUG
 521     elif args.verbose:
 522         lvl = logging.INFO
 523     else:
 524         lvl = logging.WARN
 525     logging.basicConfig(stream=sys.stdout, level=lvl, format='%(levelname)s: %(message)s')
 526
 527     if not args.debug:
 528         # Async QMP, when in use, is chatty about connection failures.
 529         # This script knowingly generates a ton of connection errors.
 530         # Silence this logger.
 531         logging.getLogger('qemu.qmp.qmp_client').setLevel(logging.CRITICAL)
 532
 533     fatal_failures = []
 534     wl_stats = {}
 535     skipped = 0
 536     total = 0
 537
 538     tc = {}
 539     dbg("testcases: %r", args.testcases)
 540     if args.testcases:
 541         for t in chain(*args.testcases):
 542             for kv in t.split():
 543                 k, v = kv.split('=', 1)
 544                 tc[k] = v
 545
 546     if len(binariesToTest(args, tc)) == 0:
 547         print("No QEMU binary found", file=sys.stderr)
 548         parser.print_usage(sys.stderr)
 549         return 1
 550
 551     for t in casesToTest(args, tc):
 552         logger.info("running test case: %s", formatTestCase(t))
 553         total += 1
 554
 555         expected_match = findExpectedResult(t)
 556         if (args.quick and
 557                 (expected_match or
 558                  not getBinaryInfo(args, t['binary']).machineInfo(t['machine'])['runnable'])):
 559             dbg("skipped: %s", formatTestCase(t))
 560             skipped += 1
 561             continue
 562
 563         if args.dry_run:
 564             continue
 565
 566         try:
 567             f = checkOneCase(args, t)
 568         except KeyboardInterrupt:
 569             break
 570
 571         if f:
 572             i, rule = checkResultRuleList(f)
 573             dbg("testcase: %r, rule list match: %r", t, rule)
 574             wl_stats.setdefault(i, []).append(f)
 575             level = rule.get('loglevel', logging.DEBUG)
 576             logFailure(f, level)
 577             if rule.get('fatal') or (args.strict and level >= logging.WARN):
 578                 fatal_failures.append(f)
 579         else:
 580             dbg("success: %s", formatTestCase(t))
 581             if expected_match:
 582                 logger.warn("Didn't fail as expected: %s", formatTestCase(t))
 583
 584     logger.info("Total: %d test cases", total)
 585     if skipped:
 586         logger.info("Skipped %d test cases", skipped)
 587
 588     if args.debug:
 589         stats = sorted([(len(wl_stats.get(i, [])), rule) for i, rule in
 590                          enumerate(ERROR_RULE_LIST)], key=lambda x: x[0])
 591         for count, rule in stats:
 592             dbg("error rule stats: %d: %r", count, rule)
 593
 594     if fatal_failures:
 595         for f in fatal_failures:
 596             t = f['testcase']
 597             logger.error("Fatal failure: %s", formatTestCase(t))
 598         logger.error("Fatal failures on some machine/device combinations")
 599         return 1
 600
 601 if __name__ == '__main__':
 602     sys.exit(main())