tests/vm/basevm.py

   1 #
   2 # VM testing base class
   3 #
   4 # Copyright 2017-2019 Red Hat Inc.
   5 #
   6 # Authors:
   7 #  Fam Zheng <famz@redhat.com>
   8 #  Gerd Hoffmann <kraxel@redhat.com>
   9 #
  10 # This code is licensed under the GPL version 2 or later.  See
  11 # the COPYING file in the top-level directory.
  12 #
  13
  14 import os
  15 import re
  16 import sys
  17 import socket
  18 import logging
  19 import time
  20 import datetime
  21 import subprocess
  22 import hashlib
  23 import argparse
  24 import atexit
  25 import tempfile
  26 import shutil
  27 import multiprocessing
  28 import traceback
  29 import shlex
  30 import json
  31
  32 from qemu.machine import QEMUMachine
  33 from qemu.utils import get_info_usernet_hostfwd_port, kvm_available
  34
  35 SSH_KEY_FILE = os.path.join(os.path.dirname(__file__),
  36                "..", "keys", "id_rsa")
  37 SSH_PUB_KEY_FILE = os.path.join(os.path.dirname(__file__),
  38                    "..", "keys", "id_rsa.pub")
  39
  40 # This is the standard configuration.
  41 # Any or all of these can be overridden by
  42 # passing in a config argument to the VM constructor.
  43 DEFAULT_CONFIG = {
  44     'cpu'             : "max",
  45     'machine'         : 'pc',
  46     'guest_user'      : "qemu",
  47     'guest_pass'      : "qemupass",
  48     'root_user'       : "root",
  49     'root_pass'       : "qemupass",
  50     'ssh_key_file'    : SSH_KEY_FILE,
  51     'ssh_pub_key_file': SSH_PUB_KEY_FILE,
  52     'memory'          : "4G",
  53     'extra_args'      : [],
  54     'qemu_args'       : "",
  55     'dns'             : "",
  56     'ssh_port'        : 0,
  57     'install_cmds'    : "",
  58     'boot_dev_type'   : "block",
  59     'ssh_timeout'     : 1,
  60 }
  61 BOOT_DEVICE = {
  62     'block' :  "-drive file={},if=none,id=drive0,cache=writeback "\
  63                "-device virtio-blk,drive=drive0,bootindex=0",
  64     'scsi'  :  "-device virtio-scsi-device,id=scsi "\
  65                "-drive file={},format=raw,if=none,id=hd0 "\
  66                "-device scsi-hd,drive=hd0,bootindex=0",
  67 }
  68 class BaseVM(object):
  69
  70     envvars = [
  71         "https_proxy",
  72         "http_proxy",
  73         "ftp_proxy",
  74         "no_proxy",
  75     ]
  76
  77     # The script to run in the guest that builds QEMU
  78     BUILD_SCRIPT = ""
  79     # The guest name, to be overridden by subclasses
  80     name = "#base"
  81     # The guest architecture, to be overridden by subclasses
  82     arch = "#arch"
  83     # command to halt the guest, can be overridden by subclasses
  84     poweroff = "poweroff"
  85     # Time to wait for shutdown to finish.
  86     shutdown_timeout_default = 30
  87     # enable IPv6 networking
  88     ipv6 = True
  89     # This is the timeout on the wait for console bytes.
  90     socket_timeout = 120
  91     # Scale up some timeouts under TCG.
  92     # 4 is arbitrary, but greater than 2,
  93     # since we found we need to wait more than twice as long.
  94     tcg_timeout_multiplier = 4
  95     def __init__(self, args, config=None):
  96         self._guest = None
  97         self._genisoimage = args.genisoimage
  98         self._build_path = args.build_path
  99         self._efi_aarch64 = args.efi_aarch64
 100         self._source_path = args.source_path
 101         # Allow input config to override defaults.
 102         self._config = DEFAULT_CONFIG.copy()
 103
 104         # 1GB per core, minimum of 4. This is only a default.
 105         mem = max(4, args.jobs)
 106         self._config['memory'] = f"{mem}G"
 107
 108         if config != None:
 109             self._config.update(config)
 110         self.validate_ssh_keys()
 111         self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-",
 112                                                          suffix=".tmp",
 113                                                          dir="."))
 114         atexit.register(shutil.rmtree, self._tmpdir)
 115         # Copy the key files to a temporary directory.
 116         # Also chmod the key file to agree with ssh requirements.
 117         self._config['ssh_key'] = \
 118             open(self._config['ssh_key_file']).read().rstrip()
 119         self._config['ssh_pub_key'] = \
 120             open(self._config['ssh_pub_key_file']).read().rstrip()
 121         self._ssh_tmp_key_file = os.path.join(self._tmpdir, "id_rsa")
 122         open(self._ssh_tmp_key_file, "w").write(self._config['ssh_key'])
 123         subprocess.check_call(["chmod", "600", self._ssh_tmp_key_file])
 124
 125         self._ssh_tmp_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub")
 126         open(self._ssh_tmp_pub_key_file,
 127              "w").write(self._config['ssh_pub_key'])
 128
 129         self.debug = args.debug
 130         self._console_log_path = None
 131         if args.log_console:
 132                 self._console_log_path = \
 133                          os.path.join(os.path.expanduser("~/.cache/qemu-vm"),
 134                                       "{}.install.log".format(self.name))
 135         self._stderr = sys.stderr
 136         self._devnull = open(os.devnull, "w")
 137         if self.debug:
 138             self._stdout = sys.stdout
 139         else:
 140             self._stdout = self._devnull
 141         netdev = "user,id=vnet,hostfwd=:127.0.0.1:{}-:22"
 142         self._args = [ \
 143             "-nodefaults", "-m", self._config['memory'],
 144             "-cpu", self._config['cpu'],
 145             "-netdev",
 146             netdev.format(self._config['ssh_port']) +
 147             (",ipv6=no" if not self.ipv6 else "") +
 148             (",dns=" + self._config['dns'] if self._config['dns'] else ""),
 149             "-device", "virtio-net-pci,netdev=vnet",
 150             "-vnc", "127.0.0.1:0,to=20"]
 151         if args.jobs and args.jobs > 1:
 152             self._args += ["-smp", "%d" % args.jobs]
 153         if kvm_available(self.arch):
 154             self._shutdown_timeout = self.shutdown_timeout_default
 155             self._args += ["-enable-kvm"]
 156         else:
 157             logging.info("KVM not available, not using -enable-kvm")
 158             self._shutdown_timeout = \
 159                 self.shutdown_timeout_default * self.tcg_timeout_multiplier
 160         self._data_args = []
 161
 162         if self._config['qemu_args'] != None:
 163             qemu_args = self._config['qemu_args']
 164             qemu_args = qemu_args.replace('\n',' ').replace('\r','')
 165             # shlex groups quoted arguments together
 166             # we need this to keep the quoted args together for when
 167             # the QEMU command is issued later.
 168             args = shlex.split(qemu_args)
 169             self._config['extra_args'] = []
 170             for arg in args:
 171                 if arg:
 172                     # Preserve quotes around arguments.
 173                     # shlex above takes them out, so add them in.
 174                     if " " in arg:
 175                         arg = '"{}"'.format(arg)
 176                     self._config['extra_args'].append(arg)
 177
 178     def validate_ssh_keys(self):
 179         """Check to see if the ssh key files exist."""
 180         if 'ssh_key_file' not in self._config or\
 181            not os.path.exists(self._config['ssh_key_file']):
 182             raise Exception("ssh key file not found.")
 183         if 'ssh_pub_key_file' not in self._config or\
 184            not os.path.exists(self._config['ssh_pub_key_file']):
 185                raise Exception("ssh pub key file not found.")
 186
 187     def wait_boot(self, wait_string=None):
 188         """Wait for the standard string we expect
 189            on completion of a normal boot.
 190            The user can also choose to override with an
 191            alternate string to wait for."""
 192         if wait_string is None:
 193             if self.login_prompt is None:
 194                 raise Exception("self.login_prompt not defined")
 195             wait_string = self.login_prompt
 196         # Intentionally bump up the default timeout under TCG,
 197         # since the console wait below takes longer.
 198         timeout = self.socket_timeout
 199         if not kvm_available(self.arch):
 200             timeout *= 8
 201         self.console_init(timeout=timeout)
 202         self.console_wait(wait_string)
 203
 204     def _download_with_cache(self, url, sha256sum=None, sha512sum=None):
 205         def check_sha256sum(fname):
 206             if not sha256sum:
 207                 return True
 208             checksum = subprocess.check_output(["sha256sum", fname]).split()[0]
 209             return sha256sum == checksum.decode("utf-8")
 210
 211         def check_sha512sum(fname):
 212             if not sha512sum:
 213                 return True
 214             checksum = subprocess.check_output(["sha512sum", fname]).split()[0]
 215             return sha512sum == checksum.decode("utf-8")
 216
 217         cache_dir = os.path.expanduser("~/.cache/qemu-vm/download")
 218         if not os.path.exists(cache_dir):
 219             os.makedirs(cache_dir)
 220         fname = os.path.join(cache_dir,
 221                              hashlib.sha1(url.encode("utf-8")).hexdigest())
 222         if os.path.exists(fname) and check_sha256sum(fname) and check_sha512sum(fname):
 223             return fname
 224         logging.debug("Downloading %s to %s...", url, fname)
 225         subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"],
 226                               stdout=self._stdout, stderr=self._stderr)
 227         os.rename(fname + ".download", fname)
 228         return fname
 229
 230     def _ssh_do(self, user, cmd, check):
 231         ssh_cmd = ["ssh",
 232                    "-t",
 233                    "-o", "StrictHostKeyChecking=no",
 234                    "-o", "UserKnownHostsFile=" + os.devnull,
 235                    "-o",
 236                    "ConnectTimeout={}".format(self._config["ssh_timeout"]),
 237                    "-p", str(self.ssh_port), "-i", self._ssh_tmp_key_file,
 238                    "-o", "IdentitiesOnly=yes"]
 239         # If not in debug mode, set ssh to quiet mode to
 240         # avoid printing the results of commands.
 241         if not self.debug:
 242             ssh_cmd.append("-q")
 243         for var in self.envvars:
 244             ssh_cmd += ['-o', "SendEnv=%s" % var ]
 245         assert not isinstance(cmd, str)
 246         ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd)
 247         logging.debug("ssh_cmd: %s", " ".join(ssh_cmd))
 248         r = subprocess.call(ssh_cmd)
 249         if check and r != 0:
 250             raise Exception("SSH command failed: %s" % cmd)
 251         return r
 252
 253     def ssh(self, *cmd):
 254         return self._ssh_do(self._config["guest_user"], cmd, False)
 255
 256     def ssh_root(self, *cmd):
 257         return self._ssh_do(self._config["root_user"], cmd, False)
 258
 259     def ssh_check(self, *cmd):
 260         self._ssh_do(self._config["guest_user"], cmd, True)
 261
 262     def ssh_root_check(self, *cmd):
 263         self._ssh_do(self._config["root_user"], cmd, True)
 264
 265     def build_image(self, img):
 266         raise NotImplementedError
 267
 268     def exec_qemu_img(self, *args):
 269         cmd = [os.environ.get("QEMU_IMG", "qemu-img")]
 270         cmd.extend(list(args))
 271         subprocess.check_call(cmd)
 272
 273     def add_source_dir(self, src_dir):
 274         name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5]
 275         tarfile = os.path.join(self._tmpdir, name + ".tar")
 276         logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir)
 277         subprocess.check_call(["./scripts/archive-source.sh", tarfile],
 278                               cwd=src_dir, stdin=self._devnull,
 279                               stdout=self._stdout, stderr=self._stderr)
 280         self._data_args += ["-drive",
 281                             "file=%s,if=none,id=%s,cache=writeback,format=raw" % \
 282                                     (tarfile, name),
 283                             "-device",
 284                             "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)]
 285
 286     def boot(self, img, extra_args=[]):
 287         boot_dev = BOOT_DEVICE[self._config['boot_dev_type']]
 288         boot_params = boot_dev.format(img)
 289         args = self._args + boot_params.split(' ')
 290         args += self._data_args + extra_args + self._config['extra_args']
 291         logging.debug("QEMU args: %s", " ".join(args))
 292         qemu_path = get_qemu_path(self.arch, self._build_path)
 293
 294         # Since console_log_path is only set when the user provides the
 295         # log_console option, we will set drain_console=True so the
 296         # console is always drained.
 297         guest = QEMUMachine(binary=qemu_path, args=args,
 298                             console_log=self._console_log_path,
 299                             drain_console=True)
 300         guest.set_machine(self._config['machine'])
 301         guest.set_console()
 302         try:
 303             guest.launch()
 304         except:
 305             logging.error("Failed to launch QEMU, command line:")
 306             logging.error(" ".join([qemu_path] + args))
 307             logging.error("Log:")
 308             logging.error(guest.get_log())
 309             logging.error("QEMU version >= 2.10 is required")
 310             raise
 311         atexit.register(self.shutdown)
 312         self._guest = guest
 313         # Init console so we can start consuming the chars.
 314         self.console_init()
 315         usernet_info = guest.qmp("human-monitor-command",
 316                                  command_line="info usernet").get("return")
 317         self.ssh_port = get_info_usernet_hostfwd_port(usernet_info)
 318         if not self.ssh_port:
 319             raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \
 320                             usernet_info)
 321
 322     def console_init(self, timeout = None):
 323         if timeout == None:
 324             timeout = self.socket_timeout
 325         vm = self._guest
 326         vm.console_socket.settimeout(timeout)
 327         self.console_raw_path = os.path.join(vm._temp_dir,
 328                                              vm._name + "-console.raw")
 329         self.console_raw_file = open(self.console_raw_path, 'wb')
 330
 331     def console_log(self, text):
 332         for line in re.split("[\r\n]", text):
 333             # filter out terminal escape sequences
 334             line = re.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line)
 335             line = re.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line)
 336             # replace unprintable chars
 337             line = re.sub("\x1b", "<esc>", line)
 338             line = re.sub("[\x00-\x1f]", ".", line)
 339             line = re.sub("[\x80-\xff]", ".", line)
 340             if line == "":
 341                 continue
 342             # log console line
 343             sys.stderr.write("con recv: %s\n" % line)
 344
 345     def console_wait(self, expect, expectalt = None):
 346         vm = self._guest
 347         output = ""
 348         while True:
 349             try:
 350                 chars = vm.console_socket.recv(1)
 351                 if self.console_raw_file:
 352                     self.console_raw_file.write(chars)
 353                     self.console_raw_file.flush()
 354             except socket.timeout:
 355                 sys.stderr.write("console: *** read timeout ***\n")
 356                 sys.stderr.write("console: waiting for: '%s'\n" % expect)
 357                 if not expectalt is None:
 358                     sys.stderr.write("console: waiting for: '%s' (alt)\n" % expectalt)
 359                 sys.stderr.write("console: line buffer:\n")
 360                 sys.stderr.write("\n")
 361                 self.console_log(output.rstrip())
 362                 sys.stderr.write("\n")
 363                 raise
 364             output += chars.decode("latin1")
 365             if expect in output:
 366                 break
 367             if not expectalt is None and expectalt in output:
 368                 break
 369             if "\r" in output or "\n" in output:
 370                 lines = re.split("[\r\n]", output)
 371                 output = lines.pop()
 372                 if self.debug:
 373                     self.console_log("\n".join(lines))
 374         if self.debug:
 375             self.console_log(output)
 376         if not expectalt is None and expectalt in output:
 377             return False
 378         return True
 379
 380     def console_consume(self):
 381         vm = self._guest
 382         output = ""
 383         vm.console_socket.setblocking(0)
 384         while True:
 385             try:
 386                 chars = vm.console_socket.recv(1)
 387             except:
 388                 break
 389             output += chars.decode("latin1")
 390             if "\r" in output or "\n" in output:
 391                 lines = re.split("[\r\n]", output)
 392                 output = lines.pop()
 393                 if self.debug:
 394                     self.console_log("\n".join(lines))
 395         if self.debug:
 396             self.console_log(output)
 397         vm.console_socket.setblocking(1)
 398
 399     def console_send(self, command):
 400         vm = self._guest
 401         if self.debug:
 402             logline = re.sub("\n", "<enter>", command)
 403             logline = re.sub("[\x00-\x1f]", ".", logline)
 404             sys.stderr.write("con send: %s\n" % logline)
 405         for char in list(command):
 406             vm.console_socket.send(char.encode("utf-8"))
 407             time.sleep(0.01)
 408
 409     def console_wait_send(self, wait, command):
 410         self.console_wait(wait)
 411         self.console_send(command)
 412
 413     def console_ssh_init(self, prompt, user, pw):
 414         sshkey_cmd = "echo '%s' > .ssh/authorized_keys\n" \
 415                      % self._config['ssh_pub_key'].rstrip()
 416         self.console_wait_send("login:",    "%s\n" % user)
 417         self.console_wait_send("Password:", "%s\n" % pw)
 418         self.console_wait_send(prompt,      "mkdir .ssh\n")
 419         self.console_wait_send(prompt,      sshkey_cmd)
 420         self.console_wait_send(prompt,      "chmod 755 .ssh\n")
 421         self.console_wait_send(prompt,      "chmod 644 .ssh/authorized_keys\n")
 422
 423     def console_sshd_config(self, prompt):
 424         self.console_wait(prompt)
 425         self.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n")
 426         for var in self.envvars:
 427             self.console_wait(prompt)
 428             self.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var)
 429
 430     def print_step(self, text):
 431         sys.stderr.write("### %s ...\n" % text)
 432
 433     def wait_ssh(self, wait_root=False, seconds=300, cmd="exit 0"):
 434         # Allow more time for VM to boot under TCG.
 435         if not kvm_available(self.arch):
 436             seconds *= self.tcg_timeout_multiplier
 437         starttime = datetime.datetime.now()
 438         endtime = starttime + datetime.timedelta(seconds=seconds)
 439         cmd_success = False
 440         while datetime.datetime.now() < endtime:
 441             if wait_root and self.ssh_root(cmd) == 0:
 442                 cmd_success = True
 443                 break
 444             elif self.ssh(cmd) == 0:
 445                 cmd_success = True
 446                 break
 447             seconds = (endtime - datetime.datetime.now()).total_seconds()
 448             logging.debug("%ds before timeout", seconds)
 449             time.sleep(1)
 450         if not cmd_success:
 451             raise Exception("Timeout while waiting for guest ssh")
 452
 453     def shutdown(self):
 454         self._guest.shutdown(timeout=self._shutdown_timeout)
 455
 456     def wait(self):
 457         self._guest.wait(timeout=self._shutdown_timeout)
 458
 459     def graceful_shutdown(self):
 460         self.ssh_root(self.poweroff)
 461         self._guest.wait(timeout=self._shutdown_timeout)
 462
 463     def qmp(self, *args, **kwargs):
 464         return self._guest.qmp(*args, **kwargs)
 465
 466     def gen_cloud_init_iso(self):
 467         cidir = self._tmpdir
 468         mdata = open(os.path.join(cidir, "meta-data"), "w")
 469         name = self.name.replace(".","-")
 470         mdata.writelines(["instance-id: {}-vm-0\n".format(name),
 471                           "local-hostname: {}-guest\n".format(name)])
 472         mdata.close()
 473         udata = open(os.path.join(cidir, "user-data"), "w")
 474         print("guest user:pw {}:{}".format(self._config['guest_user'],
 475                                            self._config['guest_pass']))
 476         udata.writelines(["#cloud-config\n",
 477                           "chpasswd:\n",
 478                           "  list: |\n",
 479                           "    root:%s\n" % self._config['root_pass'],
 480                           "    %s:%s\n" % (self._config['guest_user'],
 481                                            self._config['guest_pass']),
 482                           "  expire: False\n",
 483                           "users:\n",
 484                           "  - name: %s\n" % self._config['guest_user'],
 485                           "    sudo: ALL=(ALL) NOPASSWD:ALL\n",
 486                           "    ssh-authorized-keys:\n",
 487                           "    - %s\n" % self._config['ssh_pub_key'],
 488                           "  - name: root\n",
 489                           "    ssh-authorized-keys:\n",
 490                           "    - %s\n" % self._config['ssh_pub_key'],
 491                           "locale: en_US.UTF-8\n"])
 492         proxy = os.environ.get("http_proxy")
 493         if not proxy is None:
 494             udata.writelines(["apt:\n",
 495                               "  proxy: %s" % proxy])
 496         udata.close()
 497         subprocess.check_call([self._genisoimage, "-output", "cloud-init.iso",
 498                                "-volid", "cidata", "-joliet", "-rock",
 499                                "user-data", "meta-data"],
 500                               cwd=cidir,
 501                               stdin=self._devnull, stdout=self._stdout,
 502                               stderr=self._stdout)
 503         return os.path.join(cidir, "cloud-init.iso")
 504
 505     def get_qemu_packages_from_lcitool_json(self, json_path=None):
 506         """Parse a lcitool variables json file and return the PKGS list."""
 507         if json_path is None:
 508             json_path = os.path.join(
 509                 os.path.dirname(__file__), "generated", self.name + ".json"
 510             )
 511         with open(json_path, "r") as fh:
 512             return json.load(fh)["pkgs"]
 513
 514
 515 def get_qemu_path(arch, build_path=None):
 516     """Fetch the path to the qemu binary."""
 517     # If QEMU environment variable set, it takes precedence
 518     if "QEMU" in os.environ:
 519         qemu_path = os.environ["QEMU"]
 520     elif build_path:
 521         qemu_path = os.path.join(build_path, arch + "-softmmu")
 522         qemu_path = os.path.join(qemu_path, "qemu-system-" + arch)
 523     else:
 524         # Default is to use system path for qemu.
 525         qemu_path = "qemu-system-" + arch
 526     return qemu_path
 527
 528 def get_qemu_version(qemu_path):
 529     """Get the version number from the current QEMU,
 530        and return the major number."""
 531     output = subprocess.check_output([qemu_path, '--version'])
 532     version_line = output.decode("utf-8")
 533     version_num = re.split(' |\(', version_line)[3].split('.')[0]
 534     return int(version_num)
 535
 536 def parse_config(config, args):
 537     """ Parse yaml config and populate our config structure.
 538         The yaml config allows the user to override the
 539         defaults for VM parameters.  In many cases these
 540         defaults can be overridden without rebuilding the VM."""
 541     if args.config:
 542         config_file = args.config
 543     elif 'QEMU_CONFIG' in os.environ:
 544         config_file = os.environ['QEMU_CONFIG']
 545     else:
 546         return config
 547     if not os.path.exists(config_file):
 548         raise Exception("config file {} does not exist".format(config_file))
 549     # We gracefully handle importing the yaml module
 550     # since it might not be installed.
 551     # If we are here it means the user supplied a .yml file,
 552     # so if the yaml module is not installed we will exit with error.
 553     try:
 554         import yaml
 555     except ImportError:
 556         print("The python3-yaml package is needed "\
 557               "to support config.yaml files")
 558         # Instead of raising an exception we exit to avoid
 559         # a raft of messy (expected) errors to stdout.
 560         exit(1)
 561     with open(config_file) as f:
 562         yaml_dict = yaml.safe_load(f)
 563
 564     if 'qemu-conf' in yaml_dict:
 565         config.update(yaml_dict['qemu-conf'])
 566     else:
 567         raise Exception("config file {} is not valid"\
 568                         " missing qemu-conf".format(config_file))
 569     return config
 570
 571 def parse_args(vmcls):
 572
 573     def get_default_jobs():
 574         if multiprocessing.cpu_count() > 1:
 575             if kvm_available(vmcls.arch):
 576                 return multiprocessing.cpu_count() // 2
 577             elif os.uname().machine == "x86_64" and \
 578                  vmcls.arch in ["aarch64", "x86_64", "i386"]:
 579                 # MTTCG is available on these arches and we can allow
 580                 # more cores. but only up to a reasonable limit. User
 581                 # can always override these limits with --jobs.
 582                 return min(multiprocessing.cpu_count() // 2, 8)
 583         return 1
 584
 585     parser = argparse.ArgumentParser(
 586         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
 587         description="Utility for provisioning VMs and running builds",
 588         epilog="""Remaining arguments are passed to the command.
 589         Exit codes: 0 = success, 1 = command line error,
 590         2 = environment initialization failed,
 591         3 = test command failed""")
 592     parser.add_argument("--debug", "-D", action="store_true",
 593                         help="enable debug output")
 594     parser.add_argument("--image", "-i", default="%s.img" % vmcls.name,
 595                         help="image file name")
 596     parser.add_argument("--force", "-f", action="store_true",
 597                         help="force build image even if image exists")
 598     parser.add_argument("--jobs", type=int, default=get_default_jobs(),
 599                         help="number of virtual CPUs")
 600     parser.add_argument("--verbose", "-V", action="store_true",
 601                         help="Pass V=1 to builds within the guest")
 602     parser.add_argument("--build-image", "-b", action="store_true",
 603                         help="build image")
 604     parser.add_argument("--build-qemu",
 605                         help="build QEMU from source in guest")
 606     parser.add_argument("--build-target",
 607                         help="QEMU build target", default="check")
 608     parser.add_argument("--build-path", default=None,
 609                         help="Path of build directory, "\
 610                         "for using build tree QEMU binary. ")
 611     parser.add_argument("--source-path", default=None,
 612                         help="Path of source directory, "\
 613                         "for finding additional files. ")
 614     parser.add_argument("--interactive", "-I", action="store_true",
 615                         help="Interactively run command")
 616     parser.add_argument("--snapshot", "-s", action="store_true",
 617                         help="run tests with a snapshot")
 618     parser.add_argument("--genisoimage", default="genisoimage",
 619                         help="iso imaging tool")
 620     parser.add_argument("--config", "-c", default=None,
 621                         help="Provide config yaml for configuration. "\
 622                         "See config_example.yaml for example.")
 623     parser.add_argument("--efi-aarch64",
 624                         default="/usr/share/qemu-efi-aarch64/QEMU_EFI.fd",
 625                         help="Path to efi image for aarch64 VMs.")
 626     parser.add_argument("--log-console", action="store_true",
 627                         help="Log console to file.")
 628     parser.add_argument("commands", nargs="*", help="""Remaining
 629         commands after -- are passed to command inside the VM""")
 630
 631     return parser.parse_args()
 632
 633 def main(vmcls, config=None):
 634     try:
 635         if config == None:
 636             config = DEFAULT_CONFIG
 637         args = parse_args(vmcls)
 638         if not args.commands and not args.build_qemu and not args.build_image:
 639             print("Nothing to do?")
 640             return 1
 641         config = parse_config(config, args)
 642         logging.basicConfig(level=(logging.DEBUG if args.debug
 643                                    else logging.WARN))
 644         vm = vmcls(args, config=config)
 645         if args.build_image:
 646             if os.path.exists(args.image) and not args.force:
 647                 sys.stderr.writelines(["Image file exists: %s\n" % args.image,
 648                                       "Use --force option to overwrite\n"])
 649                 return 1
 650             return vm.build_image(args.image)
 651         if args.build_qemu:
 652             vm.add_source_dir(args.build_qemu)
 653             cmd = [vm.BUILD_SCRIPT.format(
 654                    configure_opts = " ".join(args.commands),
 655                    jobs=int(args.jobs),
 656                    target=args.build_target,
 657                    verbose = "V=1" if args.verbose else "")]
 658         else:
 659             cmd = args.commands
 660         img = args.image
 661         if args.snapshot:
 662             img += ",snapshot=on"
 663         vm.boot(img)
 664         vm.wait_ssh()
 665     except Exception as e:
 666         if isinstance(e, SystemExit) and e.code == 0:
 667             return 0
 668         sys.stderr.write("Failed to prepare guest environment\n")
 669         traceback.print_exc()
 670         return 2
 671
 672     exitcode = 0
 673     if vm.ssh(*cmd) != 0:
 674         exitcode = 3
 675     if args.interactive:
 676         vm.ssh()
 677
 678     if not args.snapshot:
 679         vm.graceful_shutdown()
 680
 681     return exitcode