]>
Commit | Line | Data |
---|---|---|
ff2ebff0 FZ |
1 | # |
2 | # VM testing base class | |
3 | # | |
8dd38334 | 4 | # Copyright 2017-2019 Red Hat Inc. |
ff2ebff0 FZ |
5 | # |
6 | # Authors: | |
7 | # Fam Zheng <famz@redhat.com> | |
8dd38334 | 8 | # Gerd Hoffmann <kraxel@redhat.com> |
ff2ebff0 FZ |
9 | # |
10 | # This code is licensed under the GPL version 2 or later. See | |
11 | # the COPYING file in the top-level directory. | |
12 | # | |
13 | ||
14 | import os | |
8dd38334 | 15 | import re |
ff2ebff0 | 16 | import sys |
8dd38334 | 17 | import socket |
ff2ebff0 FZ |
18 | import logging |
19 | import time | |
20 | import datetime | |
ff2ebff0 FZ |
21 | import subprocess |
22 | import hashlib | |
2fea3a12 | 23 | import argparse |
ff2ebff0 FZ |
24 | import atexit |
25 | import tempfile | |
26 | import shutil | |
27 | import multiprocessing | |
28 | import traceback | |
5d676197 RF |
29 | import shlex |
30 | ||
f4c66f17 JS |
31 | from qemu.machine import QEMUMachine |
32 | from qemu.utils import get_info_usernet_hostfwd_port, kvm_available | |
33 | ||
5d676197 RF |
34 | SSH_KEY_FILE = os.path.join(os.path.dirname(__file__), |
35 | "..", "keys", "id_rsa") | |
36 | SSH_PUB_KEY_FILE = os.path.join(os.path.dirname(__file__), | |
37 | "..", "keys", "id_rsa.pub") | |
38 | ||
39 | # This is the standard configuration. | |
40 | # Any or all of these can be overridden by | |
41 | # passing in a config argument to the VM constructor. | |
42 | DEFAULT_CONFIG = { | |
43 | 'cpu' : "max", | |
44 | 'machine' : 'pc', | |
45 | 'guest_user' : "qemu", | |
46 | 'guest_pass' : "qemupass", | |
9fc33bf4 | 47 | 'root_user' : "root", |
5d676197 RF |
48 | 'root_pass' : "qemupass", |
49 | 'ssh_key_file' : SSH_KEY_FILE, | |
50 | 'ssh_pub_key_file': SSH_PUB_KEY_FILE, | |
51 | 'memory' : "4G", | |
52 | 'extra_args' : [], | |
53 | 'qemu_args' : "", | |
54 | 'dns' : "", | |
55 | 'ssh_port' : 0, | |
56 | 'install_cmds' : "", | |
57 | 'boot_dev_type' : "block", | |
58 | 'ssh_timeout' : 1, | |
59 | } | |
60 | BOOT_DEVICE = { | |
61 | 'block' : "-drive file={},if=none,id=drive0,cache=writeback "\ | |
62 | "-device virtio-blk,drive=drive0,bootindex=0", | |
63 | 'scsi' : "-device virtio-scsi-device,id=scsi "\ | |
64 | "-drive file={},format=raw,if=none,id=hd0 "\ | |
65 | "-device scsi-hd,drive=hd0,bootindex=0", | |
66 | } | |
ff2ebff0 | 67 | class BaseVM(object): |
ff2ebff0 | 68 | |
b08ba163 GH |
69 | envvars = [ |
70 | "https_proxy", | |
71 | "http_proxy", | |
72 | "ftp_proxy", | |
73 | "no_proxy", | |
74 | ] | |
75 | ||
ff2ebff0 FZ |
76 | # The script to run in the guest that builds QEMU |
77 | BUILD_SCRIPT = "" | |
78 | # The guest name, to be overridden by subclasses | |
79 | name = "#base" | |
31719c37 PMD |
80 | # The guest architecture, to be overridden by subclasses |
81 | arch = "#arch" | |
b3f94b2f GH |
82 | # command to halt the guest, can be overridden by subclasses |
83 | poweroff = "poweroff" | |
4a70232b RF |
84 | # Time to wait for shutdown to finish. |
85 | shutdown_timeout_default = 30 | |
5b790481 EH |
86 | # enable IPv6 networking |
87 | ipv6 = True | |
5d676197 RF |
88 | # This is the timeout on the wait for console bytes. |
89 | socket_timeout = 120 | |
c9de3935 RF |
90 | # Scale up some timeouts under TCG. |
91 | # 4 is arbitrary, but greater than 2, | |
92 | # since we found we need to wait more than twice as long. | |
4a70232b | 93 | tcg_timeout_multiplier = 4 |
5d676197 | 94 | def __init__(self, args, config=None): |
ff2ebff0 | 95 | self._guest = None |
1f335d18 RF |
96 | self._genisoimage = args.genisoimage |
97 | self._build_path = args.build_path | |
13336606 | 98 | self._efi_aarch64 = args.efi_aarch64 |
7bb17a92 | 99 | self._source_path = args.source_path |
5d676197 RF |
100 | # Allow input config to override defaults. |
101 | self._config = DEFAULT_CONFIG.copy() | |
eaf46a65 JS |
102 | |
103 | # 1GB per core, minimum of 4. This is only a default. | |
104 | mem = max(4, args.jobs) | |
105 | self._config['memory'] = f"{mem}G" | |
106 | ||
5d676197 RF |
107 | if config != None: |
108 | self._config.update(config) | |
109 | self.validate_ssh_keys() | |
ff2ebff0 FZ |
110 | self._tmpdir = os.path.realpath(tempfile.mkdtemp(prefix="vm-test-", |
111 | suffix=".tmp", | |
112 | dir=".")) | |
113 | atexit.register(shutil.rmtree, self._tmpdir) | |
5d676197 RF |
114 | # Copy the key files to a temporary directory. |
115 | # Also chmod the key file to agree with ssh requirements. | |
116 | self._config['ssh_key'] = \ | |
117 | open(self._config['ssh_key_file']).read().rstrip() | |
118 | self._config['ssh_pub_key'] = \ | |
119 | open(self._config['ssh_pub_key_file']).read().rstrip() | |
120 | self._ssh_tmp_key_file = os.path.join(self._tmpdir, "id_rsa") | |
121 | open(self._ssh_tmp_key_file, "w").write(self._config['ssh_key']) | |
122 | subprocess.check_call(["chmod", "600", self._ssh_tmp_key_file]) | |
123 | ||
124 | self._ssh_tmp_pub_key_file = os.path.join(self._tmpdir, "id_rsa.pub") | |
125 | open(self._ssh_tmp_pub_key_file, | |
126 | "w").write(self._config['ssh_pub_key']) | |
ff2ebff0 | 127 | |
1f335d18 | 128 | self.debug = args.debug |
ff14ab0c RF |
129 | self._console_log_path = None |
130 | if args.log_console: | |
131 | self._console_log_path = \ | |
132 | os.path.join(os.path.expanduser("~/.cache/qemu-vm"), | |
133 | "{}.install.log".format(self.name)) | |
ff2ebff0 FZ |
134 | self._stderr = sys.stderr |
135 | self._devnull = open(os.devnull, "w") | |
136 | if self.debug: | |
137 | self._stdout = sys.stdout | |
138 | else: | |
139 | self._stdout = self._devnull | |
5d676197 | 140 | netdev = "user,id=vnet,hostfwd=:127.0.0.1:{}-:22" |
ff2ebff0 | 141 | self._args = [ \ |
5d676197 RF |
142 | "-nodefaults", "-m", self._config['memory'], |
143 | "-cpu", self._config['cpu'], | |
144 | "-netdev", | |
145 | netdev.format(self._config['ssh_port']) + | |
146 | (",ipv6=no" if not self.ipv6 else "") + | |
147 | (",dns=" + self._config['dns'] if self._config['dns'] else ""), | |
ff2ebff0 | 148 | "-device", "virtio-net-pci,netdev=vnet", |
8dd38334 | 149 | "-vnc", "127.0.0.1:0,to=20"] |
1f335d18 RF |
150 | if args.jobs and args.jobs > 1: |
151 | self._args += ["-smp", "%d" % args.jobs] | |
71531bb5 | 152 | if kvm_available(self.arch): |
4a70232b | 153 | self._shutdown_timeout = self.shutdown_timeout_default |
ff2ebff0 FZ |
154 | self._args += ["-enable-kvm"] |
155 | else: | |
156 | logging.info("KVM not available, not using -enable-kvm") | |
4a70232b RF |
157 | self._shutdown_timeout = \ |
158 | self.shutdown_timeout_default * self.tcg_timeout_multiplier | |
ff2ebff0 FZ |
159 | self._data_args = [] |
160 | ||
5d676197 RF |
161 | if self._config['qemu_args'] != None: |
162 | qemu_args = self._config['qemu_args'] | |
163 | qemu_args = qemu_args.replace('\n',' ').replace('\r','') | |
164 | # shlex groups quoted arguments together | |
165 | # we need this to keep the quoted args together for when | |
166 | # the QEMU command is issued later. | |
167 | args = shlex.split(qemu_args) | |
168 | self._config['extra_args'] = [] | |
169 | for arg in args: | |
170 | if arg: | |
171 | # Preserve quotes around arguments. | |
172 | # shlex above takes them out, so add them in. | |
173 | if " " in arg: | |
174 | arg = '"{}"'.format(arg) | |
175 | self._config['extra_args'].append(arg) | |
176 | ||
177 | def validate_ssh_keys(self): | |
178 | """Check to see if the ssh key files exist.""" | |
179 | if 'ssh_key_file' not in self._config or\ | |
180 | not os.path.exists(self._config['ssh_key_file']): | |
181 | raise Exception("ssh key file not found.") | |
182 | if 'ssh_pub_key_file' not in self._config or\ | |
183 | not os.path.exists(self._config['ssh_pub_key_file']): | |
184 | raise Exception("ssh pub key file not found.") | |
185 | ||
186 | def wait_boot(self, wait_string=None): | |
187 | """Wait for the standard string we expect | |
188 | on completion of a normal boot. | |
189 | The user can also choose to override with an | |
190 | alternate string to wait for.""" | |
191 | if wait_string is None: | |
192 | if self.login_prompt is None: | |
193 | raise Exception("self.login_prompt not defined") | |
194 | wait_string = self.login_prompt | |
195 | # Intentionally bump up the default timeout under TCG, | |
196 | # since the console wait below takes longer. | |
197 | timeout = self.socket_timeout | |
198 | if not kvm_available(self.arch): | |
199 | timeout *= 8 | |
200 | self.console_init(timeout=timeout) | |
201 | self.console_wait(wait_string) | |
202 | ||
5b4b4865 | 203 | def _download_with_cache(self, url, sha256sum=None, sha512sum=None): |
ff2ebff0 FZ |
204 | def check_sha256sum(fname): |
205 | if not sha256sum: | |
206 | return True | |
207 | checksum = subprocess.check_output(["sha256sum", fname]).split()[0] | |
3ace9be6 | 208 | return sha256sum == checksum.decode("utf-8") |
ff2ebff0 | 209 | |
5b4b4865 AB |
210 | def check_sha512sum(fname): |
211 | if not sha512sum: | |
212 | return True | |
213 | checksum = subprocess.check_output(["sha512sum", fname]).split()[0] | |
214 | return sha512sum == checksum.decode("utf-8") | |
215 | ||
ff2ebff0 FZ |
216 | cache_dir = os.path.expanduser("~/.cache/qemu-vm/download") |
217 | if not os.path.exists(cache_dir): | |
218 | os.makedirs(cache_dir) | |
3ace9be6 GH |
219 | fname = os.path.join(cache_dir, |
220 | hashlib.sha1(url.encode("utf-8")).hexdigest()) | |
5b4b4865 | 221 | if os.path.exists(fname) and check_sha256sum(fname) and check_sha512sum(fname): |
ff2ebff0 FZ |
222 | return fname |
223 | logging.debug("Downloading %s to %s...", url, fname) | |
224 | subprocess.check_call(["wget", "-c", url, "-O", fname + ".download"], | |
225 | stdout=self._stdout, stderr=self._stderr) | |
226 | os.rename(fname + ".download", fname) | |
227 | return fname | |
228 | ||
796471e9 | 229 | def _ssh_do(self, user, cmd, check): |
89adc5b9 RF |
230 | ssh_cmd = ["ssh", |
231 | "-t", | |
ff2ebff0 FZ |
232 | "-o", "StrictHostKeyChecking=no", |
233 | "-o", "UserKnownHostsFile=" + os.devnull, | |
5d676197 RF |
234 | "-o", |
235 | "ConnectTimeout={}".format(self._config["ssh_timeout"]), | |
339bf0c0 IL |
236 | "-p", str(self.ssh_port), "-i", self._ssh_tmp_key_file, |
237 | "-o", "IdentitiesOnly=yes"] | |
89adc5b9 RF |
238 | # If not in debug mode, set ssh to quiet mode to |
239 | # avoid printing the results of commands. | |
240 | if not self.debug: | |
241 | ssh_cmd.append("-q") | |
b08ba163 GH |
242 | for var in self.envvars: |
243 | ssh_cmd += ['-o', "SendEnv=%s" % var ] | |
ff2ebff0 FZ |
244 | assert not isinstance(cmd, str) |
245 | ssh_cmd += ["%s@127.0.0.1" % user] + list(cmd) | |
246 | logging.debug("ssh_cmd: %s", " ".join(ssh_cmd)) | |
726c9a3b | 247 | r = subprocess.call(ssh_cmd) |
ff2ebff0 FZ |
248 | if check and r != 0: |
249 | raise Exception("SSH command failed: %s" % cmd) | |
250 | return r | |
251 | ||
252 | def ssh(self, *cmd): | |
df001680 | 253 | return self._ssh_do(self._config["guest_user"], cmd, False) |
ff2ebff0 | 254 | |
ff2ebff0 | 255 | def ssh_root(self, *cmd): |
9fc33bf4 | 256 | return self._ssh_do(self._config["root_user"], cmd, False) |
ff2ebff0 FZ |
257 | |
258 | def ssh_check(self, *cmd): | |
df001680 | 259 | self._ssh_do(self._config["guest_user"], cmd, True) |
ff2ebff0 FZ |
260 | |
261 | def ssh_root_check(self, *cmd): | |
9fc33bf4 | 262 | self._ssh_do(self._config["root_user"], cmd, True) |
ff2ebff0 FZ |
263 | |
264 | def build_image(self, img): | |
265 | raise NotImplementedError | |
266 | ||
1e48931c WSM |
267 | def exec_qemu_img(self, *args): |
268 | cmd = [os.environ.get("QEMU_IMG", "qemu-img")] | |
269 | cmd.extend(list(args)) | |
270 | subprocess.check_call(cmd) | |
271 | ||
ff2ebff0 | 272 | def add_source_dir(self, src_dir): |
3ace9be6 | 273 | name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5] |
ff2ebff0 FZ |
274 | tarfile = os.path.join(self._tmpdir, name + ".tar") |
275 | logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir) | |
276 | subprocess.check_call(["./scripts/archive-source.sh", tarfile], | |
277 | cwd=src_dir, stdin=self._devnull, | |
278 | stdout=self._stdout, stderr=self._stderr) | |
279 | self._data_args += ["-drive", | |
280 | "file=%s,if=none,id=%s,cache=writeback,format=raw" % \ | |
281 | (tarfile, name), | |
282 | "-device", | |
283 | "virtio-blk,drive=%s,serial=%s,bootindex=1" % (name, name)] | |
284 | ||
285 | def boot(self, img, extra_args=[]): | |
5d676197 RF |
286 | boot_dev = BOOT_DEVICE[self._config['boot_dev_type']] |
287 | boot_params = boot_dev.format(img) | |
288 | args = self._args + boot_params.split(' ') | |
289 | args += self._data_args + extra_args + self._config['extra_args'] | |
ff2ebff0 | 290 | logging.debug("QEMU args: %s", " ".join(args)) |
e56c4504 | 291 | qemu_path = get_qemu_path(self.arch, self._build_path) |
ff14ab0c RF |
292 | |
293 | # Since console_log_path is only set when the user provides the | |
294 | # log_console option, we will set drain_console=True so the | |
295 | # console is always drained. | |
296 | guest = QEMUMachine(binary=qemu_path, args=args, | |
297 | console_log=self._console_log_path, | |
298 | drain_console=True) | |
5d676197 | 299 | guest.set_machine(self._config['machine']) |
8dd38334 | 300 | guest.set_console() |
ff2ebff0 FZ |
301 | try: |
302 | guest.launch() | |
303 | except: | |
304 | logging.error("Failed to launch QEMU, command line:") | |
e56c4504 | 305 | logging.error(" ".join([qemu_path] + args)) |
ff2ebff0 FZ |
306 | logging.error("Log:") |
307 | logging.error(guest.get_log()) | |
308 | logging.error("QEMU version >= 2.10 is required") | |
309 | raise | |
310 | atexit.register(self.shutdown) | |
311 | self._guest = guest | |
ff14ab0c RF |
312 | # Init console so we can start consuming the chars. |
313 | self.console_init() | |
ff2ebff0 | 314 | usernet_info = guest.qmp("human-monitor-command", |
976218cb CR |
315 | command_line="info usernet").get("return") |
316 | self.ssh_port = get_info_usernet_hostfwd_port(usernet_info) | |
ff2ebff0 FZ |
317 | if not self.ssh_port: |
318 | raise Exception("Cannot find ssh port from 'info usernet':\n%s" % \ | |
319 | usernet_info) | |
320 | ||
ff14ab0c RF |
321 | def console_init(self, timeout = None): |
322 | if timeout == None: | |
323 | timeout = self.socket_timeout | |
8dd38334 GH |
324 | vm = self._guest |
325 | vm.console_socket.settimeout(timeout) | |
698a64f9 GH |
326 | self.console_raw_path = os.path.join(vm._temp_dir, |
327 | vm._name + "-console.raw") | |
328 | self.console_raw_file = open(self.console_raw_path, 'wb') | |
8dd38334 GH |
329 | |
330 | def console_log(self, text): | |
331 | for line in re.split("[\r\n]", text): | |
332 | # filter out terminal escape sequences | |
333 | line = re.sub("\x1b\[[0-9;?]*[a-zA-Z]", "", line) | |
334 | line = re.sub("\x1b\([0-9;?]*[a-zA-Z]", "", line) | |
335 | # replace unprintable chars | |
336 | line = re.sub("\x1b", "<esc>", line) | |
337 | line = re.sub("[\x00-\x1f]", ".", line) | |
338 | line = re.sub("[\x80-\xff]", ".", line) | |
339 | if line == "": | |
340 | continue | |
341 | # log console line | |
342 | sys.stderr.write("con recv: %s\n" % line) | |
343 | ||
60136e06 | 344 | def console_wait(self, expect, expectalt = None): |
8dd38334 GH |
345 | vm = self._guest |
346 | output = "" | |
347 | while True: | |
348 | try: | |
349 | chars = vm.console_socket.recv(1) | |
698a64f9 GH |
350 | if self.console_raw_file: |
351 | self.console_raw_file.write(chars) | |
352 | self.console_raw_file.flush() | |
8dd38334 GH |
353 | except socket.timeout: |
354 | sys.stderr.write("console: *** read timeout ***\n") | |
355 | sys.stderr.write("console: waiting for: '%s'\n" % expect) | |
60136e06 GH |
356 | if not expectalt is None: |
357 | sys.stderr.write("console: waiting for: '%s' (alt)\n" % expectalt) | |
8dd38334 GH |
358 | sys.stderr.write("console: line buffer:\n") |
359 | sys.stderr.write("\n") | |
360 | self.console_log(output.rstrip()) | |
361 | sys.stderr.write("\n") | |
362 | raise | |
363 | output += chars.decode("latin1") | |
364 | if expect in output: | |
365 | break | |
60136e06 GH |
366 | if not expectalt is None and expectalt in output: |
367 | break | |
8dd38334 GH |
368 | if "\r" in output or "\n" in output: |
369 | lines = re.split("[\r\n]", output) | |
370 | output = lines.pop() | |
371 | if self.debug: | |
372 | self.console_log("\n".join(lines)) | |
373 | if self.debug: | |
374 | self.console_log(output) | |
60136e06 GH |
375 | if not expectalt is None and expectalt in output: |
376 | return False | |
377 | return True | |
8dd38334 | 378 | |
6c4f0416 GH |
379 | def console_consume(self): |
380 | vm = self._guest | |
381 | output = "" | |
382 | vm.console_socket.setblocking(0) | |
383 | while True: | |
384 | try: | |
385 | chars = vm.console_socket.recv(1) | |
386 | except: | |
387 | break | |
388 | output += chars.decode("latin1") | |
389 | if "\r" in output or "\n" in output: | |
390 | lines = re.split("[\r\n]", output) | |
391 | output = lines.pop() | |
392 | if self.debug: | |
393 | self.console_log("\n".join(lines)) | |
394 | if self.debug: | |
395 | self.console_log(output) | |
396 | vm.console_socket.setblocking(1) | |
397 | ||
8dd38334 GH |
398 | def console_send(self, command): |
399 | vm = self._guest | |
400 | if self.debug: | |
401 | logline = re.sub("\n", "<enter>", command) | |
402 | logline = re.sub("[\x00-\x1f]", ".", logline) | |
403 | sys.stderr.write("con send: %s\n" % logline) | |
404 | for char in list(command): | |
405 | vm.console_socket.send(char.encode("utf-8")) | |
406 | time.sleep(0.01) | |
407 | ||
408 | def console_wait_send(self, wait, command): | |
409 | self.console_wait(wait) | |
410 | self.console_send(command) | |
411 | ||
412 | def console_ssh_init(self, prompt, user, pw): | |
5d676197 RF |
413 | sshkey_cmd = "echo '%s' > .ssh/authorized_keys\n" \ |
414 | % self._config['ssh_pub_key'].rstrip() | |
8dd38334 GH |
415 | self.console_wait_send("login:", "%s\n" % user) |
416 | self.console_wait_send("Password:", "%s\n" % pw) | |
417 | self.console_wait_send(prompt, "mkdir .ssh\n") | |
418 | self.console_wait_send(prompt, sshkey_cmd) | |
419 | self.console_wait_send(prompt, "chmod 755 .ssh\n") | |
420 | self.console_wait_send(prompt, "chmod 644 .ssh/authorized_keys\n") | |
421 | ||
422 | def console_sshd_config(self, prompt): | |
423 | self.console_wait(prompt) | |
424 | self.console_send("echo 'PermitRootLogin yes' >> /etc/ssh/sshd_config\n") | |
425 | for var in self.envvars: | |
426 | self.console_wait(prompt) | |
427 | self.console_send("echo 'AcceptEnv %s' >> /etc/ssh/sshd_config\n" % var) | |
428 | ||
429 | def print_step(self, text): | |
430 | sys.stderr.write("### %s ...\n" % text) | |
431 | ||
6ee982c9 | 432 | def wait_ssh(self, wait_root=False, seconds=300, cmd="exit 0"): |
c9de3935 RF |
433 | # Allow more time for VM to boot under TCG. |
434 | if not kvm_available(self.arch): | |
4a70232b | 435 | seconds *= self.tcg_timeout_multiplier |
ff2ebff0 | 436 | starttime = datetime.datetime.now() |
f5d3d218 | 437 | endtime = starttime + datetime.timedelta(seconds=seconds) |
6ee982c9 | 438 | cmd_success = False |
f5d3d218 | 439 | while datetime.datetime.now() < endtime: |
6ee982c9 RF |
440 | if wait_root and self.ssh_root(cmd) == 0: |
441 | cmd_success = True | |
fbb3aa29 | 442 | break |
6ee982c9 RF |
443 | elif self.ssh(cmd) == 0: |
444 | cmd_success = True | |
ff2ebff0 | 445 | break |
f5d3d218 PMD |
446 | seconds = (endtime - datetime.datetime.now()).total_seconds() |
447 | logging.debug("%ds before timeout", seconds) | |
ff2ebff0 | 448 | time.sleep(1) |
6ee982c9 | 449 | if not cmd_success: |
ff2ebff0 FZ |
450 | raise Exception("Timeout while waiting for guest ssh") |
451 | ||
452 | def shutdown(self): | |
4a70232b | 453 | self._guest.shutdown(timeout=self._shutdown_timeout) |
ff2ebff0 FZ |
454 | |
455 | def wait(self): | |
4a70232b | 456 | self._guest.wait(timeout=self._shutdown_timeout) |
ff2ebff0 | 457 | |
b3f94b2f GH |
458 | def graceful_shutdown(self): |
459 | self.ssh_root(self.poweroff) | |
4a70232b | 460 | self._guest.wait(timeout=self._shutdown_timeout) |
b3f94b2f | 461 | |
ff2ebff0 FZ |
462 | def qmp(self, *args, **kwargs): |
463 | return self._guest.qmp(*args, **kwargs) | |
464 | ||
b081986c RF |
465 | def gen_cloud_init_iso(self): |
466 | cidir = self._tmpdir | |
467 | mdata = open(os.path.join(cidir, "meta-data"), "w") | |
468 | name = self.name.replace(".","-") | |
469 | mdata.writelines(["instance-id: {}-vm-0\n".format(name), | |
470 | "local-hostname: {}-guest\n".format(name)]) | |
471 | mdata.close() | |
472 | udata = open(os.path.join(cidir, "user-data"), "w") | |
5d676197 RF |
473 | print("guest user:pw {}:{}".format(self._config['guest_user'], |
474 | self._config['guest_pass'])) | |
b081986c RF |
475 | udata.writelines(["#cloud-config\n", |
476 | "chpasswd:\n", | |
477 | " list: |\n", | |
5d676197 RF |
478 | " root:%s\n" % self._config['root_pass'], |
479 | " %s:%s\n" % (self._config['guest_user'], | |
480 | self._config['guest_pass']), | |
b081986c RF |
481 | " expire: False\n", |
482 | "users:\n", | |
5d676197 | 483 | " - name: %s\n" % self._config['guest_user'], |
b081986c RF |
484 | " sudo: ALL=(ALL) NOPASSWD:ALL\n", |
485 | " ssh-authorized-keys:\n", | |
5d676197 | 486 | " - %s\n" % self._config['ssh_pub_key'], |
b081986c RF |
487 | " - name: root\n", |
488 | " ssh-authorized-keys:\n", | |
5d676197 | 489 | " - %s\n" % self._config['ssh_pub_key'], |
b081986c RF |
490 | "locale: en_US.UTF-8\n"]) |
491 | proxy = os.environ.get("http_proxy") | |
492 | if not proxy is None: | |
493 | udata.writelines(["apt:\n", | |
494 | " proxy: %s" % proxy]) | |
495 | udata.close() | |
92fecad3 | 496 | subprocess.check_call([self._genisoimage, "-output", "cloud-init.iso", |
b081986c RF |
497 | "-volid", "cidata", "-joliet", "-rock", |
498 | "user-data", "meta-data"], | |
92fecad3 AB |
499 | cwd=cidir, |
500 | stdin=self._devnull, stdout=self._stdout, | |
501 | stderr=self._stdout) | |
b081986c RF |
502 | return os.path.join(cidir, "cloud-init.iso") |
503 | ||
e56c4504 RF |
504 | def get_qemu_path(arch, build_path=None): |
505 | """Fetch the path to the qemu binary.""" | |
506 | # If QEMU environment variable set, it takes precedence | |
507 | if "QEMU" in os.environ: | |
508 | qemu_path = os.environ["QEMU"] | |
509 | elif build_path: | |
510 | qemu_path = os.path.join(build_path, arch + "-softmmu") | |
511 | qemu_path = os.path.join(qemu_path, "qemu-system-" + arch) | |
512 | else: | |
513 | # Default is to use system path for qemu. | |
514 | qemu_path = "qemu-system-" + arch | |
515 | return qemu_path | |
516 | ||
13336606 RF |
517 | def get_qemu_version(qemu_path): |
518 | """Get the version number from the current QEMU, | |
519 | and return the major number.""" | |
520 | output = subprocess.check_output([qemu_path, '--version']) | |
521 | version_line = output.decode("utf-8") | |
522 | version_num = re.split(' |\(', version_line)[3].split('.')[0] | |
523 | return int(version_num) | |
524 | ||
3f1e8137 RF |
525 | def parse_config(config, args): |
526 | """ Parse yaml config and populate our config structure. | |
527 | The yaml config allows the user to override the | |
528 | defaults for VM parameters. In many cases these | |
529 | defaults can be overridden without rebuilding the VM.""" | |
530 | if args.config: | |
531 | config_file = args.config | |
532 | elif 'QEMU_CONFIG' in os.environ: | |
533 | config_file = os.environ['QEMU_CONFIG'] | |
534 | else: | |
535 | return config | |
536 | if not os.path.exists(config_file): | |
537 | raise Exception("config file {} does not exist".format(config_file)) | |
538 | # We gracefully handle importing the yaml module | |
539 | # since it might not be installed. | |
540 | # If we are here it means the user supplied a .yml file, | |
541 | # so if the yaml module is not installed we will exit with error. | |
542 | try: | |
543 | import yaml | |
544 | except ImportError: | |
545 | print("The python3-yaml package is needed "\ | |
546 | "to support config.yaml files") | |
547 | # Instead of raising an exception we exit to avoid | |
548 | # a raft of messy (expected) errors to stdout. | |
549 | exit(1) | |
550 | with open(config_file) as f: | |
551 | yaml_dict = yaml.safe_load(f) | |
552 | ||
553 | if 'qemu-conf' in yaml_dict: | |
554 | config.update(yaml_dict['qemu-conf']) | |
555 | else: | |
556 | raise Exception("config file {} is not valid"\ | |
557 | " missing qemu-conf".format(config_file)) | |
558 | return config | |
559 | ||
63a24c5e | 560 | def parse_args(vmcls): |
8a6e007e PMD |
561 | |
562 | def get_default_jobs(): | |
b0953944 AB |
563 | if multiprocessing.cpu_count() > 1: |
564 | if kvm_available(vmcls.arch): | |
565 | return multiprocessing.cpu_count() // 2 | |
566 | elif os.uname().machine == "x86_64" and \ | |
567 | vmcls.arch in ["aarch64", "x86_64", "i386"]: | |
568 | # MTTCG is available on these arches and we can allow | |
569 | # more cores. but only up to a reasonable limit. User | |
570 | # can always override these limits with --jobs. | |
571 | return min(multiprocessing.cpu_count() // 2, 8) | |
44c8a6ab | 572 | return 1 |
8a6e007e | 573 | |
2fea3a12 AB |
574 | parser = argparse.ArgumentParser( |
575 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, | |
576 | description="Utility for provisioning VMs and running builds", | |
577 | epilog="""Remaining arguments are passed to the command. | |
578 | Exit codes: 0 = success, 1 = command line error, | |
579 | 2 = environment initialization failed, | |
580 | 3 = test command failed""") | |
581 | parser.add_argument("--debug", "-D", action="store_true", | |
582 | help="enable debug output") | |
583 | parser.add_argument("--image", "-i", default="%s.img" % vmcls.name, | |
584 | help="image file name") | |
585 | parser.add_argument("--force", "-f", action="store_true", | |
586 | help="force build image even if image exists") | |
587 | parser.add_argument("--jobs", type=int, default=get_default_jobs(), | |
588 | help="number of virtual CPUs") | |
589 | parser.add_argument("--verbose", "-V", action="store_true", | |
590 | help="Pass V=1 to builds within the guest") | |
591 | parser.add_argument("--build-image", "-b", action="store_true", | |
592 | help="build image") | |
593 | parser.add_argument("--build-qemu", | |
594 | help="build QEMU from source in guest") | |
595 | parser.add_argument("--build-target", | |
596 | help="QEMU build target", default="check") | |
597 | parser.add_argument("--build-path", default=None, | |
598 | help="Path of build directory, "\ | |
599 | "for using build tree QEMU binary. ") | |
7bb17a92 AB |
600 | parser.add_argument("--source-path", default=None, |
601 | help="Path of source directory, "\ | |
602 | "for finding additional files. ") | |
2fea3a12 AB |
603 | parser.add_argument("--interactive", "-I", action="store_true", |
604 | help="Interactively run command") | |
605 | parser.add_argument("--snapshot", "-s", action="store_true", | |
606 | help="run tests with a snapshot") | |
607 | parser.add_argument("--genisoimage", default="genisoimage", | |
608 | help="iso imaging tool") | |
609 | parser.add_argument("--config", "-c", default=None, | |
610 | help="Provide config yaml for configuration. "\ | |
611 | "See config_example.yaml for example.") | |
612 | parser.add_argument("--efi-aarch64", | |
613 | default="/usr/share/qemu-efi-aarch64/QEMU_EFI.fd", | |
614 | help="Path to efi image for aarch64 VMs.") | |
615 | parser.add_argument("--log-console", action="store_true", | |
616 | help="Log console to file.") | |
617 | parser.add_argument("commands", nargs="*", help="""Remaining | |
618 | commands after -- are passed to command inside the VM""") | |
619 | ||
ff2ebff0 FZ |
620 | return parser.parse_args() |
621 | ||
5d676197 | 622 | def main(vmcls, config=None): |
ff2ebff0 | 623 | try: |
5d676197 RF |
624 | if config == None: |
625 | config = DEFAULT_CONFIG | |
2fea3a12 AB |
626 | args = parse_args(vmcls) |
627 | if not args.commands and not args.build_qemu and not args.build_image: | |
f03868bd | 628 | print("Nothing to do?") |
ff2ebff0 | 629 | return 1 |
3f1e8137 | 630 | config = parse_config(config, args) |
fb3b4e6d EH |
631 | logging.basicConfig(level=(logging.DEBUG if args.debug |
632 | else logging.WARN)) | |
5d676197 | 633 | vm = vmcls(args, config=config) |
ff2ebff0 FZ |
634 | if args.build_image: |
635 | if os.path.exists(args.image) and not args.force: | |
636 | sys.stderr.writelines(["Image file exists: %s\n" % args.image, | |
637 | "Use --force option to overwrite\n"]) | |
638 | return 1 | |
639 | return vm.build_image(args.image) | |
640 | if args.build_qemu: | |
641 | vm.add_source_dir(args.build_qemu) | |
642 | cmd = [vm.BUILD_SCRIPT.format( | |
2fea3a12 | 643 | configure_opts = " ".join(args.commands), |
3ace9be6 | 644 | jobs=int(args.jobs), |
5c2ec9b6 | 645 | target=args.build_target, |
41e3340a | 646 | verbose = "V=1" if args.verbose else "")] |
ff2ebff0 | 647 | else: |
2fea3a12 | 648 | cmd = args.commands |
983c2a77 FZ |
649 | img = args.image |
650 | if args.snapshot: | |
651 | img += ",snapshot=on" | |
652 | vm.boot(img) | |
ff2ebff0 FZ |
653 | vm.wait_ssh() |
654 | except Exception as e: | |
655 | if isinstance(e, SystemExit) and e.code == 0: | |
656 | return 0 | |
657 | sys.stderr.write("Failed to prepare guest environment\n") | |
658 | traceback.print_exc() | |
659 | return 2 | |
660 | ||
b3f94b2f GH |
661 | exitcode = 0 |
662 | if vm.ssh(*cmd) != 0: | |
663 | exitcode = 3 | |
bcc388df | 664 | if args.interactive: |
796471e9 | 665 | vm.ssh() |
b3f94b2f GH |
666 | |
667 | if not args.snapshot: | |
668 | vm.graceful_shutdown() | |
669 | ||
670 | return exitcode |