]> git.proxmox.com Git - qemu-server.git/blob - PVE/QemuServer.pm
enable cluster mapped USB devices for guests
[qemu-server.git] / PVE / QemuServer.pm
1 package PVE::QemuServer;
2
3 use strict;
4 use warnings;
5
6 use Cwd 'abs_path';
7 use Digest::SHA;
8 use Fcntl ':flock';
9 use Fcntl;
10 use File::Basename;
11 use File::Copy qw(copy);
12 use File::Path;
13 use File::stat;
14 use Getopt::Long;
15 use IO::Dir;
16 use IO::File;
17 use IO::Handle;
18 use IO::Select;
19 use IO::Socket::UNIX;
20 use IPC::Open3;
21 use JSON;
22 use List::Util qw(first);
23 use MIME::Base64;
24 use POSIX;
25 use Storable qw(dclone);
26 use Time::HiRes qw(gettimeofday usleep);
27 use URI::Escape;
28 use UUID;
29
30 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
31 use PVE::CGroup;
32 use PVE::CpuSet;
33 use PVE::DataCenterConfig;
34 use PVE::Exception qw(raise raise_param_exc);
35 use PVE::Format qw(render_duration render_bytes);
36 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
37 use PVE::INotify;
38 use PVE::JSONSchema qw(get_standard_option parse_property_string);
39 use PVE::ProcFSTools;
40 use PVE::PBSClient;
41 use PVE::RESTEnvironment qw(log_warn);
42 use PVE::RPCEnvironment;
43 use PVE::Storage;
44 use PVE::SysFSTools;
45 use PVE::Systemd;
46 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
47
48 use PVE::QMPClient;
49 use PVE::QemuConfig;
50 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
51 use PVE::QemuServer::Cloudinit;
52 use PVE::QemuServer::CGroup;
53 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
54 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
55 use PVE::QemuServer::Machine;
56 use PVE::QemuServer::Memory;
57 use PVE::QemuServer::Monitor qw(mon_cmd);
58 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
59 use PVE::QemuServer::USB;
60
61 my $have_sdn;
62 eval {
63 require PVE::Network::SDN::Zones;
64 $have_sdn = 1;
65 };
66
67 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
68 my $OVMF = {
69 x86_64 => {
70 '4m-no-smm' => [
71 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
72 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
73 ],
74 '4m-no-smm-ms' => [
75 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
76 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
77 ],
78 '4m' => [
79 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
80 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
81 ],
82 '4m-ms' => [
83 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
84 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
85 ],
86 default => [
87 "$EDK2_FW_BASE/OVMF_CODE.fd",
88 "$EDK2_FW_BASE/OVMF_VARS.fd",
89 ],
90 },
91 aarch64 => {
92 default => [
93 "$EDK2_FW_BASE/AAVMF_CODE.fd",
94 "$EDK2_FW_BASE/AAVMF_VARS.fd",
95 ],
96 },
97 };
98
99 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
100
101 # Note about locking: we use flock on the config file protect against concurent actions.
102 # Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
103 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
104 # But you can ignore this kind of lock with the --skiplock flag.
105
106 cfs_register_file(
107 '/qemu-server/',
108 \&parse_vm_config,
109 \&write_vm_config
110 );
111
112 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
113 description => "Some command save/restore state from this location.",
114 type => 'string',
115 maxLength => 128,
116 optional => 1,
117 });
118
119 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
120 description => "Specifies the QEMU machine type.",
121 type => 'string',
122 pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
123 maxLength => 40,
124 optional => 1,
125 });
126
127 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
128 my $nodename_cache;
129 sub nodename {
130 $nodename_cache //= PVE::INotify::nodename();
131 return $nodename_cache;
132 }
133
134 my $watchdog_fmt = {
135 model => {
136 default_key => 1,
137 type => 'string',
138 enum => [qw(i6300esb ib700)],
139 description => "Watchdog type to emulate.",
140 default => 'i6300esb',
141 optional => 1,
142 },
143 action => {
144 type => 'string',
145 enum => [qw(reset shutdown poweroff pause debug none)],
146 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
147 optional => 1,
148 },
149 };
150 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
151
152 my $agent_fmt = {
153 enabled => {
154 description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
155 type => 'boolean',
156 default => 0,
157 default_key => 1,
158 },
159 fstrim_cloned_disks => {
160 description => "Run fstrim after moving a disk or migrating the VM.",
161 type => 'boolean',
162 optional => 1,
163 default => 0,
164 },
165 'freeze-fs-on-backup' => {
166 description => "Freeze/thaw guest filesystems on backup for consistency.",
167 type => 'boolean',
168 optional => 1,
169 default => 1,
170 },
171 type => {
172 description => "Select the agent type",
173 type => 'string',
174 default => 'virtio',
175 optional => 1,
176 enum => [qw(virtio isa)],
177 },
178 };
179
180 my $vga_fmt = {
181 type => {
182 description => "Select the VGA type.",
183 type => 'string',
184 default => 'std',
185 optional => 1,
186 default_key => 1,
187 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
188 },
189 memory => {
190 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
191 type => 'integer',
192 optional => 1,
193 minimum => 4,
194 maximum => 512,
195 },
196 };
197
198 my $ivshmem_fmt = {
199 size => {
200 type => 'integer',
201 minimum => 1,
202 description => "The size of the file in MB.",
203 },
204 name => {
205 type => 'string',
206 pattern => '[a-zA-Z0-9\-]+',
207 optional => 1,
208 format_description => 'string',
209 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
210 },
211 };
212
213 my $audio_fmt = {
214 device => {
215 type => 'string',
216 enum => [qw(ich9-intel-hda intel-hda AC97)],
217 description => "Configure an audio device."
218 },
219 driver => {
220 type => 'string',
221 enum => ['spice', 'none'],
222 default => 'spice',
223 optional => 1,
224 description => "Driver backend for the audio device."
225 },
226 };
227
228 my $spice_enhancements_fmt = {
229 foldersharing => {
230 type => 'boolean',
231 optional => 1,
232 default => '0',
233 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
234 },
235 videostreaming => {
236 type => 'string',
237 enum => ['off', 'all', 'filter'],
238 default => 'off',
239 optional => 1,
240 description => "Enable video streaming. Uses compression for detected video streams."
241 },
242 };
243
244 my $rng_fmt = {
245 source => {
246 type => 'string',
247 enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
248 default_key => 1,
249 description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
250 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
251 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
252 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
253 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
254 ." a hardware RNG from the host.",
255 },
256 max_bytes => {
257 type => 'integer',
258 description => "Maximum bytes of entropy allowed to get injected into the guest every"
259 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
260 ." `0` to disable limiting (potentially dangerous!).",
261 optional => 1,
262
263 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
264 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
265 # reading from /dev/urandom
266 default => 1024,
267 },
268 period => {
269 type => 'integer',
270 description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
271 ." the guest to retrieve another 'max_bytes' of entropy.",
272 optional => 1,
273 default => 1000,
274 },
275 };
276
277 my $meta_info_fmt = {
278 'ctime' => {
279 type => 'integer',
280 description => "The guest creation timestamp as UNIX epoch time",
281 minimum => 0,
282 optional => 1,
283 },
284 'creation-qemu' => {
285 type => 'string',
286 description => "The QEMU (machine) version from the time this VM was created.",
287 pattern => '\d+(\.\d+)+',
288 optional => 1,
289 },
290 };
291
292 my $confdesc = {
293 onboot => {
294 optional => 1,
295 type => 'boolean',
296 description => "Specifies whether a VM will be started during system bootup.",
297 default => 0,
298 },
299 autostart => {
300 optional => 1,
301 type => 'boolean',
302 description => "Automatic restart after crash (currently ignored).",
303 default => 0,
304 },
305 hotplug => {
306 optional => 1,
307 type => 'string', format => 'pve-hotplug-features',
308 description => "Selectively enable hotplug features. This is a comma separated list of"
309 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
310 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
311 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
312 ." windows > 7.",
313 default => 'network,disk,usb',
314 },
315 reboot => {
316 optional => 1,
317 type => 'boolean',
318 description => "Allow reboot. If set to '0' the VM exit on reboot.",
319 default => 1,
320 },
321 lock => {
322 optional => 1,
323 type => 'string',
324 description => "Lock/unlock the VM.",
325 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
326 },
327 cpulimit => {
328 optional => 1,
329 type => 'number',
330 description => "Limit of CPU usage.",
331 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
332 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
333 minimum => 0,
334 maximum => 128,
335 default => 0,
336 },
337 cpuunits => {
338 optional => 1,
339 type => 'integer',
340 description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
341 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
342 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
343 ." weights of all the other running VMs.",
344 minimum => 1,
345 maximum => 262144,
346 default => 'cgroup v1: 1024, cgroup v2: 100',
347 },
348 memory => {
349 optional => 1,
350 type => 'integer',
351 description => "Amount of RAM for the VM in MiB. This is the maximum available memory when"
352 ." you use the balloon device.",
353 minimum => 16,
354 default => 512,
355 },
356 balloon => {
357 optional => 1,
358 type => 'integer',
359 description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
360 minimum => 0,
361 },
362 shares => {
363 optional => 1,
364 type => 'integer',
365 description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
366 ." more memory this VM gets. Number is relative to weights of all other running VMs."
367 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
368 minimum => 0,
369 maximum => 50000,
370 default => 1000,
371 },
372 keyboard => {
373 optional => 1,
374 type => 'string',
375 description => "Keyboard layout for VNC server. This option is generally not required and"
376 ." is often better handled from within the guest OS.",
377 enum => PVE::Tools::kvmkeymaplist(),
378 default => undef,
379 },
380 name => {
381 optional => 1,
382 type => 'string', format => 'dns-name',
383 description => "Set a name for the VM. Only used on the configuration web interface.",
384 },
385 scsihw => {
386 optional => 1,
387 type => 'string',
388 description => "SCSI controller model",
389 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
390 default => 'lsi',
391 },
392 description => {
393 optional => 1,
394 type => 'string',
395 description => "Description for the VM. Shown in the web-interface VM's summary."
396 ." This is saved as comment inside the configuration file.",
397 maxLength => 1024 * 8,
398 },
399 ostype => {
400 optional => 1,
401 type => 'string',
402 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
403 description => "Specify guest operating system.",
404 verbose_description => <<EODESC,
405 Specify guest operating system. This is used to enable special
406 optimization/features for specific operating systems:
407
408 [horizontal]
409 other;; unspecified OS
410 wxp;; Microsoft Windows XP
411 w2k;; Microsoft Windows 2000
412 w2k3;; Microsoft Windows 2003
413 w2k8;; Microsoft Windows 2008
414 wvista;; Microsoft Windows Vista
415 win7;; Microsoft Windows 7
416 win8;; Microsoft Windows 8/2012/2012r2
417 win10;; Microsoft Windows 10/2016/2019
418 win11;; Microsoft Windows 11/2022
419 l24;; Linux 2.4 Kernel
420 l26;; Linux 2.6 - 6.X Kernel
421 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
422 EODESC
423 },
424 boot => {
425 optional => 1,
426 type => 'string', format => 'pve-qm-boot',
427 description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
428 ." key or 'legacy=' is deprecated.",
429 },
430 bootdisk => {
431 optional => 1,
432 type => 'string', format => 'pve-qm-bootdisk',
433 description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
434 pattern => '(ide|sata|scsi|virtio)\d+',
435 },
436 smp => {
437 optional => 1,
438 type => 'integer',
439 description => "The number of CPUs. Please use option -sockets instead.",
440 minimum => 1,
441 default => 1,
442 },
443 sockets => {
444 optional => 1,
445 type => 'integer',
446 description => "The number of CPU sockets.",
447 minimum => 1,
448 default => 1,
449 },
450 cores => {
451 optional => 1,
452 type => 'integer',
453 description => "The number of cores per socket.",
454 minimum => 1,
455 default => 1,
456 },
457 numa => {
458 optional => 1,
459 type => 'boolean',
460 description => "Enable/disable NUMA.",
461 default => 0,
462 },
463 hugepages => {
464 optional => 1,
465 type => 'string',
466 description => "Enable/disable hugepages memory.",
467 enum => [qw(any 2 1024)],
468 },
469 keephugepages => {
470 optional => 1,
471 type => 'boolean',
472 default => 0,
473 description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
474 ." after VM shutdown and can be used for subsequent starts.",
475 },
476 vcpus => {
477 optional => 1,
478 type => 'integer',
479 description => "Number of hotplugged vcpus.",
480 minimum => 1,
481 default => 0,
482 },
483 acpi => {
484 optional => 1,
485 type => 'boolean',
486 description => "Enable/disable ACPI.",
487 default => 1,
488 },
489 agent => {
490 optional => 1,
491 description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
492 type => 'string',
493 format => $agent_fmt,
494 },
495 kvm => {
496 optional => 1,
497 type => 'boolean',
498 description => "Enable/disable KVM hardware virtualization.",
499 default => 1,
500 },
501 tdf => {
502 optional => 1,
503 type => 'boolean',
504 description => "Enable/disable time drift fix.",
505 default => 0,
506 },
507 localtime => {
508 optional => 1,
509 type => 'boolean',
510 description => "Set the real time clock (RTC) to local time. This is enabled by default if"
511 ." the `ostype` indicates a Microsoft Windows OS.",
512 },
513 freeze => {
514 optional => 1,
515 type => 'boolean',
516 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
517 },
518 vga => {
519 optional => 1,
520 type => 'string', format => $vga_fmt,
521 description => "Configure the VGA hardware.",
522 verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
523 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
524 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
525 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
526 ." display server. For win* OS you can select how many independent displays you want,"
527 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
528 ." using a serial device as terminal.",
529 },
530 watchdog => {
531 optional => 1,
532 type => 'string', format => 'pve-qm-watchdog',
533 description => "Create a virtual hardware watchdog device.",
534 verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
535 ." action), the watchdog must be periodically polled by an agent inside the guest or"
536 ." else the watchdog will reset the guest (or execute the respective action specified)",
537 },
538 startdate => {
539 optional => 1,
540 type => 'string',
541 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
542 description => "Set the initial date of the real time clock. Valid format for date are:"
543 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
544 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
545 default => 'now',
546 },
547 startup => get_standard_option('pve-startup-order'),
548 template => {
549 optional => 1,
550 type => 'boolean',
551 description => "Enable/disable Template.",
552 default => 0,
553 },
554 args => {
555 optional => 1,
556 type => 'string',
557 description => "Arbitrary arguments passed to kvm.",
558 verbose_description => <<EODESCR,
559 Arbitrary arguments passed to kvm, for example:
560
561 args: -no-reboot -smbios 'type=0,vendor=FOO'
562
563 NOTE: this option is for experts only.
564 EODESCR
565 },
566 tablet => {
567 optional => 1,
568 type => 'boolean',
569 default => 1,
570 description => "Enable/disable the USB tablet device.",
571 verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
572 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
573 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
574 ." may consider disabling this to save some context switches. This is turned off by"
575 ." default if you use spice (`qm set <vmid> --vga qxl`).",
576 },
577 migrate_speed => {
578 optional => 1,
579 type => 'integer',
580 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
581 minimum => 0,
582 default => 0,
583 },
584 migrate_downtime => {
585 optional => 1,
586 type => 'number',
587 description => "Set maximum tolerated downtime (in seconds) for migrations.",
588 minimum => 0,
589 default => 0.1,
590 },
591 cdrom => {
592 optional => 1,
593 type => 'string', format => 'pve-qm-ide',
594 typetext => '<volume>',
595 description => "This is an alias for option -ide2",
596 },
597 cpu => {
598 optional => 1,
599 description => "Emulated CPU type.",
600 type => 'string',
601 format => 'pve-vm-cpu-conf',
602 },
603 parent => get_standard_option('pve-snapshot-name', {
604 optional => 1,
605 description => "Parent snapshot name. This is used internally, and should not be modified.",
606 }),
607 snaptime => {
608 optional => 1,
609 description => "Timestamp for snapshots.",
610 type => 'integer',
611 minimum => 0,
612 },
613 vmstate => {
614 optional => 1,
615 type => 'string', format => 'pve-volume-id',
616 description => "Reference to a volume which stores the VM state. This is used internally"
617 ." for snapshots.",
618 },
619 vmstatestorage => get_standard_option('pve-storage-id', {
620 description => "Default storage for VM state volumes/files.",
621 optional => 1,
622 }),
623 runningmachine => get_standard_option('pve-qemu-machine', {
624 description => "Specifies the QEMU machine type of the running vm. This is used internally"
625 ." for snapshots.",
626 }),
627 runningcpu => {
628 description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
629 ." internally for snapshots.",
630 optional => 1,
631 type => 'string',
632 pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
633 format_description => 'QEMU -cpu parameter'
634 },
635 machine => get_standard_option('pve-qemu-machine'),
636 arch => {
637 description => "Virtual processor architecture. Defaults to the host.",
638 optional => 1,
639 type => 'string',
640 enum => [qw(x86_64 aarch64)],
641 },
642 smbios1 => {
643 description => "Specify SMBIOS type 1 fields.",
644 type => 'string', format => 'pve-qm-smbios1',
645 maxLength => 512,
646 optional => 1,
647 },
648 protection => {
649 optional => 1,
650 type => 'boolean',
651 description => "Sets the protection flag of the VM. This will disable the remove VM and"
652 ." remove disk operations.",
653 default => 0,
654 },
655 bios => {
656 optional => 1,
657 type => 'string',
658 enum => [ qw(seabios ovmf) ],
659 description => "Select BIOS implementation.",
660 default => 'seabios',
661 },
662 vmgenid => {
663 type => 'string',
664 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
665 format_description => 'UUID',
666 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
667 ." to disable explicitly.",
668 verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
669 ." value identifier to the guest OS. This allows to notify the guest operating system"
670 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
671 ." execution or creation from a template). The guest operating system notices the"
672 ." change, and is then able to react as appropriate by marking its copies of"
673 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
674 ."Note that auto-creation only works when done through API/CLI create or update methods"
675 .", but not when manually editing the config file.",
676 default => "1 (autogenerated)",
677 optional => 1,
678 },
679 hookscript => {
680 type => 'string',
681 format => 'pve-volume-id',
682 optional => 1,
683 description => "Script that will be executed during various steps in the vms lifetime.",
684 },
685 ivshmem => {
686 type => 'string',
687 format => $ivshmem_fmt,
688 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
689 ." the host.",
690 optional => 1,
691 },
692 audio0 => {
693 type => 'string',
694 format => $audio_fmt,
695 description => "Configure a audio device, useful in combination with QXL/Spice.",
696 optional => 1
697 },
698 spice_enhancements => {
699 type => 'string',
700 format => $spice_enhancements_fmt,
701 description => "Configure additional enhancements for SPICE.",
702 optional => 1
703 },
704 tags => {
705 type => 'string', format => 'pve-tag-list',
706 description => 'Tags of the VM. This is only meta information.',
707 optional => 1,
708 },
709 rng0 => {
710 type => 'string',
711 format => $rng_fmt,
712 description => "Configure a VirtIO-based Random Number Generator.",
713 optional => 1,
714 },
715 meta => {
716 type => 'string',
717 format => $meta_info_fmt,
718 description => "Some (read-only) meta-information about this guest.",
719 optional => 1,
720 },
721 affinity => {
722 type => 'string', format => 'pve-cpuset',
723 description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
724 optional => 1,
725 },
726 };
727
728 my $cicustom_fmt = {
729 meta => {
730 type => 'string',
731 optional => 1,
732 description => 'Specify a custom file containing all meta data passed to the VM via"
733 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
734 format => 'pve-volume-id',
735 format_description => 'volume',
736 },
737 network => {
738 type => 'string',
739 optional => 1,
740 description => 'To pass a custom file containing all network data to the VM via cloud-init.',
741 format => 'pve-volume-id',
742 format_description => 'volume',
743 },
744 user => {
745 type => 'string',
746 optional => 1,
747 description => 'To pass a custom file containing all user data to the VM via cloud-init.',
748 format => 'pve-volume-id',
749 format_description => 'volume',
750 },
751 vendor => {
752 type => 'string',
753 optional => 1,
754 description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
755 format => 'pve-volume-id',
756 format_description => 'volume',
757 },
758 };
759 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
760
761 my $confdesc_cloudinit = {
762 citype => {
763 optional => 1,
764 type => 'string',
765 description => 'Specifies the cloud-init configuration format. The default depends on the'
766 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
767 .' and `configdrive2` for windows.',
768 enum => ['configdrive2', 'nocloud', 'opennebula'],
769 },
770 ciuser => {
771 optional => 1,
772 type => 'string',
773 description => "cloud-init: User name to change ssh keys and password for instead of the"
774 ." image's configured default user.",
775 },
776 cipassword => {
777 optional => 1,
778 type => 'string',
779 description => 'cloud-init: Password to assign the user. Using this is generally not'
780 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
781 .' support hashed passwords.',
782 },
783 ciupgrade => {
784 optional => 1,
785 type => 'boolean',
786 description => 'cloud-init: do an automatic package upgrade after the first boot.'
787 },
788 cicustom => {
789 optional => 1,
790 type => 'string',
791 description => 'cloud-init: Specify custom files to replace the automatically generated'
792 .' ones at start.',
793 format => 'pve-qm-cicustom',
794 },
795 searchdomain => {
796 optional => 1,
797 type => 'string',
798 description => 'cloud-init: Sets DNS search domains for a container. Create will'
799 .' automatically use the setting from the host if neither searchdomain nor nameserver'
800 .' are set.',
801 },
802 nameserver => {
803 optional => 1,
804 type => 'string', format => 'address-list',
805 description => 'cloud-init: Sets DNS server IP address for a container. Create will'
806 .' automatically use the setting from the host if neither searchdomain nor nameserver'
807 .' are set.',
808 },
809 sshkeys => {
810 optional => 1,
811 type => 'string',
812 format => 'urlencoded',
813 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
814 },
815 };
816
817 # what about other qemu settings ?
818 #cpu => 'string',
819 #machine => 'string',
820 #fda => 'file',
821 #fdb => 'file',
822 #mtdblock => 'file',
823 #sd => 'file',
824 #pflash => 'file',
825 #snapshot => 'bool',
826 #bootp => 'file',
827 ##tftp => 'dir',
828 ##smb => 'dir',
829 #kernel => 'file',
830 #append => 'string',
831 #initrd => 'file',
832 ##soundhw => 'string',
833
834 while (my ($k, $v) = each %$confdesc) {
835 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
836 }
837
838 my $MAX_NETS = 32;
839 my $MAX_SERIAL_PORTS = 4;
840 my $MAX_PARALLEL_PORTS = 3;
841 my $MAX_NUMA = 8;
842
843 my $numa_fmt = {
844 cpus => {
845 type => "string",
846 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
847 description => "CPUs accessing this NUMA node.",
848 format_description => "id[-id];...",
849 },
850 memory => {
851 type => "number",
852 description => "Amount of memory this NUMA node provides.",
853 optional => 1,
854 },
855 hostnodes => {
856 type => "string",
857 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
858 description => "Host NUMA nodes to use.",
859 format_description => "id[-id];...",
860 optional => 1,
861 },
862 policy => {
863 type => 'string',
864 enum => [qw(preferred bind interleave)],
865 description => "NUMA allocation policy.",
866 optional => 1,
867 },
868 };
869 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
870 my $numadesc = {
871 optional => 1,
872 type => 'string', format => $numa_fmt,
873 description => "NUMA topology.",
874 };
875 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
876
877 for (my $i = 0; $i < $MAX_NUMA; $i++) {
878 $confdesc->{"numa$i"} = $numadesc;
879 }
880
881 my $nic_model_list = [
882 'e1000',
883 'e1000-82540em',
884 'e1000-82544gc',
885 'e1000-82545em',
886 'e1000e',
887 'i82551',
888 'i82557b',
889 'i82559er',
890 'ne2k_isa',
891 'ne2k_pci',
892 'pcnet',
893 'rtl8139',
894 'virtio',
895 'vmxnet3',
896 ];
897 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
898
899 my $net_fmt_bridge_descr = <<__EOD__;
900 Bridge to attach the network device to. The Proxmox VE standard bridge
901 is called 'vmbr0'.
902
903 If you do not specify a bridge, we create a kvm user (NATed) network
904 device, which provides DHCP and DNS services. The following addresses
905 are used:
906
907 10.0.2.2 Gateway
908 10.0.2.3 DNS Server
909 10.0.2.4 SMB Server
910
911 The DHCP server assign addresses to the guest starting from 10.0.2.15.
912 __EOD__
913
914 my $net_fmt = {
915 macaddr => get_standard_option('mac-addr', {
916 description => "MAC address. That address must be unique withing your network. This is"
917 ." automatically generated if not specified.",
918 }),
919 model => {
920 type => 'string',
921 description => "Network Card Model. The 'virtio' model provides the best performance with"
922 ." very low CPU overhead. If your guest does not support this driver, it is usually"
923 ." best to use 'e1000'.",
924 enum => $nic_model_list,
925 default_key => 1,
926 },
927 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
928 bridge => get_standard_option('pve-bridge-id', {
929 description => $net_fmt_bridge_descr,
930 optional => 1,
931 }),
932 queues => {
933 type => 'integer',
934 minimum => 0, maximum => 64,
935 description => 'Number of packet queues to be used on the device.',
936 optional => 1,
937 },
938 rate => {
939 type => 'number',
940 minimum => 0,
941 description => "Rate limit in mbps (megabytes per second) as floating point number.",
942 optional => 1,
943 },
944 tag => {
945 type => 'integer',
946 minimum => 1, maximum => 4094,
947 description => 'VLAN tag to apply to packets on this interface.',
948 optional => 1,
949 },
950 trunks => {
951 type => 'string',
952 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
953 description => 'VLAN trunks to pass through this interface.',
954 format_description => 'vlanid[;vlanid...]',
955 optional => 1,
956 },
957 firewall => {
958 type => 'boolean',
959 description => 'Whether this interface should be protected by the firewall.',
960 optional => 1,
961 },
962 link_down => {
963 type => 'boolean',
964 description => 'Whether this interface should be disconnected (like pulling the plug).',
965 optional => 1,
966 },
967 mtu => {
968 type => 'integer',
969 minimum => 1, maximum => 65520,
970 description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
971 optional => 1,
972 },
973 };
974
975 my $netdesc = {
976 optional => 1,
977 type => 'string', format => $net_fmt,
978 description => "Specify network devices.",
979 };
980
981 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
982
983 my $ipconfig_fmt = {
984 ip => {
985 type => 'string',
986 format => 'pve-ipv4-config',
987 format_description => 'IPv4Format/CIDR',
988 description => 'IPv4 address in CIDR format.',
989 optional => 1,
990 default => 'dhcp',
991 },
992 gw => {
993 type => 'string',
994 format => 'ipv4',
995 format_description => 'GatewayIPv4',
996 description => 'Default gateway for IPv4 traffic.',
997 optional => 1,
998 requires => 'ip',
999 },
1000 ip6 => {
1001 type => 'string',
1002 format => 'pve-ipv6-config',
1003 format_description => 'IPv6Format/CIDR',
1004 description => 'IPv6 address in CIDR format.',
1005 optional => 1,
1006 default => 'dhcp',
1007 },
1008 gw6 => {
1009 type => 'string',
1010 format => 'ipv6',
1011 format_description => 'GatewayIPv6',
1012 description => 'Default gateway for IPv6 traffic.',
1013 optional => 1,
1014 requires => 'ip6',
1015 },
1016 };
1017 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1018 my $ipconfigdesc = {
1019 optional => 1,
1020 type => 'string', format => 'pve-qm-ipconfig',
1021 description => <<'EODESCR',
1022 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1023
1024 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1025
1026 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1027 gateway should be provided.
1028 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1029 cloud-init 19.4 or newer.
1030
1031 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1032 dhcp on IPv4.
1033 EODESCR
1034 };
1035 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1036
1037 for (my $i = 0; $i < $MAX_NETS; $i++) {
1038 $confdesc->{"net$i"} = $netdesc;
1039 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1040 }
1041
1042 foreach my $key (keys %$confdesc_cloudinit) {
1043 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1044 }
1045
1046 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1047 sub pve_verify_cpuset {
1048 my ($set_text, $noerr) = @_;
1049
1050 my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1051
1052 if ($@) {
1053 return if $noerr;
1054 die "unable to parse cpuset option\n";
1055 }
1056
1057 return PVE::CpuSet->new($members)->short_string();
1058 }
1059
1060 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1061 sub verify_volume_id_or_qm_path {
1062 my ($volid, $noerr) = @_;
1063
1064 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1065
1066 return verify_volume_id_or_absolute_path($volid, $noerr);
1067 }
1068
1069 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1070 sub verify_volume_id_or_absolute_path {
1071 my ($volid, $noerr) = @_;
1072
1073 return $volid if $volid =~ m|^/|;
1074
1075 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1076 if ($@) {
1077 return if $noerr;
1078 die $@;
1079 }
1080 return $volid;
1081 }
1082
1083 my $serialdesc = {
1084 optional => 1,
1085 type => 'string',
1086 pattern => '(/dev/.+|socket)',
1087 description => "Create a serial device inside the VM (n is 0 to 3)",
1088 verbose_description => <<EODESCR,
1089 Create a serial device inside the VM (n is 0 to 3), and pass through a
1090 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1091 host side (use 'qm terminal' to open a terminal connection).
1092
1093 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1094 use with special care.
1095
1096 CAUTION: Experimental! User reported problems with this option.
1097 EODESCR
1098 };
1099
1100 my $paralleldesc= {
1101 optional => 1,
1102 type => 'string',
1103 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1104 description => "Map host parallel devices (n is 0 to 2).",
1105 verbose_description => <<EODESCR,
1106 Map host parallel devices (n is 0 to 2).
1107
1108 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1109 machines - use with special care.
1110
1111 CAUTION: Experimental! User reported problems with this option.
1112 EODESCR
1113 };
1114
1115 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1116 $confdesc->{"parallel$i"} = $paralleldesc;
1117 }
1118
1119 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1120 $confdesc->{"serial$i"} = $serialdesc;
1121 }
1122
1123 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
1124 $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1125 }
1126
1127 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1128 $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1129 }
1130
1131 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
1132 $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1133 }
1134
1135 my $boot_fmt = {
1136 legacy => {
1137 optional => 1,
1138 default_key => 1,
1139 type => 'string',
1140 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1141 . " Deprecated, use 'order=' instead.",
1142 pattern => '[acdn]{1,4}',
1143 format_description => "[acdn]{1,4}",
1144
1145 # note: this is also the fallback if boot: is not given at all
1146 default => 'cdn',
1147 },
1148 order => {
1149 optional => 1,
1150 type => 'string',
1151 format => 'pve-qm-bootdev-list',
1152 format_description => "device[;device...]",
1153 description => <<EODESC,
1154 The guest will attempt to boot from devices in the order they appear here.
1155
1156 Disks, optical drives and passed-through storage USB devices will be directly
1157 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1158 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1159
1160 Note that only devices in this list will be marked as bootable and thus loaded
1161 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1162 (e.g. software-raid), you need to specify all of them here.
1163
1164 Overrides the deprecated 'legacy=[acdn]*' value when given.
1165 EODESC
1166 },
1167 };
1168 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1169
1170 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1171 sub verify_bootdev {
1172 my ($dev, $noerr) = @_;
1173
1174 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1175 return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1176
1177 my $check = sub {
1178 my ($base) = @_;
1179 return 0 if $dev !~ m/^$base\d+$/;
1180 return 0 if !$confdesc->{$dev};
1181 return 1;
1182 };
1183
1184 return $dev if $check->("net");
1185 return $dev if $check->("usb");
1186 return $dev if $check->("hostpci");
1187
1188 return if $noerr;
1189 die "invalid boot device '$dev'\n";
1190 }
1191
1192 sub print_bootorder {
1193 my ($devs) = @_;
1194 return "" if !@$devs;
1195 my $data = { order => join(';', @$devs) };
1196 return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1197 }
1198
1199 my $kvm_api_version = 0;
1200
1201 sub kvm_version {
1202 return $kvm_api_version if $kvm_api_version;
1203
1204 open my $fh, '<', '/dev/kvm' or return;
1205
1206 # 0xae00 => KVM_GET_API_VERSION
1207 $kvm_api_version = ioctl($fh, 0xae00, 0);
1208 close($fh);
1209
1210 return $kvm_api_version;
1211 }
1212
1213 my $kvm_user_version = {};
1214 my $kvm_mtime = {};
1215
1216 sub kvm_user_version {
1217 my ($binary) = @_;
1218
1219 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1220 my $st = stat($binary);
1221
1222 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1223 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1224 $cachedmtime == $st->mtime;
1225
1226 $kvm_user_version->{$binary} = 'unknown';
1227 $kvm_mtime->{$binary} = $st->mtime;
1228
1229 my $code = sub {
1230 my $line = shift;
1231 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1232 $kvm_user_version->{$binary} = $2;
1233 }
1234 };
1235
1236 eval { run_command([$binary, '--version'], outfunc => $code); };
1237 warn $@ if $@;
1238
1239 return $kvm_user_version->{$binary};
1240
1241 }
1242 my sub extract_version {
1243 my ($machine_type, $version) = @_;
1244 $version = kvm_user_version() if !defined($version);
1245 return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1246 }
1247
1248 sub kernel_has_vhost_net {
1249 return -c '/dev/vhost-net';
1250 }
1251
1252 sub option_exists {
1253 my $key = shift;
1254 return defined($confdesc->{$key});
1255 }
1256
1257 my $cdrom_path;
1258 sub get_cdrom_path {
1259
1260 return $cdrom_path if defined($cdrom_path);
1261
1262 $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1263
1264 if (!defined($cdrom_path)) {
1265 log_warn("no physical CD-ROM available, ignoring");
1266 $cdrom_path = '';
1267 }
1268
1269 return $cdrom_path;
1270 }
1271
1272 sub get_iso_path {
1273 my ($storecfg, $vmid, $cdrom) = @_;
1274
1275 if ($cdrom eq 'cdrom') {
1276 return get_cdrom_path();
1277 } elsif ($cdrom eq 'none') {
1278 return '';
1279 } elsif ($cdrom =~ m|^/|) {
1280 return $cdrom;
1281 } else {
1282 return PVE::Storage::path($storecfg, $cdrom);
1283 }
1284 }
1285
1286 # try to convert old style file names to volume IDs
1287 sub filename_to_volume_id {
1288 my ($vmid, $file, $media) = @_;
1289
1290 if (!($file eq 'none' || $file eq 'cdrom' ||
1291 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1292
1293 return if $file =~ m|/|;
1294
1295 if ($media && $media eq 'cdrom') {
1296 $file = "local:iso/$file";
1297 } else {
1298 $file = "local:$vmid/$file";
1299 }
1300 }
1301
1302 return $file;
1303 }
1304
1305 sub verify_media_type {
1306 my ($opt, $vtype, $media) = @_;
1307
1308 return if !$media;
1309
1310 my $etype;
1311 if ($media eq 'disk') {
1312 $etype = 'images';
1313 } elsif ($media eq 'cdrom') {
1314 $etype = 'iso';
1315 } else {
1316 die "internal error";
1317 }
1318
1319 return if ($vtype eq $etype);
1320
1321 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1322 }
1323
1324 sub cleanup_drive_path {
1325 my ($opt, $storecfg, $drive) = @_;
1326
1327 # try to convert filesystem paths to volume IDs
1328
1329 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1330 ($drive->{file} !~ m|^/dev/.+|) &&
1331 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1332 ($drive->{file} !~ m/^\d+$/)) {
1333 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1334 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1335 if !$vtype;
1336 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1337 verify_media_type($opt, $vtype, $drive->{media});
1338 $drive->{file} = $volid;
1339 }
1340
1341 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1342 }
1343
1344 sub parse_hotplug_features {
1345 my ($data) = @_;
1346
1347 my $res = {};
1348
1349 return $res if $data eq '0';
1350
1351 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1352
1353 foreach my $feature (PVE::Tools::split_list($data)) {
1354 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1355 $res->{$1} = 1;
1356 } else {
1357 die "invalid hotplug feature '$feature'\n";
1358 }
1359 }
1360 return $res;
1361 }
1362
1363 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1364 sub pve_verify_hotplug_features {
1365 my ($value, $noerr) = @_;
1366
1367 return $value if parse_hotplug_features($value);
1368
1369 return if $noerr;
1370
1371 die "unable to parse hotplug option\n";
1372 }
1373
1374 sub scsi_inquiry {
1375 my($fh, $noerr) = @_;
1376
1377 my $SG_IO = 0x2285;
1378 my $SG_GET_VERSION_NUM = 0x2282;
1379
1380 my $versionbuf = "\x00" x 8;
1381 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1382 if (!$ret) {
1383 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1384 return;
1385 }
1386 my $version = unpack("I", $versionbuf);
1387 if ($version < 30000) {
1388 die "scsi generic interface too old\n" if !$noerr;
1389 return;
1390 }
1391
1392 my $buf = "\x00" x 36;
1393 my $sensebuf = "\x00" x 8;
1394 my $cmd = pack("C x3 C x1", 0x12, 36);
1395
1396 # see /usr/include/scsi/sg.h
1397 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1398
1399 my $packet = pack(
1400 $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1401 );
1402
1403 $ret = ioctl($fh, $SG_IO, $packet);
1404 if (!$ret) {
1405 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1406 return;
1407 }
1408
1409 my @res = unpack($sg_io_hdr_t, $packet);
1410 if ($res[17] || $res[18]) {
1411 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1412 return;
1413 }
1414
1415 my $res = {};
1416 $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1417
1418 $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1419 $res->{type} &= 0x1F;
1420
1421 return $res;
1422 }
1423
1424 sub path_is_scsi {
1425 my ($path) = @_;
1426
1427 my $fh = IO::File->new("+<$path") || return;
1428 my $res = scsi_inquiry($fh, 1);
1429 close($fh);
1430
1431 return $res;
1432 }
1433
1434 sub print_tabletdevice_full {
1435 my ($conf, $arch) = @_;
1436
1437 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1438
1439 # we use uhci for old VMs because tablet driver was buggy in older qemu
1440 my $usbbus;
1441 if ($q35 || $arch eq 'aarch64') {
1442 $usbbus = 'ehci';
1443 } else {
1444 $usbbus = 'uhci';
1445 }
1446
1447 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1448 }
1449
1450 sub print_keyboarddevice_full {
1451 my ($conf, $arch) = @_;
1452
1453 return if $arch ne 'aarch64';
1454
1455 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1456 }
1457
1458 my sub get_drive_id {
1459 my ($drive) = @_;
1460 return "$drive->{interface}$drive->{index}";
1461 }
1462
1463 sub print_drivedevice_full {
1464 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1465
1466 my $device = '';
1467 my $maxdev = 0;
1468
1469 my $drive_id = get_drive_id($drive);
1470 if ($drive->{interface} eq 'virtio') {
1471 my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1472 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1473 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1474 } elsif ($drive->{interface} eq 'scsi') {
1475
1476 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1477 my $unit = $drive->{index} % $maxdev;
1478 my $devicetype = 'hd';
1479 my $path = '';
1480 if (drive_is_cdrom($drive)) {
1481 $devicetype = 'cd';
1482 } else {
1483 if ($drive->{file} =~ m|^/|) {
1484 $path = $drive->{file};
1485 if (my $info = path_is_scsi($path)) {
1486 if ($info->{type} == 0 && $drive->{scsiblock}) {
1487 $devicetype = 'block';
1488 } elsif ($info->{type} == 1) { # tape
1489 $devicetype = 'generic';
1490 }
1491 }
1492 } else {
1493 $path = PVE::Storage::path($storecfg, $drive->{file});
1494 }
1495
1496 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1497 my $version = extract_version($machine_type, kvm_user_version());
1498 if ($path =~ m/^iscsi\:\/\// &&
1499 !min_version($version, 4, 1)) {
1500 $devicetype = 'generic';
1501 }
1502 }
1503
1504 if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1505 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1506 } else {
1507 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1508 .",lun=$drive->{index}";
1509 }
1510 $device .= ",drive=drive-$drive_id,id=$drive_id";
1511
1512 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1513 $device .= ",rotation_rate=1";
1514 }
1515 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1516
1517 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1518 my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1519 my $controller = int($drive->{index} / $maxdev);
1520 my $unit = $drive->{index} % $maxdev;
1521 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1522
1523 $device = "ide-$devicetype";
1524 if ($drive->{interface} eq 'ide') {
1525 $device .= ",bus=ide.$controller,unit=$unit";
1526 } else {
1527 $device .= ",bus=ahci$controller.$unit";
1528 }
1529 $device .= ",drive=drive-$drive_id,id=$drive_id";
1530
1531 if ($devicetype eq 'hd') {
1532 if (my $model = $drive->{model}) {
1533 $model = URI::Escape::uri_unescape($model);
1534 $device .= ",model=$model";
1535 }
1536 if ($drive->{ssd}) {
1537 $device .= ",rotation_rate=1";
1538 }
1539 }
1540 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1541 } elsif ($drive->{interface} eq 'usb') {
1542 die "implement me";
1543 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1544 } else {
1545 die "unsupported interface type";
1546 }
1547
1548 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1549
1550 if (my $serial = $drive->{serial}) {
1551 $serial = URI::Escape::uri_unescape($serial);
1552 $device .= ",serial=$serial";
1553 }
1554
1555
1556 return $device;
1557 }
1558
1559 sub get_initiator_name {
1560 my $initiator;
1561
1562 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1563 while (defined(my $line = <$fh>)) {
1564 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1565 $initiator = $1;
1566 last;
1567 }
1568 $fh->close();
1569
1570 return $initiator;
1571 }
1572
1573 my sub storage_allows_io_uring_default {
1574 my ($scfg, $cache_direct) = @_;
1575
1576 # io_uring with cache mode writeback or writethrough on krbd will hang...
1577 return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1578
1579 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1580 # sometimes, just plain disable...
1581 return if $scfg && $scfg->{type} eq 'lvm';
1582
1583 # io_uring causes problems when used with CIFS since kernel 5.15
1584 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1585 return if $scfg && $scfg->{type} eq 'cifs';
1586
1587 return 1;
1588 }
1589
1590 my sub drive_uses_cache_direct {
1591 my ($drive, $scfg) = @_;
1592
1593 my $cache_direct = 0;
1594
1595 if (my $cache = $drive->{cache}) {
1596 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1597 } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1598 $cache_direct = 1;
1599 }
1600
1601 return $cache_direct;
1602 }
1603
1604 sub print_drive_commandline_full {
1605 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1606
1607 my $path;
1608 my $volid = $drive->{file};
1609 my $format = $drive->{format};
1610 my $drive_id = get_drive_id($drive);
1611
1612 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1613 my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1614
1615 if (drive_is_cdrom($drive)) {
1616 $path = get_iso_path($storecfg, $vmid, $volid);
1617 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1618 } else {
1619 if ($storeid) {
1620 $path = PVE::Storage::path($storecfg, $volid);
1621 $format //= qemu_img_format($scfg, $volname);
1622 } else {
1623 $path = $volid;
1624 $format //= "raw";
1625 }
1626 }
1627
1628 my $is_rbd = $path =~ m/^rbd:/;
1629
1630 my $opts = '';
1631 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1632 foreach my $o (@qemu_drive_options) {
1633 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1634 }
1635
1636 # snapshot only accepts on|off
1637 if (defined($drive->{snapshot})) {
1638 my $v = $drive->{snapshot} ? 'on' : 'off';
1639 $opts .= ",snapshot=$v";
1640 }
1641
1642 if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1643 $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1644 }
1645
1646 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1647 my ($dir, $qmpname) = @$type;
1648 if (my $v = $drive->{"mbps$dir"}) {
1649 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1650 }
1651 if (my $v = $drive->{"mbps${dir}_max"}) {
1652 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1653 }
1654 if (my $v = $drive->{"bps${dir}_max_length"}) {
1655 $opts .= ",throttling.bps$qmpname-max-length=$v";
1656 }
1657 if (my $v = $drive->{"iops${dir}"}) {
1658 $opts .= ",throttling.iops$qmpname=$v";
1659 }
1660 if (my $v = $drive->{"iops${dir}_max"}) {
1661 $opts .= ",throttling.iops$qmpname-max=$v";
1662 }
1663 if (my $v = $drive->{"iops${dir}_max_length"}) {
1664 $opts .= ",throttling.iops$qmpname-max-length=$v";
1665 }
1666 }
1667
1668 if ($pbs_name) {
1669 $format = "rbd" if $is_rbd;
1670 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1671 if !$format;
1672 $opts .= ",format=alloc-track,file.driver=$format";
1673 } elsif ($format) {
1674 $opts .= ",format=$format";
1675 }
1676
1677 my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1678
1679 $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1680
1681 if (!$drive->{aio}) {
1682 if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1683 # io_uring supports all cache modes
1684 $opts .= ",aio=io_uring";
1685 } else {
1686 # aio native works only with O_DIRECT
1687 if($cache_direct) {
1688 $opts .= ",aio=native";
1689 } else {
1690 $opts .= ",aio=threads";
1691 }
1692 }
1693 }
1694
1695 if (!drive_is_cdrom($drive)) {
1696 my $detectzeroes;
1697 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1698 $detectzeroes = 'off';
1699 } elsif ($drive->{discard}) {
1700 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1701 } else {
1702 # This used to be our default with discard not being specified:
1703 $detectzeroes = 'on';
1704 }
1705
1706 # note: 'detect-zeroes' works per blockdev and we want it to persist
1707 # after the alloc-track is removed, so put it on 'file' directly
1708 my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1709 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1710 }
1711
1712 if ($pbs_name) {
1713 $opts .= ",backing=$pbs_name";
1714 $opts .= ",auto-remove=on";
1715 }
1716
1717 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1718 my $file_param = "file";
1719 if ($pbs_name) {
1720 # non-rbd drivers require the underlying file to be a seperate block
1721 # node, so add a second .file indirection
1722 $file_param .= ".file" if !$is_rbd;
1723 $file_param .= ".filename";
1724 }
1725 my $pathinfo = $path ? "$file_param=$path," : '';
1726
1727 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1728 }
1729
1730 sub print_pbs_blockdev {
1731 my ($pbs_conf, $pbs_name) = @_;
1732 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1733 $blockdev .= ",repository=$pbs_conf->{repository}";
1734 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1735 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1736 $blockdev .= ",archive=$pbs_conf->{archive}";
1737 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1738 return $blockdev;
1739 }
1740
1741 sub print_netdevice_full {
1742 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1743
1744 my $device = $net->{model};
1745 if ($net->{model} eq 'virtio') {
1746 $device = 'virtio-net-pci';
1747 };
1748
1749 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1750 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1751 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1752 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1753 # and out of each queue plus one config interrupt and control vector queue
1754 my $vectors = $net->{queues} * 2 + 2;
1755 $tmpstr .= ",vectors=$vectors,mq=on";
1756 if (min_version($machine_version, 7, 1)) {
1757 $tmpstr .= ",packed=on";
1758 }
1759 }
1760
1761 if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1762 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1763 }
1764
1765 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1766
1767 if (my $mtu = $net->{mtu}) {
1768 if ($net->{model} eq 'virtio' && $net->{bridge}) {
1769 my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1770 if ($mtu == 1) {
1771 $mtu = $bridge_mtu;
1772 } elsif ($mtu < 576) {
1773 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1774 } elsif ($mtu > $bridge_mtu) {
1775 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1776 }
1777 $tmpstr .= ",host_mtu=$mtu";
1778 } else {
1779 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1780 }
1781 }
1782
1783 if ($use_old_bios_files) {
1784 my $romfile;
1785 if ($device eq 'virtio-net-pci') {
1786 $romfile = 'pxe-virtio.rom';
1787 } elsif ($device eq 'e1000') {
1788 $romfile = 'pxe-e1000.rom';
1789 } elsif ($device eq 'e1000e') {
1790 $romfile = 'pxe-e1000e.rom';
1791 } elsif ($device eq 'ne2k') {
1792 $romfile = 'pxe-ne2k_pci.rom';
1793 } elsif ($device eq 'pcnet') {
1794 $romfile = 'pxe-pcnet.rom';
1795 } elsif ($device eq 'rtl8139') {
1796 $romfile = 'pxe-rtl8139.rom';
1797 }
1798 $tmpstr .= ",romfile=$romfile" if $romfile;
1799 }
1800
1801 return $tmpstr;
1802 }
1803
1804 sub print_netdev_full {
1805 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1806
1807 my $i = '';
1808 if ($netid =~ m/^net(\d+)$/) {
1809 $i = int($1);
1810 }
1811
1812 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1813
1814 my $ifname = "tap${vmid}i$i";
1815
1816 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1817 die "interface name '$ifname' is too long (max 15 character)\n"
1818 if length($ifname) >= 16;
1819
1820 my $vhostparam = '';
1821 if (is_native($arch)) {
1822 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1823 }
1824
1825 my $vmname = $conf->{name} || "vm$vmid";
1826
1827 my $netdev = "";
1828 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1829
1830 if ($net->{bridge}) {
1831 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1832 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1833 } else {
1834 $netdev = "type=user,id=$netid,hostname=$vmname";
1835 }
1836
1837 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1838
1839 return $netdev;
1840 }
1841
1842 my $vga_map = {
1843 'cirrus' => 'cirrus-vga',
1844 'std' => 'VGA',
1845 'vmware' => 'vmware-svga',
1846 'virtio' => 'virtio-vga',
1847 'virtio-gl' => 'virtio-vga-gl',
1848 };
1849
1850 sub print_vga_device {
1851 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1852
1853 my $type = $vga_map->{$vga->{type}};
1854 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1855 $type = 'virtio-gpu';
1856 }
1857 my $vgamem_mb = $vga->{memory};
1858
1859 my $max_outputs = '';
1860 if ($qxlnum) {
1861 $type = $id ? 'qxl' : 'qxl-vga';
1862
1863 if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1864 # set max outputs so linux can have up to 4 qxl displays with one device
1865 if (min_version($machine_version, 4, 1)) {
1866 $max_outputs = ",max_outputs=4";
1867 }
1868 }
1869 }
1870
1871 die "no devicetype for $vga->{type}\n" if !$type;
1872
1873 my $memory = "";
1874 if ($vgamem_mb) {
1875 if ($vga->{type} =~ /^virtio/) {
1876 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1877 $memory = ",max_hostmem=$bytes";
1878 } elsif ($qxlnum) {
1879 # from https://www.spice-space.org/multiple-monitors.html
1880 $memory = ",vgamem_mb=$vga->{memory}";
1881 my $ram = $vgamem_mb * 4;
1882 my $vram = $vgamem_mb * 2;
1883 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1884 } else {
1885 $memory = ",vgamem_mb=$vga->{memory}";
1886 }
1887 } elsif ($qxlnum && $id) {
1888 $memory = ",ram_size=67108864,vram_size=33554432";
1889 }
1890
1891 my $edidoff = "";
1892 if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1893 $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1894 }
1895
1896 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1897 my $vgaid = "vga" . ($id // '');
1898 my $pciaddr;
1899 if ($q35 && $vgaid eq 'vga') {
1900 # the first display uses pcie.0 bus on q35 machines
1901 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1902 } else {
1903 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1904 }
1905
1906 if ($vga->{type} eq 'virtio-gl') {
1907 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1908 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1909 if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1910
1911 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1912 if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1913 }
1914
1915 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1916 }
1917
1918 sub parse_number_sets {
1919 my ($set) = @_;
1920 my $res = [];
1921 foreach my $part (split(/;/, $set)) {
1922 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1923 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1924 push @$res, [ $1, $2 ];
1925 } else {
1926 die "invalid range: $part\n";
1927 }
1928 }
1929 return $res;
1930 }
1931
1932 sub parse_numa {
1933 my ($data) = @_;
1934
1935 my $res = parse_property_string($numa_fmt, $data);
1936 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1937 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1938 return $res;
1939 }
1940
1941 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1942 sub parse_net {
1943 my ($data, $disable_mac_autogen) = @_;
1944
1945 my $res = eval { parse_property_string($net_fmt, $data) };
1946 if ($@) {
1947 warn $@;
1948 return;
1949 }
1950 if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1951 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1952 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1953 }
1954 return $res;
1955 }
1956
1957 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1958 sub parse_ipconfig {
1959 my ($data) = @_;
1960
1961 my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1962 if ($@) {
1963 warn $@;
1964 return;
1965 }
1966
1967 if ($res->{gw} && !$res->{ip}) {
1968 warn 'gateway specified without specifying an IP address';
1969 return;
1970 }
1971 if ($res->{gw6} && !$res->{ip6}) {
1972 warn 'IPv6 gateway specified without specifying an IPv6 address';
1973 return;
1974 }
1975 if ($res->{gw} && $res->{ip} eq 'dhcp') {
1976 warn 'gateway specified together with DHCP';
1977 return;
1978 }
1979 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1980 # gw6 + auto/dhcp
1981 warn "IPv6 gateway specified together with $res->{ip6} address";
1982 return;
1983 }
1984
1985 if (!$res->{ip} && !$res->{ip6}) {
1986 return { ip => 'dhcp', ip6 => 'dhcp' };
1987 }
1988
1989 return $res;
1990 }
1991
1992 sub print_net {
1993 my $net = shift;
1994
1995 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1996 }
1997
1998 sub add_random_macs {
1999 my ($settings) = @_;
2000
2001 foreach my $opt (keys %$settings) {
2002 next if $opt !~ m/^net(\d+)$/;
2003 my $net = parse_net($settings->{$opt});
2004 next if !$net;
2005 $settings->{$opt} = print_net($net);
2006 }
2007 }
2008
2009 sub vm_is_volid_owner {
2010 my ($storecfg, $vmid, $volid) = @_;
2011
2012 if ($volid !~ m|^/|) {
2013 my ($path, $owner);
2014 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2015 if ($owner && ($owner == $vmid)) {
2016 return 1;
2017 }
2018 }
2019
2020 return;
2021 }
2022
2023 sub vmconfig_register_unused_drive {
2024 my ($storecfg, $vmid, $conf, $drive) = @_;
2025
2026 if (drive_is_cloudinit($drive)) {
2027 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2028 warn $@ if $@;
2029 delete $conf->{cloudinit};
2030 } elsif (!drive_is_cdrom($drive)) {
2031 my $volid = $drive->{file};
2032 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2033 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2034 }
2035 }
2036 }
2037
2038 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2039 my $smbios1_fmt = {
2040 uuid => {
2041 type => 'string',
2042 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2043 format_description => 'UUID',
2044 description => "Set SMBIOS1 UUID.",
2045 optional => 1,
2046 },
2047 version => {
2048 type => 'string',
2049 pattern => '[A-Za-z0-9+\/]+={0,2}',
2050 format_description => 'Base64 encoded string',
2051 description => "Set SMBIOS1 version.",
2052 optional => 1,
2053 },
2054 serial => {
2055 type => 'string',
2056 pattern => '[A-Za-z0-9+\/]+={0,2}',
2057 format_description => 'Base64 encoded string',
2058 description => "Set SMBIOS1 serial number.",
2059 optional => 1,
2060 },
2061 manufacturer => {
2062 type => 'string',
2063 pattern => '[A-Za-z0-9+\/]+={0,2}',
2064 format_description => 'Base64 encoded string',
2065 description => "Set SMBIOS1 manufacturer.",
2066 optional => 1,
2067 },
2068 product => {
2069 type => 'string',
2070 pattern => '[A-Za-z0-9+\/]+={0,2}',
2071 format_description => 'Base64 encoded string',
2072 description => "Set SMBIOS1 product ID.",
2073 optional => 1,
2074 },
2075 sku => {
2076 type => 'string',
2077 pattern => '[A-Za-z0-9+\/]+={0,2}',
2078 format_description => 'Base64 encoded string',
2079 description => "Set SMBIOS1 SKU string.",
2080 optional => 1,
2081 },
2082 family => {
2083 type => 'string',
2084 pattern => '[A-Za-z0-9+\/]+={0,2}',
2085 format_description => 'Base64 encoded string',
2086 description => "Set SMBIOS1 family string.",
2087 optional => 1,
2088 },
2089 base64 => {
2090 type => 'boolean',
2091 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2092 optional => 1,
2093 },
2094 };
2095
2096 sub parse_smbios1 {
2097 my ($data) = @_;
2098
2099 my $res = eval { parse_property_string($smbios1_fmt, $data) };
2100 warn $@ if $@;
2101 return $res;
2102 }
2103
2104 sub print_smbios1 {
2105 my ($smbios1) = @_;
2106 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2107 }
2108
2109 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2110
2111 sub parse_watchdog {
2112 my ($value) = @_;
2113
2114 return if !$value;
2115
2116 my $res = eval { parse_property_string($watchdog_fmt, $value) };
2117 warn $@ if $@;
2118 return $res;
2119 }
2120
2121 sub parse_guest_agent {
2122 my ($conf) = @_;
2123
2124 return {} if !defined($conf->{agent});
2125
2126 my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2127 warn $@ if $@;
2128
2129 # if the agent is disabled ignore the other potentially set properties
2130 return {} if !$res->{enabled};
2131 return $res;
2132 }
2133
2134 sub get_qga_key {
2135 my ($conf, $key) = @_;
2136 return undef if !defined($conf->{agent});
2137
2138 my $agent = parse_guest_agent($conf);
2139 return $agent->{$key};
2140 }
2141
2142 sub parse_vga {
2143 my ($value) = @_;
2144
2145 return {} if !$value;
2146 my $res = eval { parse_property_string($vga_fmt, $value) };
2147 warn $@ if $@;
2148 return $res;
2149 }
2150
2151 sub parse_rng {
2152 my ($value) = @_;
2153
2154 return if !$value;
2155
2156 my $res = eval { parse_property_string($rng_fmt, $value) };
2157 warn $@ if $@;
2158 return $res;
2159 }
2160
2161 sub parse_meta_info {
2162 my ($value) = @_;
2163
2164 return if !$value;
2165
2166 my $res = eval { parse_property_string($meta_info_fmt, $value) };
2167 warn $@ if $@;
2168 return $res;
2169 }
2170
2171 sub new_meta_info_string {
2172 my () = @_; # for now do not allow to override any value
2173
2174 return PVE::JSONSchema::print_property_string(
2175 {
2176 'creation-qemu' => kvm_user_version(),
2177 ctime => "". int(time()),
2178 },
2179 $meta_info_fmt
2180 );
2181 }
2182
2183 sub qemu_created_version_fixups {
2184 my ($conf, $forcemachine, $kvmver) = @_;
2185
2186 my $meta = parse_meta_info($conf->{meta}) // {};
2187 my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2188
2189 # check if we need to apply some handling for VMs that always use the latest machine version but
2190 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2191 # would be required (we do not want to pin machine version for non-windows OS type)
2192 if (
2193 (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2194 && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2195 && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2196 && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2197 ) {
2198 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2199 if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2200 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2201 # and thus with the predictable interface naming of systemd
2202 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2203 }
2204 }
2205 return;
2206 }
2207
2208 # add JSON properties for create and set function
2209 sub json_config_properties {
2210 my ($prop, $with_disk_alloc) = @_;
2211
2212 my $skip_json_config_opts = {
2213 parent => 1,
2214 snaptime => 1,
2215 vmstate => 1,
2216 runningmachine => 1,
2217 runningcpu => 1,
2218 meta => 1,
2219 };
2220
2221 foreach my $opt (keys %$confdesc) {
2222 next if $skip_json_config_opts->{$opt};
2223
2224 if ($with_disk_alloc && is_valid_drivename($opt)) {
2225 $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2226 } else {
2227 $prop->{$opt} = $confdesc->{$opt};
2228 }
2229 }
2230
2231 return $prop;
2232 }
2233
2234 # Properties that we can read from an OVF file
2235 sub json_ovf_properties {
2236 my $prop = {};
2237
2238 for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2239 $prop->{$device} = {
2240 type => 'string',
2241 format => 'pve-volume-id-or-absolute-path',
2242 description => "Disk image that gets imported to $device",
2243 optional => 1,
2244 };
2245 }
2246
2247 $prop->{cores} = {
2248 type => 'integer',
2249 description => "The number of CPU cores.",
2250 optional => 1,
2251 };
2252 $prop->{memory} = {
2253 type => 'integer',
2254 description => "Amount of RAM for the VM in MB.",
2255 optional => 1,
2256 };
2257 $prop->{name} = {
2258 type => 'string',
2259 description => "Name of the VM.",
2260 optional => 1,
2261 };
2262
2263 return $prop;
2264 }
2265
2266 # return copy of $confdesc_cloudinit to generate documentation
2267 sub cloudinit_config_properties {
2268
2269 return dclone($confdesc_cloudinit);
2270 }
2271
2272 sub cloudinit_pending_properties {
2273 my $p = {
2274 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2275 name => 1,
2276 };
2277 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2278 return $p;
2279 }
2280
2281 sub check_type {
2282 my ($key, $value) = @_;
2283
2284 die "unknown setting '$key'\n" if !$confdesc->{$key};
2285
2286 my $type = $confdesc->{$key}->{type};
2287
2288 if (!defined($value)) {
2289 die "got undefined value\n";
2290 }
2291
2292 if ($value =~ m/[\n\r]/) {
2293 die "property contains a line feed\n";
2294 }
2295
2296 if ($type eq 'boolean') {
2297 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2298 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2299 die "type check ('boolean') failed - got '$value'\n";
2300 } elsif ($type eq 'integer') {
2301 return int($1) if $value =~ m/^(\d+)$/;
2302 die "type check ('integer') failed - got '$value'\n";
2303 } elsif ($type eq 'number') {
2304 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2305 die "type check ('number') failed - got '$value'\n";
2306 } elsif ($type eq 'string') {
2307 if (my $fmt = $confdesc->{$key}->{format}) {
2308 PVE::JSONSchema::check_format($fmt, $value);
2309 return $value;
2310 }
2311 $value =~ s/^\"(.*)\"$/$1/;
2312 return $value;
2313 } else {
2314 die "internal error"
2315 }
2316 }
2317
2318 sub destroy_vm {
2319 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2320
2321 my $conf = PVE::QemuConfig->load_config($vmid);
2322
2323 if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2324 PVE::QemuConfig->check_lock($conf);
2325 }
2326
2327 if ($conf->{template}) {
2328 # check if any base image is still used by a linked clone
2329 PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2330 my ($ds, $drive) = @_;
2331 return if drive_is_cdrom($drive);
2332
2333 my $volid = $drive->{file};
2334 return if !$volid || $volid =~ m|^/|;
2335
2336 die "base volume '$volid' is still in use by linked cloned\n"
2337 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2338
2339 });
2340 }
2341
2342 my $volids = {};
2343 my $remove_owned_drive = sub {
2344 my ($ds, $drive) = @_;
2345 return if drive_is_cdrom($drive, 1);
2346
2347 my $volid = $drive->{file};
2348 return if !$volid || $volid =~ m|^/|;
2349 return if $volids->{$volid};
2350
2351 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2352 return if !$path || !$owner || ($owner != $vmid);
2353
2354 $volids->{$volid} = 1;
2355 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2356 warn "Could not remove disk '$volid', check manually: $@" if $@;
2357 };
2358
2359 # only remove disks owned by this VM (referenced in the config)
2360 my $include_opts = {
2361 include_unused => 1,
2362 extra_keys => ['vmstate'],
2363 };
2364 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2365
2366 for my $snap (values %{$conf->{snapshots}}) {
2367 next if !defined($snap->{vmstate});
2368 my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2369 next if !defined($drive);
2370 $remove_owned_drive->('vmstate', $drive);
2371 }
2372
2373 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2374
2375 if ($purge_unreferenced) { # also remove unreferenced disk
2376 my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2377 PVE::Storage::foreach_volid($vmdisks, sub {
2378 my ($volid, $sid, $volname, $d) = @_;
2379 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2380 warn $@ if $@;
2381 });
2382 }
2383
2384 if (defined $replacement_conf) {
2385 PVE::QemuConfig->write_config($vmid, $replacement_conf);
2386 } else {
2387 PVE::QemuConfig->destroy_config($vmid);
2388 }
2389 }
2390
2391 sub parse_vm_config {
2392 my ($filename, $raw, $strict) = @_;
2393
2394 return if !defined($raw);
2395
2396 my $res = {
2397 digest => Digest::SHA::sha1_hex($raw),
2398 snapshots => {},
2399 pending => {},
2400 cloudinit => {},
2401 };
2402
2403 my $handle_error = sub {
2404 my ($msg) = @_;
2405
2406 if ($strict) {
2407 die $msg;
2408 } else {
2409 warn $msg;
2410 }
2411 };
2412
2413 $filename =~ m|/qemu-server/(\d+)\.conf$|
2414 || die "got strange filename '$filename'";
2415
2416 my $vmid = $1;
2417
2418 my $conf = $res;
2419 my $descr;
2420 my $finish_description = sub {
2421 if (defined($descr)) {
2422 $descr =~ s/\s+$//;
2423 $conf->{description} = $descr;
2424 }
2425 $descr = undef;
2426 };
2427 my $section = '';
2428
2429 my @lines = split(/\n/, $raw);
2430 foreach my $line (@lines) {
2431 next if $line =~ m/^\s*$/;
2432
2433 if ($line =~ m/^\[PENDING\]\s*$/i) {
2434 $section = 'pending';
2435 $finish_description->();
2436 $conf = $res->{$section} = {};
2437 next;
2438 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2439 $section = 'cloudinit';
2440 $finish_description->();
2441 $conf = $res->{$section} = {};
2442 next;
2443
2444 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2445 $section = $1;
2446 $finish_description->();
2447 $conf = $res->{snapshots}->{$section} = {};
2448 next;
2449 }
2450
2451 if ($line =~ m/^\#(.*)$/) {
2452 $descr = '' if !defined($descr);
2453 $descr .= PVE::Tools::decode_text($1) . "\n";
2454 next;
2455 }
2456
2457 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2458 $descr = '' if !defined($descr);
2459 $descr .= PVE::Tools::decode_text($2);
2460 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2461 $conf->{snapstate} = $1;
2462 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2463 my $key = $1;
2464 my $value = $2;
2465 $conf->{$key} = $value;
2466 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2467 my $value = $1;
2468 if ($section eq 'pending') {
2469 $conf->{delete} = $value; # we parse this later
2470 } else {
2471 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2472 }
2473 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2474 my $key = $1;
2475 my $value = $2;
2476 if ($section eq 'cloudinit') {
2477 # ignore validation only used for informative purpose
2478 $conf->{$key} = $value;
2479 next;
2480 }
2481 eval { $value = check_type($key, $value); };
2482 if ($@) {
2483 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2484 } else {
2485 $key = 'ide2' if $key eq 'cdrom';
2486 my $fmt = $confdesc->{$key}->{format};
2487 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2488 my $v = parse_drive($key, $value);
2489 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2490 $v->{file} = $volid;
2491 $value = print_drive($v);
2492 } else {
2493 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2494 next;
2495 }
2496 }
2497
2498 $conf->{$key} = $value;
2499 }
2500 } else {
2501 $handle_error->("vm $vmid - unable to parse config: $line\n");
2502 }
2503 }
2504
2505 $finish_description->();
2506 delete $res->{snapstate}; # just to be sure
2507
2508 return $res;
2509 }
2510
2511 sub write_vm_config {
2512 my ($filename, $conf) = @_;
2513
2514 delete $conf->{snapstate}; # just to be sure
2515
2516 if ($conf->{cdrom}) {
2517 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2518 $conf->{ide2} = $conf->{cdrom};
2519 delete $conf->{cdrom};
2520 }
2521
2522 # we do not use 'smp' any longer
2523 if ($conf->{sockets}) {
2524 delete $conf->{smp};
2525 } elsif ($conf->{smp}) {
2526 $conf->{sockets} = $conf->{smp};
2527 delete $conf->{cores};
2528 delete $conf->{smp};
2529 }
2530
2531 my $used_volids = {};
2532
2533 my $cleanup_config = sub {
2534 my ($cref, $pending, $snapname) = @_;
2535
2536 foreach my $key (keys %$cref) {
2537 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2538 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2539 my $value = $cref->{$key};
2540 if ($key eq 'delete') {
2541 die "propertry 'delete' is only allowed in [PENDING]\n"
2542 if !$pending;
2543 # fixme: check syntax?
2544 next;
2545 }
2546 eval { $value = check_type($key, $value); };
2547 die "unable to parse value of '$key' - $@" if $@;
2548
2549 $cref->{$key} = $value;
2550
2551 if (!$snapname && is_valid_drivename($key)) {
2552 my $drive = parse_drive($key, $value);
2553 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2554 }
2555 }
2556 };
2557
2558 &$cleanup_config($conf);
2559
2560 &$cleanup_config($conf->{pending}, 1);
2561
2562 foreach my $snapname (keys %{$conf->{snapshots}}) {
2563 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2564 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2565 }
2566
2567 # remove 'unusedX' settings if we re-add a volume
2568 foreach my $key (keys %$conf) {
2569 my $value = $conf->{$key};
2570 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2571 delete $conf->{$key};
2572 }
2573 }
2574
2575 my $generate_raw_config = sub {
2576 my ($conf, $pending) = @_;
2577
2578 my $raw = '';
2579
2580 # add description as comment to top of file
2581 if (defined(my $descr = $conf->{description})) {
2582 if ($descr) {
2583 foreach my $cl (split(/\n/, $descr)) {
2584 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2585 }
2586 } else {
2587 $raw .= "#\n" if $pending;
2588 }
2589 }
2590
2591 foreach my $key (sort keys %$conf) {
2592 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2593 $raw .= "$key: $conf->{$key}\n";
2594 }
2595 return $raw;
2596 };
2597
2598 my $raw = &$generate_raw_config($conf);
2599
2600 if (scalar(keys %{$conf->{pending}})){
2601 $raw .= "\n[PENDING]\n";
2602 $raw .= &$generate_raw_config($conf->{pending}, 1);
2603 }
2604
2605 if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2606 $raw .= "\n[special:cloudinit]\n";
2607 $raw .= &$generate_raw_config($conf->{cloudinit});
2608 }
2609
2610 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2611 $raw .= "\n[$snapname]\n";
2612 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2613 }
2614
2615 return $raw;
2616 }
2617
2618 sub load_defaults {
2619
2620 my $res = {};
2621
2622 # we use static defaults from our JSON schema configuration
2623 foreach my $key (keys %$confdesc) {
2624 if (defined(my $default = $confdesc->{$key}->{default})) {
2625 $res->{$key} = $default;
2626 }
2627 }
2628
2629 return $res;
2630 }
2631
2632 sub config_list {
2633 my $vmlist = PVE::Cluster::get_vmlist();
2634 my $res = {};
2635 return $res if !$vmlist || !$vmlist->{ids};
2636 my $ids = $vmlist->{ids};
2637 my $nodename = nodename();
2638
2639 foreach my $vmid (keys %$ids) {
2640 my $d = $ids->{$vmid};
2641 next if !$d->{node} || $d->{node} ne $nodename;
2642 next if !$d->{type} || $d->{type} ne 'qemu';
2643 $res->{$vmid}->{exists} = 1;
2644 }
2645 return $res;
2646 }
2647
2648 # test if VM uses local resources (to prevent migration)
2649 sub check_local_resources {
2650 my ($conf, $noerr) = @_;
2651
2652 my @loc_res = ();
2653
2654 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2655 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2656
2657 push @loc_res, "ivshmem" if $conf->{ivshmem};
2658
2659 foreach my $k (keys %$conf) {
2660 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2661 # sockets are safe: they will recreated be on the target side post-migrate
2662 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2663 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2664 }
2665
2666 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2667
2668 return \@loc_res;
2669 }
2670
2671 # check if used storages are available on all nodes (use by migrate)
2672 sub check_storage_availability {
2673 my ($storecfg, $conf, $node) = @_;
2674
2675 PVE::QemuConfig->foreach_volume($conf, sub {
2676 my ($ds, $drive) = @_;
2677
2678 my $volid = $drive->{file};
2679 return if !$volid;
2680
2681 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2682 return if !$sid;
2683
2684 # check if storage is available on both nodes
2685 my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2686 PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2687
2688 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2689
2690 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2691 if !$scfg->{content}->{$vtype};
2692 });
2693 }
2694
2695 # list nodes where all VM images are available (used by has_feature API)
2696 sub shared_nodes {
2697 my ($conf, $storecfg) = @_;
2698
2699 my $nodelist = PVE::Cluster::get_nodelist();
2700 my $nodehash = { map { $_ => 1 } @$nodelist };
2701 my $nodename = nodename();
2702
2703 PVE::QemuConfig->foreach_volume($conf, sub {
2704 my ($ds, $drive) = @_;
2705
2706 my $volid = $drive->{file};
2707 return if !$volid;
2708
2709 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2710 if ($storeid) {
2711 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2712 if ($scfg->{disable}) {
2713 $nodehash = {};
2714 } elsif (my $avail = $scfg->{nodes}) {
2715 foreach my $node (keys %$nodehash) {
2716 delete $nodehash->{$node} if !$avail->{$node};
2717 }
2718 } elsif (!$scfg->{shared}) {
2719 foreach my $node (keys %$nodehash) {
2720 delete $nodehash->{$node} if $node ne $nodename
2721 }
2722 }
2723 }
2724 });
2725
2726 return $nodehash
2727 }
2728
2729 sub check_local_storage_availability {
2730 my ($conf, $storecfg) = @_;
2731
2732 my $nodelist = PVE::Cluster::get_nodelist();
2733 my $nodehash = { map { $_ => {} } @$nodelist };
2734
2735 PVE::QemuConfig->foreach_volume($conf, sub {
2736 my ($ds, $drive) = @_;
2737
2738 my $volid = $drive->{file};
2739 return if !$volid;
2740
2741 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2742 if ($storeid) {
2743 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2744
2745 if ($scfg->{disable}) {
2746 foreach my $node (keys %$nodehash) {
2747 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2748 }
2749 } elsif (my $avail = $scfg->{nodes}) {
2750 foreach my $node (keys %$nodehash) {
2751 if (!$avail->{$node}) {
2752 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2753 }
2754 }
2755 }
2756 }
2757 });
2758
2759 foreach my $node (values %$nodehash) {
2760 if (my $unavail = $node->{unavailable_storages}) {
2761 $node->{unavailable_storages} = [ sort keys %$unavail ];
2762 }
2763 }
2764
2765 return $nodehash
2766 }
2767
2768 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2769 sub check_running {
2770 my ($vmid, $nocheck, $node) = @_;
2771
2772 # $nocheck is set when called during a migration, in which case the config
2773 # file might still or already reside on the *other* node
2774 # - because rename has already happened, and current node is source
2775 # - because rename hasn't happened yet, and current node is target
2776 # - because rename has happened, current node is target, but hasn't yet
2777 # processed it yet
2778 PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2779 return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2780 }
2781
2782 sub vzlist {
2783
2784 my $vzlist = config_list();
2785
2786 my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2787
2788 while (defined(my $de = $fd->read)) {
2789 next if $de !~ m/^(\d+)\.pid$/;
2790 my $vmid = $1;
2791 next if !defined($vzlist->{$vmid});
2792 if (my $pid = check_running($vmid)) {
2793 $vzlist->{$vmid}->{pid} = $pid;
2794 }
2795 }
2796
2797 return $vzlist;
2798 }
2799
2800 our $vmstatus_return_properties = {
2801 vmid => get_standard_option('pve-vmid'),
2802 status => {
2803 description => "QEMU process status.",
2804 type => 'string',
2805 enum => ['stopped', 'running'],
2806 },
2807 maxmem => {
2808 description => "Maximum memory in bytes.",
2809 type => 'integer',
2810 optional => 1,
2811 renderer => 'bytes',
2812 },
2813 maxdisk => {
2814 description => "Root disk size in bytes.",
2815 type => 'integer',
2816 optional => 1,
2817 renderer => 'bytes',
2818 },
2819 name => {
2820 description => "VM name.",
2821 type => 'string',
2822 optional => 1,
2823 },
2824 qmpstatus => {
2825 description => "VM run state from the 'query-status' QMP monitor command.",
2826 type => 'string',
2827 optional => 1,
2828 },
2829 pid => {
2830 description => "PID of running qemu process.",
2831 type => 'integer',
2832 optional => 1,
2833 },
2834 uptime => {
2835 description => "Uptime.",
2836 type => 'integer',
2837 optional => 1,
2838 renderer => 'duration',
2839 },
2840 cpus => {
2841 description => "Maximum usable CPUs.",
2842 type => 'number',
2843 optional => 1,
2844 },
2845 lock => {
2846 description => "The current config lock, if any.",
2847 type => 'string',
2848 optional => 1,
2849 },
2850 tags => {
2851 description => "The current configured tags, if any",
2852 type => 'string',
2853 optional => 1,
2854 },
2855 'running-machine' => {
2856 description => "The currently running machine type (if running).",
2857 type => 'string',
2858 optional => 1,
2859 },
2860 'running-qemu' => {
2861 description => "The currently running QEMU version (if running).",
2862 type => 'string',
2863 optional => 1,
2864 },
2865 };
2866
2867 my $last_proc_pid_stat;
2868
2869 # get VM status information
2870 # This must be fast and should not block ($full == false)
2871 # We only query KVM using QMP if $full == true (this can be slow)
2872 sub vmstatus {
2873 my ($opt_vmid, $full) = @_;
2874
2875 my $res = {};
2876
2877 my $storecfg = PVE::Storage::config();
2878
2879 my $list = vzlist();
2880 my $defaults = load_defaults();
2881
2882 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2883
2884 my $cpucount = $cpuinfo->{cpus} || 1;
2885
2886 foreach my $vmid (keys %$list) {
2887 next if $opt_vmid && ($vmid ne $opt_vmid);
2888
2889 my $conf = PVE::QemuConfig->load_config($vmid);
2890
2891 my $d = { vmid => int($vmid) };
2892 $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2893
2894 # fixme: better status?
2895 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2896
2897 my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2898 if (defined($size)) {
2899 $d->{disk} = 0; # no info available
2900 $d->{maxdisk} = $size;
2901 } else {
2902 $d->{disk} = 0;
2903 $d->{maxdisk} = 0;
2904 }
2905
2906 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2907 * ($conf->{cores} || $defaults->{cores});
2908 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2909 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2910
2911 $d->{name} = $conf->{name} || "VM $vmid";
2912 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2913 : $defaults->{memory}*(1024*1024);
2914
2915 if ($conf->{balloon}) {
2916 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2917 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2918 : $defaults->{shares};
2919 }
2920
2921 $d->{uptime} = 0;
2922 $d->{cpu} = 0;
2923 $d->{mem} = 0;
2924
2925 $d->{netout} = 0;
2926 $d->{netin} = 0;
2927
2928 $d->{diskread} = 0;
2929 $d->{diskwrite} = 0;
2930
2931 $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2932
2933 $d->{serial} = 1 if conf_has_serial($conf);
2934 $d->{lock} = $conf->{lock} if $conf->{lock};
2935 $d->{tags} = $conf->{tags} if defined($conf->{tags});
2936
2937 $res->{$vmid} = $d;
2938 }
2939
2940 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2941 foreach my $dev (keys %$netdev) {
2942 next if $dev !~ m/^tap([1-9]\d*)i/;
2943 my $vmid = $1;
2944 my $d = $res->{$vmid};
2945 next if !$d;
2946
2947 $d->{netout} += $netdev->{$dev}->{receive};
2948 $d->{netin} += $netdev->{$dev}->{transmit};
2949
2950 if ($full) {
2951 $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2952 $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2953 }
2954
2955 }
2956
2957 my $ctime = gettimeofday;
2958
2959 foreach my $vmid (keys %$list) {
2960
2961 my $d = $res->{$vmid};
2962 my $pid = $d->{pid};
2963 next if !$pid;
2964
2965 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2966 next if !$pstat; # not running
2967
2968 my $used = $pstat->{utime} + $pstat->{stime};
2969
2970 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2971
2972 if ($pstat->{vsize}) {
2973 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2974 }
2975
2976 my $old = $last_proc_pid_stat->{$pid};
2977 if (!$old) {
2978 $last_proc_pid_stat->{$pid} = {
2979 time => $ctime,
2980 used => $used,
2981 cpu => 0,
2982 };
2983 next;
2984 }
2985
2986 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2987
2988 if ($dtime > 1000) {
2989 my $dutime = $used - $old->{used};
2990
2991 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2992 $last_proc_pid_stat->{$pid} = {
2993 time => $ctime,
2994 used => $used,
2995 cpu => $d->{cpu},
2996 };
2997 } else {
2998 $d->{cpu} = $old->{cpu};
2999 }
3000 }
3001
3002 return $res if !$full;
3003
3004 my $qmpclient = PVE::QMPClient->new();
3005
3006 my $ballooncb = sub {
3007 my ($vmid, $resp) = @_;
3008
3009 my $info = $resp->{'return'};
3010 return if !$info->{max_mem};
3011
3012 my $d = $res->{$vmid};
3013
3014 # use memory assigned to VM
3015 $d->{maxmem} = $info->{max_mem};
3016 $d->{balloon} = $info->{actual};
3017
3018 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3019 $d->{mem} = $info->{total_mem} - $info->{free_mem};
3020 $d->{freemem} = $info->{free_mem};
3021 }
3022
3023 $d->{ballooninfo} = $info;
3024 };
3025
3026 my $blockstatscb = sub {
3027 my ($vmid, $resp) = @_;
3028 my $data = $resp->{'return'} || [];
3029 my $totalrdbytes = 0;
3030 my $totalwrbytes = 0;
3031
3032 for my $blockstat (@$data) {
3033 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3034 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3035
3036 $blockstat->{device} =~ s/drive-//;
3037 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3038 }
3039 $res->{$vmid}->{diskread} = $totalrdbytes;
3040 $res->{$vmid}->{diskwrite} = $totalwrbytes;
3041 };
3042
3043 my $machinecb = sub {
3044 my ($vmid, $resp) = @_;
3045 my $data = $resp->{'return'} || [];
3046
3047 $res->{$vmid}->{'running-machine'} =
3048 PVE::QemuServer::Machine::current_from_query_machines($data);
3049 };
3050
3051 my $versioncb = sub {
3052 my ($vmid, $resp) = @_;
3053 my $data = $resp->{'return'} // {};
3054 my $version = 'unknown';
3055
3056 if (my $v = $data->{qemu}) {
3057 $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3058 }
3059
3060 $res->{$vmid}->{'running-qemu'} = $version;
3061 };
3062
3063 my $statuscb = sub {
3064 my ($vmid, $resp) = @_;
3065
3066 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3067 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3068 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3069 # this fails if ballon driver is not loaded, so this must be
3070 # the last commnand (following command are aborted if this fails).
3071 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3072
3073 my $status = 'unknown';
3074 if (!defined($status = $resp->{'return'}->{status})) {
3075 warn "unable to get VM status\n";
3076 return;
3077 }
3078
3079 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3080 };
3081
3082 foreach my $vmid (keys %$list) {
3083 next if $opt_vmid && ($vmid ne $opt_vmid);
3084 next if !$res->{$vmid}->{pid}; # not running
3085 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3086 }
3087
3088 $qmpclient->queue_execute(undef, 2);
3089
3090 foreach my $vmid (keys %$list) {
3091 next if $opt_vmid && ($vmid ne $opt_vmid);
3092 next if !$res->{$vmid}->{pid}; #not running
3093
3094 # we can't use the $qmpclient since it might have already aborted on
3095 # 'query-balloon', but this might also fail for older versions...
3096 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3097 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3098 }
3099
3100 foreach my $vmid (keys %$list) {
3101 next if $opt_vmid && ($vmid ne $opt_vmid);
3102 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3103 }
3104
3105 return $res;
3106 }
3107
3108 sub conf_has_serial {
3109 my ($conf) = @_;
3110
3111 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3112 if ($conf->{"serial$i"}) {
3113 return 1;
3114 }
3115 }
3116
3117 return 0;
3118 }
3119
3120 sub conf_has_audio {
3121 my ($conf, $id) = @_;
3122
3123 $id //= 0;
3124 my $audio = $conf->{"audio$id"};
3125 return if !defined($audio);
3126
3127 my $audioproperties = parse_property_string($audio_fmt, $audio);
3128 my $audiodriver = $audioproperties->{driver} // 'spice';
3129
3130 return {
3131 dev => $audioproperties->{device},
3132 dev_id => "audiodev$id",
3133 backend => $audiodriver,
3134 backend_id => "$audiodriver-backend${id}",
3135 };
3136 }
3137
3138 sub audio_devs {
3139 my ($audio, $audiopciaddr, $machine_version) = @_;
3140
3141 my $devs = [];
3142
3143 my $id = $audio->{dev_id};
3144 my $audiodev = "";
3145 if (min_version($machine_version, 4, 2)) {
3146 $audiodev = ",audiodev=$audio->{backend_id}";
3147 }
3148
3149 if ($audio->{dev} eq 'AC97') {
3150 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3151 } elsif ($audio->{dev} =~ /intel\-hda$/) {
3152 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3153 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3154 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3155 } else {
3156 die "unkown audio device '$audio->{dev}', implement me!";
3157 }
3158
3159 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3160
3161 return $devs;
3162 }
3163
3164 sub get_tpm_paths {
3165 my ($vmid) = @_;
3166 return {
3167 socket => "/var/run/qemu-server/$vmid.swtpm",
3168 pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3169 };
3170 }
3171
3172 sub add_tpm_device {
3173 my ($vmid, $devices, $conf) = @_;
3174
3175 return if !$conf->{tpmstate0};
3176
3177 my $paths = get_tpm_paths($vmid);
3178
3179 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3180 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3181 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3182 }
3183
3184 sub start_swtpm {
3185 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3186
3187 return if !$tpmdrive;
3188
3189 my $state;
3190 my $tpm = parse_drive("tpmstate0", $tpmdrive);
3191 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3192 if ($storeid) {
3193 $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3194 } else {
3195 $state = $tpm->{file};
3196 }
3197
3198 my $paths = get_tpm_paths($vmid);
3199
3200 # during migration, we will get state from remote
3201 #
3202 if (!$migration) {
3203 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3204 my $setup_cmd = [
3205 "swtpm_setup",
3206 "--tpmstate",
3207 "file://$state",
3208 "--createek",
3209 "--create-ek-cert",
3210 "--create-platform-cert",
3211 "--lock-nvram",
3212 "--config",
3213 "/etc/swtpm_setup.conf", # do not use XDG configs
3214 "--runas",
3215 "0", # force creation as root, error if not possible
3216 "--not-overwrite", # ignore existing state, do not modify
3217 ];
3218
3219 push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3220 # TPM 2.0 supports ECC crypto, use if possible
3221 push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3222
3223 run_command($setup_cmd, outfunc => sub {
3224 print "swtpm_setup: $1\n";
3225 });
3226 }
3227
3228 # Used to distinguish different invocations in the log.
3229 my $log_prefix = "[id=" . int(time()) . "] ";
3230
3231 my $emulator_cmd = [
3232 "swtpm",
3233 "socket",
3234 "--tpmstate",
3235 "backend-uri=file://$state,mode=0600",
3236 "--ctrl",
3237 "type=unixio,path=$paths->{socket},mode=0600",
3238 "--pid",
3239 "file=$paths->{pid}",
3240 "--terminate", # terminate on QEMU disconnect
3241 "--daemon",
3242 "--log",
3243 "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3244 ];
3245 push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3246 run_command($emulator_cmd, outfunc => sub { print $1; });
3247
3248 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3249 while (! -e $paths->{pid}) {
3250 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3251 usleep(50_000);
3252 }
3253
3254 # return untainted PID of swtpm daemon so it can be killed on error
3255 file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3256 return $1;
3257 }
3258
3259 sub vga_conf_has_spice {
3260 my ($vga) = @_;
3261
3262 my $vgaconf = parse_vga($vga);
3263 my $vgatype = $vgaconf->{type};
3264 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3265
3266 return $1 || 1;
3267 }
3268
3269 sub is_native($) {
3270 my ($arch) = @_;
3271 return get_host_arch() eq $arch;
3272 }
3273
3274 sub get_vm_arch {
3275 my ($conf) = @_;
3276 return $conf->{arch} // get_host_arch();
3277 }
3278
3279 my $default_machines = {
3280 x86_64 => 'pc',
3281 aarch64 => 'virt',
3282 };
3283
3284 sub get_installed_machine_version {
3285 my ($kvmversion) = @_;
3286 $kvmversion = kvm_user_version() if !defined($kvmversion);
3287 $kvmversion =~ m/^(\d+\.\d+)/;
3288 return $1;
3289 }
3290
3291 sub windows_get_pinned_machine_version {
3292 my ($machine, $base_version, $kvmversion) = @_;
3293
3294 my $pin_version = $base_version;
3295 if (!defined($base_version) ||
3296 !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3297 ) {
3298 $pin_version = get_installed_machine_version($kvmversion);
3299 }
3300 if (!$machine || $machine eq 'pc') {
3301 $machine = "pc-i440fx-$pin_version";
3302 } elsif ($machine eq 'q35') {
3303 $machine = "pc-q35-$pin_version";
3304 } elsif ($machine eq 'virt') {
3305 $machine = "virt-$pin_version";
3306 } else {
3307 warn "unknown machine type '$machine', not touching that!\n";
3308 }
3309
3310 return $machine;
3311 }
3312
3313 sub get_vm_machine {
3314 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3315
3316 my $machine = $forcemachine || $conf->{machine};
3317
3318 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3319 $kvmversion //= kvm_user_version();
3320 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3321 # layout which confuses windows quite a bit and may result in various regressions..
3322 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3323 if (windows_version($conf->{ostype})) {
3324 $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3325 }
3326 $arch //= 'x86_64';
3327 $machine ||= $default_machines->{$arch};
3328 if ($add_pve_version) {
3329 my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3330 $machine .= "+pve$pvever";
3331 }
3332 }
3333
3334 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3335 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3336 $machine = $1 if $is_pxe;
3337
3338 # for version-pinned machines that do not include a pve-version (e.g.
3339 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3340 $machine .= '+pve0';
3341
3342 $machine .= '.pxe' if $is_pxe;
3343 }
3344
3345 return $machine;
3346 }
3347
3348 sub get_ovmf_files($$$) {
3349 my ($arch, $efidisk, $smm) = @_;
3350
3351 my $types = $OVMF->{$arch}
3352 or die "no OVMF images known for architecture '$arch'\n";
3353
3354 my $type = 'default';
3355 if ($arch ne "aarch64" && defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3356 $type = $smm ? "4m" : "4m-no-smm";
3357 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3358 }
3359
3360 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3361 die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3362 die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3363
3364 return ($ovmf_code, $ovmf_vars);
3365 }
3366
3367 my $Arch2Qemu = {
3368 aarch64 => '/usr/bin/qemu-system-aarch64',
3369 x86_64 => '/usr/bin/qemu-system-x86_64',
3370 };
3371 sub get_command_for_arch($) {
3372 my ($arch) = @_;
3373 return '/usr/bin/kvm' if is_native($arch);
3374
3375 my $cmd = $Arch2Qemu->{$arch}
3376 or die "don't know how to emulate architecture '$arch'\n";
3377 return $cmd;
3378 }
3379
3380 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3381 # to use in a QEMU command line (-cpu element), first array_intersect the result
3382 # of query_supported_ with query_understood_. This is necessary because:
3383 #
3384 # a) query_understood_ returns flags the host cannot use and
3385 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3386 # flags, but CPU settings - with most of them being flags. Those settings
3387 # (and some flags, curiously) cannot be specified as a "-cpu" argument.
3388 #
3389 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3390 # expensive. If you need the value returned from this, you can get it much
3391 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3392 # $accel being 'kvm' or 'tcg'.
3393 #
3394 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3395 # changes, automatically populating pmxcfs.
3396 #
3397 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3398 # since kvm and tcg machines support different flags
3399 #
3400 sub query_supported_cpu_flags {
3401 my ($arch) = @_;
3402
3403 $arch //= get_host_arch();
3404 my $default_machine = $default_machines->{$arch};
3405
3406 my $flags = {};
3407
3408 # FIXME: Once this is merged, the code below should work for ARM as well:
3409 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3410 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3411 $arch eq "aarch64";
3412
3413 my $kvm_supported = defined(kvm_version());
3414 my $qemu_cmd = get_command_for_arch($arch);
3415 my $fakevmid = -1;
3416 my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3417
3418 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3419 my $query_supported_run_qemu = sub {
3420 my ($kvm) = @_;
3421
3422 my $flags = {};
3423 my $cmd = [
3424 $qemu_cmd,
3425 '-machine', $default_machine,
3426 '-display', 'none',
3427 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3428 '-mon', 'chardev=qmp,mode=control',
3429 '-pidfile', $pidfile,
3430 '-S', '-daemonize'
3431 ];
3432
3433 if (!$kvm) {
3434 push @$cmd, '-accel', 'tcg';
3435 }
3436
3437 my $rc = run_command($cmd, noerr => 1, quiet => 0);
3438 die "QEMU flag querying VM exited with code " . $rc if $rc;
3439
3440 eval {
3441 my $cmd_result = mon_cmd(
3442 $fakevmid,
3443 'query-cpu-model-expansion',
3444 type => 'full',
3445 model => { name => 'host' }
3446 );
3447
3448 my $props = $cmd_result->{model}->{props};
3449 foreach my $prop (keys %$props) {
3450 next if $props->{$prop} ne '1';
3451 # QEMU returns some flags multiple times, with '_', '.' or '-'
3452 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3453 # We only keep those with underscores, to match /proc/cpuinfo
3454 $prop =~ s/\.|-/_/g;
3455 $flags->{$prop} = 1;
3456 }
3457 };
3458 my $err = $@;
3459
3460 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3461 vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3462
3463 die $err if $err;
3464
3465 return [ sort keys %$flags ];
3466 };
3467
3468 # We need to query QEMU twice, since KVM and TCG have different supported flags
3469 PVE::QemuConfig->lock_config($fakevmid, sub {
3470 $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3471 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3472
3473 if ($kvm_supported) {
3474 $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3475 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3476 }
3477 });
3478
3479 return $flags;
3480 }
3481
3482 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3483 my $understood_cpu_flag_dir = "/usr/share/kvm";
3484 sub query_understood_cpu_flags {
3485 my $arch = get_host_arch();
3486 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3487
3488 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3489 if ! -e $filepath;
3490
3491 my $raw = file_get_contents($filepath);
3492 $raw =~ s/^\s+|\s+$//g;
3493 my @flags = split(/\s+/, $raw);
3494
3495 return \@flags;
3496 }
3497
3498 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3499 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3500 my sub should_disable_smm {
3501 my ($conf, $vga, $machine) = @_;
3502
3503 return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3504
3505 return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3506 $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3507 }
3508
3509 my sub print_ovmf_drive_commandlines {
3510 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3511
3512 my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3513
3514 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3515
3516 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3517 if ($d) {
3518 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3519 my ($path, $format) = $d->@{'file', 'format'};
3520 if ($storeid) {
3521 $path = PVE::Storage::path($storecfg, $d->{file});
3522 if (!defined($format)) {
3523 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3524 $format = qemu_img_format($scfg, $volname);
3525 }
3526 } elsif (!defined($format)) {
3527 die "efidisk format must be specified\n";
3528 }
3529 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3530 if ($path =~ m/^rbd:/) {
3531 $var_drive_str .= ',cache=writeback';
3532 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3533 }
3534 $var_drive_str .= ",format=$format,file=$path";
3535
3536 $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3537 $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3538 } else {
3539 log_warn("no efidisk configured! Using temporary efivars disk.");
3540 my $path = "/tmp/$vmid-ovmf.fd";
3541 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3542 $var_drive_str .= ",format=raw,file=$path";
3543 $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3544 }
3545
3546 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3547 }
3548
3549 sub config_to_command {
3550 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3551 $pbs_backing) = @_;
3552
3553 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3554 my $devices = [];
3555 my $bridges = {};
3556 my $ostype = $conf->{ostype};
3557 my $winversion = windows_version($ostype);
3558 my $kvm = $conf->{kvm};
3559 my $nodename = nodename();
3560
3561 my $arch = get_vm_arch($conf);
3562 my $kvm_binary = get_command_for_arch($arch);
3563 my $kvmver = kvm_user_version($kvm_binary);
3564
3565 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3566 $kvmver //= "undefined";
3567 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3568 }
3569
3570 my $add_pve_version = min_version($kvmver, 4, 1);
3571
3572 my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3573 my $machine_version = extract_version($machine_type, $kvmver);
3574 $kvm //= 1 if is_native($arch);
3575
3576 $machine_version =~ m/(\d+)\.(\d+)/;
3577 my ($machine_major, $machine_minor) = ($1, $2);
3578
3579 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3580 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3581 } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3582 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3583 ." please upgrade node '$nodename'\n"
3584 } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3585 my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3586 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3587 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3588 ." node '$nodename'\n";
3589 }
3590
3591 # if a specific +pve version is required for a feature, use $version_guard
3592 # instead of min_version to allow machines to be run with the minimum
3593 # required version
3594 my $required_pve_version = 0;
3595 my $version_guard = sub {
3596 my ($major, $minor, $pve) = @_;
3597 return 0 if !min_version($machine_version, $major, $minor, $pve);
3598 my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3599 return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3600 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3601 return 1;
3602 };
3603
3604 if ($kvm && !defined kvm_version()) {
3605 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3606 ." or enable in BIOS.\n";
3607 }
3608
3609 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3610 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3611 my $use_old_bios_files = undef;
3612 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3613
3614 my $cmd = [];
3615 if ($conf->{affinity}) {
3616 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3617 }
3618
3619 push @$cmd, $kvm_binary;
3620
3621 push @$cmd, '-id', $vmid;
3622
3623 my $vmname = $conf->{name} || "vm$vmid";
3624
3625 push @$cmd, '-name', "$vmname,debug-threads=on";
3626
3627 push @$cmd, '-no-shutdown';
3628
3629 my $use_virtio = 0;
3630
3631 my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3632 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3633 push @$cmd, '-mon', "chardev=qmp,mode=control";
3634
3635 if (min_version($machine_version, 2, 12)) {
3636 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3637 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3638 }
3639
3640 push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3641
3642 push @$cmd, '-daemonize';
3643
3644 if ($conf->{smbios1}) {
3645 my $smbios_conf = parse_smbios1($conf->{smbios1});
3646 if ($smbios_conf->{base64}) {
3647 # Do not pass base64 flag to qemu
3648 delete $smbios_conf->{base64};
3649 my $smbios_string = "";
3650 foreach my $key (keys %$smbios_conf) {
3651 my $value;
3652 if ($key eq "uuid") {
3653 $value = $smbios_conf->{uuid}
3654 } else {
3655 $value = decode_base64($smbios_conf->{$key});
3656 }
3657 # qemu accepts any binary data, only commas need escaping by double comma
3658 $value =~ s/,/,,/g;
3659 $smbios_string .= "," . $key . "=" . $value if $value;
3660 }
3661 push @$cmd, '-smbios', "type=1" . $smbios_string;
3662 } else {
3663 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3664 }
3665 }
3666
3667 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3668 my ($code_drive_str, $var_drive_str) =
3669 print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3670 push $cmd->@*, '-drive', $code_drive_str;
3671 push $cmd->@*, '-drive', $var_drive_str;
3672 }
3673
3674 if ($q35) { # tell QEMU to load q35 config early
3675 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3676 if (min_version($machine_version, 4, 0)) {
3677 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3678 } else {
3679 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3680 }
3681 }
3682
3683 if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3684 push @$cmd, $fixups->@*;
3685 }
3686
3687 if ($conf->{vmgenid}) {
3688 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3689 }
3690
3691 # add usb controllers
3692 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3693 $conf, $bridges, $arch, $machine_type, $machine_version);
3694 push @$devices, @usbcontrollers if @usbcontrollers;
3695 my $vga = parse_vga($conf->{vga});
3696
3697 my $qxlnum = vga_conf_has_spice($conf->{vga});
3698 $vga->{type} = 'qxl' if $qxlnum;
3699
3700 if (!$vga->{type}) {
3701 if ($arch eq 'aarch64') {
3702 $vga->{type} = 'virtio';
3703 } elsif (min_version($machine_version, 2, 9)) {
3704 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3705 } else {
3706 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3707 }
3708 }
3709
3710 # enable absolute mouse coordinates (needed by vnc)
3711 my $tablet = $conf->{tablet};
3712 if (!defined($tablet)) {
3713 $tablet = $defaults->{tablet};
3714 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3715 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3716 }
3717
3718 if ($tablet) {
3719 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3720 my $kbd = print_keyboarddevice_full($conf, $arch);
3721 push @$devices, '-device', $kbd if defined($kbd);
3722 }
3723
3724 my $bootorder = device_bootorder($conf);
3725
3726 # host pci device passthrough
3727 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3728 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3729
3730 # usb devices
3731 my $usb_dev_features = {};
3732 $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3733
3734 my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3735 $conf, $usb_dev_features, $bootorder, $machine_version);
3736 push @$devices, @usbdevices if @usbdevices;
3737
3738 # serial devices
3739 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3740 my $path = $conf->{"serial$i"} or next;
3741 if ($path eq 'socket') {
3742 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3743 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3744 # On aarch64, serial0 is the UART device. QEMU only allows
3745 # connecting UART devices via the '-serial' command line, as
3746 # the device has a fixed slot on the hardware...
3747 if ($arch eq 'aarch64' && $i == 0) {
3748 push @$devices, '-serial', "chardev:serial$i";
3749 } else {
3750 push @$devices, '-device', "isa-serial,chardev=serial$i";
3751 }
3752 } else {
3753 die "no such serial device\n" if ! -c $path;
3754 push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3755 push @$devices, '-device', "isa-serial,chardev=serial$i";
3756 }
3757 }
3758
3759 # parallel devices
3760 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3761 if (my $path = $conf->{"parallel$i"}) {
3762 die "no such parallel device\n" if ! -c $path;
3763 my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3764 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3765 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3766 }
3767 }
3768
3769 if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3770 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3771 my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3772 push @$devices, @$audio_devs;
3773 }
3774
3775 add_tpm_device($vmid, $devices, $conf);
3776
3777 my $sockets = 1;
3778 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3779 $sockets = $conf->{sockets} if $conf->{sockets};
3780
3781 my $cores = $conf->{cores} || 1;
3782
3783 my $maxcpus = $sockets * $cores;
3784
3785 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3786
3787 my $allowed_vcpus = $cpuinfo->{cpus};
3788
3789 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3790
3791 if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3792 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3793 for (my $i = 2; $i <= $vcpus; $i++) {
3794 my $cpustr = print_cpu_device($conf,$i);
3795 push @$cmd, '-device', $cpustr;
3796 }
3797
3798 } else {
3799
3800 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3801 }
3802 push @$cmd, '-nodefaults';
3803
3804 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3805
3806 push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3807
3808 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
3809
3810 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3811 push @$devices, '-device', print_vga_device(
3812 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3813
3814 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3815
3816 my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3817 push @$cmd, '-vnc', "unix:$socket,password=on";
3818 } else {
3819 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3820 push @$cmd, '-nographic';
3821 }
3822
3823 # time drift fix
3824 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3825 my $useLocaltime = $conf->{localtime};
3826
3827 if ($winversion >= 5) { # windows
3828 $useLocaltime = 1 if !defined($conf->{localtime});
3829
3830 # use time drift fix when acpi is enabled
3831 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3832 $tdf = 1 if !defined($conf->{tdf});
3833 }
3834 }
3835
3836 if ($winversion >= 6) {
3837 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3838 push @$machineFlags, 'hpet=off';
3839 }
3840
3841 push @$rtcFlags, 'driftfix=slew' if $tdf;
3842
3843 if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3844 push @$rtcFlags, "base=$conf->{startdate}";
3845 } elsif ($useLocaltime) {
3846 push @$rtcFlags, 'base=localtime';
3847 }
3848
3849 if ($forcecpu) {
3850 push @$cmd, '-cpu', $forcecpu;
3851 } else {
3852 push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3853 }
3854
3855 PVE::QemuServer::Memory::config(
3856 $conf, $vmid, $sockets, $cores, $defaults, $hotplug_features->{memory}, $cmd);
3857
3858 push @$cmd, '-S' if $conf->{freeze};
3859
3860 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3861
3862 my $guest_agent = parse_guest_agent($conf);
3863
3864 if ($guest_agent->{enabled}) {
3865 my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3866 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3867
3868 if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3869 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3870 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3871 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3872 } elsif ($guest_agent->{type} eq 'isa') {
3873 push @$devices, '-device', "isa-serial,chardev=qga0";
3874 }
3875 }
3876
3877 my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3878 if ($rng && $version_guard->(4, 1, 2)) {
3879 check_rng_source($rng->{source});
3880
3881 my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3882 my $period = $rng->{period} // $rng_fmt->{period}->{default};
3883 my $limiter_str = "";
3884 if ($max_bytes) {
3885 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3886 }
3887
3888 my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3889 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3890 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3891 }
3892
3893 my $spice_port;
3894
3895 if ($qxlnum || $vga->{type} =~ /^virtio/) {
3896 if ($qxlnum > 1) {
3897 if ($winversion){
3898 for (my $i = 1; $i < $qxlnum; $i++){
3899 push @$devices, '-device', print_vga_device(
3900 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3901 }
3902 } else {
3903 # assume other OS works like Linux
3904 my ($ram, $vram) = ("134217728", "67108864");
3905 if ($vga->{memory}) {
3906 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3907 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3908 }
3909 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3910 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3911 }
3912 }
3913
3914 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3915
3916 my $pfamily = PVE::Tools::get_host_address_family($nodename);
3917 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3918 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3919
3920 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3921 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3922 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3923
3924 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3925 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3926
3927 my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3928 my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3929 if ($spice_enhancement->{foldersharing}) {
3930 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3931 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3932 }
3933
3934 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3935 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3936 if $spice_enhancement->{videostreaming};
3937
3938 push @$devices, '-spice', "$spice_opts";
3939 }
3940
3941 # enable balloon by default, unless explicitly disabled
3942 if (!defined($conf->{balloon}) || $conf->{balloon}) {
3943 my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3944 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3945 $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3946 push @$devices, '-device', $ballooncmd;
3947 }
3948
3949 if ($conf->{watchdog}) {
3950 my $wdopts = parse_watchdog($conf->{watchdog});
3951 my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3952 my $watchdog = $wdopts->{model} || 'i6300esb';
3953 push @$devices, '-device', "$watchdog$pciaddr";
3954 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3955 }
3956
3957 my $vollist = [];
3958 my $scsicontroller = {};
3959 my $ahcicontroller = {};
3960 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3961
3962 # Add iscsi initiator name if available
3963 if (my $initiator = get_initiator_name()) {
3964 push @$devices, '-iscsi', "initiator-name=$initiator";
3965 }
3966
3967 PVE::QemuConfig->foreach_volume($conf, sub {
3968 my ($ds, $drive) = @_;
3969
3970 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3971 check_volume_storage_type($storecfg, $drive->{file});
3972 push @$vollist, $drive->{file};
3973 }
3974
3975 # ignore efidisk here, already added in bios/fw handling code above
3976 return if $drive->{interface} eq 'efidisk';
3977 # similar for TPM
3978 return if $drive->{interface} eq 'tpmstate';
3979
3980 $use_virtio = 1 if $ds =~ m/^virtio/;
3981
3982 $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3983
3984 if ($drive->{interface} eq 'virtio'){
3985 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3986 }
3987
3988 if ($drive->{interface} eq 'scsi') {
3989
3990 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3991
3992 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3993 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3994
3995 my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3996 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3997
3998 my $iothread = '';
3999 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4000 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4001 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4002 } elsif ($drive->{iothread}) {
4003 log_warn(
4004 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4005 );
4006 }
4007
4008 my $queues = '';
4009 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4010 $queues = ",num_queues=$drive->{queues}";
4011 }
4012
4013 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4014 if !$scsicontroller->{$controller};
4015 $scsicontroller->{$controller}=1;
4016 }
4017
4018 if ($drive->{interface} eq 'sata') {
4019 my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4020 my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4021 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4022 if !$ahcicontroller->{$controller};
4023 $ahcicontroller->{$controller}=1;
4024 }
4025
4026 my $pbs_conf = $pbs_backing->{$ds};
4027 my $pbs_name = undef;
4028 if ($pbs_conf) {
4029 $pbs_name = "drive-$ds-pbs";
4030 push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4031 }
4032
4033 my $drive_cmd = print_drive_commandline_full(
4034 $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4035
4036 # extra protection for templates, but SATA and IDE don't support it..
4037 $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4038
4039 push @$devices, '-drive',$drive_cmd;
4040 push @$devices, '-device', print_drivedevice_full(
4041 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4042 });
4043
4044 for (my $i = 0; $i < $MAX_NETS; $i++) {
4045 my $netname = "net$i";
4046
4047 next if !$conf->{$netname};
4048 my $d = parse_net($conf->{$netname});
4049 next if !$d;
4050 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4051
4052 $use_virtio = 1 if $d->{model} eq 'virtio';
4053
4054 $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4055
4056 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4057 push @$devices, '-netdev', $netdevfull;
4058
4059 my $netdevicefull = print_netdevice_full(
4060 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4061
4062 push @$devices, '-device', $netdevicefull;
4063 }
4064
4065 if ($conf->{ivshmem}) {
4066 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4067
4068 my $bus;
4069 if ($q35) {
4070 $bus = print_pcie_addr("ivshmem");
4071 } else {
4072 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4073 }
4074
4075 my $ivshmem_name = $ivshmem->{name} // $vmid;
4076 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4077
4078 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4079 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4080 .",size=$ivshmem->{size}M";
4081 }
4082
4083 # pci.4 is nested in pci.1
4084 $bridges->{1} = 1 if $bridges->{4};
4085
4086 if (!$q35) { # add pci bridges
4087 if (min_version($machine_version, 2, 3)) {
4088 $bridges->{1} = 1;
4089 $bridges->{2} = 1;
4090 }
4091 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4092 }
4093
4094 for my $k (sort {$b cmp $a} keys %$bridges) {
4095 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4096
4097 my $k_name = $k;
4098 if ($k == 2 && $legacy_igd) {
4099 $k_name = "$k-igd";
4100 }
4101 my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4102 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4103
4104 if ($q35) { # add after -readconfig pve-q35.cfg
4105 splice @$devices, 2, 0, '-device', $devstr;
4106 } else {
4107 unshift @$devices, '-device', $devstr if $k > 0;
4108 }
4109 }
4110
4111 if (!$kvm) {
4112 push @$machineFlags, 'accel=tcg';
4113 }
4114
4115 push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4116
4117 my $machine_type_min = $machine_type;
4118 if ($add_pve_version) {
4119 $machine_type_min =~ s/\+pve\d+$//;
4120 $machine_type_min .= "+pve$required_pve_version";
4121 }
4122 push @$machineFlags, "type=${machine_type_min}";
4123
4124 push @$cmd, @$devices;
4125 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4126 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4127 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4128
4129 if (my $vmstate = $conf->{vmstate}) {
4130 my $statepath = PVE::Storage::path($storecfg, $vmstate);
4131 push @$vollist, $vmstate;
4132 push @$cmd, '-loadstate', $statepath;
4133 print "activating and using '$vmstate' as vmstate\n";
4134 }
4135
4136 if (PVE::QemuConfig->is_template($conf)) {
4137 # needed to workaround base volumes being read-only
4138 push @$cmd, '-snapshot';
4139 }
4140
4141 # add custom args
4142 if ($conf->{args}) {
4143 my $aa = PVE::Tools::split_args($conf->{args});
4144 push @$cmd, @$aa;
4145 }
4146
4147 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4148 }
4149
4150 sub check_rng_source {
4151 my ($source) = @_;
4152
4153 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4154 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4155 if ! -e $source;
4156
4157 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4158 if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4159 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4160 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4161 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4162 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4163 ." to the host.\n";
4164 }
4165 }
4166
4167 sub spice_port {
4168 my ($vmid) = @_;
4169
4170 my $res = mon_cmd($vmid, 'query-spice');
4171
4172 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4173 }
4174
4175 sub vm_devices_list {
4176 my ($vmid) = @_;
4177
4178 my $res = mon_cmd($vmid, 'query-pci');
4179 my $devices_to_check = [];
4180 my $devices = {};
4181 foreach my $pcibus (@$res) {
4182 push @$devices_to_check, @{$pcibus->{devices}},
4183 }
4184
4185 while (@$devices_to_check) {
4186 my $to_check = [];
4187 for my $d (@$devices_to_check) {
4188 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4189 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4190
4191 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4192 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4193 }
4194 $devices_to_check = $to_check;
4195 }
4196
4197 my $resblock = mon_cmd($vmid, 'query-block');
4198 foreach my $block (@$resblock) {
4199 if($block->{device} =~ m/^drive-(\S+)/){
4200 $devices->{$1} = 1;
4201 }
4202 }
4203
4204 my $resmice = mon_cmd($vmid, 'query-mice');
4205 foreach my $mice (@$resmice) {
4206 if ($mice->{name} eq 'QEMU HID Tablet') {
4207 $devices->{tablet} = 1;
4208 last;
4209 }
4210 }
4211
4212 # for usb devices there is no query-usb
4213 # but we can iterate over the entries in
4214 # qom-list path=/machine/peripheral
4215 my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4216 foreach my $per (@$resperipheral) {
4217 if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4218 $devices->{$per->{name}} = 1;
4219 }
4220 }
4221
4222 return $devices;
4223 }
4224
4225 sub vm_deviceplug {
4226 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4227
4228 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4229
4230 my $devices_list = vm_devices_list($vmid);
4231 return 1 if defined($devices_list->{$deviceid});
4232
4233 # add PCI bridge if we need it for the device
4234 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4235
4236 if ($deviceid eq 'tablet') {
4237 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4238 } elsif ($deviceid eq 'keyboard') {
4239 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4240 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4241 my $id = $1;
4242 qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4243 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4244 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4245 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4246 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4247 qemu_iothread_add($vmid, $deviceid, $device);
4248
4249 qemu_driveadd($storecfg, $vmid, $device);
4250 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4251
4252 qemu_deviceadd($vmid, $devicefull);
4253 eval { qemu_deviceaddverify($vmid, $deviceid); };
4254 if (my $err = $@) {
4255 eval { qemu_drivedel($vmid, $deviceid); };
4256 warn $@ if $@;
4257 die $err;
4258 }
4259 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4260 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4261 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4262 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4263
4264 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4265
4266 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4267 qemu_iothread_add($vmid, $deviceid, $device);
4268 $devicefull .= ",iothread=iothread-$deviceid";
4269 }
4270
4271 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4272 $devicefull .= ",num_queues=$device->{queues}";
4273 }
4274
4275 qemu_deviceadd($vmid, $devicefull);
4276 qemu_deviceaddverify($vmid, $deviceid);
4277 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4278 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4279 qemu_driveadd($storecfg, $vmid, $device);
4280
4281 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4282 eval { qemu_deviceadd($vmid, $devicefull); };
4283 if (my $err = $@) {
4284 eval { qemu_drivedel($vmid, $deviceid); };
4285 warn $@ if $@;
4286 die $err;
4287 }
4288 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4289 return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4290
4291 my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4292 my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4293 my $use_old_bios_files = undef;
4294 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4295
4296 my $netdevicefull = print_netdevice_full(
4297 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4298 qemu_deviceadd($vmid, $netdevicefull);
4299 eval {
4300 qemu_deviceaddverify($vmid, $deviceid);
4301 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4302 };
4303 if (my $err = $@) {
4304 eval { qemu_netdevdel($vmid, $deviceid); };
4305 warn $@ if $@;
4306 die $err;
4307 }
4308 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4309 my $bridgeid = $2;
4310 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4311 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4312
4313 qemu_deviceadd($vmid, $devicefull);
4314 qemu_deviceaddverify($vmid, $deviceid);
4315 } else {
4316 die "can't hotplug device '$deviceid'\n";
4317 }
4318
4319 return 1;
4320 }
4321
4322 # fixme: this should raise exceptions on error!
4323 sub vm_deviceunplug {
4324 my ($vmid, $conf, $deviceid) = @_;
4325
4326 my $devices_list = vm_devices_list($vmid);
4327 return 1 if !defined($devices_list->{$deviceid});
4328
4329 my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4330 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4331
4332 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4333 qemu_devicedel($vmid, $deviceid);
4334 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4335 qemu_devicedel($vmid, $deviceid);
4336 qemu_devicedelverify($vmid, $deviceid);
4337 } elsif ($deviceid =~ m/^usb\d+$/) {
4338 qemu_devicedel($vmid, $deviceid);
4339 qemu_devicedelverify($vmid, $deviceid);
4340 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4341 my $device = parse_drive($deviceid, $conf->{$deviceid});
4342
4343 qemu_devicedel($vmid, $deviceid);
4344 qemu_devicedelverify($vmid, $deviceid);
4345 qemu_drivedel($vmid, $deviceid);
4346 qemu_iothread_del($vmid, $deviceid, $device);
4347 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4348 qemu_devicedel($vmid, $deviceid);
4349 qemu_devicedelverify($vmid, $deviceid);
4350 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4351 my $device = parse_drive($deviceid, $conf->{$deviceid});
4352
4353 qemu_devicedel($vmid, $deviceid);
4354 qemu_devicedelverify($vmid, $deviceid);
4355 qemu_drivedel($vmid, $deviceid);
4356 qemu_deletescsihw($conf, $vmid, $deviceid);
4357
4358 qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4359 if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4360 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4361 qemu_devicedel($vmid, $deviceid);
4362 qemu_devicedelverify($vmid, $deviceid);
4363 qemu_netdevdel($vmid, $deviceid);
4364 } else {
4365 die "can't unplug device '$deviceid'\n";
4366 }
4367
4368 return 1;
4369 }
4370
4371 sub qemu_spice_usbredir_chardev_add {
4372 my ($vmid, $id) = @_;
4373
4374 mon_cmd($vmid, "chardev-add" , (
4375 id => $id,
4376 backend => {
4377 type => 'spicevmc',
4378 data => {
4379 type => "usbredir",
4380 },
4381 },
4382 ));
4383 }
4384
4385 sub qemu_deviceadd {
4386 my ($vmid, $devicefull) = @_;
4387
4388 $devicefull = "driver=".$devicefull;
4389 my %options = split(/[=,]/, $devicefull);
4390
4391 mon_cmd($vmid, "device_add" , %options);
4392 }
4393
4394 sub qemu_devicedel {
4395 my ($vmid, $deviceid) = @_;
4396
4397 my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4398 }
4399
4400 sub qemu_iothread_add {
4401 my ($vmid, $deviceid, $device) = @_;
4402
4403 if ($device->{iothread}) {
4404 my $iothreads = vm_iothreads_list($vmid);
4405 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4406 }
4407 }
4408
4409 sub qemu_iothread_del {
4410 my ($vmid, $deviceid, $device) = @_;
4411
4412 if ($device->{iothread}) {
4413 my $iothreads = vm_iothreads_list($vmid);
4414 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4415 }
4416 }
4417
4418 sub qemu_objectadd {
4419 my ($vmid, $objectid, $qomtype) = @_;
4420
4421 mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4422
4423 return 1;
4424 }
4425
4426 sub qemu_objectdel {
4427 my ($vmid, $objectid) = @_;
4428
4429 mon_cmd($vmid, "object-del", id => $objectid);
4430
4431 return 1;
4432 }
4433
4434 sub qemu_driveadd {
4435 my ($storecfg, $vmid, $device) = @_;
4436
4437 my $kvmver = get_running_qemu_version($vmid);
4438 my $io_uring = min_version($kvmver, 6, 0);
4439 my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4440 $drive =~ s/\\/\\\\/g;
4441 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4442
4443 # If the command succeeds qemu prints: "OK"
4444 return 1 if $ret =~ m/OK/s;
4445
4446 die "adding drive failed: $ret\n";
4447 }
4448
4449 sub qemu_drivedel {
4450 my ($vmid, $deviceid) = @_;
4451
4452 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4453 $ret =~ s/^\s+//;
4454
4455 return 1 if $ret eq "";
4456
4457 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4458 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4459
4460 die "deleting drive $deviceid failed : $ret\n";
4461 }
4462
4463 sub qemu_deviceaddverify {
4464 my ($vmid, $deviceid) = @_;
4465
4466 for (my $i = 0; $i <= 5; $i++) {
4467 my $devices_list = vm_devices_list($vmid);
4468 return 1 if defined($devices_list->{$deviceid});
4469 sleep 1;
4470 }
4471
4472 die "error on hotplug device '$deviceid'\n";
4473 }
4474
4475
4476 sub qemu_devicedelverify {
4477 my ($vmid, $deviceid) = @_;
4478
4479 # need to verify that the device is correctly removed as device_del
4480 # is async and empty return is not reliable
4481
4482 for (my $i = 0; $i <= 5; $i++) {
4483 my $devices_list = vm_devices_list($vmid);
4484 return 1 if !defined($devices_list->{$deviceid});
4485 sleep 1;
4486 }
4487
4488 die "error on hot-unplugging device '$deviceid'\n";
4489 }
4490
4491 sub qemu_findorcreatescsihw {
4492 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4493
4494 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4495
4496 my $scsihwid="$controller_prefix$controller";
4497 my $devices_list = vm_devices_list($vmid);
4498
4499 if (!defined($devices_list->{$scsihwid})) {
4500 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4501 }
4502
4503 return 1;
4504 }
4505
4506 sub qemu_deletescsihw {
4507 my ($conf, $vmid, $opt) = @_;
4508
4509 my $device = parse_drive($opt, $conf->{$opt});
4510
4511 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4512 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4513 return 1;
4514 }
4515
4516 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4517
4518 my $devices_list = vm_devices_list($vmid);
4519 foreach my $opt (keys %{$devices_list}) {
4520 if (is_valid_drivename($opt)) {
4521 my $drive = parse_drive($opt, $conf->{$opt});
4522 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4523 return 1;
4524 }
4525 }
4526 }
4527
4528 my $scsihwid="scsihw$controller";
4529
4530 vm_deviceunplug($vmid, $conf, $scsihwid);
4531
4532 return 1;
4533 }
4534
4535 sub qemu_add_pci_bridge {
4536 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4537
4538 my $bridges = {};
4539
4540 my $bridgeid;
4541
4542 print_pci_addr($device, $bridges, $arch, $machine_type);
4543
4544 while (my ($k, $v) = each %$bridges) {
4545 $bridgeid = $k;
4546 }
4547 return 1 if !defined($bridgeid) || $bridgeid < 1;
4548
4549 my $bridge = "pci.$bridgeid";
4550 my $devices_list = vm_devices_list($vmid);
4551
4552 if (!defined($devices_list->{$bridge})) {
4553 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4554 }
4555
4556 return 1;
4557 }
4558
4559 sub qemu_set_link_status {
4560 my ($vmid, $device, $up) = @_;
4561
4562 mon_cmd($vmid, "set_link", name => $device,
4563 up => $up ? JSON::true : JSON::false);
4564 }
4565
4566 sub qemu_netdevadd {
4567 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4568
4569 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4570 my %options = split(/[=,]/, $netdev);
4571
4572 if (defined(my $vhost = $options{vhost})) {
4573 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4574 }
4575
4576 if (defined(my $queues = $options{queues})) {
4577 $options{queues} = $queues + 0;
4578 }
4579
4580 mon_cmd($vmid, "netdev_add", %options);
4581 return 1;
4582 }
4583
4584 sub qemu_netdevdel {
4585 my ($vmid, $deviceid) = @_;
4586
4587 mon_cmd($vmid, "netdev_del", id => $deviceid);
4588 }
4589
4590 sub qemu_usb_hotplug {
4591 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4592
4593 return if !$device;
4594
4595 # remove the old one first
4596 vm_deviceunplug($vmid, $conf, $deviceid);
4597
4598 # check if xhci controller is necessary and available
4599 my $devicelist = vm_devices_list($vmid);
4600
4601 if (!$devicelist->{xhci}) {
4602 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4603 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4604 }
4605
4606 # add the new one
4607 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4608 }
4609
4610 sub qemu_cpu_hotplug {
4611 my ($vmid, $conf, $vcpus) = @_;
4612
4613 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4614
4615 my $sockets = 1;
4616 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4617 $sockets = $conf->{sockets} if $conf->{sockets};
4618 my $cores = $conf->{cores} || 1;
4619 my $maxcpus = $sockets * $cores;
4620
4621 $vcpus = $maxcpus if !$vcpus;
4622
4623 die "you can't add more vcpus than maxcpus\n"
4624 if $vcpus > $maxcpus;
4625
4626 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4627
4628 if ($vcpus < $currentvcpus) {
4629
4630 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4631
4632 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4633 qemu_devicedel($vmid, "cpu$i");
4634 my $retry = 0;
4635 my $currentrunningvcpus = undef;
4636 while (1) {
4637 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4638 last if scalar(@{$currentrunningvcpus}) == $i-1;
4639 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4640 $retry++;
4641 sleep 1;
4642 }
4643 #update conf after each succesfull cpu unplug
4644 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4645 PVE::QemuConfig->write_config($vmid, $conf);
4646 }
4647 } else {
4648 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4649 }
4650
4651 return;
4652 }
4653
4654 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4655 die "vcpus in running vm does not match its configuration\n"
4656 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4657
4658 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4659
4660 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4661 my $cpustr = print_cpu_device($conf, $i);
4662 qemu_deviceadd($vmid, $cpustr);
4663
4664 my $retry = 0;
4665 my $currentrunningvcpus = undef;
4666 while (1) {
4667 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4668 last if scalar(@{$currentrunningvcpus}) == $i;
4669 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4670 sleep 1;
4671 $retry++;
4672 }
4673 #update conf after each succesfull cpu hotplug
4674 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4675 PVE::QemuConfig->write_config($vmid, $conf);
4676 }
4677 } else {
4678
4679 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4680 mon_cmd($vmid, "cpu-add", id => int($i));
4681 }
4682 }
4683 }
4684
4685 sub qemu_block_set_io_throttle {
4686 my ($vmid, $deviceid,
4687 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4688 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4689 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4690 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4691
4692 return if !check_running($vmid) ;
4693
4694 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4695 bps => int($bps),
4696 bps_rd => int($bps_rd),
4697 bps_wr => int($bps_wr),
4698 iops => int($iops),
4699 iops_rd => int($iops_rd),
4700 iops_wr => int($iops_wr),
4701 bps_max => int($bps_max),
4702 bps_rd_max => int($bps_rd_max),
4703 bps_wr_max => int($bps_wr_max),
4704 iops_max => int($iops_max),
4705 iops_rd_max => int($iops_rd_max),
4706 iops_wr_max => int($iops_wr_max),
4707 bps_max_length => int($bps_max_length),
4708 bps_rd_max_length => int($bps_rd_max_length),
4709 bps_wr_max_length => int($bps_wr_max_length),
4710 iops_max_length => int($iops_max_length),
4711 iops_rd_max_length => int($iops_rd_max_length),
4712 iops_wr_max_length => int($iops_wr_max_length),
4713 );
4714
4715 }
4716
4717 sub qemu_block_resize {
4718 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4719
4720 my $running = check_running($vmid);
4721
4722 PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4723
4724 return if !$running;
4725
4726 my $padding = (1024 - $size % 1024) % 1024;
4727 $size = $size + $padding;
4728
4729 mon_cmd(
4730 $vmid,
4731 "block_resize",
4732 device => $deviceid,
4733 size => int($size),
4734 timeout => 60,
4735 );
4736 }
4737
4738 sub qemu_volume_snapshot {
4739 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4740
4741 my $running = check_running($vmid);
4742
4743 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4744 mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4745 } else {
4746 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4747 }
4748 }
4749
4750 sub qemu_volume_snapshot_delete {
4751 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4752
4753 my $running = check_running($vmid);
4754
4755 if($running) {
4756
4757 $running = undef;
4758 my $conf = PVE::QemuConfig->load_config($vmid);
4759 PVE::QemuConfig->foreach_volume($conf, sub {
4760 my ($ds, $drive) = @_;
4761 $running = 1 if $drive->{file} eq $volid;
4762 });
4763 }
4764
4765 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4766 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4767 } else {
4768 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4769 }
4770 }
4771
4772 sub set_migration_caps {
4773 my ($vmid, $savevm) = @_;
4774
4775 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4776
4777 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4778 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4779
4780 my $cap_ref = [];
4781
4782 my $enabled_cap = {
4783 "auto-converge" => 1,
4784 "xbzrle" => 1,
4785 "x-rdma-pin-all" => 0,
4786 "zero-blocks" => 0,
4787 "compress" => 0,
4788 "dirty-bitmaps" => $dirty_bitmaps,
4789 };
4790
4791 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4792
4793 for my $supported_capability (@$supported_capabilities) {
4794 push @$cap_ref, {
4795 capability => $supported_capability->{capability},
4796 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4797 };
4798 }
4799
4800 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4801 }
4802
4803 sub foreach_volid {
4804 my ($conf, $func, @param) = @_;
4805
4806 my $volhash = {};
4807
4808 my $test_volid = sub {
4809 my ($key, $drive, $snapname) = @_;
4810
4811 my $volid = $drive->{file};
4812 return if !$volid;
4813
4814 $volhash->{$volid}->{cdrom} //= 1;
4815 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4816
4817 my $replicate = $drive->{replicate} // 1;
4818 $volhash->{$volid}->{replicate} //= 0;
4819 $volhash->{$volid}->{replicate} = 1 if $replicate;
4820
4821 $volhash->{$volid}->{shared} //= 0;
4822 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4823
4824 $volhash->{$volid}->{referenced_in_config} //= 0;
4825 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4826
4827 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4828 if defined($snapname);
4829
4830 my $size = $drive->{size};
4831 $volhash->{$volid}->{size} //= $size if $size;
4832
4833 $volhash->{$volid}->{is_vmstate} //= 0;
4834 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4835
4836 $volhash->{$volid}->{is_tpmstate} //= 0;
4837 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4838
4839 $volhash->{$volid}->{is_unused} //= 0;
4840 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4841
4842 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4843 };
4844
4845 my $include_opts = {
4846 extra_keys => ['vmstate'],
4847 include_unused => 1,
4848 };
4849
4850 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4851 foreach my $snapname (keys %{$conf->{snapshots}}) {
4852 my $snap = $conf->{snapshots}->{$snapname};
4853 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4854 }
4855
4856 foreach my $volid (keys %$volhash) {
4857 &$func($volid, $volhash->{$volid}, @param);
4858 }
4859 }
4860
4861 my $fast_plug_option = {
4862 'description' => 1,
4863 'hookscript' => 1,
4864 'lock' => 1,
4865 'migrate_downtime' => 1,
4866 'migrate_speed' => 1,
4867 'name' => 1,
4868 'onboot' => 1,
4869 'protection' => 1,
4870 'shares' => 1,
4871 'startup' => 1,
4872 'tags' => 1,
4873 'vmstatestorage' => 1,
4874 };
4875
4876 for my $opt (keys %$confdesc_cloudinit) {
4877 $fast_plug_option->{$opt} = 1;
4878 };
4879
4880 # hotplug changes in [PENDING]
4881 # $selection hash can be used to only apply specified options, for
4882 # example: { cores => 1 } (only apply changed 'cores')
4883 # $errors ref is used to return error messages
4884 sub vmconfig_hotplug_pending {
4885 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4886
4887 my $defaults = load_defaults();
4888 my $arch = get_vm_arch($conf);
4889 my $machine_type = get_vm_machine($conf, undef, $arch);
4890
4891 # commit values which do not have any impact on running VM first
4892 # Note: those option cannot raise errors, we we do not care about
4893 # $selection and always apply them.
4894
4895 my $add_error = sub {
4896 my ($opt, $msg) = @_;
4897 $errors->{$opt} = "hotplug problem - $msg";
4898 };
4899
4900 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4901
4902 my $cloudinit_record_changed = sub {
4903 my ($conf, $opt, $old, $new) = @_;
4904 return if !$cloudinit_pending_properties->{$opt};
4905
4906 my $ci = ($conf->{cloudinit} //= {});
4907
4908 my $recorded = $ci->{$opt};
4909 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4910
4911 if (defined($new)) {
4912 if (defined($old)) {
4913 # an existing value is being modified
4914 if (defined($recorded)) {
4915 # the value was already not in sync
4916 if ($new eq $recorded) {
4917 # a value is being reverted to the cloud-init state:
4918 delete $ci->{$opt};
4919 delete $added{$opt};
4920 } else {
4921 # the value was changed multiple times, do nothing
4922 }
4923 } elsif ($added{$opt}) {
4924 # the value had been marked as added and is being changed, do nothing
4925 } else {
4926 # the value is new, record it:
4927 $ci->{$opt} = $old;
4928 }
4929 } else {
4930 # a new value is being added
4931 if (defined($recorded)) {
4932 # it was already not in sync
4933 if ($new eq $recorded) {
4934 # a value is being reverted to the cloud-init state:
4935 delete $ci->{$opt};
4936 delete $added{$opt};
4937 } else {
4938 # the value had temporarily been removed, do nothing
4939 }
4940 } elsif ($added{$opt}) {
4941 # the value had been marked as added already, do nothing
4942 } else {
4943 # the value is new, add it
4944 $added{$opt} = 1;
4945 }
4946 }
4947 } elsif (!defined($old)) {
4948 # a non-existent value is being removed? ignore...
4949 } else {
4950 # a value is being deleted
4951 if (defined($recorded)) {
4952 # a value was already recorded, just keep it
4953 } elsif ($added{$opt}) {
4954 # the value was marked as added, remove it
4955 delete $added{$opt};
4956 } else {
4957 # a previously unrecorded value is being removed, record the old value:
4958 $ci->{$opt} = $old;
4959 }
4960 }
4961
4962 my $added = join(',', sort keys %added);
4963 $ci->{added} = $added if length($added);
4964 };
4965
4966 my $changes = 0;
4967 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4968 if ($fast_plug_option->{$opt}) {
4969 my $new = delete $conf->{pending}->{$opt};
4970 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4971 $conf->{$opt} = $new;
4972 $changes = 1;
4973 }
4974 }
4975
4976 if ($changes) {
4977 PVE::QemuConfig->write_config($vmid, $conf);
4978 }
4979
4980 my $ostype = $conf->{ostype};
4981 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4982 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4983 my $usb_hotplug = $hotplug_features->{usb}
4984 && min_version($version, 7, 1)
4985 && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4986
4987 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4988 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4989
4990 foreach my $opt (sort keys %$pending_delete_hash) {
4991 next if $selection && !$selection->{$opt};
4992 my $force = $pending_delete_hash->{$opt}->{force};
4993 eval {
4994 if ($opt eq 'hotplug') {
4995 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4996 } elsif ($opt eq 'tablet') {
4997 die "skip\n" if !$hotplug_features->{usb};
4998 if ($defaults->{tablet}) {
4999 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5000 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5001 if $arch eq 'aarch64';
5002 } else {
5003 vm_deviceunplug($vmid, $conf, 'tablet');
5004 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5005 }
5006 } elsif ($opt =~ m/^usb(\d+)$/) {
5007 my $index = $1;
5008 die "skip\n" if !$usb_hotplug;
5009 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5010 vm_deviceunplug($vmid, $conf, $opt);
5011 } elsif ($opt eq 'vcpus') {
5012 die "skip\n" if !$hotplug_features->{cpu};
5013 qemu_cpu_hotplug($vmid, $conf, undef);
5014 } elsif ($opt eq 'balloon') {
5015 # enable balloon device is not hotpluggable
5016 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5017 # here we reset the ballooning value to memory
5018 my $balloon = $conf->{memory} || $defaults->{memory};
5019 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5020 } elsif ($fast_plug_option->{$opt}) {
5021 # do nothing
5022 } elsif ($opt =~ m/^net(\d+)$/) {
5023 die "skip\n" if !$hotplug_features->{network};
5024 vm_deviceunplug($vmid, $conf, $opt);
5025 } elsif (is_valid_drivename($opt)) {
5026 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5027 vm_deviceunplug($vmid, $conf, $opt);
5028 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5029 } elsif ($opt =~ m/^memory$/) {
5030 die "skip\n" if !$hotplug_features->{memory};
5031 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults);
5032 } elsif ($opt eq 'cpuunits') {
5033 $cgroup->change_cpu_shares(undef);
5034 } elsif ($opt eq 'cpulimit') {
5035 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5036 } else {
5037 die "skip\n";
5038 }
5039 };
5040 if (my $err = $@) {
5041 &$add_error($opt, $err) if $err ne "skip\n";
5042 } else {
5043 my $old = delete $conf->{$opt};
5044 $cloudinit_record_changed->($conf, $opt, $old, undef);
5045 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5046 }
5047 }
5048
5049 my $cloudinit_opt;
5050 foreach my $opt (keys %{$conf->{pending}}) {
5051 next if $selection && !$selection->{$opt};
5052 my $value = $conf->{pending}->{$opt};
5053 eval {
5054 if ($opt eq 'hotplug') {
5055 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5056 } elsif ($opt eq 'tablet') {
5057 die "skip\n" if !$hotplug_features->{usb};
5058 if ($value == 1) {
5059 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5060 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5061 if $arch eq 'aarch64';
5062 } elsif ($value == 0) {
5063 vm_deviceunplug($vmid, $conf, 'tablet');
5064 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5065 }
5066 } elsif ($opt =~ m/^usb(\d+)$/) {
5067 my $index = $1;
5068 die "skip\n" if !$usb_hotplug;
5069 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5070 my $id = $opt;
5071 if ($d->{host} =~ m/^spice$/i) {
5072 $id = "usbredirdev$index";
5073 }
5074 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5075 } elsif ($opt eq 'vcpus') {
5076 die "skip\n" if !$hotplug_features->{cpu};
5077 qemu_cpu_hotplug($vmid, $conf, $value);
5078 } elsif ($opt eq 'balloon') {
5079 # enable/disable balloning device is not hotpluggable
5080 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
5081 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5082 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5083
5084 # allow manual ballooning if shares is set to zero
5085 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5086 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5087 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5088 }
5089 } elsif ($opt =~ m/^net(\d+)$/) {
5090 # some changes can be done without hotplug
5091 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5092 $vmid, $opt, $value, $arch, $machine_type);
5093 } elsif (is_valid_drivename($opt)) {
5094 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5095 # some changes can be done without hotplug
5096 my $drive = parse_drive($opt, $value);
5097 if (drive_is_cloudinit($drive)) {
5098 $cloudinit_opt = [$opt, $drive];
5099 # apply all the other changes first, then generate the cloudinit disk
5100 die "skip\n";
5101 }
5102 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5103 $vmid, $opt, $value, $arch, $machine_type);
5104 } elsif ($opt =~ m/^memory$/) { #dimms
5105 die "skip\n" if !$hotplug_features->{memory};
5106 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $value);
5107 } elsif ($opt eq 'cpuunits') {
5108 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5109 $cgroup->change_cpu_shares($new_cpuunits);
5110 } elsif ($opt eq 'cpulimit') {
5111 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5112 $cgroup->change_cpu_quota($cpulimit, 100000);
5113 } elsif ($opt eq 'agent') {
5114 vmconfig_update_agent($conf, $opt, $value);
5115 } else {
5116 die "skip\n"; # skip non-hot-pluggable options
5117 }
5118 };
5119 if (my $err = $@) {
5120 &$add_error($opt, $err) if $err ne "skip\n";
5121 } else {
5122 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5123 $conf->{$opt} = $value;
5124 delete $conf->{pending}->{$opt};
5125 }
5126 }
5127
5128 if (defined($cloudinit_opt)) {
5129 my ($opt, $drive) = @$cloudinit_opt;
5130 my $value = $conf->{pending}->{$opt};
5131 eval {
5132 my $temp = {%$conf, $opt => $value};
5133 PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5134 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5135 $vmid, $opt, $value, $arch, $machine_type);
5136 };
5137 if (my $err = $@) {
5138 &$add_error($opt, $err) if $err ne "skip\n";
5139 } else {
5140 $conf->{$opt} = $value;
5141 delete $conf->{pending}->{$opt};
5142 }
5143 }
5144
5145 # unplug xhci controller if no usb device is left
5146 if ($usb_hotplug) {
5147 my $has_usb = 0;
5148 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5149 next if !defined($conf->{"usb$i"});
5150 $has_usb = 1;
5151 last;
5152 }
5153 if (!$has_usb) {
5154 vm_deviceunplug($vmid, $conf, 'xhci');
5155 }
5156 }
5157
5158 PVE::QemuConfig->write_config($vmid, $conf);
5159
5160 if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5161 PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5162 }
5163 }
5164
5165 sub try_deallocate_drive {
5166 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5167
5168 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5169 my $volid = $drive->{file};
5170 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5171 my $sid = PVE::Storage::parse_volume_id($volid);
5172 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5173
5174 # check if the disk is really unused
5175 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5176 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5177 PVE::Storage::vdisk_free($storecfg, $volid);
5178 return 1;
5179 } else {
5180 # If vm is not owner of this disk remove from config
5181 return 1;
5182 }
5183 }
5184
5185 return;
5186 }
5187
5188 sub vmconfig_delete_or_detach_drive {
5189 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5190
5191 my $drive = parse_drive($opt, $conf->{$opt});
5192
5193 my $rpcenv = PVE::RPCEnvironment::get();
5194 my $authuser = $rpcenv->get_user();
5195
5196 if ($force) {
5197 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5198 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5199 } else {
5200 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5201 }
5202 }
5203
5204
5205
5206 sub vmconfig_apply_pending {
5207 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5208
5209 return if !scalar(keys %{$conf->{pending}});
5210
5211 my $add_apply_error = sub {
5212 my ($opt, $msg) = @_;
5213 my $err_msg = "unable to apply pending change $opt : $msg";
5214 $errors->{$opt} = $err_msg;
5215 warn $err_msg;
5216 };
5217
5218 # cold plug
5219
5220 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5221 foreach my $opt (sort keys %$pending_delete_hash) {
5222 my $force = $pending_delete_hash->{$opt}->{force};
5223 eval {
5224 if ($opt =~ m/^unused/) {
5225 die "internal error";
5226 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5227 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5228 }
5229 };
5230 if (my $err = $@) {
5231 $add_apply_error->($opt, $err);
5232 } else {
5233 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5234 delete $conf->{$opt};
5235 }
5236 }
5237
5238 PVE::QemuConfig->cleanup_pending($conf);
5239
5240 my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5241
5242 foreach my $opt (keys %{$conf->{pending}}) { # add/change
5243 next if $opt eq 'delete'; # just to be sure
5244 eval {
5245 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5246 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5247 }
5248 };
5249 if (my $err = $@) {
5250 $add_apply_error->($opt, $err);
5251 } else {
5252
5253 if (is_valid_drivename($opt)) {
5254 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5255 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5256 }
5257
5258 $conf->{$opt} = delete $conf->{pending}->{$opt};
5259 }
5260 }
5261
5262 # write all changes at once to avoid unnecessary i/o
5263 PVE::QemuConfig->write_config($vmid, $conf);
5264 if ($generate_cloudinit) {
5265 if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5266 # After successful generation and if there were changes to be applied, update the
5267 # config to drop the {cloudinit} entry.
5268 PVE::QemuConfig->write_config($vmid, $conf);
5269 }
5270 }
5271 }
5272
5273 sub vmconfig_update_net {
5274 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5275
5276 my $newnet = parse_net($value);
5277
5278 if ($conf->{$opt}) {
5279 my $oldnet = parse_net($conf->{$opt});
5280
5281 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5282 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5283 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5284 safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5285 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5286
5287 # for non online change, we try to hot-unplug
5288 die "skip\n" if !$hotplug;
5289 vm_deviceunplug($vmid, $conf, $opt);
5290 } else {
5291
5292 die "internal error" if $opt !~ m/net(\d+)/;
5293 my $iface = "tap${vmid}i$1";
5294
5295 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5296 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5297 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5298 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5299 PVE::Network::tap_unplug($iface);
5300
5301 if ($have_sdn) {
5302 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5303 } else {
5304 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5305 }
5306 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5307 # Rate can be applied on its own but any change above needs to
5308 # include the rate in tap_plug since OVS resets everything.
5309 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5310 }
5311
5312 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5313 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5314 }
5315
5316 return 1;
5317 }
5318 }
5319
5320 if ($hotplug) {
5321 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5322 } else {
5323 die "skip\n";
5324 }
5325 }
5326
5327 sub vmconfig_update_agent {
5328 my ($conf, $opt, $value) = @_;
5329
5330 die "skip\n" if !$conf->{$opt};
5331
5332 my $hotplug_options = { fstrim_cloned_disks => 1 };
5333
5334 my $old_agent = parse_guest_agent($conf);
5335 my $agent = parse_guest_agent({$opt => $value});
5336
5337 for my $option (keys %$agent) { # added/changed options
5338 next if defined($hotplug_options->{$option});
5339 die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5340 }
5341
5342 for my $option (keys %$old_agent) { # removed options
5343 next if defined($hotplug_options->{$option});
5344 die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5345 }
5346
5347 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5348 }
5349
5350 sub vmconfig_update_disk {
5351 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5352
5353 my $drive = parse_drive($opt, $value);
5354
5355 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5356 my $media = $drive->{media} || 'disk';
5357 my $oldmedia = $old_drive->{media} || 'disk';
5358 die "unable to change media type\n" if $media ne $oldmedia;
5359
5360 if (!drive_is_cdrom($old_drive)) {
5361
5362 if ($drive->{file} ne $old_drive->{file}) {
5363
5364 die "skip\n" if !$hotplug;
5365
5366 # unplug and register as unused
5367 vm_deviceunplug($vmid, $conf, $opt);
5368 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5369
5370 } else {
5371 # update existing disk
5372
5373 # skip non hotpluggable value
5374 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5375 safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5376 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5377 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5378 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5379 safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5380 safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5381 die "skip\n";
5382 }
5383
5384 # apply throttle
5385 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5386 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5387 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5388 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5389 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5390 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5391 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5392 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5393 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5394 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5395 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5396 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5397 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5398 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5399 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5400 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5401 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5402 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5403
5404 qemu_block_set_io_throttle(
5405 $vmid,"drive-$opt",
5406 ($drive->{mbps} || 0)*1024*1024,
5407 ($drive->{mbps_rd} || 0)*1024*1024,
5408 ($drive->{mbps_wr} || 0)*1024*1024,
5409 $drive->{iops} || 0,
5410 $drive->{iops_rd} || 0,
5411 $drive->{iops_wr} || 0,
5412 ($drive->{mbps_max} || 0)*1024*1024,
5413 ($drive->{mbps_rd_max} || 0)*1024*1024,
5414 ($drive->{mbps_wr_max} || 0)*1024*1024,
5415 $drive->{iops_max} || 0,
5416 $drive->{iops_rd_max} || 0,
5417 $drive->{iops_wr_max} || 0,
5418 $drive->{bps_max_length} || 1,
5419 $drive->{bps_rd_max_length} || 1,
5420 $drive->{bps_wr_max_length} || 1,
5421 $drive->{iops_max_length} || 1,
5422 $drive->{iops_rd_max_length} || 1,
5423 $drive->{iops_wr_max_length} || 1,
5424 );
5425
5426 }
5427
5428 return 1;
5429 }
5430
5431 } else { # cdrom
5432
5433 if ($drive->{file} eq 'none') {
5434 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5435 if (drive_is_cloudinit($old_drive)) {
5436 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5437 }
5438 } else {
5439 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5440
5441 # force eject if locked
5442 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5443
5444 if ($path) {
5445 mon_cmd($vmid, "blockdev-change-medium",
5446 id => "$opt", filename => "$path");
5447 }
5448 }
5449
5450 return 1;
5451 }
5452 }
5453
5454 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5455 # hotplug new disks
5456 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5457 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5458 }
5459
5460 sub vmconfig_update_cloudinit_drive {
5461 my ($storecfg, $conf, $vmid) = @_;
5462
5463 my $cloudinit_ds = undef;
5464 my $cloudinit_drive = undef;
5465
5466 PVE::QemuConfig->foreach_volume($conf, sub {
5467 my ($ds, $drive) = @_;
5468 if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5469 $cloudinit_ds = $ds;
5470 $cloudinit_drive = $drive;
5471 }
5472 });
5473
5474 return if !$cloudinit_drive;
5475
5476 if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5477 PVE::QemuConfig->write_config($vmid, $conf);
5478 }
5479
5480 my $running = PVE::QemuServer::check_running($vmid);
5481
5482 if ($running) {
5483 my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5484 if ($path) {
5485 mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5486 mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5487 }
5488 }
5489 }
5490
5491 # called in locked context by incoming migration
5492 sub vm_migrate_get_nbd_disks {
5493 my ($storecfg, $conf, $replicated_volumes) = @_;
5494
5495 my $local_volumes = {};
5496 PVE::QemuConfig->foreach_volume($conf, sub {
5497 my ($ds, $drive) = @_;
5498
5499 return if drive_is_cdrom($drive);
5500 return if $ds eq 'tpmstate0';
5501
5502 my $volid = $drive->{file};
5503
5504 return if !$volid;
5505
5506 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5507
5508 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5509 return if $scfg->{shared};
5510
5511 # replicated disks re-use existing state via bitmap
5512 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5513 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5514 });
5515 return $local_volumes;
5516 }
5517
5518 # called in locked context by incoming migration
5519 sub vm_migrate_alloc_nbd_disks {
5520 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5521
5522 my $nbd = {};
5523 foreach my $opt (sort keys %$source_volumes) {
5524 my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5525
5526 if ($use_existing) {
5527 $nbd->{$opt}->{drivestr} = print_drive($drive);
5528 $nbd->{$opt}->{volid} = $volid;
5529 $nbd->{$opt}->{replicated} = 1;
5530 next;
5531 }
5532
5533 # storage mapping + volname = regular migration
5534 # storage mapping + format = remote migration
5535 # order of precedence, filtered by whether storage supports it:
5536 # 1. explicit requested format
5537 # 2. format of current volume
5538 # 3. default format of storage
5539 if (!$storagemap->{identity}) {
5540 $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5541 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5542 if (!$format || !grep { $format eq $_ } @$validFormats) {
5543 if ($volname) {
5544 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5545 my $fileFormat = qemu_img_format($scfg, $volname);
5546 $format = $fileFormat
5547 if grep { $fileFormat eq $_ } @$validFormats;
5548 }
5549 $format //= $defFormat;
5550 }
5551 } else {
5552 # can't happen for remote migration, so $volname is always defined
5553 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5554 $format = qemu_img_format($scfg, $volname);
5555 }
5556
5557 my $size = $drive->{size} / 1024;
5558 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5559 my $newdrive = $drive;
5560 $newdrive->{format} = $format;
5561 $newdrive->{file} = $newvolid;
5562 my $drivestr = print_drive($newdrive);
5563 $nbd->{$opt}->{drivestr} = $drivestr;
5564 $nbd->{$opt}->{volid} = $newvolid;
5565 }
5566
5567 return $nbd;
5568 }
5569
5570 # see vm_start_nolock for parameters, additionally:
5571 # migrate_opts:
5572 # storagemap = parsed storage map for allocating NBD disks
5573 sub vm_start {
5574 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5575
5576 return PVE::QemuConfig->lock_config($vmid, sub {
5577 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5578
5579 die "you can't start a vm if it's a template\n"
5580 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5581
5582 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5583 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5584
5585 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5586
5587 if ($has_backup_lock && $running) {
5588 # a backup is currently running, attempt to start the guest in the
5589 # existing QEMU instance
5590 return vm_resume($vmid);
5591 }
5592
5593 PVE::QemuConfig->check_lock($conf)
5594 if !($params->{skiplock} || $has_suspended_lock);
5595
5596 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5597
5598 die "VM $vmid already running\n" if $running;
5599
5600 if (my $storagemap = $migrate_opts->{storagemap}) {
5601 my $replicated = $migrate_opts->{replicated_volumes};
5602 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5603 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5604
5605 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5606 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5607 }
5608 }
5609
5610 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5611 });
5612 }
5613
5614
5615 # params:
5616 # statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5617 # skiplock => 0/1, skip checking for config lock
5618 # skiptemplate => 0/1, skip checking whether VM is template
5619 # forcemachine => to force QEMU machine (rollback/migration)
5620 # forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5621 # timeout => in seconds
5622 # paused => start VM in paused state (backup)
5623 # resume => resume from hibernation
5624 # pbs-backing => {
5625 # sata0 => {
5626 # repository
5627 # snapshot
5628 # keyfile
5629 # archive
5630 # },
5631 # virtio2 => ...
5632 # }
5633 # migrate_opts:
5634 # nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5635 # migratedfrom => source node
5636 # spice_ticket => used for spice migration, passed via tunnel/stdin
5637 # network => CIDR of migration network
5638 # type => secure/insecure - tunnel over encrypted connection or plain-text
5639 # nbd_proto_version => int, 0 for TCP, 1 for UNIX
5640 # replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5641 # offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5642 # contained in config
5643 sub vm_start_nolock {
5644 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5645
5646 my $statefile = $params->{statefile};
5647 my $resume = $params->{resume};
5648
5649 my $migratedfrom = $migrate_opts->{migratedfrom};
5650 my $migration_type = $migrate_opts->{type};
5651
5652 my $res = {};
5653
5654 # clean up leftover reboot request files
5655 eval { clear_reboot_request($vmid); };
5656 warn $@ if $@;
5657
5658 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5659 vmconfig_apply_pending($vmid, $conf, $storecfg);
5660 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5661 }
5662
5663 # don't regenerate the ISO if the VM is started as part of a live migration
5664 # this way we can reuse the old ISO with the correct config
5665 if (!$migratedfrom) {
5666 if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5667 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5668 # $conf->{cloudinit}, so we could just not do this?
5669 # But we do it above, so for now let's be consistent.
5670 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5671 }
5672 }
5673
5674 # override offline migrated volumes, conf is out of date still
5675 if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5676 for my $key (sort keys $offline_volumes->%*) {
5677 my $parsed = parse_drive($key, $conf->{$key});
5678 $parsed->{file} = $offline_volumes->{$key};
5679 $conf->{$key} = print_drive($parsed);
5680 }
5681 }
5682
5683 my $defaults = load_defaults();
5684
5685 # set environment variable useful inside network script
5686 # for remote migration the config is available on the target node!
5687 if (!$migrate_opts->{remote_node}) {
5688 $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5689 }
5690
5691 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5692
5693 my $forcemachine = $params->{forcemachine};
5694 my $forcecpu = $params->{forcecpu};
5695 if ($resume) {
5696 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5697 $forcemachine = $conf->{runningmachine};
5698 $forcecpu = $conf->{runningcpu};
5699 print "Resuming suspended VM\n";
5700 }
5701
5702 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5703 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5704
5705 my $migration_ip;
5706 my $get_migration_ip = sub {
5707 my ($nodename) = @_;
5708
5709 return $migration_ip if defined($migration_ip);
5710
5711 my $cidr = $migrate_opts->{network};
5712
5713 if (!defined($cidr)) {
5714 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5715 $cidr = $dc_conf->{migration}->{network};
5716 }
5717
5718 if (defined($cidr)) {
5719 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5720
5721 die "could not get IP: no address configured on local " .
5722 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5723
5724 die "could not get IP: multiple addresses configured on local " .
5725 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5726
5727 $migration_ip = @$ips[0];
5728 }
5729
5730 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5731 if !defined($migration_ip);
5732
5733 return $migration_ip;
5734 };
5735
5736 if ($statefile) {
5737 if ($statefile eq 'tcp') {
5738 my $migrate = $res->{migrate} = { proto => 'tcp' };
5739 $migrate->{addr} = "localhost";
5740 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5741 my $nodename = nodename();
5742
5743 if (!defined($migration_type)) {
5744 if (defined($datacenterconf->{migration}->{type})) {
5745 $migration_type = $datacenterconf->{migration}->{type};
5746 } else {
5747 $migration_type = 'secure';
5748 }
5749 }
5750
5751 if ($migration_type eq 'insecure') {
5752 $migrate->{addr} = $get_migration_ip->($nodename);
5753 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5754 }
5755
5756 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5757 $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5758 $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5759 push @$cmd, '-incoming', $migrate->{uri};
5760 push @$cmd, '-S';
5761
5762 } elsif ($statefile eq 'unix') {
5763 # should be default for secure migrations as a ssh TCP forward
5764 # tunnel is not deterministic reliable ready and fails regurarly
5765 # to set up in time, so use UNIX socket forwards
5766 my $migrate = $res->{migrate} = { proto => 'unix' };
5767 $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5768 unlink $migrate->{addr};
5769
5770 $migrate->{uri} = "unix:$migrate->{addr}";
5771 push @$cmd, '-incoming', $migrate->{uri};
5772 push @$cmd, '-S';
5773
5774 } elsif (-e $statefile) {
5775 push @$cmd, '-loadstate', $statefile;
5776 } else {
5777 my $statepath = PVE::Storage::path($storecfg, $statefile);
5778 push @$vollist, $statefile;
5779 push @$cmd, '-loadstate', $statepath;
5780 }
5781 } elsif ($params->{paused}) {
5782 push @$cmd, '-S';
5783 }
5784
5785 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5786
5787 my $pci_devices = {}; # host pci devices
5788 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5789 my $dev = $conf->{"hostpci$i"} or next;
5790 $pci_devices->{$i} = parse_hostpci($dev);
5791 }
5792
5793 # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5794 my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5795
5796 # map to a flat list of pci ids
5797 my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5798
5799 # reserve all PCI IDs before actually doing anything with them
5800 PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5801
5802 eval {
5803 my $uuid;
5804 for my $id (sort keys %$pci_devices) {
5805 my $d = $pci_devices->{$id};
5806 for my $dev ($d->{pciid}->@*) {
5807 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5808
5809 # nvidia grid needs the qemu parameter '-uuid' set
5810 # use smbios uuid or mdev uuid as fallback for that
5811 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5812 if (defined($conf->{smbios1})) {
5813 my $smbios_conf = parse_smbios1($conf->{smbios1});
5814 $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5815 }
5816 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id) if !defined($uuid);
5817 }
5818 }
5819 }
5820 push @$cmd, '-uuid', $uuid if defined($uuid);
5821 };
5822 if (my $err = $@) {
5823 eval { cleanup_pci_devices($vmid, $conf) };
5824 warn $@ if $@;
5825 die $err;
5826 }
5827
5828 PVE::Storage::activate_volumes($storecfg, $vollist);
5829
5830 eval {
5831 run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5832 };
5833 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5834 # timeout should be more than enough here...
5835 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5836
5837 my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5838
5839 my %run_params = (
5840 timeout => $statefile ? undef : $start_timeout,
5841 umask => 0077,
5842 noerr => 1,
5843 );
5844
5845 # when migrating, prefix QEMU output so other side can pick up any
5846 # errors that might occur and show the user
5847 if ($migratedfrom) {
5848 $run_params{quiet} = 1;
5849 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5850 }
5851
5852 my %systemd_properties = (
5853 Slice => 'qemu.slice',
5854 KillMode => 'process',
5855 SendSIGKILL => 0,
5856 TimeoutStopUSec => ULONG_MAX, # infinity
5857 );
5858
5859 if (PVE::CGroup::cgroup_mode() == 2) {
5860 $systemd_properties{CPUWeight} = $cpuunits;
5861 } else {
5862 $systemd_properties{CPUShares} = $cpuunits;
5863 }
5864
5865 if (my $cpulimit = $conf->{cpulimit}) {
5866 $systemd_properties{CPUQuota} = int($cpulimit * 100);
5867 }
5868 $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5869
5870 my $run_qemu = sub {
5871 PVE::Tools::run_fork sub {
5872 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5873
5874 my $tpmpid;
5875 if (my $tpm = $conf->{tpmstate0}) {
5876 # start the TPM emulator so QEMU can connect on start
5877 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5878 }
5879
5880 my $exitcode = run_command($cmd, %run_params);
5881 if ($exitcode) {
5882 if ($tpmpid) {
5883 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5884 kill 'TERM', $tpmpid;
5885 }
5886 die "QEMU exited with code $exitcode\n";
5887 }
5888 };
5889 };
5890
5891 if ($conf->{hugepages}) {
5892
5893 my $code = sub {
5894 my $hotplug_features =
5895 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5896 my $hugepages_topology =
5897 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5898
5899 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5900
5901 PVE::QemuServer::Memory::hugepages_mount();
5902 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5903
5904 eval { $run_qemu->() };
5905 if (my $err = $@) {
5906 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5907 if !$conf->{keephugepages};
5908 die $err;
5909 }
5910
5911 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5912 if !$conf->{keephugepages};
5913 };
5914 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5915
5916 } else {
5917 eval { $run_qemu->() };
5918 }
5919
5920 if (my $err = $@) {
5921 # deactivate volumes if start fails
5922 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5923 warn $@ if $@;
5924 eval { cleanup_pci_devices($vmid, $conf) };
5925 warn $@ if $@;
5926
5927 die "start failed: $err";
5928 }
5929
5930 # re-reserve all PCI IDs now that we can know the actual VM PID
5931 my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5932 eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5933 warn $@ if $@;
5934
5935 if (defined($res->{migrate})) {
5936 print "migration listens on $res->{migrate}->{uri}\n";
5937 } elsif ($statefile) {
5938 eval { mon_cmd($vmid, "cont"); };
5939 warn $@ if $@;
5940 }
5941
5942 #start nbd server for storage migration
5943 if (my $nbd = $migrate_opts->{nbd}) {
5944 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5945
5946 my $migrate_storage_uri;
5947 # nbd_protocol_version > 0 for unix socket support
5948 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5949 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5950 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5951 $migrate_storage_uri = "nbd:unix:$socket_path";
5952 $res->{migrate}->{unix_sockets} = [$socket_path];
5953 } else {
5954 my $nodename = nodename();
5955 my $localip = $get_migration_ip->($nodename);
5956 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5957 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5958
5959 mon_cmd($vmid, "nbd-server-start", addr => {
5960 type => 'inet',
5961 data => {
5962 host => "${localip}",
5963 port => "${storage_migrate_port}",
5964 },
5965 });
5966 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5967 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5968 }
5969
5970 my $block_info = mon_cmd($vmid, "query-block");
5971 $block_info = { map { $_->{device} => $_ } $block_info->@* };
5972
5973 foreach my $opt (sort keys %$nbd) {
5974 my $drivestr = $nbd->{$opt}->{drivestr};
5975 my $volid = $nbd->{$opt}->{volid};
5976
5977 my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5978
5979 mon_cmd(
5980 $vmid,
5981 "block-export-add",
5982 id => "drive-$opt",
5983 'node-name' => $block_node,
5984 writable => JSON::true,
5985 type => "nbd",
5986 name => "drive-$opt", # NBD export name
5987 );
5988
5989 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5990 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5991 print "re-using replicated volume: $opt - $volid\n"
5992 if $nbd->{$opt}->{replicated};
5993
5994 $res->{drives}->{$opt} = $nbd->{$opt};
5995 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5996 }
5997 }
5998
5999 if ($migratedfrom) {
6000 eval {
6001 set_migration_caps($vmid);
6002 };
6003 warn $@ if $@;
6004
6005 if ($spice_port) {
6006 print "spice listens on port $spice_port\n";
6007 $res->{spice_port} = $spice_port;
6008 if ($migrate_opts->{spice_ticket}) {
6009 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6010 $migrate_opts->{spice_ticket});
6011 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6012 }
6013 }
6014
6015 } else {
6016 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6017 if !$statefile && $conf->{balloon};
6018
6019 foreach my $opt (keys %$conf) {
6020 next if $opt !~ m/^net\d+$/;
6021 my $nicconf = parse_net($conf->{$opt});
6022 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6023 }
6024 add_nets_bridge_fdb($conf, $vmid);
6025 }
6026
6027 if (!defined($conf->{balloon}) || $conf->{balloon}) {
6028 eval {
6029 mon_cmd(
6030 $vmid,
6031 'qom-set',
6032 path => "machine/peripheral/balloon0",
6033 property => "guest-stats-polling-interval",
6034 value => 2
6035 );
6036 };
6037 log_warn("could not set polling interval for ballooning - $@") if $@;
6038 }
6039
6040 if ($resume) {
6041 print "Resumed VM, removing state\n";
6042 if (my $vmstate = $conf->{vmstate}) {
6043 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6044 PVE::Storage::vdisk_free($storecfg, $vmstate);
6045 }
6046 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6047 PVE::QemuConfig->write_config($vmid, $conf);
6048 }
6049
6050 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6051
6052 return $res;
6053 }
6054
6055 sub vm_commandline {
6056 my ($storecfg, $vmid, $snapname) = @_;
6057
6058 my $conf = PVE::QemuConfig->load_config($vmid);
6059
6060 my ($forcemachine, $forcecpu);
6061 if ($snapname) {
6062 my $snapshot = $conf->{snapshots}->{$snapname};
6063 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6064
6065 # check for machine or CPU overrides in snapshot
6066 $forcemachine = $snapshot->{runningmachine};
6067 $forcecpu = $snapshot->{runningcpu};
6068
6069 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6070
6071 $conf = $snapshot;
6072 }
6073
6074 my $defaults = load_defaults();
6075
6076 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6077
6078 return PVE::Tools::cmd2string($cmd);
6079 }
6080
6081 sub vm_reset {
6082 my ($vmid, $skiplock) = @_;
6083
6084 PVE::QemuConfig->lock_config($vmid, sub {
6085
6086 my $conf = PVE::QemuConfig->load_config($vmid);
6087
6088 PVE::QemuConfig->check_lock($conf) if !$skiplock;
6089
6090 mon_cmd($vmid, "system_reset");
6091 });
6092 }
6093
6094 sub get_vm_volumes {
6095 my ($conf) = @_;
6096
6097 my $vollist = [];
6098 foreach_volid($conf, sub {
6099 my ($volid, $attr) = @_;
6100
6101 return if $volid =~ m|^/|;
6102
6103 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6104 return if !$sid;
6105
6106 push @$vollist, $volid;
6107 });
6108
6109 return $vollist;
6110 }
6111
6112 sub cleanup_pci_devices {
6113 my ($vmid, $conf) = @_;
6114
6115 foreach my $key (keys %$conf) {
6116 next if $key !~ m/^hostpci(\d+)$/;
6117 my $hostpciindex = $1;
6118 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6119 my $d = parse_hostpci($conf->{$key});
6120 if ($d->{mdev}) {
6121 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6122 # don't want to break ABI just for this two liner
6123 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6124
6125 # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6126 # out when we do it first. so wait for 10 seconds and then try it
6127 my $pciid = $d->{pciid}->[0]->{id};
6128 my $info = PVE::SysFSTools::pci_device_info("$pciid");
6129 if ($info->{vendor} eq '10de') {
6130 sleep 10;
6131 }
6132
6133 PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6134 }
6135 }
6136 PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6137 }
6138
6139 sub vm_stop_cleanup {
6140 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6141
6142 eval {
6143
6144 if (!$keepActive) {
6145 my $vollist = get_vm_volumes($conf);
6146 PVE::Storage::deactivate_volumes($storecfg, $vollist);
6147
6148 if (my $tpmdrive = $conf->{tpmstate0}) {
6149 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6150 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6151 if ($storeid) {
6152 PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6153 }
6154 }
6155 }
6156
6157 foreach my $ext (qw(mon qmp pid vnc qga)) {
6158 unlink "/var/run/qemu-server/${vmid}.$ext";
6159 }
6160
6161 if ($conf->{ivshmem}) {
6162 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6163 # just delete it for now, VMs which have this already open do not
6164 # are affected, but new VMs will get a separated one. If this
6165 # becomes an issue we either add some sort of ref-counting or just
6166 # add a "don't delete on stop" flag to the ivshmem format.
6167 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6168 }
6169
6170 cleanup_pci_devices($vmid, $conf);
6171
6172 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6173 };
6174 warn $@ if $@; # avoid errors - just warn
6175 }
6176
6177 # call only in locked context
6178 sub _do_vm_stop {
6179 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6180
6181 my $pid = check_running($vmid, $nocheck);
6182 return if !$pid;
6183
6184 my $conf;
6185 if (!$nocheck) {
6186 $conf = PVE::QemuConfig->load_config($vmid);
6187 PVE::QemuConfig->check_lock($conf) if !$skiplock;
6188 if (!defined($timeout) && $shutdown && $conf->{startup}) {
6189 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6190 $timeout = $opts->{down} if $opts->{down};
6191 }
6192 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6193 }
6194
6195 eval {
6196 if ($shutdown) {
6197 if (defined($conf) && get_qga_key($conf, 'enabled')) {
6198 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6199 } else {
6200 mon_cmd($vmid, "system_powerdown");
6201 }
6202 } else {
6203 mon_cmd($vmid, "quit");
6204 }
6205 };
6206 my $err = $@;
6207
6208 if (!$err) {
6209 $timeout = 60 if !defined($timeout);
6210
6211 my $count = 0;
6212 while (($count < $timeout) && check_running($vmid, $nocheck)) {
6213 $count++;
6214 sleep 1;
6215 }
6216
6217 if ($count >= $timeout) {
6218 if ($force) {
6219 warn "VM still running - terminating now with SIGTERM\n";
6220 kill 15, $pid;
6221 } else {
6222 die "VM quit/powerdown failed - got timeout\n";
6223 }
6224 } else {
6225 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6226 return;
6227 }
6228 } else {
6229 if (!check_running($vmid, $nocheck)) {
6230 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6231 return;
6232 }
6233 if ($force) {
6234 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6235 kill 15, $pid;
6236 } else {
6237 die "VM quit/powerdown failed\n";
6238 }
6239 }
6240
6241 # wait again
6242 $timeout = 10;
6243
6244 my $count = 0;
6245 while (($count < $timeout) && check_running($vmid, $nocheck)) {
6246 $count++;
6247 sleep 1;
6248 }
6249
6250 if ($count >= $timeout) {
6251 warn "VM still running - terminating now with SIGKILL\n";
6252 kill 9, $pid;
6253 sleep 1;
6254 }
6255
6256 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6257 }
6258
6259 # Note: use $nocheck to skip tests if VM configuration file exists.
6260 # We need that when migration VMs to other nodes (files already moved)
6261 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6262 sub vm_stop {
6263 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6264
6265 $force = 1 if !defined($force) && !$shutdown;
6266
6267 if ($migratedfrom){
6268 my $pid = check_running($vmid, $nocheck, $migratedfrom);
6269 kill 15, $pid if $pid;
6270 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6271 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6272 return;
6273 }
6274
6275 PVE::QemuConfig->lock_config($vmid, sub {
6276 _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6277 });
6278 }
6279
6280 sub vm_reboot {
6281 my ($vmid, $timeout) = @_;
6282
6283 PVE::QemuConfig->lock_config($vmid, sub {
6284 eval {
6285
6286 # only reboot if running, as qmeventd starts it again on a stop event
6287 return if !check_running($vmid);
6288
6289 create_reboot_request($vmid);
6290
6291 my $storecfg = PVE::Storage::config();
6292 _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6293
6294 };
6295 if (my $err = $@) {
6296 # avoid that the next normal shutdown will be confused for a reboot
6297 clear_reboot_request($vmid);
6298 die $err;
6299 }
6300 });
6301 }
6302
6303 # note: if using the statestorage parameter, the caller has to check privileges
6304 sub vm_suspend {
6305 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6306
6307 my $conf;
6308 my $path;
6309 my $storecfg;
6310 my $vmstate;
6311
6312 PVE::QemuConfig->lock_config($vmid, sub {
6313
6314 $conf = PVE::QemuConfig->load_config($vmid);
6315
6316 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6317 PVE::QemuConfig->check_lock($conf)
6318 if !($skiplock || $is_backing_up);
6319
6320 die "cannot suspend to disk during backup\n"
6321 if $is_backing_up && $includestate;
6322
6323 if ($includestate) {
6324 $conf->{lock} = 'suspending';
6325 my $date = strftime("%Y-%m-%d", localtime(time()));
6326 $storecfg = PVE::Storage::config();
6327 if (!$statestorage) {
6328 $statestorage = find_vmstate_storage($conf, $storecfg);
6329 # check permissions for the storage
6330 my $rpcenv = PVE::RPCEnvironment::get();
6331 if ($rpcenv->{type} ne 'cli') {
6332 my $authuser = $rpcenv->get_user();
6333 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6334 }
6335 }
6336
6337
6338 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6339 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6340 $path = PVE::Storage::path($storecfg, $vmstate);
6341 PVE::QemuConfig->write_config($vmid, $conf);
6342 } else {
6343 mon_cmd($vmid, "stop");
6344 }
6345 });
6346
6347 if ($includestate) {
6348 # save vm state
6349 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6350
6351 eval {
6352 set_migration_caps($vmid, 1);
6353 mon_cmd($vmid, "savevm-start", statefile => $path);
6354 for(;;) {
6355 my $state = mon_cmd($vmid, "query-savevm");
6356 if (!$state->{status}) {
6357 die "savevm not active\n";
6358 } elsif ($state->{status} eq 'active') {
6359 sleep(1);
6360 next;
6361 } elsif ($state->{status} eq 'completed') {
6362 print "State saved, quitting\n";
6363 last;
6364 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6365 die "query-savevm failed with error '$state->{error}'\n"
6366 } else {
6367 die "query-savevm returned status '$state->{status}'\n";
6368 }
6369 }
6370 };
6371 my $err = $@;
6372
6373 PVE::QemuConfig->lock_config($vmid, sub {
6374 $conf = PVE::QemuConfig->load_config($vmid);
6375 if ($err) {
6376 # cleanup, but leave suspending lock, to indicate something went wrong
6377 eval {
6378 mon_cmd($vmid, "savevm-end");
6379 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6380 PVE::Storage::vdisk_free($storecfg, $vmstate);
6381 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6382 PVE::QemuConfig->write_config($vmid, $conf);
6383 };
6384 warn $@ if $@;
6385 die $err;
6386 }
6387
6388 die "lock changed unexpectedly\n"
6389 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6390
6391 mon_cmd($vmid, "quit");
6392 $conf->{lock} = 'suspended';
6393 PVE::QemuConfig->write_config($vmid, $conf);
6394 });
6395 }
6396 }
6397
6398 # $nocheck is set when called as part of a migration - in this context the
6399 # location of the config file (source or target node) is not deterministic,
6400 # since migration cannot wait for pmxcfs to process the rename
6401 sub vm_resume {
6402 my ($vmid, $skiplock, $nocheck) = @_;
6403
6404 PVE::QemuConfig->lock_config($vmid, sub {
6405 my $res = mon_cmd($vmid, 'query-status');
6406 my $resume_cmd = 'cont';
6407 my $reset = 0;
6408 my $conf;
6409 if ($nocheck) {
6410 $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6411 if ($@) {
6412 my $vmlist = PVE::Cluster::get_vmlist();
6413 if (exists($vmlist->{ids}->{$vmid})) {
6414 my $node = $vmlist->{ids}->{$vmid}->{node};
6415 $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6416 }
6417 if (!$conf) {
6418 PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6419 $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6420 }
6421 }
6422 } else {
6423 $conf = PVE::QemuConfig->load_config($vmid);
6424 }
6425
6426 if ($res->{status}) {
6427 return if $res->{status} eq 'running'; # job done, go home
6428 $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6429 $reset = 1 if $res->{status} eq 'shutdown';
6430 }
6431
6432 if (!$nocheck) {
6433 PVE::QemuConfig->check_lock($conf)
6434 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6435 }
6436
6437 if ($reset) {
6438 # required if a VM shuts down during a backup and we get a resume
6439 # request before the backup finishes for example
6440 mon_cmd($vmid, "system_reset");
6441 }
6442
6443 add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6444
6445 mon_cmd($vmid, $resume_cmd);
6446 });
6447 }
6448
6449 sub vm_sendkey {
6450 my ($vmid, $skiplock, $key) = @_;
6451
6452 PVE::QemuConfig->lock_config($vmid, sub {
6453
6454 my $conf = PVE::QemuConfig->load_config($vmid);
6455
6456 # there is no qmp command, so we use the human monitor command
6457 my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6458 die $res if $res ne '';
6459 });
6460 }
6461
6462 sub check_bridge_access {
6463 my ($rpcenv, $authuser, $conf) = @_;
6464
6465 return 1 if $authuser eq 'root@pam';
6466
6467 for my $opt (sort keys $conf->%*) {
6468 next if $opt !~ m/^net\d+$/;
6469 my $net = parse_net($conf->{$opt});
6470 my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6471 PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6472 }
6473 return 1;
6474 };
6475
6476 sub check_mapping_access {
6477 my ($rpcenv, $user, $conf) = @_;
6478
6479 for my $opt (keys $conf->%*) {
6480 if ($opt =~ m/^usb\d+$/) {
6481 my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6482 if (my $host = $device->{host}) {
6483 die "only root can set '$opt' config for real devices\n"
6484 if $host !~ m/^spice$/i && $user ne 'root@pam';
6485 } elsif ($device->{mapping}) {
6486 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6487 } else {
6488 die "either 'host' or 'mapping' must be set.\n";
6489 }
6490 }
6491 }
6492 };
6493
6494 # FIXME: improve checks on restore by checking before actually extracing and
6495 # merging the new config
6496 sub check_restore_permissions {
6497 my ($rpcenv, $user, $conf) = @_;
6498 check_bridge_access($rpcenv, $user, $conf);
6499 check_mapping_access($rpcenv, $user, $conf);
6500 }
6501 # vzdump restore implementaion
6502
6503 sub tar_archive_read_firstfile {
6504 my $archive = shift;
6505
6506 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6507
6508 # try to detect archive type first
6509 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6510 die "unable to open file '$archive'\n";
6511 my $firstfile = <$fh>;
6512 kill 15, $pid;
6513 close $fh;
6514
6515 die "ERROR: archive contaions no data\n" if !$firstfile;
6516 chomp $firstfile;
6517
6518 return $firstfile;
6519 }
6520
6521 sub tar_restore_cleanup {
6522 my ($storecfg, $statfile) = @_;
6523
6524 print STDERR "starting cleanup\n";
6525
6526 if (my $fd = IO::File->new($statfile, "r")) {
6527 while (defined(my $line = <$fd>)) {
6528 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6529 my $volid = $2;
6530 eval {
6531 if ($volid =~ m|^/|) {
6532 unlink $volid || die 'unlink failed\n';
6533 } else {
6534 PVE::Storage::vdisk_free($storecfg, $volid);
6535 }
6536 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6537 };
6538 print STDERR "unable to cleanup '$volid' - $@" if $@;
6539 } else {
6540 print STDERR "unable to parse line in statfile - $line";
6541 }
6542 }
6543 $fd->close();
6544 }
6545 }
6546
6547 sub restore_file_archive {
6548 my ($archive, $vmid, $user, $opts) = @_;
6549
6550 return restore_vma_archive($archive, $vmid, $user, $opts)
6551 if $archive eq '-';
6552
6553 my $info = PVE::Storage::archive_info($archive);
6554 my $format = $opts->{format} // $info->{format};
6555 my $comp = $info->{compression};
6556
6557 # try to detect archive format
6558 if ($format eq 'tar') {
6559 return restore_tar_archive($archive, $vmid, $user, $opts);
6560 } else {
6561 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6562 }
6563 }
6564
6565 # hepler to remove disks that will not be used after restore
6566 my $restore_cleanup_oldconf = sub {
6567 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6568
6569 my $kept_disks = {};
6570
6571 PVE::QemuConfig->foreach_volume($oldconf, sub {
6572 my ($ds, $drive) = @_;
6573
6574 return if drive_is_cdrom($drive, 1);
6575
6576 my $volid = $drive->{file};
6577 return if !$volid || $volid =~ m|^/|;
6578
6579 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6580 return if !$path || !$owner || ($owner != $vmid);
6581
6582 # Note: only delete disk we want to restore
6583 # other volumes will become unused
6584 if ($virtdev_hash->{$ds}) {
6585 eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6586 if (my $err = $@) {
6587 warn $err;
6588 }
6589 } else {
6590 $kept_disks->{$volid} = 1;
6591 }
6592 });
6593
6594 # after the restore we have no snapshots anymore
6595 for my $snapname (keys $oldconf->{snapshots}->%*) {
6596 my $snap = $oldconf->{snapshots}->{$snapname};
6597 if ($snap->{vmstate}) {
6598 eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6599 if (my $err = $@) {
6600 warn $err;
6601 }
6602 }
6603
6604 for my $volid (keys $kept_disks->%*) {
6605 eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6606 warn $@ if $@;
6607 }
6608 }
6609 };
6610
6611 # Helper to parse vzdump backup device hints
6612 #
6613 # $rpcenv: Environment, used to ckeck storage permissions
6614 # $user: User ID, to check storage permissions
6615 # $storecfg: Storage configuration
6616 # $fh: the file handle for reading the configuration
6617 # $devinfo: should contain device sizes for all backu-up'ed devices
6618 # $options: backup options (pool, default storage)
6619 #
6620 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6621 my $parse_backup_hints = sub {
6622 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6623
6624 my $check_storage = sub { # assert if an image can be allocate
6625 my ($storeid, $scfg) = @_;
6626 die "Content type 'images' is not available on storage '$storeid'\n"
6627 if !$scfg->{content}->{images};
6628 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6629 if $user ne 'root@pam';
6630 };
6631
6632 my $virtdev_hash = {};
6633 while (defined(my $line = <$fh>)) {
6634 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6635 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6636 die "archive does not contain data for drive '$virtdev'\n"
6637 if !$devinfo->{$devname};
6638
6639 if (defined($options->{storage})) {
6640 $storeid = $options->{storage} || 'local';
6641 } elsif (!$storeid) {
6642 $storeid = 'local';
6643 }
6644 $format = 'raw' if !$format;
6645 $devinfo->{$devname}->{devname} = $devname;
6646 $devinfo->{$devname}->{virtdev} = $virtdev;
6647 $devinfo->{$devname}->{format} = $format;
6648 $devinfo->{$devname}->{storeid} = $storeid;
6649
6650 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6651 $check_storage->($storeid, $scfg); # permission and content type check
6652
6653 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6654 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6655 my $virtdev = $1;
6656 my $drive = parse_drive($virtdev, $2);
6657
6658 if (drive_is_cloudinit($drive)) {
6659 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6660 $storeid = $options->{storage} if defined ($options->{storage});
6661 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6662 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6663
6664 $check_storage->($storeid, $scfg); # permission and content type check
6665
6666 $virtdev_hash->{$virtdev} = {
6667 format => $format,
6668 storeid => $storeid,
6669 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6670 is_cloudinit => 1,
6671 };
6672 }
6673 }
6674 }
6675
6676 return $virtdev_hash;
6677 };
6678
6679 # Helper to allocate and activate all volumes required for a restore
6680 #
6681 # $storecfg: Storage configuration
6682 # $virtdev_hash: as returned by parse_backup_hints()
6683 #
6684 # Returns: { $virtdev => $volid }
6685 my $restore_allocate_devices = sub {
6686 my ($storecfg, $virtdev_hash, $vmid) = @_;
6687
6688 my $map = {};
6689 foreach my $virtdev (sort keys %$virtdev_hash) {
6690 my $d = $virtdev_hash->{$virtdev};
6691 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6692 my $storeid = $d->{storeid};
6693 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6694
6695 # test if requested format is supported
6696 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6697 my $supported = grep { $_ eq $d->{format} } @$validFormats;
6698 $d->{format} = $defFormat if !$supported;
6699
6700 my $name;
6701 if ($d->{is_cloudinit}) {
6702 $name = "vm-$vmid-cloudinit";
6703 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6704 if ($scfg->{path}) {
6705 $name .= ".$d->{format}";
6706 }
6707 }
6708
6709 my $volid = PVE::Storage::vdisk_alloc(
6710 $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6711
6712 print STDERR "new volume ID is '$volid'\n";
6713 $d->{volid} = $volid;
6714
6715 PVE::Storage::activate_volumes($storecfg, [$volid]);
6716
6717 $map->{$virtdev} = $volid;
6718 }
6719
6720 return $map;
6721 };
6722
6723 sub restore_update_config_line {
6724 my ($cookie, $map, $line, $unique) = @_;
6725
6726 return '' if $line =~ m/^\#qmdump\#/;
6727 return '' if $line =~ m/^\#vzdump\#/;
6728 return '' if $line =~ m/^lock:/;
6729 return '' if $line =~ m/^unused\d+:/;
6730 return '' if $line =~ m/^parent:/;
6731
6732 my $res = '';
6733
6734 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6735 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6736 # try to convert old 1.X settings
6737 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6738 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6739 my ($model, $macaddr) = split(/\=/, $devconfig);
6740 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6741 my $net = {
6742 model => $model,
6743 bridge => "vmbr$ind",
6744 macaddr => $macaddr,
6745 };
6746 my $netstr = print_net($net);
6747
6748 $res .= "net$cookie->{netcount}: $netstr\n";
6749 $cookie->{netcount}++;
6750 }
6751 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6752 my ($id, $netstr) = ($1, $2);
6753 my $net = parse_net($netstr);
6754 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6755 $netstr = print_net($net);
6756 $res .= "$id: $netstr\n";
6757 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6758 my $virtdev = $1;
6759 my $value = $3;
6760 my $di = parse_drive($virtdev, $value);
6761 if (defined($di->{backup}) && !$di->{backup}) {
6762 $res .= "#$line";
6763 } elsif ($map->{$virtdev}) {
6764 delete $di->{format}; # format can change on restore
6765 $di->{file} = $map->{$virtdev};
6766 $value = print_drive($di);
6767 $res .= "$virtdev: $value\n";
6768 } else {
6769 $res .= $line;
6770 }
6771 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6772 my $vmgenid = $1;
6773 if ($vmgenid ne '0') {
6774 # always generate a new vmgenid if there was a valid one setup
6775 $vmgenid = generate_uuid();
6776 }
6777 $res .= "vmgenid: $vmgenid\n";
6778 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6779 my ($uuid, $uuid_str);
6780 UUID::generate($uuid);
6781 UUID::unparse($uuid, $uuid_str);
6782 my $smbios1 = parse_smbios1($2);
6783 $smbios1->{uuid} = $uuid_str;
6784 $res .= $1.print_smbios1($smbios1)."\n";
6785 } else {
6786 $res .= $line;
6787 }
6788
6789 return $res;
6790 }
6791
6792 my $restore_deactivate_volumes = sub {
6793 my ($storecfg, $virtdev_hash) = @_;
6794
6795 my $vollist = [];
6796 for my $dev (values $virtdev_hash->%*) {
6797 push $vollist->@*, $dev->{volid} if $dev->{volid};
6798 }
6799
6800 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6801 print STDERR $@ if $@;
6802 };
6803
6804 my $restore_destroy_volumes = sub {
6805 my ($storecfg, $virtdev_hash) = @_;
6806
6807 for my $dev (values $virtdev_hash->%*) {
6808 my $volid = $dev->{volid} or next;
6809 eval {
6810 PVE::Storage::vdisk_free($storecfg, $volid);
6811 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6812 };
6813 print STDERR "unable to cleanup '$volid' - $@" if $@;
6814 }
6815 };
6816
6817 my $restore_merge_config = sub {
6818 my ($filename, $backup_conf_raw, $override_conf) = @_;
6819
6820 my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6821 for my $key (keys $override_conf->%*) {
6822 $backup_conf->{$key} = $override_conf->{$key};
6823 }
6824
6825 return $backup_conf;
6826 };
6827
6828 sub scan_volids {
6829 my ($cfg, $vmid) = @_;
6830
6831 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6832
6833 my $volid_hash = {};
6834 foreach my $storeid (keys %$info) {
6835 foreach my $item (@{$info->{$storeid}}) {
6836 next if !($item->{volid} && $item->{size});
6837 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6838 $volid_hash->{$item->{volid}} = $item;
6839 }
6840 }
6841
6842 return $volid_hash;
6843 }
6844
6845 sub update_disk_config {
6846 my ($vmid, $conf, $volid_hash) = @_;
6847
6848 my $changes;
6849 my $prefix = "VM $vmid";
6850
6851 # used and unused disks
6852 my $referenced = {};
6853
6854 # Note: it is allowed to define multiple storages with same path (alias), so
6855 # we need to check both 'volid' and real 'path' (two different volid can point
6856 # to the same path).
6857
6858 my $referencedpath = {};
6859
6860 # update size info
6861 PVE::QemuConfig->foreach_volume($conf, sub {
6862 my ($opt, $drive) = @_;
6863
6864 my $volid = $drive->{file};
6865 return if !$volid;
6866 my $volume = $volid_hash->{$volid};
6867
6868 # mark volid as "in-use" for next step
6869 $referenced->{$volid} = 1;
6870 if ($volume && (my $path = $volume->{path})) {
6871 $referencedpath->{$path} = 1;
6872 }
6873
6874 return if drive_is_cdrom($drive);
6875 return if !$volume;
6876
6877 my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6878 if (defined($updated)) {
6879 $changes = 1;
6880 $conf->{$opt} = print_drive($updated);
6881 print "$prefix ($opt): $msg\n";
6882 }
6883 });
6884
6885 # remove 'unusedX' entry if volume is used
6886 PVE::QemuConfig->foreach_unused_volume($conf, sub {
6887 my ($opt, $drive) = @_;
6888
6889 my $volid = $drive->{file};
6890 return if !$volid;
6891
6892 my $path;
6893 $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6894 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6895 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6896 $changes = 1;
6897 delete $conf->{$opt};
6898 }
6899
6900 $referenced->{$volid} = 1;
6901 $referencedpath->{$path} = 1 if $path;
6902 });
6903
6904 foreach my $volid (sort keys %$volid_hash) {
6905 next if $volid =~ m/vm-$vmid-state-/;
6906 next if $referenced->{$volid};
6907 my $path = $volid_hash->{$volid}->{path};
6908 next if !$path; # just to be sure
6909 next if $referencedpath->{$path};
6910 $changes = 1;
6911 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6912 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6913 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6914 }
6915
6916 return $changes;
6917 }
6918
6919 sub rescan {
6920 my ($vmid, $nolock, $dryrun) = @_;
6921
6922 my $cfg = PVE::Storage::config();
6923
6924 print "rescan volumes...\n";
6925 my $volid_hash = scan_volids($cfg, $vmid);
6926
6927 my $updatefn = sub {
6928 my ($vmid) = @_;
6929
6930 my $conf = PVE::QemuConfig->load_config($vmid);
6931
6932 PVE::QemuConfig->check_lock($conf);
6933
6934 my $vm_volids = {};
6935 foreach my $volid (keys %$volid_hash) {
6936 my $info = $volid_hash->{$volid};
6937 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6938 }
6939
6940 my $changes = update_disk_config($vmid, $conf, $vm_volids);
6941
6942 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6943 };
6944
6945 if (defined($vmid)) {
6946 if ($nolock) {
6947 &$updatefn($vmid);
6948 } else {
6949 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6950 }
6951 } else {
6952 my $vmlist = config_list();
6953 foreach my $vmid (keys %$vmlist) {
6954 if ($nolock) {
6955 &$updatefn($vmid);
6956 } else {
6957 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6958 }
6959 }
6960 }
6961 }
6962
6963 sub restore_proxmox_backup_archive {
6964 my ($archive, $vmid, $user, $options) = @_;
6965
6966 my $storecfg = PVE::Storage::config();
6967
6968 my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6969 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6970
6971 my $fingerprint = $scfg->{fingerprint};
6972 my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6973
6974 my $repo = PVE::PBSClient::get_repository($scfg);
6975 my $namespace = $scfg->{namespace};
6976
6977 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6978 my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6979 local $ENV{PBS_PASSWORD} = $password;
6980 local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6981
6982 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6983 PVE::Storage::parse_volname($storecfg, $archive);
6984
6985 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6986
6987 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6988
6989 my $tmpdir = "/var/tmp/vzdumptmp$$";
6990 rmtree $tmpdir;
6991 mkpath $tmpdir;
6992
6993 my $conffile = PVE::QemuConfig->config_file($vmid);
6994 # disable interrupts (always do cleanups)
6995 local $SIG{INT} =
6996 local $SIG{TERM} =
6997 local $SIG{QUIT} =
6998 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6999
7000 # Note: $oldconf is undef if VM does not exists
7001 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7002 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7003 my $new_conf_raw = '';
7004
7005 my $rpcenv = PVE::RPCEnvironment::get();
7006 my $devinfo = {}; # info about drives included in backup
7007 my $virtdev_hash = {}; # info about allocated drives
7008
7009 eval {
7010 # enable interrupts
7011 local $SIG{INT} =
7012 local $SIG{TERM} =
7013 local $SIG{QUIT} =
7014 local $SIG{HUP} =
7015 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7016
7017 my $cfgfn = "$tmpdir/qemu-server.conf";
7018 my $firewall_config_fn = "$tmpdir/fw.conf";
7019 my $index_fn = "$tmpdir/index.json";
7020
7021 my $cmd = "restore";
7022
7023 my $param = [$pbs_backup_name, "index.json", $index_fn];
7024 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7025 my $index = PVE::Tools::file_get_contents($index_fn);
7026 $index = decode_json($index);
7027
7028 foreach my $info (@{$index->{files}}) {
7029 if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7030 my $devname = $1;
7031 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7032 $devinfo->{$devname}->{size} = $1;
7033 } else {
7034 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7035 }
7036 }
7037 }
7038
7039 my $is_qemu_server_backup = scalar(
7040 grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7041 );
7042 if (!$is_qemu_server_backup) {
7043 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7044 }
7045 my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7046
7047 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7048 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7049
7050 if ($has_firewall_config) {
7051 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7052 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7053
7054 my $pve_firewall_dir = '/etc/pve/firewall';
7055 mkdir $pve_firewall_dir; # make sure the dir exists
7056 PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7057 }
7058
7059 my $fh = IO::File->new($cfgfn, "r") ||
7060 die "unable to read qemu-server.conf - $!\n";
7061
7062 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7063
7064 # fixme: rate limit?
7065
7066 # create empty/temp config
7067 PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7068
7069 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7070
7071 # allocate volumes
7072 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7073
7074 foreach my $virtdev (sort keys %$virtdev_hash) {
7075 my $d = $virtdev_hash->{$virtdev};
7076 next if $d->{is_cloudinit}; # no need to restore cloudinit
7077
7078 # this fails if storage is unavailable
7079 my $volid = $d->{volid};
7080 my $path = PVE::Storage::path($storecfg, $volid);
7081
7082 # for live-restore we only want to preload the efidisk and TPM state
7083 next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7084
7085 my @ns_arg;
7086 if (defined(my $ns = $scfg->{namespace})) {
7087 @ns_arg = ('--ns', $ns);
7088 }
7089
7090 my $pbs_restore_cmd = [
7091 '/usr/bin/pbs-restore',
7092 '--repository', $repo,
7093 @ns_arg,
7094 $pbs_backup_name,
7095 "$d->{devname}.img.fidx",
7096 $path,
7097 '--verbose',
7098 ];
7099
7100 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7101 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7102
7103 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7104 push @$pbs_restore_cmd, '--skip-zero';
7105 }
7106
7107 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7108 print "restore proxmox backup image: $dbg_cmdstring\n";
7109 run_command($pbs_restore_cmd);
7110 }
7111
7112 $fh->seek(0, 0) || die "seek failed - $!\n";
7113
7114 my $cookie = { netcount => 0 };
7115 while (defined(my $line = <$fh>)) {
7116 $new_conf_raw .= restore_update_config_line(
7117 $cookie,
7118 $map,
7119 $line,
7120 $options->{unique},
7121 );
7122 }
7123
7124 $fh->close();
7125 };
7126 my $err = $@;
7127
7128 if ($err || !$options->{live}) {
7129 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7130 }
7131
7132 rmtree $tmpdir;
7133
7134 if ($err) {
7135 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7136 die $err;
7137 }
7138
7139 if ($options->{live}) {
7140 # keep lock during live-restore
7141 $new_conf_raw .= "\nlock: create";
7142 }
7143
7144 my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7145 check_restore_permissions($rpcenv, $user, $new_conf);
7146 PVE::QemuConfig->write_config($vmid, $new_conf);
7147
7148 eval { rescan($vmid, 1); };
7149 warn $@ if $@;
7150
7151 PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7152
7153 if ($options->{live}) {
7154 # enable interrupts
7155 local $SIG{INT} =
7156 local $SIG{TERM} =
7157 local $SIG{QUIT} =
7158 local $SIG{HUP} =
7159 local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7160
7161 my $conf = PVE::QemuConfig->load_config($vmid);
7162 die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7163
7164 # these special drives are already restored before start
7165 delete $devinfo->{'drive-efidisk0'};
7166 delete $devinfo->{'drive-tpmstate0-backup'};
7167
7168 my $pbs_opts = {
7169 repo => $repo,
7170 keyfile => $keyfile,
7171 snapshot => $pbs_backup_name,
7172 namespace => $namespace,
7173 };
7174 pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7175
7176 PVE::QemuConfig->remove_lock($vmid, "create");
7177 }
7178 }
7179
7180 sub pbs_live_restore {
7181 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7182
7183 print "starting VM for live-restore\n";
7184 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7185
7186 my $pbs_backing = {};
7187 for my $ds (keys %$restored_disks) {
7188 $ds =~ m/^drive-(.*)$/;
7189 my $confname = $1;
7190 $pbs_backing->{$confname} = {
7191 repository => $opts->{repo},
7192 snapshot => $opts->{snapshot},
7193 archive => "$ds.img.fidx",
7194 };
7195 $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7196 $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7197
7198 my $drive = parse_drive($confname, $conf->{$confname});
7199 print "restoring '$ds' to '$drive->{file}'\n";
7200 }
7201
7202 my $drives_streamed = 0;
7203 eval {
7204 # make sure HA doesn't interrupt our restore by stopping the VM
7205 if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7206 run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7207 }
7208
7209 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7210 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7211 vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7212
7213 my $qmeventd_fd = register_qmeventd_handle($vmid);
7214
7215 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7216 # this will effectively collapse the backing image chain consisting of
7217 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7218 # removes itself once all backing images vanish with 'auto-remove=on')
7219 my $jobs = {};
7220 for my $ds (sort keys %$restored_disks) {
7221 my $job_id = "restore-$ds";
7222 mon_cmd($vmid, 'block-stream',
7223 'job-id' => $job_id,
7224 device => "$ds",
7225 );
7226 $jobs->{$job_id} = {};
7227 }
7228
7229 mon_cmd($vmid, 'cont');
7230 qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7231
7232 print "restore-drive jobs finished successfully, removing all tracking block devices"
7233 ." to disconnect from Proxmox Backup Server\n";
7234
7235 for my $ds (sort keys %$restored_disks) {
7236 mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7237 }
7238
7239 close($qmeventd_fd);
7240 };
7241
7242 my $err = $@;
7243
7244 if ($err) {
7245 warn "An error occurred during live-restore: $err\n";
7246 _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7247 die "live-restore failed\n";
7248 }
7249 }
7250
7251 sub restore_vma_archive {
7252 my ($archive, $vmid, $user, $opts, $comp) = @_;
7253
7254 my $readfrom = $archive;
7255
7256 my $cfg = PVE::Storage::config();
7257 my $commands = [];
7258 my $bwlimit = $opts->{bwlimit};
7259
7260 my $dbg_cmdstring = '';
7261 my $add_pipe = sub {
7262 my ($cmd) = @_;
7263 push @$commands, $cmd;
7264 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7265 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7266 $readfrom = '-';
7267 };
7268
7269 my $input = undef;
7270 if ($archive eq '-') {
7271 $input = '<&STDIN';
7272 } else {
7273 # If we use a backup from a PVE defined storage we also consider that
7274 # storage's rate limit:
7275 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7276 if (defined($volid)) {
7277 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7278 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7279 if ($readlimit) {
7280 print STDERR "applying read rate limit: $readlimit\n";
7281 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7282 $add_pipe->($cstream);
7283 }
7284 }
7285 }
7286
7287 if ($comp) {
7288 my $info = PVE::Storage::decompressor_info('vma', $comp);
7289 my $cmd = $info->{decompressor};
7290 push @$cmd, $readfrom;
7291 $add_pipe->($cmd);
7292 }
7293
7294 my $tmpdir = "/var/tmp/vzdumptmp$$";
7295 rmtree $tmpdir;
7296
7297 # disable interrupts (always do cleanups)
7298 local $SIG{INT} =
7299 local $SIG{TERM} =
7300 local $SIG{QUIT} =
7301 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7302
7303 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7304 POSIX::mkfifo($mapfifo, 0600);
7305 my $fifofh;
7306 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7307
7308 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7309
7310 my $oldtimeout;
7311 my $timeout = 5;
7312
7313 my $devinfo = {}; # info about drives included in backup
7314 my $virtdev_hash = {}; # info about allocated drives
7315
7316 my $rpcenv = PVE::RPCEnvironment::get();
7317
7318 my $conffile = PVE::QemuConfig->config_file($vmid);
7319
7320 # Note: $oldconf is undef if VM does not exist
7321 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7322 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7323 my $new_conf_raw = '';
7324
7325 my %storage_limits;
7326
7327 my $print_devmap = sub {
7328 my $cfgfn = "$tmpdir/qemu-server.conf";
7329
7330 # we can read the config - that is already extracted
7331 my $fh = IO::File->new($cfgfn, "r") ||
7332 die "unable to read qemu-server.conf - $!\n";
7333
7334 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7335 if (-f $fwcfgfn) {
7336 my $pve_firewall_dir = '/etc/pve/firewall';
7337 mkdir $pve_firewall_dir; # make sure the dir exists
7338 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7339 }
7340
7341 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7342
7343 foreach my $info (values %{$virtdev_hash}) {
7344 my $storeid = $info->{storeid};
7345 next if defined($storage_limits{$storeid});
7346
7347 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7348 print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7349 $storage_limits{$storeid} = $limit * 1024;
7350 }
7351
7352 foreach my $devname (keys %$devinfo) {
7353 die "found no device mapping information for device '$devname'\n"
7354 if !$devinfo->{$devname}->{virtdev};
7355 }
7356
7357 # create empty/temp config
7358 if ($oldconf) {
7359 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7360 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7361 }
7362
7363 # allocate volumes
7364 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7365
7366 # print restore information to $fifofh
7367 foreach my $virtdev (sort keys %$virtdev_hash) {
7368 my $d = $virtdev_hash->{$virtdev};
7369 next if $d->{is_cloudinit}; # no need to restore cloudinit
7370
7371 my $storeid = $d->{storeid};
7372 my $volid = $d->{volid};
7373
7374 my $map_opts = '';
7375 if (my $limit = $storage_limits{$storeid}) {
7376 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7377 }
7378
7379 my $write_zeros = 1;
7380 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7381 $write_zeros = 0;
7382 }
7383
7384 my $path = PVE::Storage::path($cfg, $volid);
7385
7386 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7387
7388 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7389 }
7390
7391 $fh->seek(0, 0) || die "seek failed - $!\n";
7392
7393 my $cookie = { netcount => 0 };
7394 while (defined(my $line = <$fh>)) {
7395 $new_conf_raw .= restore_update_config_line(
7396 $cookie,
7397 $map,
7398 $line,
7399 $opts->{unique},
7400 );
7401 }
7402
7403 $fh->close();
7404 };
7405
7406 eval {
7407 # enable interrupts
7408 local $SIG{INT} =
7409 local $SIG{TERM} =
7410 local $SIG{QUIT} =
7411 local $SIG{HUP} =
7412 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7413 local $SIG{ALRM} = sub { die "got timeout\n"; };
7414
7415 $oldtimeout = alarm($timeout);
7416
7417 my $parser = sub {
7418 my $line = shift;
7419
7420 print "$line\n";
7421
7422 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7423 my ($dev_id, $size, $devname) = ($1, $2, $3);
7424 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7425 } elsif ($line =~ m/^CTIME: /) {
7426 # we correctly received the vma config, so we can disable
7427 # the timeout now for disk allocation (set to 10 minutes, so
7428 # that we always timeout if something goes wrong)
7429 alarm(600);
7430 &$print_devmap();
7431 print $fifofh "done\n";
7432 my $tmp = $oldtimeout || 0;
7433 $oldtimeout = undef;
7434 alarm($tmp);
7435 close($fifofh);
7436 $fifofh = undef;
7437 }
7438 };
7439
7440 print "restore vma archive: $dbg_cmdstring\n";
7441 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7442 };
7443 my $err = $@;
7444
7445 alarm($oldtimeout) if $oldtimeout;
7446
7447 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7448
7449 close($fifofh) if $fifofh;
7450 unlink $mapfifo;
7451 rmtree $tmpdir;
7452
7453 if ($err) {
7454 $restore_destroy_volumes->($cfg, $virtdev_hash);
7455 die $err;
7456 }
7457
7458 my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7459 check_restore_permissions($rpcenv, $user, $new_conf);
7460 PVE::QemuConfig->write_config($vmid, $new_conf);
7461
7462 eval { rescan($vmid, 1); };
7463 warn $@ if $@;
7464
7465 PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7466 }
7467
7468 sub restore_tar_archive {
7469 my ($archive, $vmid, $user, $opts) = @_;
7470
7471 if (scalar(keys $opts->{override_conf}->%*) > 0) {
7472 my $keystring = join(' ', keys $opts->{override_conf}->%*);
7473 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7474 }
7475
7476 if ($archive ne '-') {
7477 my $firstfile = tar_archive_read_firstfile($archive);
7478 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7479 if $firstfile ne 'qemu-server.conf';
7480 }
7481
7482 my $storecfg = PVE::Storage::config();
7483
7484 # avoid zombie disks when restoring over an existing VM -> cleanup first
7485 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7486 # skiplock=1 because qmrestore has set the 'create' lock itself already
7487 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7488 destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7489
7490 my $tocmd = "/usr/lib/qemu-server/qmextract";
7491
7492 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7493 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7494 $tocmd .= ' --prealloc' if $opts->{prealloc};
7495 $tocmd .= ' --info' if $opts->{info};
7496
7497 # tar option "xf" does not autodetect compression when read from STDIN,
7498 # so we pipe to zcat
7499 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7500 PVE::Tools::shellquote("--to-command=$tocmd");
7501
7502 my $tmpdir = "/var/tmp/vzdumptmp$$";
7503 mkpath $tmpdir;
7504
7505 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7506 local $ENV{VZDUMP_VMID} = $vmid;
7507 local $ENV{VZDUMP_USER} = $user;
7508
7509 my $conffile = PVE::QemuConfig->config_file($vmid);
7510 my $new_conf_raw = '';
7511
7512 # disable interrupts (always do cleanups)
7513 local $SIG{INT} =
7514 local $SIG{TERM} =
7515 local $SIG{QUIT} =
7516 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7517
7518 eval {
7519 # enable interrupts
7520 local $SIG{INT} =
7521 local $SIG{TERM} =
7522 local $SIG{QUIT} =
7523 local $SIG{HUP} =
7524 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7525
7526 if ($archive eq '-') {
7527 print "extracting archive from STDIN\n";
7528 run_command($cmd, input => "<&STDIN");
7529 } else {
7530 print "extracting archive '$archive'\n";
7531 run_command($cmd);
7532 }
7533
7534 return if $opts->{info};
7535
7536 # read new mapping
7537 my $map = {};
7538 my $statfile = "$tmpdir/qmrestore.stat";
7539 if (my $fd = IO::File->new($statfile, "r")) {
7540 while (defined (my $line = <$fd>)) {
7541 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7542 $map->{$1} = $2 if $1;
7543 } else {
7544 print STDERR "unable to parse line in statfile - $line\n";
7545 }
7546 }
7547 $fd->close();
7548 }
7549
7550 my $confsrc = "$tmpdir/qemu-server.conf";
7551
7552 my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7553
7554 my $cookie = { netcount => 0 };
7555 while (defined (my $line = <$srcfd>)) {
7556 $new_conf_raw .= restore_update_config_line(
7557 $cookie,
7558 $map,
7559 $line,
7560 $opts->{unique},
7561 );
7562 }
7563
7564 $srcfd->close();
7565 };
7566 if (my $err = $@) {
7567 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7568 die $err;
7569 }
7570
7571 rmtree $tmpdir;
7572
7573 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7574
7575 PVE::Cluster::cfs_update(); # make sure we read new file
7576
7577 eval { rescan($vmid, 1); };
7578 warn $@ if $@;
7579 };
7580
7581 sub foreach_storage_used_by_vm {
7582 my ($conf, $func) = @_;
7583
7584 my $sidhash = {};
7585
7586 PVE::QemuConfig->foreach_volume($conf, sub {
7587 my ($ds, $drive) = @_;
7588 return if drive_is_cdrom($drive);
7589
7590 my $volid = $drive->{file};
7591
7592 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7593 $sidhash->{$sid} = $sid if $sid;
7594 });
7595
7596 foreach my $sid (sort keys %$sidhash) {
7597 &$func($sid);
7598 }
7599 }
7600
7601 my $qemu_snap_storage = {
7602 rbd => 1,
7603 };
7604 sub do_snapshots_with_qemu {
7605 my ($storecfg, $volid, $deviceid) = @_;
7606
7607 return if $deviceid =~ m/tpmstate0/;
7608
7609 my $storage_name = PVE::Storage::parse_volume_id($volid);
7610 my $scfg = $storecfg->{ids}->{$storage_name};
7611 die "could not find storage '$storage_name'\n" if !defined($scfg);
7612
7613 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7614 return 1;
7615 }
7616
7617 if ($volid =~ m/\.(qcow2|qed)$/){
7618 return 1;
7619 }
7620
7621 return;
7622 }
7623
7624 sub qga_check_running {
7625 my ($vmid, $nowarn) = @_;
7626
7627 eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7628 if ($@) {
7629 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7630 return 0;
7631 }
7632 return 1;
7633 }
7634
7635 sub template_create {
7636 my ($vmid, $conf, $disk) = @_;
7637
7638 my $storecfg = PVE::Storage::config();
7639
7640 PVE::QemuConfig->foreach_volume($conf, sub {
7641 my ($ds, $drive) = @_;
7642
7643 return if drive_is_cdrom($drive);
7644 return if $disk && $ds ne $disk;
7645
7646 my $volid = $drive->{file};
7647 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7648
7649 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7650 $drive->{file} = $voliddst;
7651 $conf->{$ds} = print_drive($drive);
7652 PVE::QemuConfig->write_config($vmid, $conf);
7653 });
7654 }
7655
7656 sub convert_iscsi_path {
7657 my ($path) = @_;
7658
7659 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7660 my $portal = $1;
7661 my $target = $2;
7662 my $lun = $3;
7663
7664 my $initiator_name = get_initiator_name();
7665
7666 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7667 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7668 }
7669
7670 die "cannot convert iscsi path '$path', unkown format\n";
7671 }
7672
7673 sub qemu_img_convert {
7674 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7675
7676 my $storecfg = PVE::Storage::config();
7677 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7678 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7679
7680 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7681
7682 my $cachemode;
7683 my $src_path;
7684 my $src_is_iscsi = 0;
7685 my $src_format;
7686
7687 if ($src_storeid) {
7688 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7689 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7690 $src_format = qemu_img_format($src_scfg, $src_volname);
7691 $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7692 $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7693 $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7694 } elsif (-f $src_volid || -b $src_volid) {
7695 $src_path = $src_volid;
7696 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7697 $src_format = $1;
7698 }
7699 }
7700
7701 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7702
7703 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7704 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7705 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7706 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7707
7708 my $cmd = [];
7709 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7710 push @$cmd, '-l', "snapshot.name=$snapname"
7711 if $snapname && $src_format && $src_format eq "qcow2";
7712 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7713 push @$cmd, '-T', $cachemode if defined($cachemode);
7714 push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7715
7716 if ($src_is_iscsi) {
7717 push @$cmd, '--image-opts';
7718 $src_path = convert_iscsi_path($src_path);
7719 } elsif ($src_format) {
7720 push @$cmd, '-f', $src_format;
7721 }
7722
7723 if ($dst_is_iscsi) {
7724 push @$cmd, '--target-image-opts';
7725 $dst_path = convert_iscsi_path($dst_path);
7726 } else {
7727 push @$cmd, '-O', $dst_format;
7728 }
7729
7730 push @$cmd, $src_path;
7731
7732 if (!$dst_is_iscsi && $is_zero_initialized) {
7733 push @$cmd, "zeroinit:$dst_path";
7734 } else {
7735 push @$cmd, $dst_path;
7736 }
7737
7738 my $parser = sub {
7739 my $line = shift;
7740 if($line =~ m/\((\S+)\/100\%\)/){
7741 my $percent = $1;
7742 my $transferred = int($size * $percent / 100);
7743 my $total_h = render_bytes($size, 1);
7744 my $transferred_h = render_bytes($transferred, 1);
7745
7746 print "transferred $transferred_h of $total_h ($percent%)\n";
7747 }
7748
7749 };
7750
7751 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
7752 my $err = $@;
7753 die "copy failed: $err" if $err;
7754 }
7755
7756 sub qemu_img_format {
7757 my ($scfg, $volname) = @_;
7758
7759 if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7760 return $1;
7761 } else {
7762 return "raw";
7763 }
7764 }
7765
7766 sub qemu_drive_mirror {
7767 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7768
7769 $jobs = {} if !$jobs;
7770
7771 my $qemu_target;
7772 my $format;
7773 $jobs->{"drive-$drive"} = {};
7774
7775 if ($dst_volid =~ /^nbd:/) {
7776 $qemu_target = $dst_volid;
7777 $format = "nbd";
7778 } else {
7779 my $storecfg = PVE::Storage::config();
7780 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7781
7782 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7783
7784 $format = qemu_img_format($dst_scfg, $dst_volname);
7785
7786 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7787
7788 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7789 }
7790
7791 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7792 $opts->{format} = $format if $format;
7793
7794 if (defined($src_bitmap)) {
7795 $opts->{sync} = 'incremental';
7796 $opts->{bitmap} = $src_bitmap;
7797 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7798 }
7799
7800 if (defined($bwlimit)) {
7801 $opts->{speed} = $bwlimit * 1024;
7802 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7803 } else {
7804 print "drive mirror is starting for drive-$drive\n";
7805 }
7806
7807 # if a job already runs for this device we get an error, catch it for cleanup
7808 eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7809 if (my $err = $@) {
7810 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7811 warn "$@\n" if $@;
7812 die "mirroring error: $err\n";
7813 }
7814
7815 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7816 }
7817
7818 # $completion can be either
7819 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7820 # 'cancel': wait until all jobs are ready, block-job-cancel them
7821 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7822 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7823 sub qemu_drive_mirror_monitor {
7824 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7825
7826 $completion //= 'complete';
7827 $op //= "mirror";
7828
7829 eval {
7830 my $err_complete = 0;
7831
7832 my $starttime = time ();
7833 while (1) {
7834 die "block job ('$op') timed out\n" if $err_complete > 300;
7835
7836 my $stats = mon_cmd($vmid, "query-block-jobs");
7837 my $ctime = time();
7838
7839 my $running_jobs = {};
7840 for my $stat (@$stats) {
7841 next if $stat->{type} ne $op;
7842 $running_jobs->{$stat->{device}} = $stat;
7843 }
7844
7845 my $readycounter = 0;
7846
7847 for my $job_id (sort keys %$jobs) {
7848 my $job = $running_jobs->{$job_id};
7849
7850 my $vanished = !defined($job);
7851 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7852 if($complete || ($vanished && $completion eq 'auto')) {
7853 print "$job_id: $op-job finished\n";
7854 delete $jobs->{$job_id};
7855 next;
7856 }
7857
7858 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7859
7860 my $busy = $job->{busy};
7861 my $ready = $job->{ready};
7862 if (my $total = $job->{len}) {
7863 my $transferred = $job->{offset} || 0;
7864 my $remaining = $total - $transferred;
7865 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7866
7867 my $duration = $ctime - $starttime;
7868 my $total_h = render_bytes($total, 1);
7869 my $transferred_h = render_bytes($transferred, 1);
7870
7871 my $status = sprintf(
7872 "transferred $transferred_h of $total_h ($percent%%) in %s",
7873 render_duration($duration),
7874 );
7875
7876 if ($ready) {
7877 if ($busy) {
7878 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7879 } else {
7880 $status .= ", ready";
7881 }
7882 }
7883 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7884 $jobs->{$job_id}->{ready} = $ready;
7885 }
7886
7887 $readycounter++ if $job->{ready};
7888 }
7889
7890 last if scalar(keys %$jobs) == 0;
7891
7892 if ($readycounter == scalar(keys %$jobs)) {
7893 print "all '$op' jobs are ready\n";
7894
7895 # do the complete later (or has already been done)
7896 last if $completion eq 'skip' || $completion eq 'auto';
7897
7898 if ($vmiddst && $vmiddst != $vmid) {
7899 my $agent_running = $qga && qga_check_running($vmid);
7900 if ($agent_running) {
7901 print "freeze filesystem\n";
7902 eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7903 warn $@ if $@;
7904 } else {
7905 print "suspend vm\n";
7906 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7907 warn $@ if $@;
7908 }
7909
7910 # if we clone a disk for a new target vm, we don't switch the disk
7911 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7912
7913 if ($agent_running) {
7914 print "unfreeze filesystem\n";
7915 eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7916 warn $@ if $@;
7917 } else {
7918 print "resume vm\n";
7919 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7920 warn $@ if $@;
7921 }
7922
7923 last;
7924 } else {
7925
7926 for my $job_id (sort keys %$jobs) {
7927 # try to switch the disk if source and destination are on the same guest
7928 print "$job_id: Completing block job_id...\n";
7929
7930 my $op;
7931 if ($completion eq 'complete') {
7932 $op = 'block-job-complete';
7933 } elsif ($completion eq 'cancel') {
7934 $op = 'block-job-cancel';
7935 } else {
7936 die "invalid completion value: $completion\n";
7937 }
7938 eval { mon_cmd($vmid, $op, device => $job_id) };
7939 if ($@ =~ m/cannot be completed/) {
7940 print "$job_id: block job cannot be completed, trying again.\n";
7941 $err_complete++;
7942 }else {
7943 print "$job_id: Completed successfully.\n";
7944 $jobs->{$job_id}->{complete} = 1;
7945 }
7946 }
7947 }
7948 }
7949 sleep 1;
7950 }
7951 };
7952 my $err = $@;
7953
7954 if ($err) {
7955 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7956 die "block job ($op) error: $err";
7957 }
7958 }
7959
7960 sub qemu_blockjobs_cancel {
7961 my ($vmid, $jobs) = @_;
7962
7963 foreach my $job (keys %$jobs) {
7964 print "$job: Cancelling block job\n";
7965 eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7966 $jobs->{$job}->{cancel} = 1;
7967 }
7968
7969 while (1) {
7970 my $stats = mon_cmd($vmid, "query-block-jobs");
7971
7972 my $running_jobs = {};
7973 foreach my $stat (@$stats) {
7974 $running_jobs->{$stat->{device}} = $stat;
7975 }
7976
7977 foreach my $job (keys %$jobs) {
7978
7979 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7980 print "$job: Done.\n";
7981 delete $jobs->{$job};
7982 }
7983 }
7984
7985 last if scalar(keys %$jobs) == 0;
7986
7987 sleep 1;
7988 }
7989 }
7990
7991 # Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
7992 # source, but some storages have problems with io_uring, sometimes even leading to crashes.
7993 my sub clone_disk_check_io_uring {
7994 my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
7995
7996 return if !$use_drive_mirror;
7997
7998 # Don't complain when not changing storage.
7999 # Assume if it works for the source, it'll work for the target too.
8000 return if $src_storeid eq $dst_storeid;
8001
8002 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8003 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8004
8005 my $cache_direct = drive_uses_cache_direct($src_drive);
8006
8007 my $src_uses_io_uring;
8008 if ($src_drive->{aio}) {
8009 $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8010 } else {
8011 $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8012 }
8013
8014 die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8015 if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8016 }
8017
8018 sub clone_disk {
8019 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8020
8021 my ($vmid, $running) = $source->@{qw(vmid running)};
8022 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8023
8024 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8025 my ($storage, $format) = $dest->@{qw(storage format)};
8026
8027 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
8028
8029 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8030 die "cloning from/to EFI disk requires EFI disk\n"
8031 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8032 die "cloning from/to TPM state requires TPM state\n"
8033 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8034
8035 # This would lead to two device nodes in QEMU pointing to the same backing image!
8036 die "cannot change drive name when cloning disk from/to the same VM\n"
8037 if $use_drive_mirror && $vmid == $newvmid;
8038 }
8039
8040 die "cannot move TPM state while VM is running\n"
8041 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8042
8043 my $newvolid;
8044
8045 print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8046 print "$src_drivename " if $src_drivename;
8047 print "($drive->{file})\n";
8048
8049 if (!$full) {
8050 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
8051 push @$newvollist, $newvolid;
8052 } else {
8053 my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8054 my $storeid = $storage || $src_storeid;
8055
8056 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8057
8058 my $name = undef;
8059 my $size = undef;
8060 if (drive_is_cloudinit($drive)) {
8061 $name = "vm-$newvmid-cloudinit";
8062 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8063 if ($scfg->{path}) {
8064 $name .= ".$dst_format";
8065 }
8066 $snapname = undef;
8067 $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8068 } elsif ($dst_drivename eq 'efidisk0') {
8069 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8070 } elsif ($dst_drivename eq 'tpmstate0') {
8071 $dst_format = 'raw';
8072 $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8073 } else {
8074 clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8075
8076 $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8077 }
8078 $newvolid = PVE::Storage::vdisk_alloc(
8079 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8080 );
8081 push @$newvollist, $newvolid;
8082
8083 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8084
8085 if (drive_is_cloudinit($drive)) {
8086 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8087 # if this is the case, we have to complete any block-jobs still there from
8088 # previous drive-mirrors
8089 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8090 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8091 }
8092 goto no_data_clone;
8093 }
8094
8095 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8096 if ($use_drive_mirror) {
8097 qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8098 $completion, $qga, $bwlimit);
8099 } else {
8100 if ($dst_drivename eq 'efidisk0') {
8101 # the relevant data on the efidisk may be smaller than the source
8102 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8103 # that is given by the OVMF_VARS.fd
8104 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8105 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8106
8107 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8108
8109 # better for Ceph if block size is not too small, see bug #3324
8110 my $bs = 1024*1024;
8111
8112 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8113
8114 if ($src_format eq 'qcow2' && $snapname) {
8115 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8116 if !min_version(kvm_user_version(), 6, 2);
8117 push $cmd->@*, '-l', $snapname;
8118 }
8119 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8120 run_command($cmd);
8121 } else {
8122 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8123 }
8124 }
8125 }
8126
8127 no_data_clone:
8128 my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8129
8130 my $disk = dclone($drive);
8131 delete $disk->{format};
8132 $disk->{file} = $newvolid;
8133 $disk->{size} = $size if defined($size);
8134
8135 return $disk;
8136 }
8137
8138 sub get_running_qemu_version {
8139 my ($vmid) = @_;
8140 my $res = mon_cmd($vmid, "query-version");
8141 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8142 }
8143
8144 sub qemu_use_old_bios_files {
8145 my ($machine_type) = @_;
8146
8147 return if !$machine_type;
8148
8149 my $use_old_bios_files = undef;
8150
8151 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8152 $machine_type = $1;
8153 $use_old_bios_files = 1;
8154 } else {
8155 my $version = extract_version($machine_type, kvm_user_version());
8156 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8157 # load new efi bios files on migration. So this hack is required to allow
8158 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8159 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8160 $use_old_bios_files = !min_version($version, 2, 4);
8161 }
8162
8163 return ($use_old_bios_files, $machine_type);
8164 }
8165
8166 sub get_efivars_size {
8167 my ($conf, $efidisk) = @_;
8168
8169 my $arch = get_vm_arch($conf);
8170 $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8171 my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8172 my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8173 return -s $ovmf_vars;
8174 }
8175
8176 sub update_efidisk_size {
8177 my ($conf) = @_;
8178
8179 return if !defined($conf->{efidisk0});
8180
8181 my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8182 $disk->{size} = get_efivars_size($conf);
8183 $conf->{efidisk0} = print_drive($disk);
8184
8185 return;
8186 }
8187
8188 sub update_tpmstate_size {
8189 my ($conf) = @_;
8190
8191 my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8192 $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8193 $conf->{tpmstate0} = print_drive($disk);
8194 }
8195
8196 sub create_efidisk($$$$$$$) {
8197 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8198
8199 my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8200
8201 my $vars_size_b = -s $ovmf_vars;
8202 my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8203 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8204 PVE::Storage::activate_volumes($storecfg, [$volid]);
8205
8206 qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8207 my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8208
8209 return ($volid, $size/1024);
8210 }
8211
8212 sub vm_iothreads_list {
8213 my ($vmid) = @_;
8214
8215 my $res = mon_cmd($vmid, 'query-iothreads');
8216
8217 my $iothreads = {};
8218 foreach my $iothread (@$res) {
8219 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8220 }
8221
8222 return $iothreads;
8223 }
8224
8225 sub scsihw_infos {
8226 my ($conf, $drive) = @_;
8227
8228 my $maxdev = 0;
8229
8230 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8231 $maxdev = 7;
8232 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8233 $maxdev = 1;
8234 } else {
8235 $maxdev = 256;
8236 }
8237
8238 my $controller = int($drive->{index} / $maxdev);
8239 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8240 ? "virtioscsi"
8241 : "scsihw";
8242
8243 return ($maxdev, $controller, $controller_prefix);
8244 }
8245
8246 sub resolve_dst_disk_format {
8247 my ($storecfg, $storeid, $src_volname, $format) = @_;
8248 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8249
8250 if (!$format) {
8251 # if no target format is specified, use the source disk format as hint
8252 if ($src_volname) {
8253 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8254 $format = qemu_img_format($scfg, $src_volname);
8255 } else {
8256 return $defFormat;
8257 }
8258 }
8259
8260 # test if requested format is supported - else use default
8261 my $supported = grep { $_ eq $format } @$validFormats;
8262 $format = $defFormat if !$supported;
8263 return $format;
8264 }
8265
8266 # NOTE: if this logic changes, please update docs & possibly gui logic
8267 sub find_vmstate_storage {
8268 my ($conf, $storecfg) = @_;
8269
8270 # first, return storage from conf if set
8271 return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8272
8273 my ($target, $shared, $local);
8274
8275 foreach_storage_used_by_vm($conf, sub {
8276 my ($sid) = @_;
8277 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8278 my $dst = $scfg->{shared} ? \$shared : \$local;
8279 $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8280 });
8281
8282 # second, use shared storage where VM has at least one disk
8283 # third, use local storage where VM has at least one disk
8284 # fall back to local storage
8285 $target = $shared // $local // 'local';
8286
8287 return $target;
8288 }
8289
8290 sub generate_uuid {
8291 my ($uuid, $uuid_str);
8292 UUID::generate($uuid);
8293 UUID::unparse($uuid, $uuid_str);
8294 return $uuid_str;
8295 }
8296
8297 sub generate_smbios1_uuid {
8298 return "uuid=".generate_uuid();
8299 }
8300
8301 sub nbd_stop {
8302 my ($vmid) = @_;
8303
8304 mon_cmd($vmid, 'nbd-server-stop');
8305 }
8306
8307 sub create_reboot_request {
8308 my ($vmid) = @_;
8309 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8310 or die "failed to create reboot trigger file: $!\n";
8311 close($fh);
8312 }
8313
8314 sub clear_reboot_request {
8315 my ($vmid) = @_;
8316 my $path = "/run/qemu-server/$vmid.reboot";
8317 my $res = 0;
8318
8319 $res = unlink($path);
8320 die "could not remove reboot request for $vmid: $!"
8321 if !$res && $! != POSIX::ENOENT;
8322
8323 return $res;
8324 }
8325
8326 sub bootorder_from_legacy {
8327 my ($conf, $bootcfg) = @_;
8328
8329 my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8330 my $bootindex_hash = {};
8331 my $i = 1;
8332 foreach my $o (split(//, $boot)) {
8333 $bootindex_hash->{$o} = $i*100;
8334 $i++;
8335 }
8336
8337 my $bootorder = {};
8338
8339 PVE::QemuConfig->foreach_volume($conf, sub {
8340 my ($ds, $drive) = @_;
8341
8342 if (drive_is_cdrom ($drive, 1)) {
8343 if ($bootindex_hash->{d}) {
8344 $bootorder->{$ds} = $bootindex_hash->{d};
8345 $bootindex_hash->{d} += 1;
8346 }
8347 } elsif ($bootindex_hash->{c}) {
8348 $bootorder->{$ds} = $bootindex_hash->{c}
8349 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8350 $bootindex_hash->{c} += 1;
8351 }
8352 });
8353
8354 if ($bootindex_hash->{n}) {
8355 for (my $i = 0; $i < $MAX_NETS; $i++) {
8356 my $netname = "net$i";
8357 next if !$conf->{$netname};
8358 $bootorder->{$netname} = $bootindex_hash->{n};
8359 $bootindex_hash->{n} += 1;
8360 }
8361 }
8362
8363 return $bootorder;
8364 }
8365
8366 # Generate default device list for 'boot: order=' property. Matches legacy
8367 # default boot order, but with explicit device names. This is important, since
8368 # the fallback for when neither 'order' nor the old format is specified relies
8369 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8370 sub get_default_bootdevices {
8371 my ($conf) = @_;
8372
8373 my @ret = ();
8374
8375 # harddisk
8376 my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8377 push @ret, $first if $first;
8378
8379 # cdrom
8380 $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8381 push @ret, $first if $first;
8382
8383 # network
8384 for (my $i = 0; $i < $MAX_NETS; $i++) {
8385 my $netname = "net$i";
8386 next if !$conf->{$netname};
8387 push @ret, $netname;
8388 last;
8389 }
8390
8391 return \@ret;
8392 }
8393
8394 sub device_bootorder {
8395 my ($conf) = @_;
8396
8397 return bootorder_from_legacy($conf) if !defined($conf->{boot});
8398
8399 my $boot = parse_property_string($boot_fmt, $conf->{boot});
8400
8401 my $bootorder = {};
8402 if (!defined($boot) || $boot->{legacy}) {
8403 $bootorder = bootorder_from_legacy($conf, $boot);
8404 } elsif ($boot->{order}) {
8405 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8406 for my $dev (PVE::Tools::split_list($boot->{order})) {
8407 $bootorder->{$dev} = $i++;
8408 }
8409 }
8410
8411 return $bootorder;
8412 }
8413
8414 sub register_qmeventd_handle {
8415 my ($vmid) = @_;
8416
8417 my $fh;
8418 my $peer = "/var/run/qmeventd.sock";
8419 my $count = 0;
8420
8421 for (;;) {
8422 $count++;
8423 $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8424 last if $fh;
8425 if ($! != EINTR && $! != EAGAIN) {
8426 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8427 }
8428 if ($count > 4) {
8429 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8430 . "after $count retries\n";
8431 }
8432 usleep(25000);
8433 }
8434
8435 # send handshake to mark VM as backing up
8436 print $fh to_json({vzdump => {vmid => "$vmid"}});
8437
8438 # return handle to be closed later when inhibit is no longer required
8439 return $fh;
8440 }
8441
8442 # bash completion helper
8443
8444 sub complete_backup_archives {
8445 my ($cmdname, $pname, $cvalue) = @_;
8446
8447 my $cfg = PVE::Storage::config();
8448
8449 my $storeid;
8450
8451 if ($cvalue =~ m/^([^:]+):/) {
8452 $storeid = $1;
8453 }
8454
8455 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8456
8457 my $res = [];
8458 foreach my $id (keys %$data) {
8459 foreach my $item (@{$data->{$id}}) {
8460 next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8461 push @$res, $item->{volid} if defined($item->{volid});
8462 }
8463 }
8464
8465 return $res;
8466 }
8467
8468 my $complete_vmid_full = sub {
8469 my ($running) = @_;
8470
8471 my $idlist = vmstatus();
8472
8473 my $res = [];
8474
8475 foreach my $id (keys %$idlist) {
8476 my $d = $idlist->{$id};
8477 if (defined($running)) {
8478 next if $d->{template};
8479 next if $running && $d->{status} ne 'running';
8480 next if !$running && $d->{status} eq 'running';
8481 }
8482 push @$res, $id;
8483
8484 }
8485 return $res;
8486 };
8487
8488 sub complete_vmid {
8489 return &$complete_vmid_full();
8490 }
8491
8492 sub complete_vmid_stopped {
8493 return &$complete_vmid_full(0);
8494 }
8495
8496 sub complete_vmid_running {
8497 return &$complete_vmid_full(1);
8498 }
8499
8500 sub complete_storage {
8501
8502 my $cfg = PVE::Storage::config();
8503 my $ids = $cfg->{ids};
8504
8505 my $res = [];
8506 foreach my $sid (keys %$ids) {
8507 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8508 next if !$ids->{$sid}->{content}->{images};
8509 push @$res, $sid;
8510 }
8511
8512 return $res;
8513 }
8514
8515 sub complete_migration_storage {
8516 my ($cmd, $param, $current_value, $all_args) = @_;
8517
8518 my $targetnode = @$all_args[1];
8519
8520 my $cfg = PVE::Storage::config();
8521 my $ids = $cfg->{ids};
8522
8523 my $res = [];
8524 foreach my $sid (keys %$ids) {
8525 next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8526 next if !$ids->{$sid}->{content}->{images};
8527 push @$res, $sid;
8528 }
8529
8530 return $res;
8531 }
8532
8533 sub vm_is_paused {
8534 my ($vmid) = @_;
8535 my $qmpstatus = eval {
8536 PVE::QemuConfig::assert_config_exists_on_node($vmid);
8537 mon_cmd($vmid, "query-status");
8538 };
8539 warn "$@\n" if $@;
8540 return $qmpstatus && $qmpstatus->{status} eq "paused";
8541 }
8542
8543 sub check_volume_storage_type {
8544 my ($storecfg, $vol) = @_;
8545
8546 my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8547 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8548 my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8549
8550 die "storage '$storeid' does not support content-type '$vtype'\n"
8551 if !$scfg->{content}->{$vtype};
8552
8553 return 1;
8554 }
8555
8556 sub add_nets_bridge_fdb {
8557 my ($conf, $vmid) = @_;
8558
8559 for my $opt (keys %$conf) {
8560 next if $opt !~ m/^net(\d+)$/;
8561 my $iface = "tap${vmid}i$1";
8562 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8563 my $net = parse_net($conf->{$opt}, 1) or next;
8564
8565 my $mac = $net->{macaddr};
8566 if (!$mac) {
8567 log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8568 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8569 next;
8570 }
8571
8572 my $bridge = $net->{bridge};
8573 if (!$bridge) {
8574 log_warn("Interface '$iface' not attached to any bridge.");
8575 next;
8576 }
8577 if ($have_sdn) {
8578 PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8579 } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8580 PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8581 }
8582 }
8583 }
8584
8585 sub del_nets_bridge_fdb {
8586 my ($conf, $vmid) = @_;
8587
8588 for my $opt (keys %$conf) {
8589 next if $opt !~ m/^net(\d+)$/;
8590 my $iface = "tap${vmid}i$1";
8591
8592 my $net = parse_net($conf->{$opt}) or next;
8593 my $mac = $net->{macaddr} or next;
8594
8595 my $bridge = $net->{bridge};
8596 if ($have_sdn) {
8597 PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8598 } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8599 PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8600 }
8601 }
8602 }
8603
8604 1;