]> git.proxmox.com Git - qemu-server.git/blame_incremental - PVE/QemuServer.pm
projects / qemu-server.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
migration: add missing use statements
[qemu-server.git] / PVE / QemuServer.pm
... / ...
CommitLineData
1package PVE::QemuServer;
2
3use strict;
4use warnings;
5
6use Cwd 'abs_path';
7use Digest::SHA;
8use Fcntl ':flock';
9use Fcntl;
10use File::Basename;
11use File::Copy qw(copy);
12use File::Path;
13use File::stat;
14use Getopt::Long;
15use IO::Dir;
16use IO::File;
17use IO::Handle;
18use IO::Select;
19use IO::Socket::UNIX;
20use IPC::Open3;
21use JSON;
22use List::Util qw(first);
23use MIME::Base64;
24use POSIX;
25use Storable qw(dclone);
26use Time::HiRes qw(gettimeofday usleep);
27use URI::Escape;
28use UUID;
29
30use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
31use PVE::CGroup;
32use PVE::CpuSet;
33use PVE::DataCenterConfig;
34use PVE::Exception qw(raise raise_param_exc);
35use PVE::Format qw(render_duration render_bytes);
36use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
37use PVE::HA::Config;
38use PVE::Mapping::PCI;
39use PVE::Mapping::USB;
40use PVE::INotify;
41use PVE::JSONSchema qw(get_standard_option parse_property_string);
42use PVE::ProcFSTools;
43use PVE::PBSClient;
44use PVE::RESTEnvironment qw(log_warn);
45use PVE::RPCEnvironment;
46use PVE::Storage;
47use PVE::SysFSTools;
48use PVE::Systemd;
49use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
50
51use PVE::QMPClient;
52use PVE::QemuConfig;
53use PVE::QemuServer::Helpers qw(config_aware_timeout min_version windows_version);
54use PVE::QemuServer::Cloudinit;
55use PVE::QemuServer::CGroup;
56use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options get_cpu_bitness is_native_arch);
57use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
58use PVE::QemuServer::Machine;
59use PVE::QemuServer::Memory qw(get_current_memory);
60use PVE::QemuServer::Monitor qw(mon_cmd);
61use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
62use PVE::QemuServer::QMPHelpers qw(qemu_deviceadd qemu_devicedel qemu_objectadd qemu_objectdel);
63use PVE::QemuServer::USB;
64
65my $have_sdn;
66eval {
67 require PVE::Network::SDN::Zones;
68 require PVE::Network::SDN::Vnets;
69 $have_sdn = 1;
70};
71
72my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
73my $OVMF = {
74 x86_64 => {
75 '4m-no-smm' => [
76 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
77 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
78 ],
79 '4m-no-smm-ms' => [
80 "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
81 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
82 ],
83 '4m' => [
84 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
85 "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
86 ],
87 '4m-ms' => [
88 "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
89 "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
90 ],
91 # FIXME: These are legacy 2MB-sized images that modern OVMF doesn't supports to build
92 # anymore. how can we deperacate this sanely without breaking existing instances, or using
93 # older backups and snapshot?
94 default => [
95 "$EDK2_FW_BASE/OVMF_CODE.fd",
96 "$EDK2_FW_BASE/OVMF_VARS.fd",
97 ],
98 },
99 aarch64 => {
100 default => [
101 "$EDK2_FW_BASE/AAVMF_CODE.fd",
102 "$EDK2_FW_BASE/AAVMF_VARS.fd",
103 ],
104 },
105};
106
107my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
108
109# Note about locking: we use flock on the config file protect against concurent actions.
110# Aditionaly, we have a 'lock' setting in the config file. This can be set to 'migrate',
111# 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
112# But you can ignore this kind of lock with the --skiplock flag.
113
114cfs_register_file(
115 '/qemu-server/',
116 \&parse_vm_config,
117 \&write_vm_config
118);
119
120PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
121 description => "Some command save/restore state from this location.",
122 type => 'string',
123 maxLength => 128,
124 optional => 1,
125});
126
127# FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
128my $nodename_cache;
129sub nodename {
130 $nodename_cache //= PVE::INotify::nodename();
131 return $nodename_cache;
132}
133
134my $watchdog_fmt = {
135 model => {
136 default_key => 1,
137 type => 'string',
138 enum => [qw(i6300esb ib700)],
139 description => "Watchdog type to emulate.",
140 default => 'i6300esb',
141 optional => 1,
142 },
143 action => {
144 type => 'string',
145 enum => [qw(reset shutdown poweroff pause debug none)],
146 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
147 optional => 1,
148 },
149};
150PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
151
152my $agent_fmt = {
153 enabled => {
154 description => "Enable/disable communication with a QEMU Guest Agent (QGA) running in the VM.",
155 type => 'boolean',
156 default => 0,
157 default_key => 1,
158 },
159 fstrim_cloned_disks => {
160 description => "Run fstrim after moving a disk or migrating the VM.",
161 type => 'boolean',
162 optional => 1,
163 default => 0,
164 },
165 'freeze-fs-on-backup' => {
166 description => "Freeze/thaw guest filesystems on backup for consistency.",
167 type => 'boolean',
168 optional => 1,
169 default => 1,
170 },
171 type => {
172 description => "Select the agent type",
173 type => 'string',
174 default => 'virtio',
175 optional => 1,
176 enum => [qw(virtio isa)],
177 },
178};
179
180my $vga_fmt = {
181 type => {
182 description => "Select the VGA type.",
183 type => 'string',
184 default => 'std',
185 optional => 1,
186 default_key => 1,
187 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
188 },
189 memory => {
190 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
191 type => 'integer',
192 optional => 1,
193 minimum => 4,
194 maximum => 512,
195 },
196 clipboard => {
197 description => 'Enable a specific clipboard. If not set, depending on the display type the'
198 .' SPICE one will be added. Migration with VNC clipboard is not yet supported!',
199 type => 'string',
200 enum => ['vnc'],
201 optional => 1,
202 },
203};
204
205my $ivshmem_fmt = {
206 size => {
207 type => 'integer',
208 minimum => 1,
209 description => "The size of the file in MB.",
210 },
211 name => {
212 type => 'string',
213 pattern => '[a-zA-Z0-9\-]+',
214 optional => 1,
215 format_description => 'string',
216 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
217 },
218};
219
220my $audio_fmt = {
221 device => {
222 type => 'string',
223 enum => [qw(ich9-intel-hda intel-hda AC97)],
224 description => "Configure an audio device."
225 },
226 driver => {
227 type => 'string',
228 enum => ['spice', 'none'],
229 default => 'spice',
230 optional => 1,
231 description => "Driver backend for the audio device."
232 },
233};
234
235my $spice_enhancements_fmt = {
236 foldersharing => {
237 type => 'boolean',
238 optional => 1,
239 default => '0',
240 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
241 },
242 videostreaming => {
243 type => 'string',
244 enum => ['off', 'all', 'filter'],
245 default => 'off',
246 optional => 1,
247 description => "Enable video streaming. Uses compression for detected video streams."
248 },
249};
250
251my $rng_fmt = {
252 source => {
253 type => 'string',
254 enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
255 default_key => 1,
256 description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
257 ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
258 ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
259 ." still seeded from real entropy, and the bytes provided will most likely be mixed"
260 ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
261 ." a hardware RNG from the host.",
262 },
263 max_bytes => {
264 type => 'integer',
265 description => "Maximum bytes of entropy allowed to get injected into the guest every"
266 ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
267 ." `0` to disable limiting (potentially dangerous!).",
268 optional => 1,
269
270 # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
271 # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
272 # reading from /dev/urandom
273 default => 1024,
274 },
275 period => {
276 type => 'integer',
277 description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
278 ." the guest to retrieve another 'max_bytes' of entropy.",
279 optional => 1,
280 default => 1000,
281 },
282};
283
284my $meta_info_fmt = {
285 'ctime' => {
286 type => 'integer',
287 description => "The guest creation timestamp as UNIX epoch time",
288 minimum => 0,
289 optional => 1,
290 },
291 'creation-qemu' => {
292 type => 'string',
293 description => "The QEMU (machine) version from the time this VM was created.",
294 pattern => '\d+(\.\d+)+',
295 optional => 1,
296 },
297};
298
299my $confdesc = {
300 onboot => {
301 optional => 1,
302 type => 'boolean',
303 description => "Specifies whether a VM will be started during system bootup.",
304 default => 0,
305 },
306 autostart => {
307 optional => 1,
308 type => 'boolean',
309 description => "Automatic restart after crash (currently ignored).",
310 default => 0,
311 },
312 hotplug => {
313 optional => 1,
314 type => 'string', format => 'pve-hotplug-features',
315 description => "Selectively enable hotplug features. This is a comma separated list of"
316 ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
317 ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
318 ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
319 ." windows > 7.",
320 default => 'network,disk,usb',
321 },
322 reboot => {
323 optional => 1,
324 type => 'boolean',
325 description => "Allow reboot. If set to '0' the VM exit on reboot.",
326 default => 1,
327 },
328 lock => {
329 optional => 1,
330 type => 'string',
331 description => "Lock/unlock the VM.",
332 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
333 },
334 cpulimit => {
335 optional => 1,
336 type => 'number',
337 description => "Limit of CPU usage.",
338 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
339 ." total of '2' CPU time. Value '0' indicates no CPU limit.",
340 minimum => 0,
341 maximum => 128,
342 default => 0,
343 },
344 cpuunits => {
345 optional => 1,
346 type => 'integer',
347 description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
348 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
349 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
350 ." weights of all the other running VMs.",
351 minimum => 1,
352 maximum => 262144,
353 default => 'cgroup v1: 1024, cgroup v2: 100',
354 },
355 memory => {
356 optional => 1,
357 type => 'string',
358 description => "Memory properties.",
359 format => $PVE::QemuServer::Memory::memory_fmt
360 },
361 balloon => {
362 optional => 1,
363 type => 'integer',
364 description => "Amount of target RAM for the VM in MiB. Using zero disables the ballon driver.",
365 minimum => 0,
366 },
367 shares => {
368 optional => 1,
369 type => 'integer',
370 description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
371 ." more memory this VM gets. Number is relative to weights of all other running VMs."
372 ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
373 minimum => 0,
374 maximum => 50000,
375 default => 1000,
376 },
377 keyboard => {
378 optional => 1,
379 type => 'string',
380 description => "Keyboard layout for VNC server. This option is generally not required and"
381 ." is often better handled from within the guest OS.",
382 enum => PVE::Tools::kvmkeymaplist(),
383 default => undef,
384 },
385 name => {
386 optional => 1,
387 type => 'string', format => 'dns-name',
388 description => "Set a name for the VM. Only used on the configuration web interface.",
389 },
390 scsihw => {
391 optional => 1,
392 type => 'string',
393 description => "SCSI controller model",
394 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
395 default => 'lsi',
396 },
397 description => {
398 optional => 1,
399 type => 'string',
400 description => "Description for the VM. Shown in the web-interface VM's summary."
401 ." This is saved as comment inside the configuration file.",
402 maxLength => 1024 * 8,
403 },
404 ostype => {
405 optional => 1,
406 type => 'string',
407 # NOTE: When extending, also consider extending `%guest_types` in `Import/ESXi.pm`.
408 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
409 description => "Specify guest operating system.",
410 verbose_description => <<EODESC,
411Specify guest operating system. This is used to enable special
412optimization/features for specific operating systems:
413
414[horizontal]
415other;; unspecified OS
416wxp;; Microsoft Windows XP
417w2k;; Microsoft Windows 2000
418w2k3;; Microsoft Windows 2003
419w2k8;; Microsoft Windows 2008
420wvista;; Microsoft Windows Vista
421win7;; Microsoft Windows 7
422win8;; Microsoft Windows 8/2012/2012r2
423win10;; Microsoft Windows 10/2016/2019
424win11;; Microsoft Windows 11/2022/2025
425l24;; Linux 2.4 Kernel
426l26;; Linux 2.6 - 6.X Kernel
427solaris;; Solaris/OpenSolaris/OpenIndiania kernel
428EODESC
429 },
430 boot => {
431 optional => 1,
432 type => 'string', format => 'pve-qm-boot',
433 description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
434 ." key or 'legacy=' is deprecated.",
435 },
436 bootdisk => {
437 optional => 1,
438 type => 'string', format => 'pve-qm-bootdisk',
439 description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
440 pattern => '(ide|sata|scsi|virtio)\d+',
441 },
442 smp => {
443 optional => 1,
444 type => 'integer',
445 description => "The number of CPUs. Please use option -sockets instead.",
446 minimum => 1,
447 default => 1,
448 },
449 sockets => {
450 optional => 1,
451 type => 'integer',
452 description => "The number of CPU sockets.",
453 minimum => 1,
454 default => 1,
455 },
456 cores => {
457 optional => 1,
458 type => 'integer',
459 description => "The number of cores per socket.",
460 minimum => 1,
461 default => 1,
462 },
463 numa => {
464 optional => 1,
465 type => 'boolean',
466 description => "Enable/disable NUMA.",
467 default => 0,
468 },
469 hugepages => {
470 optional => 1,
471 type => 'string',
472 description => "Enable/disable hugepages memory.",
473 enum => [qw(any 2 1024)],
474 },
475 keephugepages => {
476 optional => 1,
477 type => 'boolean',
478 default => 0,
479 description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
480 ." after VM shutdown and can be used for subsequent starts.",
481 },
482 vcpus => {
483 optional => 1,
484 type => 'integer',
485 description => "Number of hotplugged vcpus.",
486 minimum => 1,
487 default => 0,
488 },
489 acpi => {
490 optional => 1,
491 type => 'boolean',
492 description => "Enable/disable ACPI.",
493 default => 1,
494 },
495 agent => {
496 optional => 1,
497 description => "Enable/disable communication with the QEMU Guest Agent and its properties.",
498 type => 'string',
499 format => $agent_fmt,
500 },
501 kvm => {
502 optional => 1,
503 type => 'boolean',
504 description => "Enable/disable KVM hardware virtualization.",
505 default => 1,
506 },
507 tdf => {
508 optional => 1,
509 type => 'boolean',
510 description => "Enable/disable time drift fix.",
511 default => 0,
512 },
513 localtime => {
514 optional => 1,
515 type => 'boolean',
516 description => "Set the real time clock (RTC) to local time. This is enabled by default if"
517 ." the `ostype` indicates a Microsoft Windows OS.",
518 },
519 freeze => {
520 optional => 1,
521 type => 'boolean',
522 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
523 },
524 vga => {
525 optional => 1,
526 type => 'string', format => $vga_fmt,
527 description => "Configure the VGA hardware.",
528 verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
529 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
530 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
531 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
532 ." display server. For win* OS you can select how many independent displays you want,"
533 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
534 ." using a serial device as terminal.",
535 },
536 watchdog => {
537 optional => 1,
538 type => 'string', format => 'pve-qm-watchdog',
539 description => "Create a virtual hardware watchdog device.",
540 verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
541 ." action), the watchdog must be periodically polled by an agent inside the guest or"
542 ." else the watchdog will reset the guest (or execute the respective action specified)",
543 },
544 startdate => {
545 optional => 1,
546 type => 'string',
547 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
548 description => "Set the initial date of the real time clock. Valid format for date are:"
549 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
550 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
551 default => 'now',
552 },
553 startup => get_standard_option('pve-startup-order'),
554 template => {
555 optional => 1,
556 type => 'boolean',
557 description => "Enable/disable Template.",
558 default => 0,
559 },
560 args => {
561 optional => 1,
562 type => 'string',
563 description => "Arbitrary arguments passed to kvm.",
564 verbose_description => <<EODESCR,
565Arbitrary arguments passed to kvm, for example:
566
567args: -no-reboot -smbios 'type=0,vendor=FOO'
568
569NOTE: this option is for experts only.
570EODESCR
571 },
572 tablet => {
573 optional => 1,
574 type => 'boolean',
575 default => 1,
576 description => "Enable/disable the USB tablet device.",
577 verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
578 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
579 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
580 ." may consider disabling this to save some context switches. This is turned off by"
581 ." default if you use spice (`qm set <vmid> --vga qxl`).",
582 },
583 migrate_speed => {
584 optional => 1,
585 type => 'integer',
586 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
587 minimum => 0,
588 default => 0,
589 },
590 migrate_downtime => {
591 optional => 1,
592 type => 'number',
593 description => "Set maximum tolerated downtime (in seconds) for migrations. Should the"
594 ." migration not be able to converge in the very end, because too much newly dirtied"
595 ." RAM needs to be transferred, the limit will be increased automatically step-by-step"
596 ." until migration can converge.",
597 minimum => 0,
598 default => 0.1,
599 },
600 cdrom => {
601 optional => 1,
602 type => 'string', format => 'pve-qm-ide',
603 typetext => '<volume>',
604 description => "This is an alias for option -ide2",
605 },
606 cpu => {
607 optional => 1,
608 description => "Emulated CPU type.",
609 type => 'string',
610 format => 'pve-vm-cpu-conf',
611 },
612 parent => get_standard_option('pve-snapshot-name', {
613 optional => 1,
614 description => "Parent snapshot name. This is used internally, and should not be modified.",
615 }),
616 snaptime => {
617 optional => 1,
618 description => "Timestamp for snapshots.",
619 type => 'integer',
620 minimum => 0,
621 },
622 vmstate => {
623 optional => 1,
624 type => 'string', format => 'pve-volume-id',
625 description => "Reference to a volume which stores the VM state. This is used internally"
626 ." for snapshots.",
627 },
628 vmstatestorage => get_standard_option('pve-storage-id', {
629 description => "Default storage for VM state volumes/files.",
630 optional => 1,
631 }),
632 runningmachine => get_standard_option('pve-qemu-machine', {
633 description => "Specifies the QEMU machine type of the running vm. This is used internally"
634 ." for snapshots.",
635 }),
636 runningcpu => {
637 description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
638 ." internally for snapshots.",
639 optional => 1,
640 type => 'string',
641 pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
642 format_description => 'QEMU -cpu parameter'
643 },
644 machine => get_standard_option('pve-qemu-machine'),
645 arch => {
646 description => "Virtual processor architecture. Defaults to the host.",
647 optional => 1,
648 type => 'string',
649 enum => [qw(x86_64 aarch64)],
650 },
651 smbios1 => {
652 description => "Specify SMBIOS type 1 fields.",
653 type => 'string', format => 'pve-qm-smbios1',
654 maxLength => 512,
655 optional => 1,
656 },
657 protection => {
658 optional => 1,
659 type => 'boolean',
660 description => "Sets the protection flag of the VM. This will disable the remove VM and"
661 ." remove disk operations.",
662 default => 0,
663 },
664 bios => {
665 optional => 1,
666 type => 'string',
667 enum => [ qw(seabios ovmf) ],
668 description => "Select BIOS implementation.",
669 default => 'seabios',
670 },
671 vmgenid => {
672 type => 'string',
673 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
674 format_description => 'UUID',
675 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
676 ." to disable explicitly.",
677 verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
678 ." value identifier to the guest OS. This allows to notify the guest operating system"
679 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
680 ." execution or creation from a template). The guest operating system notices the"
681 ." change, and is then able to react as appropriate by marking its copies of"
682 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
683 ."Note that auto-creation only works when done through API/CLI create or update methods"
684 .", but not when manually editing the config file.",
685 default => "1 (autogenerated)",
686 optional => 1,
687 },
688 hookscript => {
689 type => 'string',
690 format => 'pve-volume-id',
691 optional => 1,
692 description => "Script that will be executed during various steps in the vms lifetime.",
693 },
694 ivshmem => {
695 type => 'string',
696 format => $ivshmem_fmt,
697 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
698 ." the host.",
699 optional => 1,
700 },
701 audio0 => {
702 type => 'string',
703 format => $audio_fmt,
704 description => "Configure a audio device, useful in combination with QXL/Spice.",
705 optional => 1
706 },
707 spice_enhancements => {
708 type => 'string',
709 format => $spice_enhancements_fmt,
710 description => "Configure additional enhancements for SPICE.",
711 optional => 1
712 },
713 tags => {
714 type => 'string', format => 'pve-tag-list',
715 description => 'Tags of the VM. This is only meta information.',
716 optional => 1,
717 },
718 rng0 => {
719 type => 'string',
720 format => $rng_fmt,
721 description => "Configure a VirtIO-based Random Number Generator.",
722 optional => 1,
723 },
724 meta => {
725 type => 'string',
726 format => $meta_info_fmt,
727 description => "Some (read-only) meta-information about this guest.",
728 optional => 1,
729 },
730 affinity => {
731 type => 'string', format => 'pve-cpuset',
732 description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
733 optional => 1,
734 },
735};
736
737my $cicustom_fmt = {
738 meta => {
739 type => 'string',
740 optional => 1,
741 description => 'Specify a custom file containing all meta data passed to the VM via"
742 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
743 format => 'pve-volume-id',
744 format_description => 'volume',
745 },
746 network => {
747 type => 'string',
748 optional => 1,
749 description => 'To pass a custom file containing all network data to the VM via cloud-init.',
750 format => 'pve-volume-id',
751 format_description => 'volume',
752 },
753 user => {
754 type => 'string',
755 optional => 1,
756 description => 'To pass a custom file containing all user data to the VM via cloud-init.',
757 format => 'pve-volume-id',
758 format_description => 'volume',
759 },
760 vendor => {
761 type => 'string',
762 optional => 1,
763 description => 'To pass a custom file containing all vendor data to the VM via cloud-init.',
764 format => 'pve-volume-id',
765 format_description => 'volume',
766 },
767};
768PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
769
770# any new option might need to be added to $cloudinitoptions in PVE::API2::Qemu
771my $confdesc_cloudinit = {
772 citype => {
773 optional => 1,
774 type => 'string',
775 description => 'Specifies the cloud-init configuration format. The default depends on the'
776 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
777 .' and `configdrive2` for windows.',
778 enum => ['configdrive2', 'nocloud', 'opennebula'],
779 },
780 ciuser => {
781 optional => 1,
782 type => 'string',
783 description => "cloud-init: User name to change ssh keys and password for instead of the"
784 ." image's configured default user.",
785 },
786 cipassword => {
787 optional => 1,
788 type => 'string',
789 description => 'cloud-init: Password to assign the user. Using this is generally not'
790 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
791 .' support hashed passwords.',
792 },
793 ciupgrade => {
794 optional => 1,
795 type => 'boolean',
796 description => 'cloud-init: do an automatic package upgrade after the first boot.',
797 default => 1,
798 },
799 cicustom => {
800 optional => 1,
801 type => 'string',
802 description => 'cloud-init: Specify custom files to replace the automatically generated'
803 .' ones at start.',
804 format => 'pve-qm-cicustom',
805 },
806 searchdomain => {
807 optional => 1,
808 type => 'string',
809 description => 'cloud-init: Sets DNS search domains for a container. Create will'
810 .' automatically use the setting from the host if neither searchdomain nor nameserver'
811 .' are set.',
812 },
813 nameserver => {
814 optional => 1,
815 type => 'string', format => 'address-list',
816 description => 'cloud-init: Sets DNS server IP address for a container. Create will'
817 .' automatically use the setting from the host if neither searchdomain nor nameserver'
818 .' are set.',
819 },
820 sshkeys => {
821 optional => 1,
822 type => 'string',
823 format => 'urlencoded',
824 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
825 },
826};
827
828# what about other qemu settings ?
829#cpu => 'string',
830#machine => 'string',
831#fda => 'file',
832#fdb => 'file',
833#mtdblock => 'file',
834#sd => 'file',
835#pflash => 'file',
836#snapshot => 'bool',
837#bootp => 'file',
838##tftp => 'dir',
839##smb => 'dir',
840#kernel => 'file',
841#append => 'string',
842#initrd => 'file',
843##soundhw => 'string',
844
845while (my ($k, $v) = each %$confdesc) {
846 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
847}
848
849my $MAX_NETS = 32;
850my $MAX_SERIAL_PORTS = 4;
851my $MAX_PARALLEL_PORTS = 3;
852
853for (my $i = 0; $i < $PVE::QemuServer::Memory::MAX_NUMA; $i++) {
854 $confdesc->{"numa$i"} = $PVE::QemuServer::Memory::numadesc;
855}
856
857my $nic_model_list = [
858 'e1000',
859 'e1000-82540em',
860 'e1000-82544gc',
861 'e1000-82545em',
862 'e1000e',
863 'i82551',
864 'i82557b',
865 'i82559er',
866 'ne2k_isa',
867 'ne2k_pci',
868 'pcnet',
869 'rtl8139',
870 'virtio',
871 'vmxnet3',
872];
873my $nic_model_list_txt = join(' ', sort @$nic_model_list);
874
875my $net_fmt_bridge_descr = <<__EOD__;
876Bridge to attach the network device to. The Proxmox VE standard bridge
877is called 'vmbr0'.
878
879If you do not specify a bridge, we create a kvm user (NATed) network
880device, which provides DHCP and DNS services. The following addresses
881are used:
882
883 10.0.2.2 Gateway
884 10.0.2.3 DNS Server
885 10.0.2.4 SMB Server
886
887The DHCP server assign addresses to the guest starting from 10.0.2.15.
888__EOD__
889
890my $net_fmt = {
891 macaddr => get_standard_option('mac-addr', {
892 description => "MAC address. That address must be unique withing your network. This is"
893 ." automatically generated if not specified.",
894 }),
895 model => {
896 type => 'string',
897 description => "Network Card Model. The 'virtio' model provides the best performance with"
898 ." very low CPU overhead. If your guest does not support this driver, it is usually"
899 ." best to use 'e1000'.",
900 enum => $nic_model_list,
901 default_key => 1,
902 },
903 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
904 bridge => get_standard_option('pve-bridge-id', {
905 description => $net_fmt_bridge_descr,
906 optional => 1,
907 }),
908 queues => {
909 type => 'integer',
910 minimum => 0, maximum => 64,
911 description => 'Number of packet queues to be used on the device.',
912 optional => 1,
913 },
914 rate => {
915 type => 'number',
916 minimum => 0,
917 description => "Rate limit in mbps (megabytes per second) as floating point number.",
918 optional => 1,
919 },
920 tag => {
921 type => 'integer',
922 minimum => 1, maximum => 4094,
923 description => 'VLAN tag to apply to packets on this interface.',
924 optional => 1,
925 },
926 trunks => {
927 type => 'string',
928 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
929 description => 'VLAN trunks to pass through this interface.',
930 format_description => 'vlanid[;vlanid...]',
931 optional => 1,
932 },
933 firewall => {
934 type => 'boolean',
935 description => 'Whether this interface should be protected by the firewall.',
936 optional => 1,
937 },
938 link_down => {
939 type => 'boolean',
940 description => 'Whether this interface should be disconnected (like pulling the plug).',
941 optional => 1,
942 },
943 mtu => {
944 type => 'integer',
945 minimum => 1, maximum => 65520,
946 description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
947 optional => 1,
948 },
949};
950
951my $netdesc = {
952 optional => 1,
953 type => 'string', format => $net_fmt,
954 description => "Specify network devices.",
955};
956
957PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
958
959my $ipconfig_fmt = {
960 ip => {
961 type => 'string',
962 format => 'pve-ipv4-config',
963 format_description => 'IPv4Format/CIDR',
964 description => 'IPv4 address in CIDR format.',
965 optional => 1,
966 default => 'dhcp',
967 },
968 gw => {
969 type => 'string',
970 format => 'ipv4',
971 format_description => 'GatewayIPv4',
972 description => 'Default gateway for IPv4 traffic.',
973 optional => 1,
974 requires => 'ip',
975 },
976 ip6 => {
977 type => 'string',
978 format => 'pve-ipv6-config',
979 format_description => 'IPv6Format/CIDR',
980 description => 'IPv6 address in CIDR format.',
981 optional => 1,
982 default => 'dhcp',
983 },
984 gw6 => {
985 type => 'string',
986 format => 'ipv6',
987 format_description => 'GatewayIPv6',
988 description => 'Default gateway for IPv6 traffic.',
989 optional => 1,
990 requires => 'ip6',
991 },
992};
993PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
994my $ipconfigdesc = {
995 optional => 1,
996 type => 'string', format => 'pve-qm-ipconfig',
997 description => <<'EODESCR',
998cloud-init: Specify IP addresses and gateways for the corresponding interface.
999
1000IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1001
1002The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1003gateway should be provided.
1004For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1005cloud-init 19.4 or newer.
1006
1007If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1008dhcp on IPv4.
1009EODESCR
1010};
1011PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1012
1013for (my $i = 0; $i < $MAX_NETS; $i++) {
1014 $confdesc->{"net$i"} = $netdesc;
1015 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1016}
1017
1018foreach my $key (keys %$confdesc_cloudinit) {
1019 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1020}
1021
1022PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1023sub pve_verify_cpuset {
1024 my ($set_text, $noerr) = @_;
1025
1026 my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1027
1028 if ($@) {
1029 return if $noerr;
1030 die "unable to parse cpuset option\n";
1031 }
1032
1033 return PVE::CpuSet->new($members)->short_string();
1034}
1035
1036PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1037sub verify_volume_id_or_qm_path {
1038 my ($volid, $noerr) = @_;
1039
1040 return $volid if $volid eq 'none' || $volid eq 'cdrom';
1041
1042 return verify_volume_id_or_absolute_path($volid, $noerr);
1043}
1044
1045PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1046sub verify_volume_id_or_absolute_path {
1047 my ($volid, $noerr) = @_;
1048
1049 return $volid if $volid =~ m|^/|;
1050
1051 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1052 if ($@) {
1053 return if $noerr;
1054 die $@;
1055 }
1056 return $volid;
1057}
1058
1059my $serialdesc = {
1060 optional => 1,
1061 type => 'string',
1062 pattern => '(/dev/.+|socket)',
1063 description => "Create a serial device inside the VM (n is 0 to 3)",
1064 verbose_description => <<EODESCR,
1065Create a serial device inside the VM (n is 0 to 3), and pass through a
1066host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1067host side (use 'qm terminal' to open a terminal connection).
1068
1069NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1070use with special care.
1071
1072CAUTION: Experimental! User reported problems with this option.
1073EODESCR
1074};
1075
1076my $paralleldesc= {
1077 optional => 1,
1078 type => 'string',
1079 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1080 description => "Map host parallel devices (n is 0 to 2).",
1081 verbose_description => <<EODESCR,
1082Map host parallel devices (n is 0 to 2).
1083
1084NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1085machines - use with special care.
1086
1087CAUTION: Experimental! User reported problems with this option.
1088EODESCR
1089};
1090
1091for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1092 $confdesc->{"parallel$i"} = $paralleldesc;
1093}
1094
1095for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1096 $confdesc->{"serial$i"} = $serialdesc;
1097}
1098
1099for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
1100 $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1101}
1102
1103for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1104 $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1105}
1106
1107for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
1108 $confdesc->{"usb$i"} = $PVE::QemuServer::USB::usbdesc;
1109}
1110
1111my $boot_fmt = {
1112 legacy => {
1113 optional => 1,
1114 default_key => 1,
1115 type => 'string',
1116 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1117 . " Deprecated, use 'order=' instead.",
1118 pattern => '[acdn]{1,4}',
1119 format_description => "[acdn]{1,4}",
1120
1121 # note: this is also the fallback if boot: is not given at all
1122 default => 'cdn',
1123 },
1124 order => {
1125 optional => 1,
1126 type => 'string',
1127 format => 'pve-qm-bootdev-list',
1128 format_description => "device[;device...]",
1129 description => <<EODESC,
1130The guest will attempt to boot from devices in the order they appear here.
1131
1132Disks, optical drives and passed-through storage USB devices will be directly
1133booted from, NICs will load PXE, and PCIe devices will either behave like disks
1134(e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1135
1136Note that only devices in this list will be marked as bootable and thus loaded
1137by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1138(e.g. software-raid), you need to specify all of them here.
1139
1140Overrides the deprecated 'legacy=[acdn]*' value when given.
1141EODESC
1142 },
1143};
1144PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1145
1146PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1147sub verify_bootdev {
1148 my ($dev, $noerr) = @_;
1149
1150 my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1151 return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1152
1153 my $check = sub {
1154 my ($base) = @_;
1155 return 0 if $dev !~ m/^$base\d+$/;
1156 return 0 if !$confdesc->{$dev};
1157 return 1;
1158 };
1159
1160 return $dev if $check->("net");
1161 return $dev if $check->("usb");
1162 return $dev if $check->("hostpci");
1163
1164 return if $noerr;
1165 die "invalid boot device '$dev'\n";
1166}
1167
1168sub print_bootorder {
1169 my ($devs) = @_;
1170 return "" if !@$devs;
1171 my $data = { order => join(';', @$devs) };
1172 return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1173}
1174
1175my $kvm_api_version = 0;
1176
1177sub kvm_version {
1178 return $kvm_api_version if $kvm_api_version;
1179
1180 open my $fh, '<', '/dev/kvm' or return;
1181
1182 # 0xae00 => KVM_GET_API_VERSION
1183 $kvm_api_version = ioctl($fh, 0xae00, 0);
1184 close($fh);
1185
1186 return $kvm_api_version;
1187}
1188
1189my $kvm_user_version = {};
1190my $kvm_mtime = {};
1191
1192sub kvm_user_version {
1193 my ($binary) = @_;
1194
1195 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1196 my $st = stat($binary);
1197
1198 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1199 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1200 $cachedmtime == $st->mtime;
1201
1202 $kvm_user_version->{$binary} = 'unknown';
1203 $kvm_mtime->{$binary} = $st->mtime;
1204
1205 my $code = sub {
1206 my $line = shift;
1207 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1208 $kvm_user_version->{$binary} = $2;
1209 }
1210 };
1211
1212 eval { run_command([$binary, '--version'], outfunc => $code); };
1213 warn $@ if $@;
1214
1215 return $kvm_user_version->{$binary};
1216
1217}
1218my sub extract_version {
1219 my ($machine_type, $version) = @_;
1220 $version = kvm_user_version() if !defined($version);
1221 return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1222}
1223
1224sub kernel_has_vhost_net {
1225 return -c '/dev/vhost-net';
1226}
1227
1228sub option_exists {
1229 my $key = shift;
1230 return defined($confdesc->{$key});
1231}
1232
1233my $cdrom_path;
1234sub get_cdrom_path {
1235
1236 return $cdrom_path if defined($cdrom_path);
1237
1238 $cdrom_path = first { -l $_ } map { "/dev/cdrom$_" } ('', '1', '2');
1239
1240 if (!defined($cdrom_path)) {
1241 log_warn("no physical CD-ROM available, ignoring");
1242 $cdrom_path = '';
1243 }
1244
1245 return $cdrom_path;
1246}
1247
1248sub get_iso_path {
1249 my ($storecfg, $vmid, $cdrom) = @_;
1250
1251 if ($cdrom eq 'cdrom') {
1252 return get_cdrom_path();
1253 } elsif ($cdrom eq 'none') {
1254 return '';
1255 } elsif ($cdrom =~ m|^/|) {
1256 return $cdrom;
1257 } else {
1258 return PVE::Storage::path($storecfg, $cdrom);
1259 }
1260}
1261
1262# try to convert old style file names to volume IDs
1263sub filename_to_volume_id {
1264 my ($vmid, $file, $media) = @_;
1265
1266 if (!($file eq 'none' || $file eq 'cdrom' ||
1267 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1268
1269 return if $file =~ m|/|;
1270
1271 if ($media && $media eq 'cdrom') {
1272 $file = "local:iso/$file";
1273 } else {
1274 $file = "local:$vmid/$file";
1275 }
1276 }
1277
1278 return $file;
1279}
1280
1281sub verify_media_type {
1282 my ($opt, $vtype, $media) = @_;
1283
1284 return if !$media;
1285
1286 my $etype;
1287 if ($media eq 'disk') {
1288 $etype = 'images';
1289 } elsif ($media eq 'cdrom') {
1290 $etype = 'iso';
1291 } else {
1292 die "internal error";
1293 }
1294
1295 return if ($vtype eq $etype);
1296
1297 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1298}
1299
1300sub cleanup_drive_path {
1301 my ($opt, $storecfg, $drive) = @_;
1302
1303 # try to convert filesystem paths to volume IDs
1304
1305 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1306 ($drive->{file} !~ m|^/dev/.+|) &&
1307 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1308 ($drive->{file} !~ m/^\d+$/)) {
1309 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1310 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1311 if !$vtype;
1312 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1313 verify_media_type($opt, $vtype, $drive->{media});
1314 $drive->{file} = $volid;
1315 }
1316
1317 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1318}
1319
1320sub parse_hotplug_features {
1321 my ($data) = @_;
1322
1323 my $res = {};
1324
1325 return $res if $data eq '0';
1326
1327 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1328
1329 foreach my $feature (PVE::Tools::split_list($data)) {
1330 if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1331 $res->{$1} = 1;
1332 } else {
1333 die "invalid hotplug feature '$feature'\n";
1334 }
1335 }
1336 return $res;
1337}
1338
1339PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1340sub pve_verify_hotplug_features {
1341 my ($value, $noerr) = @_;
1342
1343 return $value if parse_hotplug_features($value);
1344
1345 return if $noerr;
1346
1347 die "unable to parse hotplug option\n";
1348}
1349
1350sub assert_clipboard_config {
1351 my ($vga) = @_;
1352
1353 my $clipboard_regex = qr/^(std|cirrus|vmware|virtio|qxl)/;
1354
1355 if (
1356 $vga->{'clipboard'}
1357 && $vga->{'clipboard'} eq 'vnc'
1358 && $vga->{type}
1359 && $vga->{type} !~ $clipboard_regex
1360 ) {
1361 die "vga type $vga->{type} is not compatible with VNC clipboard\n";
1362 }
1363}
1364
1365sub print_tabletdevice_full {
1366 my ($conf, $arch) = @_;
1367
1368 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1369
1370 # we use uhci for old VMs because tablet driver was buggy in older qemu
1371 my $usbbus;
1372 if ($q35 || $arch eq 'aarch64') {
1373 $usbbus = 'ehci';
1374 } else {
1375 $usbbus = 'uhci';
1376 }
1377
1378 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1379}
1380
1381sub print_keyboarddevice_full {
1382 my ($conf, $arch) = @_;
1383
1384 return if $arch ne 'aarch64';
1385
1386 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1387}
1388
1389my sub get_drive_id {
1390 my ($drive) = @_;
1391 return "$drive->{interface}$drive->{index}";
1392}
1393
1394sub print_drivedevice_full {
1395 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1396
1397 my $device = '';
1398 my $maxdev = 0;
1399
1400 my $drive_id = get_drive_id($drive);
1401 if ($drive->{interface} eq 'virtio') {
1402 my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1403 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1404 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1405 } elsif ($drive->{interface} eq 'scsi') {
1406
1407 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1408 my $unit = $drive->{index} % $maxdev;
1409
1410 my $machine_version = extract_version($machine_type, kvm_user_version());
1411 my $device_type = PVE::QemuServer::Drive::get_scsi_device_type(
1412 $drive, $storecfg, $machine_version);
1413
1414 if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1415 $device = "scsi-$device_type,bus=$controller_prefix$controller.0,scsi-id=$unit";
1416 } else {
1417 $device = "scsi-$device_type,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1418 .",lun=$drive->{index}";
1419 }
1420 $device .= ",drive=drive-$drive_id,id=$drive_id";
1421
1422 if ($drive->{ssd} && ($device_type eq 'block' || $device_type eq 'hd')) {
1423 $device .= ",rotation_rate=1";
1424 }
1425 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1426
1427 # only scsi-hd and scsi-cd support passing vendor and product information
1428 if ($device_type eq 'hd' || $device_type eq 'cd') {
1429 if (my $vendor = $drive->{vendor}) {
1430 $device .= ",vendor=$vendor";
1431 }
1432 if (my $product = $drive->{product}) {
1433 $device .= ",product=$product";
1434 }
1435 }
1436
1437 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1438 my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1439 my $controller = int($drive->{index} / $maxdev);
1440 my $unit = $drive->{index} % $maxdev;
1441
1442 # machine type q35 only supports unit=0 for IDE rather than 2 units. This wasn't handled
1443 # correctly before, so e.g. index=2 was mapped to controller=1,unit=0 rather than
1444 # controller=2,unit=0. Note that odd indices never worked, as they would be mapped to
1445 # unit=1, so to keep backwards compat for migration, it suffices to keep even ones as they
1446 # were before. Move odd ones up by 2 where they don't clash.
1447 if (PVE::QemuServer::Machine::machine_type_is_q35($conf) && $drive->{interface} eq 'ide') {
1448 $controller += 2 * ($unit % 2);
1449 $unit = 0;
1450 }
1451
1452 my $device_type = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1453
1454 $device = "ide-$device_type";
1455 if ($drive->{interface} eq 'ide') {
1456 $device .= ",bus=ide.$controller,unit=$unit";
1457 } else {
1458 $device .= ",bus=ahci$controller.$unit";
1459 }
1460 $device .= ",drive=drive-$drive_id,id=$drive_id";
1461
1462 if ($device_type eq 'hd') {
1463 if (my $model = $drive->{model}) {
1464 $model = URI::Escape::uri_unescape($model);
1465 $device .= ",model=$model";
1466 }
1467 if ($drive->{ssd}) {
1468 $device .= ",rotation_rate=1";
1469 }
1470 }
1471 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1472 } elsif ($drive->{interface} eq 'usb') {
1473 die "implement me";
1474 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1475 } else {
1476 die "unsupported interface type";
1477 }
1478
1479 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1480
1481 if (my $serial = $drive->{serial}) {
1482 $serial = URI::Escape::uri_unescape($serial);
1483 $device .= ",serial=$serial";
1484 }
1485
1486
1487 return $device;
1488}
1489
1490sub get_initiator_name {
1491 my $initiator;
1492
1493 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1494 while (defined(my $line = <$fh>)) {
1495 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1496 $initiator = $1;
1497 last;
1498 }
1499 $fh->close();
1500
1501 return $initiator;
1502}
1503
1504my sub storage_allows_io_uring_default {
1505 my ($scfg, $cache_direct) = @_;
1506
1507 # io_uring with cache mode writeback or writethrough on krbd will hang...
1508 return if $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1509
1510 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1511 # sometimes, just plain disable...
1512 return if $scfg && $scfg->{type} eq 'lvm';
1513
1514 # io_uring causes problems when used with CIFS since kernel 5.15
1515 # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1516 return if $scfg && $scfg->{type} eq 'cifs';
1517
1518 return 1;
1519}
1520
1521my sub drive_uses_cache_direct {
1522 my ($drive, $scfg) = @_;
1523
1524 my $cache_direct = 0;
1525
1526 if (my $cache = $drive->{cache}) {
1527 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1528 } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1529 $cache_direct = 1;
1530 }
1531
1532 return $cache_direct;
1533}
1534
1535sub print_drive_commandline_full {
1536 my ($storecfg, $vmid, $drive, $live_restore_name, $io_uring) = @_;
1537
1538 my $path;
1539 my $volid = $drive->{file};
1540 my $format = $drive->{format};
1541 my $drive_id = get_drive_id($drive);
1542
1543 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1544 my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1545
1546 if (drive_is_cdrom($drive)) {
1547 $path = get_iso_path($storecfg, $vmid, $volid);
1548 die "$drive_id: cannot back cdrom drive with a live restore image\n" if $live_restore_name;
1549 } else {
1550 if ($storeid) {
1551 $path = PVE::Storage::path($storecfg, $volid);
1552 $format //= qemu_img_format($scfg, $volname);
1553 } else {
1554 $path = $volid;
1555 $format //= "raw";
1556 }
1557 }
1558
1559 my $is_rbd = $path =~ m/^rbd:/;
1560
1561 my $opts = '';
1562 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1563 foreach my $o (@qemu_drive_options) {
1564 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1565 }
1566
1567 # snapshot only accepts on|off
1568 if (defined($drive->{snapshot})) {
1569 my $v = $drive->{snapshot} ? 'on' : 'off';
1570 $opts .= ",snapshot=$v";
1571 }
1572
1573 if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1574 $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1575 }
1576
1577 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1578 my ($dir, $qmpname) = @$type;
1579 if (my $v = $drive->{"mbps$dir"}) {
1580 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1581 }
1582 if (my $v = $drive->{"mbps${dir}_max"}) {
1583 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1584 }
1585 if (my $v = $drive->{"bps${dir}_max_length"}) {
1586 $opts .= ",throttling.bps$qmpname-max-length=$v";
1587 }
1588 if (my $v = $drive->{"iops${dir}"}) {
1589 $opts .= ",throttling.iops$qmpname=$v";
1590 }
1591 if (my $v = $drive->{"iops${dir}_max"}) {
1592 $opts .= ",throttling.iops$qmpname-max=$v";
1593 }
1594 if (my $v = $drive->{"iops${dir}_max_length"}) {
1595 $opts .= ",throttling.iops$qmpname-max-length=$v";
1596 }
1597 }
1598
1599 if ($live_restore_name) {
1600 $format = "rbd" if $is_rbd;
1601 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1602 if !$format;
1603 $opts .= ",format=alloc-track,file.driver=$format";
1604 } elsif ($format) {
1605 $opts .= ",format=$format";
1606 }
1607
1608 my $cache_direct = drive_uses_cache_direct($drive, $scfg);
1609
1610 $opts .= ",cache=none" if !$drive->{cache} && $cache_direct;
1611
1612 if (!$drive->{aio}) {
1613 if ($io_uring && storage_allows_io_uring_default($scfg, $cache_direct)) {
1614 # io_uring supports all cache modes
1615 $opts .= ",aio=io_uring";
1616 } else {
1617 # aio native works only with O_DIRECT
1618 if($cache_direct) {
1619 $opts .= ",aio=native";
1620 } else {
1621 $opts .= ",aio=threads";
1622 }
1623 }
1624 }
1625
1626 if (!drive_is_cdrom($drive)) {
1627 my $detectzeroes;
1628 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1629 $detectzeroes = 'off';
1630 } elsif ($drive->{discard}) {
1631 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1632 } else {
1633 # This used to be our default with discard not being specified:
1634 $detectzeroes = 'on';
1635 }
1636
1637 # note: 'detect-zeroes' works per blockdev and we want it to persist
1638 # after the alloc-track is removed, so put it on 'file' directly
1639 my $dz_param = $live_restore_name ? "file.detect-zeroes" : "detect-zeroes";
1640 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1641 }
1642
1643 if ($live_restore_name) {
1644 $opts .= ",backing=$live_restore_name";
1645 $opts .= ",auto-remove=on";
1646 }
1647
1648 # my $file_param = $live_restore_name ? "file.file.filename" : "file";
1649 my $file_param = "file";
1650 if ($live_restore_name) {
1651 # non-rbd drivers require the underlying file to be a seperate block
1652 # node, so add a second .file indirection
1653 $file_param .= ".file" if !$is_rbd;
1654 $file_param .= ".filename";
1655 }
1656 my $pathinfo = $path ? "$file_param=$path," : '';
1657
1658 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1659}
1660
1661sub print_pbs_blockdev {
1662 my ($pbs_conf, $pbs_name) = @_;
1663 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1664 $blockdev .= ",repository=$pbs_conf->{repository}";
1665 $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1666 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1667 $blockdev .= ",archive=$pbs_conf->{archive}";
1668 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1669 return $blockdev;
1670}
1671
1672sub print_netdevice_full {
1673 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1674
1675 my $device = $net->{model};
1676 if ($net->{model} eq 'virtio') {
1677 $device = 'virtio-net-pci';
1678 };
1679
1680 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1681 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1682 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1683 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1684 # and out of each queue plus one config interrupt and control vector queue
1685 my $vectors = $net->{queues} * 2 + 2;
1686 $tmpstr .= ",vectors=$vectors,mq=on";
1687 if (min_version($machine_version, 7, 1)) {
1688 $tmpstr .= ",packed=on";
1689 }
1690 }
1691
1692 if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1693 $tmpstr .= ",rx_queue_size=1024,tx_queue_size=256";
1694 }
1695
1696 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1697
1698 if (my $mtu = $net->{mtu}) {
1699 if ($net->{model} eq 'virtio' && $net->{bridge}) {
1700 my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1701 if ($mtu == 1) {
1702 $mtu = $bridge_mtu;
1703 } elsif ($mtu < 576) {
1704 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1705 } elsif ($mtu > $bridge_mtu) {
1706 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1707 }
1708 $tmpstr .= ",host_mtu=$mtu";
1709 } else {
1710 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1711 }
1712 }
1713
1714 if ($use_old_bios_files) {
1715 my $romfile;
1716 if ($device eq 'virtio-net-pci') {
1717 $romfile = 'pxe-virtio.rom';
1718 } elsif ($device eq 'e1000') {
1719 $romfile = 'pxe-e1000.rom';
1720 } elsif ($device eq 'e1000e') {
1721 $romfile = 'pxe-e1000e.rom';
1722 } elsif ($device eq 'ne2k') {
1723 $romfile = 'pxe-ne2k_pci.rom';
1724 } elsif ($device eq 'pcnet') {
1725 $romfile = 'pxe-pcnet.rom';
1726 } elsif ($device eq 'rtl8139') {
1727 $romfile = 'pxe-rtl8139.rom';
1728 }
1729 $tmpstr .= ",romfile=$romfile" if $romfile;
1730 }
1731
1732 return $tmpstr;
1733}
1734
1735sub print_netdev_full {
1736 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1737
1738 my $i = '';
1739 if ($netid =~ m/^net(\d+)$/) {
1740 $i = int($1);
1741 }
1742
1743 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1744
1745 my $ifname = "tap${vmid}i$i";
1746
1747 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1748 die "interface name '$ifname' is too long (max 15 character)\n"
1749 if length($ifname) >= 16;
1750
1751 my $vhostparam = '';
1752 if (is_native_arch($arch)) {
1753 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1754 }
1755
1756 my $vmname = $conf->{name} || "vm$vmid";
1757
1758 my $netdev = "";
1759 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1760
1761 if ($net->{bridge}) {
1762 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1763 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1764 } else {
1765 $netdev = "type=user,id=$netid,hostname=$vmname";
1766 }
1767
1768 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1769
1770 return $netdev;
1771}
1772
1773my $vga_map = {
1774 'cirrus' => 'cirrus-vga',
1775 'std' => 'VGA',
1776 'vmware' => 'vmware-svga',
1777 'virtio' => 'virtio-vga',
1778 'virtio-gl' => 'virtio-vga-gl',
1779};
1780
1781sub print_vga_device {
1782 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1783
1784 my $type = $vga_map->{$vga->{type}};
1785 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1786 $type = 'virtio-gpu';
1787 }
1788 my $vgamem_mb = $vga->{memory};
1789
1790 my $max_outputs = '';
1791 if ($qxlnum) {
1792 $type = $id ? 'qxl' : 'qxl-vga';
1793
1794 if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1795 # set max outputs so linux can have up to 4 qxl displays with one device
1796 if (min_version($machine_version, 4, 1)) {
1797 $max_outputs = ",max_outputs=4";
1798 }
1799 }
1800 }
1801
1802 die "no device-type for $vga->{type}\n" if !$type;
1803
1804 my $memory = "";
1805 if ($vgamem_mb) {
1806 if ($vga->{type} =~ /^virtio/) {
1807 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1808 $memory = ",max_hostmem=$bytes";
1809 } elsif ($qxlnum) {
1810 # from https://www.spice-space.org/multiple-monitors.html
1811 $memory = ",vgamem_mb=$vga->{memory}";
1812 my $ram = $vgamem_mb * 4;
1813 my $vram = $vgamem_mb * 2;
1814 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1815 } else {
1816 $memory = ",vgamem_mb=$vga->{memory}";
1817 }
1818 } elsif ($qxlnum && $id) {
1819 $memory = ",ram_size=67108864,vram_size=33554432";
1820 }
1821
1822 my $edidoff = "";
1823 if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1824 $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1825 }
1826
1827 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1828 my $vgaid = "vga" . ($id // '');
1829 my $pciaddr;
1830 if ($q35 && $vgaid eq 'vga') {
1831 # the first display uses pcie.0 bus on q35 machines
1832 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1833 } else {
1834 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1835 }
1836
1837 if ($vga->{type} eq 'virtio-gl') {
1838 my $base = '/usr/lib/x86_64-linux-gnu/lib';
1839 die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1840 if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1841
1842 die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1843 if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1844 }
1845
1846 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1847}
1848
1849# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1850sub parse_net {
1851 my ($data, $disable_mac_autogen) = @_;
1852
1853 my $res = eval { parse_property_string($net_fmt, $data) };
1854 if ($@) {
1855 warn $@;
1856 return;
1857 }
1858 if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1859 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1860 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1861 }
1862 return $res;
1863}
1864
1865# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1866sub parse_ipconfig {
1867 my ($data) = @_;
1868
1869 my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1870 if ($@) {
1871 warn $@;
1872 return;
1873 }
1874
1875 if ($res->{gw} && !$res->{ip}) {
1876 warn 'gateway specified without specifying an IP address';
1877 return;
1878 }
1879 if ($res->{gw6} && !$res->{ip6}) {
1880 warn 'IPv6 gateway specified without specifying an IPv6 address';
1881 return;
1882 }
1883 if ($res->{gw} && $res->{ip} eq 'dhcp') {
1884 warn 'gateway specified together with DHCP';
1885 return;
1886 }
1887 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1888 # gw6 + auto/dhcp
1889 warn "IPv6 gateway specified together with $res->{ip6} address";
1890 return;
1891 }
1892
1893 if (!$res->{ip} && !$res->{ip6}) {
1894 return { ip => 'dhcp', ip6 => 'dhcp' };
1895 }
1896
1897 return $res;
1898}
1899
1900sub print_net {
1901 my $net = shift;
1902
1903 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1904}
1905
1906sub add_random_macs {
1907 my ($settings) = @_;
1908
1909 foreach my $opt (keys %$settings) {
1910 next if $opt !~ m/^net(\d+)$/;
1911 my $net = parse_net($settings->{$opt});
1912 next if !$net;
1913 $settings->{$opt} = print_net($net);
1914 }
1915}
1916
1917sub vm_is_volid_owner {
1918 my ($storecfg, $vmid, $volid) = @_;
1919
1920 if ($volid !~ m|^/|) {
1921 my ($path, $owner);
1922 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1923 if ($owner && ($owner == $vmid)) {
1924 return 1;
1925 }
1926 }
1927
1928 return;
1929}
1930
1931sub vmconfig_register_unused_drive {
1932 my ($storecfg, $vmid, $conf, $drive) = @_;
1933
1934 if (drive_is_cloudinit($drive)) {
1935 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1936 warn $@ if $@;
1937 delete $conf->{cloudinit};
1938 } elsif (!drive_is_cdrom($drive)) {
1939 my $volid = $drive->{file};
1940 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1941 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1942 }
1943 }
1944}
1945
1946# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1947my $smbios1_fmt = {
1948 uuid => {
1949 type => 'string',
1950 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1951 format_description => 'UUID',
1952 description => "Set SMBIOS1 UUID.",
1953 optional => 1,
1954 },
1955 version => {
1956 type => 'string',
1957 pattern => '[A-Za-z0-9+\/]+={0,2}',
1958 format_description => 'Base64 encoded string',
1959 description => "Set SMBIOS1 version.",
1960 optional => 1,
1961 },
1962 serial => {
1963 type => 'string',
1964 pattern => '[A-Za-z0-9+\/]+={0,2}',
1965 format_description => 'Base64 encoded string',
1966 description => "Set SMBIOS1 serial number.",
1967 optional => 1,
1968 },
1969 manufacturer => {
1970 type => 'string',
1971 pattern => '[A-Za-z0-9+\/]+={0,2}',
1972 format_description => 'Base64 encoded string',
1973 description => "Set SMBIOS1 manufacturer.",
1974 optional => 1,
1975 },
1976 product => {
1977 type => 'string',
1978 pattern => '[A-Za-z0-9+\/]+={0,2}',
1979 format_description => 'Base64 encoded string',
1980 description => "Set SMBIOS1 product ID.",
1981 optional => 1,
1982 },
1983 sku => {
1984 type => 'string',
1985 pattern => '[A-Za-z0-9+\/]+={0,2}',
1986 format_description => 'Base64 encoded string',
1987 description => "Set SMBIOS1 SKU string.",
1988 optional => 1,
1989 },
1990 family => {
1991 type => 'string',
1992 pattern => '[A-Za-z0-9+\/]+={0,2}',
1993 format_description => 'Base64 encoded string',
1994 description => "Set SMBIOS1 family string.",
1995 optional => 1,
1996 },
1997 base64 => {
1998 type => 'boolean',
1999 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2000 optional => 1,
2001 },
2002};
2003
2004sub parse_smbios1 {
2005 my ($data) = @_;
2006
2007 my $res = eval { parse_property_string($smbios1_fmt, $data) };
2008 warn $@ if $@;
2009 return $res;
2010}
2011
2012sub print_smbios1 {
2013 my ($smbios1) = @_;
2014 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2015}
2016
2017PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2018
2019sub parse_watchdog {
2020 my ($value) = @_;
2021
2022 return if !$value;
2023
2024 my $res = eval { parse_property_string($watchdog_fmt, $value) };
2025 warn $@ if $@;
2026 return $res;
2027}
2028
2029sub parse_guest_agent {
2030 my ($conf) = @_;
2031
2032 return {} if !defined($conf->{agent});
2033
2034 my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2035 warn $@ if $@;
2036
2037 # if the agent is disabled ignore the other potentially set properties
2038 return {} if !$res->{enabled};
2039 return $res;
2040}
2041
2042sub get_qga_key {
2043 my ($conf, $key) = @_;
2044 return undef if !defined($conf->{agent});
2045
2046 my $agent = parse_guest_agent($conf);
2047 return $agent->{$key};
2048}
2049
2050sub parse_vga {
2051 my ($value) = @_;
2052
2053 return {} if !$value;
2054 my $res = eval { parse_property_string($vga_fmt, $value) };
2055 warn $@ if $@;
2056 return $res;
2057}
2058
2059sub parse_rng {
2060 my ($value) = @_;
2061
2062 return if !$value;
2063
2064 my $res = eval { parse_property_string($rng_fmt, $value) };
2065 warn $@ if $@;
2066 return $res;
2067}
2068
2069sub parse_meta_info {
2070 my ($value) = @_;
2071
2072 return if !$value;
2073
2074 my $res = eval { parse_property_string($meta_info_fmt, $value) };
2075 warn $@ if $@;
2076 return $res;
2077}
2078
2079sub new_meta_info_string {
2080 my () = @_; # for now do not allow to override any value
2081
2082 return PVE::JSONSchema::print_property_string(
2083 {
2084 'creation-qemu' => kvm_user_version(),
2085 ctime => "". int(time()),
2086 },
2087 $meta_info_fmt
2088 );
2089}
2090
2091sub qemu_created_version_fixups {
2092 my ($conf, $forcemachine, $kvmver) = @_;
2093
2094 my $meta = parse_meta_info($conf->{meta}) // {};
2095 my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2096
2097 # check if we need to apply some handling for VMs that always use the latest machine version but
2098 # had a machine version transition happen that affected HW such that, e.g., an OS config change
2099 # would be required (we do not want to pin machine version for non-windows OS type)
2100 my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
2101 if (
2102 (!defined($machine_conf->{type}) || $machine_conf->{type} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2103 && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2104 && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2105 && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2106 ) {
2107 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2108 if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2109 # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2110 # and thus with the predictable interface naming of systemd
2111 return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2112 }
2113 }
2114 return;
2115}
2116
2117# add JSON properties for create and set function
2118sub json_config_properties {
2119 my ($prop, $with_disk_alloc) = @_;
2120
2121 my $skip_json_config_opts = {
2122 parent => 1,
2123 snaptime => 1,
2124 vmstate => 1,
2125 runningmachine => 1,
2126 runningcpu => 1,
2127 meta => 1,
2128 };
2129
2130 foreach my $opt (keys %$confdesc) {
2131 next if $skip_json_config_opts->{$opt};
2132
2133 if ($with_disk_alloc && is_valid_drivename($opt)) {
2134 $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2135 } else {
2136 $prop->{$opt} = $confdesc->{$opt};
2137 }
2138 }
2139
2140 return $prop;
2141}
2142
2143# Properties that we can read from an OVF file
2144sub json_ovf_properties {
2145 my $prop = {};
2146
2147 for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2148 $prop->{$device} = {
2149 type => 'string',
2150 format => 'pve-volume-id-or-absolute-path',
2151 description => "Disk image that gets imported to $device",
2152 optional => 1,
2153 };
2154 }
2155
2156 $prop->{cores} = {
2157 type => 'integer',
2158 description => "The number of CPU cores.",
2159 optional => 1,
2160 };
2161 $prop->{memory} = {
2162 type => 'integer',
2163 description => "Amount of RAM for the VM in MB.",
2164 optional => 1,
2165 };
2166 $prop->{name} = {
2167 type => 'string',
2168 description => "Name of the VM.",
2169 optional => 1,
2170 };
2171
2172 return $prop;
2173}
2174
2175# return copy of $confdesc_cloudinit to generate documentation
2176sub cloudinit_config_properties {
2177
2178 return dclone($confdesc_cloudinit);
2179}
2180
2181sub cloudinit_pending_properties {
2182 my $p = {
2183 map { $_ => 1 } keys $confdesc_cloudinit->%*,
2184 name => 1,
2185 };
2186 $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2187 return $p;
2188}
2189
2190sub check_type {
2191 my ($key, $value) = @_;
2192
2193 die "unknown setting '$key'\n" if !$confdesc->{$key};
2194
2195 my $type = $confdesc->{$key}->{type};
2196
2197 if (!defined($value)) {
2198 die "got undefined value\n";
2199 }
2200
2201 if ($value =~ m/[\n\r]/) {
2202 die "property contains a line feed\n";
2203 }
2204
2205 if ($type eq 'boolean') {
2206 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2207 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2208 die "type check ('boolean') failed - got '$value'\n";
2209 } elsif ($type eq 'integer') {
2210 return int($1) if $value =~ m/^(\d+)$/;
2211 die "type check ('integer') failed - got '$value'\n";
2212 } elsif ($type eq 'number') {
2213 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2214 die "type check ('number') failed - got '$value'\n";
2215 } elsif ($type eq 'string') {
2216 if (my $fmt = $confdesc->{$key}->{format}) {
2217 PVE::JSONSchema::check_format($fmt, $value);
2218 return $value;
2219 }
2220 $value =~ s/^\"(.*)\"$/$1/;
2221 return $value;
2222 } else {
2223 die "internal error"
2224 }
2225}
2226
2227sub destroy_vm {
2228 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2229
2230 my $conf = PVE::QemuConfig->load_config($vmid);
2231
2232 if (!$skiplock && !PVE::QemuConfig->has_lock($conf, 'suspended')) {
2233 PVE::QemuConfig->check_lock($conf);
2234 }
2235
2236 if ($conf->{template}) {
2237 # check if any base image is still used by a linked clone
2238 PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2239 my ($ds, $drive) = @_;
2240 return if drive_is_cdrom($drive);
2241
2242 my $volid = $drive->{file};
2243 return if !$volid || $volid =~ m|^/|;
2244
2245 die "base volume '$volid' is still in use by linked cloned\n"
2246 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2247
2248 });
2249 }
2250
2251 my $volids = {};
2252 my $remove_owned_drive = sub {
2253 my ($ds, $drive) = @_;
2254 return if drive_is_cdrom($drive, 1);
2255
2256 my $volid = $drive->{file};
2257 return if !$volid || $volid =~ m|^/|;
2258 return if $volids->{$volid};
2259
2260 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2261 return if !$path || !$owner || ($owner != $vmid);
2262
2263 $volids->{$volid} = 1;
2264 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2265 warn "Could not remove disk '$volid', check manually: $@" if $@;
2266 };
2267
2268 # only remove disks owned by this VM (referenced in the config)
2269 my $include_opts = {
2270 include_unused => 1,
2271 extra_keys => ['vmstate'],
2272 };
2273 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2274
2275 for my $snap (values %{$conf->{snapshots}}) {
2276 next if !defined($snap->{vmstate});
2277 my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2278 next if !defined($drive);
2279 $remove_owned_drive->('vmstate', $drive);
2280 }
2281
2282 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2283
2284 if ($purge_unreferenced) { # also remove unreferenced disk
2285 my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2286 PVE::Storage::foreach_volid($vmdisks, sub {
2287 my ($volid, $sid, $volname, $d) = @_;
2288 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2289 warn $@ if $@;
2290 });
2291 }
2292
2293 eval { delete_ifaces_ipams_ips($conf, $vmid)};
2294 warn $@ if $@;
2295
2296 if (defined $replacement_conf) {
2297 PVE::QemuConfig->write_config($vmid, $replacement_conf);
2298 } else {
2299 PVE::QemuConfig->destroy_config($vmid);
2300 }
2301}
2302
2303sub parse_vm_config {
2304 my ($filename, $raw, $strict) = @_;
2305
2306 return if !defined($raw);
2307
2308 my $res = {
2309 digest => Digest::SHA::sha1_hex($raw),
2310 snapshots => {},
2311 pending => {},
2312 cloudinit => {},
2313 };
2314
2315 my $handle_error = sub {
2316 my ($msg) = @_;
2317
2318 if ($strict) {
2319 die $msg;
2320 } else {
2321 warn $msg;
2322 }
2323 };
2324
2325 $filename =~ m|/qemu-server/(\d+)\.conf$|
2326 || die "got strange filename '$filename'";
2327
2328 my $vmid = $1;
2329
2330 my $conf = $res;
2331 my $descr;
2332 my $finish_description = sub {
2333 if (defined($descr)) {
2334 $descr =~ s/\s+$//;
2335 $conf->{description} = $descr;
2336 }
2337 $descr = undef;
2338 };
2339 my $section = '';
2340
2341 my @lines = split(/\n/, $raw);
2342 foreach my $line (@lines) {
2343 next if $line =~ m/^\s*$/;
2344
2345 if ($line =~ m/^\[PENDING\]\s*$/i) {
2346 $section = 'pending';
2347 $finish_description->();
2348 $conf = $res->{$section} = {};
2349 next;
2350 } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2351 $section = 'cloudinit';
2352 $finish_description->();
2353 $conf = $res->{$section} = {};
2354 next;
2355
2356 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2357 $section = $1;
2358 $finish_description->();
2359 $conf = $res->{snapshots}->{$section} = {};
2360 next;
2361 }
2362
2363 if ($line =~ m/^\#(.*)$/) {
2364 $descr = '' if !defined($descr);
2365 $descr .= PVE::Tools::decode_text($1) . "\n";
2366 next;
2367 }
2368
2369 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2370 $descr = '' if !defined($descr);
2371 $descr .= PVE::Tools::decode_text($2);
2372 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2373 $conf->{snapstate} = $1;
2374 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2375 my $key = $1;
2376 my $value = $2;
2377 $conf->{$key} = $value;
2378 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2379 my $value = $1;
2380 if ($section eq 'pending') {
2381 $conf->{delete} = $value; # we parse this later
2382 } else {
2383 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2384 }
2385 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2386 my $key = $1;
2387 my $value = $2;
2388 if ($section eq 'cloudinit') {
2389 # ignore validation only used for informative purpose
2390 $conf->{$key} = $value;
2391 next;
2392 }
2393 eval { $value = check_type($key, $value); };
2394 if ($@) {
2395 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2396 } else {
2397 $key = 'ide2' if $key eq 'cdrom';
2398 my $fmt = $confdesc->{$key}->{format};
2399 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2400 my $v = parse_drive($key, $value);
2401 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2402 $v->{file} = $volid;
2403 $value = print_drive($v);
2404 } else {
2405 $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2406 next;
2407 }
2408 }
2409
2410 $conf->{$key} = $value;
2411 }
2412 } else {
2413 $handle_error->("vm $vmid - unable to parse config: $line\n");
2414 }
2415 }
2416
2417 $finish_description->();
2418 delete $res->{snapstate}; # just to be sure
2419
2420 return $res;
2421}
2422
2423sub write_vm_config {
2424 my ($filename, $conf) = @_;
2425
2426 delete $conf->{snapstate}; # just to be sure
2427
2428 if ($conf->{cdrom}) {
2429 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2430 $conf->{ide2} = $conf->{cdrom};
2431 delete $conf->{cdrom};
2432 }
2433
2434 # we do not use 'smp' any longer
2435 if ($conf->{sockets}) {
2436 delete $conf->{smp};
2437 } elsif ($conf->{smp}) {
2438 $conf->{sockets} = $conf->{smp};
2439 delete $conf->{cores};
2440 delete $conf->{smp};
2441 }
2442
2443 my $used_volids = {};
2444
2445 my $cleanup_config = sub {
2446 my ($cref, $pending, $snapname) = @_;
2447
2448 foreach my $key (keys %$cref) {
2449 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2450 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2451 my $value = $cref->{$key};
2452 if ($key eq 'delete') {
2453 die "propertry 'delete' is only allowed in [PENDING]\n"
2454 if !$pending;
2455 # fixme: check syntax?
2456 next;
2457 }
2458 eval { $value = check_type($key, $value); };
2459 die "unable to parse value of '$key' - $@" if $@;
2460
2461 $cref->{$key} = $value;
2462
2463 if (!$snapname && is_valid_drivename($key)) {
2464 my $drive = parse_drive($key, $value);
2465 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2466 }
2467 }
2468 };
2469
2470 &$cleanup_config($conf);
2471
2472 &$cleanup_config($conf->{pending}, 1);
2473
2474 foreach my $snapname (keys %{$conf->{snapshots}}) {
2475 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2476 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2477 }
2478
2479 # remove 'unusedX' settings if we re-add a volume
2480 foreach my $key (keys %$conf) {
2481 my $value = $conf->{$key};
2482 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2483 delete $conf->{$key};
2484 }
2485 }
2486
2487 my $generate_raw_config = sub {
2488 my ($conf, $pending) = @_;
2489
2490 my $raw = '';
2491
2492 # add description as comment to top of file
2493 if (defined(my $descr = $conf->{description})) {
2494 if ($descr) {
2495 foreach my $cl (split(/\n/, $descr)) {
2496 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2497 }
2498 } else {
2499 $raw .= "#\n" if $pending;
2500 }
2501 }
2502
2503 foreach my $key (sort keys %$conf) {
2504 next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2505 $raw .= "$key: $conf->{$key}\n";
2506 }
2507 return $raw;
2508 };
2509
2510 my $raw = &$generate_raw_config($conf);
2511
2512 if (scalar(keys %{$conf->{pending}})){
2513 $raw .= "\n[PENDING]\n";
2514 $raw .= &$generate_raw_config($conf->{pending}, 1);
2515 }
2516
2517 if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2518 $raw .= "\n[special:cloudinit]\n";
2519 $raw .= &$generate_raw_config($conf->{cloudinit});
2520 }
2521
2522 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2523 $raw .= "\n[$snapname]\n";
2524 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2525 }
2526
2527 return $raw;
2528}
2529
2530sub load_defaults {
2531
2532 my $res = {};
2533
2534 # we use static defaults from our JSON schema configuration
2535 foreach my $key (keys %$confdesc) {
2536 if (defined(my $default = $confdesc->{$key}->{default})) {
2537 $res->{$key} = $default;
2538 }
2539 }
2540
2541 return $res;
2542}
2543
2544sub config_list {
2545 my $vmlist = PVE::Cluster::get_vmlist();
2546 my $res = {};
2547 return $res if !$vmlist || !$vmlist->{ids};
2548 my $ids = $vmlist->{ids};
2549 my $nodename = nodename();
2550
2551 foreach my $vmid (keys %$ids) {
2552 my $d = $ids->{$vmid};
2553 next if !$d->{node} || $d->{node} ne $nodename;
2554 next if !$d->{type} || $d->{type} ne 'qemu';
2555 $res->{$vmid}->{exists} = 1;
2556 }
2557 return $res;
2558}
2559
2560# test if VM uses local resources (to prevent migration)
2561sub check_local_resources {
2562 my ($conf, $noerr) = @_;
2563
2564 my @loc_res = ();
2565 my $mapped_res = [];
2566
2567 my $nodelist = PVE::Cluster::get_nodelist();
2568 my $pci_map = PVE::Mapping::PCI::config();
2569 my $usb_map = PVE::Mapping::USB::config();
2570
2571 my $missing_mappings_by_node = { map { $_ => [] } @$nodelist };
2572
2573 my $add_missing_mapping = sub {
2574 my ($type, $key, $id) = @_;
2575 for my $node (@$nodelist) {
2576 my $entry;
2577 if ($type eq 'pci') {
2578 $entry = PVE::Mapping::PCI::get_node_mapping($pci_map, $id, $node);
2579 } elsif ($type eq 'usb') {
2580 $entry = PVE::Mapping::USB::get_node_mapping($usb_map, $id, $node);
2581 }
2582 if (!scalar($entry->@*)) {
2583 push @{$missing_mappings_by_node->{$node}}, $key;
2584 }
2585 }
2586 };
2587
2588 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2589 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2590
2591 push @loc_res, "ivshmem" if $conf->{ivshmem};
2592
2593 foreach my $k (keys %$conf) {
2594 if ($k =~ m/^usb/) {
2595 my $entry = parse_property_string('pve-qm-usb', $conf->{$k});
2596 next if $entry->{host} && $entry->{host} =~ m/^spice$/i;
2597 if ($entry->{mapping}) {
2598 $add_missing_mapping->('usb', $k, $entry->{mapping});
2599 push @$mapped_res, $k;
2600 }
2601 }
2602 if ($k =~ m/^hostpci/) {
2603 my $entry = parse_property_string('pve-qm-hostpci', $conf->{$k});
2604 if ($entry->{mapping}) {
2605 $add_missing_mapping->('pci', $k, $entry->{mapping});
2606 push @$mapped_res, $k;
2607 }
2608 }
2609 # sockets are safe: they will recreated be on the target side post-migrate
2610 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2611 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2612 }
2613
2614 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2615
2616 return wantarray ? (\@loc_res, $mapped_res, $missing_mappings_by_node) : \@loc_res;
2617}
2618
2619# check if used storages are available on all nodes (use by migrate)
2620sub check_storage_availability {
2621 my ($storecfg, $conf, $node) = @_;
2622
2623 PVE::QemuConfig->foreach_volume($conf, sub {
2624 my ($ds, $drive) = @_;
2625
2626 my $volid = $drive->{file};
2627 return if !$volid;
2628
2629 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2630 return if !$sid;
2631
2632 # check if storage is available on both nodes
2633 my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2634 PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2635
2636 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2637
2638 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2639 if !$scfg->{content}->{$vtype};
2640 });
2641}
2642
2643# list nodes where all VM images are available (used by has_feature API)
2644sub shared_nodes {
2645 my ($conf, $storecfg) = @_;
2646
2647 my $nodelist = PVE::Cluster::get_nodelist();
2648 my $nodehash = { map { $_ => 1 } @$nodelist };
2649 my $nodename = nodename();
2650
2651 PVE::QemuConfig->foreach_volume($conf, sub {
2652 my ($ds, $drive) = @_;
2653
2654 my $volid = $drive->{file};
2655 return if !$volid;
2656
2657 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2658 if ($storeid) {
2659 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2660 if ($scfg->{disable}) {
2661 $nodehash = {};
2662 } elsif (my $avail = $scfg->{nodes}) {
2663 foreach my $node (keys %$nodehash) {
2664 delete $nodehash->{$node} if !$avail->{$node};
2665 }
2666 } elsif (!$scfg->{shared}) {
2667 foreach my $node (keys %$nodehash) {
2668 delete $nodehash->{$node} if $node ne $nodename
2669 }
2670 }
2671 }
2672 });
2673
2674 return $nodehash
2675}
2676
2677sub check_local_storage_availability {
2678 my ($conf, $storecfg) = @_;
2679
2680 my $nodelist = PVE::Cluster::get_nodelist();
2681 my $nodehash = { map { $_ => {} } @$nodelist };
2682
2683 PVE::QemuConfig->foreach_volume($conf, sub {
2684 my ($ds, $drive) = @_;
2685
2686 my $volid = $drive->{file};
2687 return if !$volid;
2688
2689 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2690 if ($storeid) {
2691 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2692
2693 if ($scfg->{disable}) {
2694 foreach my $node (keys %$nodehash) {
2695 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2696 }
2697 } elsif (my $avail = $scfg->{nodes}) {
2698 foreach my $node (keys %$nodehash) {
2699 if (!$avail->{$node}) {
2700 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2701 }
2702 }
2703 }
2704 }
2705 });
2706
2707 foreach my $node (values %$nodehash) {
2708 if (my $unavail = $node->{unavailable_storages}) {
2709 $node->{unavailable_storages} = [ sort keys %$unavail ];
2710 }
2711 }
2712
2713 return $nodehash
2714}
2715
2716# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2717sub check_running {
2718 my ($vmid, $nocheck, $node) = @_;
2719
2720 # $nocheck is set when called during a migration, in which case the config
2721 # file might still or already reside on the *other* node
2722 # - because rename has already happened, and current node is source
2723 # - because rename hasn't happened yet, and current node is target
2724 # - because rename has happened, current node is target, but hasn't yet
2725 # processed it yet
2726 PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2727 return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2728}
2729
2730sub vzlist {
2731
2732 my $vzlist = config_list();
2733
2734 my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2735
2736 while (defined(my $de = $fd->read)) {
2737 next if $de !~ m/^(\d+)\.pid$/;
2738 my $vmid = $1;
2739 next if !defined($vzlist->{$vmid});
2740 if (my $pid = check_running($vmid)) {
2741 $vzlist->{$vmid}->{pid} = $pid;
2742 }
2743 }
2744
2745 return $vzlist;
2746}
2747
2748our $vmstatus_return_properties = {
2749 vmid => get_standard_option('pve-vmid'),
2750 status => {
2751 description => "QEMU process status.",
2752 type => 'string',
2753 enum => ['stopped', 'running'],
2754 },
2755 maxmem => {
2756 description => "Maximum memory in bytes.",
2757 type => 'integer',
2758 optional => 1,
2759 renderer => 'bytes',
2760 },
2761 maxdisk => {
2762 description => "Root disk size in bytes.",
2763 type => 'integer',
2764 optional => 1,
2765 renderer => 'bytes',
2766 },
2767 name => {
2768 description => "VM name.",
2769 type => 'string',
2770 optional => 1,
2771 },
2772 qmpstatus => {
2773 description => "VM run state from the 'query-status' QMP monitor command.",
2774 type => 'string',
2775 optional => 1,
2776 },
2777 pid => {
2778 description => "PID of running qemu process.",
2779 type => 'integer',
2780 optional => 1,
2781 },
2782 uptime => {
2783 description => "Uptime.",
2784 type => 'integer',
2785 optional => 1,
2786 renderer => 'duration',
2787 },
2788 cpus => {
2789 description => "Maximum usable CPUs.",
2790 type => 'number',
2791 optional => 1,
2792 },
2793 lock => {
2794 description => "The current config lock, if any.",
2795 type => 'string',
2796 optional => 1,
2797 },
2798 tags => {
2799 description => "The current configured tags, if any",
2800 type => 'string',
2801 optional => 1,
2802 },
2803 'running-machine' => {
2804 description => "The currently running machine type (if running).",
2805 type => 'string',
2806 optional => 1,
2807 },
2808 'running-qemu' => {
2809 description => "The currently running QEMU version (if running).",
2810 type => 'string',
2811 optional => 1,
2812 },
2813};
2814
2815my $last_proc_pid_stat;
2816
2817# get VM status information
2818# This must be fast and should not block ($full == false)
2819# We only query KVM using QMP if $full == true (this can be slow)
2820sub vmstatus {
2821 my ($opt_vmid, $full) = @_;
2822
2823 my $res = {};
2824
2825 my $storecfg = PVE::Storage::config();
2826
2827 my $list = vzlist();
2828 my $defaults = load_defaults();
2829
2830 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2831
2832 my $cpucount = $cpuinfo->{cpus} || 1;
2833
2834 foreach my $vmid (keys %$list) {
2835 next if $opt_vmid && ($vmid ne $opt_vmid);
2836
2837 my $conf = PVE::QemuConfig->load_config($vmid);
2838
2839 my $d = { vmid => int($vmid) };
2840 $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2841
2842 # fixme: better status?
2843 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2844
2845 my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2846 if (defined($size)) {
2847 $d->{disk} = 0; # no info available
2848 $d->{maxdisk} = $size;
2849 } else {
2850 $d->{disk} = 0;
2851 $d->{maxdisk} = 0;
2852 }
2853
2854 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2855 * ($conf->{cores} || $defaults->{cores});
2856 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2857 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2858
2859 $d->{name} = $conf->{name} || "VM $vmid";
2860 $d->{maxmem} = get_current_memory($conf->{memory})*(1024*1024);
2861
2862 if ($conf->{balloon}) {
2863 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2864 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2865 : $defaults->{shares};
2866 }
2867
2868 $d->{uptime} = 0;
2869 $d->{cpu} = 0;
2870 $d->{mem} = 0;
2871
2872 $d->{netout} = 0;
2873 $d->{netin} = 0;
2874
2875 $d->{diskread} = 0;
2876 $d->{diskwrite} = 0;
2877
2878 $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2879
2880 $d->{serial} = 1 if conf_has_serial($conf);
2881 $d->{lock} = $conf->{lock} if $conf->{lock};
2882 $d->{tags} = $conf->{tags} if defined($conf->{tags});
2883
2884 $res->{$vmid} = $d;
2885 }
2886
2887 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2888 foreach my $dev (keys %$netdev) {
2889 next if $dev !~ m/^tap([1-9]\d*)i/;
2890 my $vmid = $1;
2891 my $d = $res->{$vmid};
2892 next if !$d;
2893
2894 $d->{netout} += $netdev->{$dev}->{receive};
2895 $d->{netin} += $netdev->{$dev}->{transmit};
2896
2897 if ($full) {
2898 $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2899 $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2900 }
2901
2902 }
2903
2904 my $ctime = gettimeofday;
2905
2906 foreach my $vmid (keys %$list) {
2907
2908 my $d = $res->{$vmid};
2909 my $pid = $d->{pid};
2910 next if !$pid;
2911
2912 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2913 next if !$pstat; # not running
2914
2915 my $used = $pstat->{utime} + $pstat->{stime};
2916
2917 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2918
2919 if ($pstat->{vsize}) {
2920 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2921 }
2922
2923 my $old = $last_proc_pid_stat->{$pid};
2924 if (!$old) {
2925 $last_proc_pid_stat->{$pid} = {
2926 time => $ctime,
2927 used => $used,
2928 cpu => 0,
2929 };
2930 next;
2931 }
2932
2933 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2934
2935 if ($dtime > 1000) {
2936 my $dutime = $used - $old->{used};
2937
2938 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2939 $last_proc_pid_stat->{$pid} = {
2940 time => $ctime,
2941 used => $used,
2942 cpu => $d->{cpu},
2943 };
2944 } else {
2945 $d->{cpu} = $old->{cpu};
2946 }
2947 }
2948
2949 return $res if !$full;
2950
2951 my $qmpclient = PVE::QMPClient->new();
2952
2953 my $ballooncb = sub {
2954 my ($vmid, $resp) = @_;
2955
2956 my $info = $resp->{'return'};
2957 return if !$info->{max_mem};
2958
2959 my $d = $res->{$vmid};
2960
2961 # use memory assigned to VM
2962 $d->{maxmem} = $info->{max_mem};
2963 $d->{balloon} = $info->{actual};
2964
2965 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2966 $d->{mem} = $info->{total_mem} - $info->{free_mem};
2967 $d->{freemem} = $info->{free_mem};
2968 }
2969
2970 $d->{ballooninfo} = $info;
2971 };
2972
2973 my $blockstatscb = sub {
2974 my ($vmid, $resp) = @_;
2975 my $data = $resp->{'return'} || [];
2976 my $totalrdbytes = 0;
2977 my $totalwrbytes = 0;
2978
2979 for my $blockstat (@$data) {
2980 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2981 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2982
2983 $blockstat->{device} =~ s/drive-//;
2984 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2985 }
2986 $res->{$vmid}->{diskread} = $totalrdbytes;
2987 $res->{$vmid}->{diskwrite} = $totalwrbytes;
2988 };
2989
2990 my $machinecb = sub {
2991 my ($vmid, $resp) = @_;
2992 my $data = $resp->{'return'} || [];
2993
2994 $res->{$vmid}->{'running-machine'} =
2995 PVE::QemuServer::Machine::current_from_query_machines($data);
2996 };
2997
2998 my $versioncb = sub {
2999 my ($vmid, $resp) = @_;
3000 my $data = $resp->{'return'} // {};
3001 my $version = 'unknown';
3002
3003 if (my $v = $data->{qemu}) {
3004 $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3005 }
3006
3007 $res->{$vmid}->{'running-qemu'} = $version;
3008 };
3009
3010 my $statuscb = sub {
3011 my ($vmid, $resp) = @_;
3012
3013 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3014 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3015 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3016 # this fails if ballon driver is not loaded, so this must be
3017 # the last commnand (following command are aborted if this fails).
3018 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3019
3020 my $status = 'unknown';
3021 if (!defined($status = $resp->{'return'}->{status})) {
3022 warn "unable to get VM status\n";
3023 return;
3024 }
3025
3026 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3027 };
3028
3029 foreach my $vmid (keys %$list) {
3030 next if $opt_vmid && ($vmid ne $opt_vmid);
3031 next if !$res->{$vmid}->{pid}; # not running
3032 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3033 }
3034
3035 $qmpclient->queue_execute(undef, 2);
3036
3037 foreach my $vmid (keys %$list) {
3038 next if $opt_vmid && ($vmid ne $opt_vmid);
3039 next if !$res->{$vmid}->{pid}; #not running
3040
3041 # we can't use the $qmpclient since it might have already aborted on
3042 # 'query-balloon', but this might also fail for older versions...
3043 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3044 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3045 }
3046
3047 foreach my $vmid (keys %$list) {
3048 next if $opt_vmid && ($vmid ne $opt_vmid);
3049 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3050 }
3051
3052 return $res;
3053}
3054
3055sub conf_has_serial {
3056 my ($conf) = @_;
3057
3058 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3059 if ($conf->{"serial$i"}) {
3060 return 1;
3061 }
3062 }
3063
3064 return 0;
3065}
3066
3067sub conf_has_audio {
3068 my ($conf, $id) = @_;
3069
3070 $id //= 0;
3071 my $audio = $conf->{"audio$id"};
3072 return if !defined($audio);
3073
3074 my $audioproperties = parse_property_string($audio_fmt, $audio);
3075 my $audiodriver = $audioproperties->{driver} // 'spice';
3076
3077 return {
3078 dev => $audioproperties->{device},
3079 dev_id => "audiodev$id",
3080 backend => $audiodriver,
3081 backend_id => "$audiodriver-backend${id}",
3082 };
3083}
3084
3085sub audio_devs {
3086 my ($audio, $audiopciaddr, $machine_version) = @_;
3087
3088 my $devs = [];
3089
3090 my $id = $audio->{dev_id};
3091 my $audiodev = "";
3092 if (min_version($machine_version, 4, 2)) {
3093 $audiodev = ",audiodev=$audio->{backend_id}";
3094 }
3095
3096 if ($audio->{dev} eq 'AC97') {
3097 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3098 } elsif ($audio->{dev} =~ /intel\-hda$/) {
3099 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3100 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3101 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3102 } else {
3103 die "unkown audio device '$audio->{dev}', implement me!";
3104 }
3105
3106 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3107
3108 return $devs;
3109}
3110
3111sub get_tpm_paths {
3112 my ($vmid) = @_;
3113 return {
3114 socket => "/var/run/qemu-server/$vmid.swtpm",
3115 pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3116 };
3117}
3118
3119sub add_tpm_device {
3120 my ($vmid, $devices, $conf) = @_;
3121
3122 return if !$conf->{tpmstate0};
3123
3124 my $paths = get_tpm_paths($vmid);
3125
3126 push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3127 push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3128 push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3129}
3130
3131sub start_swtpm {
3132 my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3133
3134 return if !$tpmdrive;
3135
3136 my $state;
3137 my $tpm = parse_drive("tpmstate0", $tpmdrive);
3138 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3139 if ($storeid) {
3140 $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3141 } else {
3142 $state = $tpm->{file};
3143 }
3144
3145 my $paths = get_tpm_paths($vmid);
3146
3147 # during migration, we will get state from remote
3148 #
3149 if (!$migration) {
3150 # run swtpm_setup to create a new TPM state if it doesn't exist yet
3151 my $setup_cmd = [
3152 "swtpm_setup",
3153 "--tpmstate",
3154 "file://$state",
3155 "--createek",
3156 "--create-ek-cert",
3157 "--create-platform-cert",
3158 "--lock-nvram",
3159 "--config",
3160 "/etc/swtpm_setup.conf", # do not use XDG configs
3161 "--runas",
3162 "0", # force creation as root, error if not possible
3163 "--not-overwrite", # ignore existing state, do not modify
3164 ];
3165
3166 push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3167 # TPM 2.0 supports ECC crypto, use if possible
3168 push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3169
3170 run_command($setup_cmd, outfunc => sub {
3171 print "swtpm_setup: $1\n";
3172 });
3173 }
3174
3175 # Used to distinguish different invocations in the log.
3176 my $log_prefix = "[id=" . int(time()) . "] ";
3177
3178 my $emulator_cmd = [
3179 "swtpm",
3180 "socket",
3181 "--tpmstate",
3182 "backend-uri=file://$state,mode=0600",
3183 "--ctrl",
3184 "type=unixio,path=$paths->{socket},mode=0600",
3185 "--pid",
3186 "file=$paths->{pid}",
3187 "--terminate", # terminate on QEMU disconnect
3188 "--daemon",
3189 "--log",
3190 "file=/run/qemu-server/$vmid-swtpm.log,level=1,prefix=$log_prefix",
3191 ];
3192 push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3193 run_command($emulator_cmd, outfunc => sub { print $1; });
3194
3195 my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3196 while (! -e $paths->{pid}) {
3197 die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3198 usleep(50_000);
3199 }
3200
3201 # return untainted PID of swtpm daemon so it can be killed on error
3202 file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3203 return $1;
3204}
3205
3206sub vga_conf_has_spice {
3207 my ($vga) = @_;
3208
3209 my $vgaconf = parse_vga($vga);
3210 my $vgatype = $vgaconf->{type};
3211 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3212
3213 return $1 || 1;
3214}
3215
3216sub get_vm_arch {
3217 my ($conf) = @_;
3218 return $conf->{arch} // get_host_arch();
3219}
3220
3221my $default_machines = {
3222 x86_64 => 'pc',
3223 aarch64 => 'virt',
3224};
3225
3226sub get_installed_machine_version {
3227 my ($kvmversion) = @_;
3228 $kvmversion = kvm_user_version() if !defined($kvmversion);
3229 $kvmversion =~ m/^(\d+\.\d+)/;
3230 return $1;
3231}
3232
3233sub windows_get_pinned_machine_version {
3234 my ($machine, $base_version, $kvmversion) = @_;
3235
3236 my $pin_version = $base_version;
3237 if (!defined($base_version) ||
3238 !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3239 ) {
3240 $pin_version = get_installed_machine_version($kvmversion);
3241 }
3242 if (!$machine || $machine eq 'pc') {
3243 $machine = "pc-i440fx-$pin_version";
3244 } elsif ($machine eq 'q35') {
3245 $machine = "pc-q35-$pin_version";
3246 } elsif ($machine eq 'virt') {
3247 $machine = "virt-$pin_version";
3248 } else {
3249 warn "unknown machine type '$machine', not touching that!\n";
3250 }
3251
3252 return $machine;
3253}
3254
3255sub get_vm_machine {
3256 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3257
3258 my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
3259 my $machine = $forcemachine || $machine_conf->{type};
3260
3261 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3262 $kvmversion //= kvm_user_version();
3263 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3264 # layout which confuses windows quite a bit and may result in various regressions..
3265 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3266 if (windows_version($conf->{ostype})) {
3267 $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3268 }
3269 $arch //= 'x86_64';
3270 $machine ||= $default_machines->{$arch};
3271 if ($add_pve_version) {
3272 my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3273 $machine .= "+pve$pvever";
3274 }
3275 }
3276
3277 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3278 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3279 $machine = $1 if $is_pxe;
3280
3281 # for version-pinned machines that do not include a pve-version (e.g.
3282 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3283 $machine .= '+pve0';
3284
3285 $machine .= '.pxe' if $is_pxe;
3286 }
3287
3288 return $machine;
3289}
3290
3291sub get_ovmf_files($$$) {
3292 my ($arch, $efidisk, $smm) = @_;
3293
3294 my $types = $OVMF->{$arch}
3295 or die "no OVMF images known for architecture '$arch'\n";
3296
3297 my $type = 'default';
3298 if ($arch eq 'x86_64') {
3299 if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3300 $type = $smm ? "4m" : "4m-no-smm";
3301 $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3302 } else {
3303 # TODO: log_warn about use of legacy images for x86_64 with Promxox VE 9
3304 }
3305 }
3306
3307 my ($ovmf_code, $ovmf_vars) = $types->{$type}->@*;
3308 die "EFI base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3309 die "EFI vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
3310
3311 return ($ovmf_code, $ovmf_vars);
3312}
3313
3314my $Arch2Qemu = {
3315 aarch64 => '/usr/bin/qemu-system-aarch64',
3316 x86_64 => '/usr/bin/qemu-system-x86_64',
3317};
3318sub get_command_for_arch($) {
3319 my ($arch) = @_;
3320 return '/usr/bin/kvm' if is_native_arch($arch);
3321
3322 my $cmd = $Arch2Qemu->{$arch}
3323 or die "don't know how to emulate architecture '$arch'\n";
3324 return $cmd;
3325}
3326
3327# To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3328# to use in a QEMU command line (-cpu element), first array_intersect the result
3329# of query_supported_ with query_understood_. This is necessary because:
3330#
3331# a) query_understood_ returns flags the host cannot use and
3332# b) query_supported_ (rather the QMP call) doesn't actually return CPU
3333# flags, but CPU settings - with most of them being flags. Those settings
3334# (and some flags, curiously) cannot be specified as a "-cpu" argument.
3335#
3336# query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3337# expensive. If you need the value returned from this, you can get it much
3338# cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3339# $accel being 'kvm' or 'tcg'.
3340#
3341# pvestatd calls this function on startup and whenever the QEMU/KVM version
3342# changes, automatically populating pmxcfs.
3343#
3344# Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3345# since kvm and tcg machines support different flags
3346#
3347sub query_supported_cpu_flags {
3348 my ($arch) = @_;
3349
3350 $arch //= get_host_arch();
3351 my $default_machine = $default_machines->{$arch};
3352
3353 my $flags = {};
3354
3355 # FIXME: Once this is merged, the code below should work for ARM as well:
3356 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3357 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3358 $arch eq "aarch64";
3359
3360 my $kvm_supported = defined(kvm_version());
3361 my $qemu_cmd = get_command_for_arch($arch);
3362 my $fakevmid = -1;
3363 my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3364
3365 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3366 my $query_supported_run_qemu = sub {
3367 my ($kvm) = @_;
3368
3369 my $flags = {};
3370 my $cmd = [
3371 $qemu_cmd,
3372 '-machine', $default_machine,
3373 '-display', 'none',
3374 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3375 '-mon', 'chardev=qmp,mode=control',
3376 '-pidfile', $pidfile,
3377 '-S', '-daemonize'
3378 ];
3379
3380 if (!$kvm) {
3381 push @$cmd, '-accel', 'tcg';
3382 }
3383
3384 my $rc = run_command($cmd, noerr => 1, quiet => 0);
3385 die "QEMU flag querying VM exited with code " . $rc if $rc;
3386
3387 eval {
3388 my $cmd_result = mon_cmd(
3389 $fakevmid,
3390 'query-cpu-model-expansion',
3391 type => 'full',
3392 model => { name => 'host' }
3393 );
3394
3395 my $props = $cmd_result->{model}->{props};
3396 foreach my $prop (keys %$props) {
3397 next if $props->{$prop} ne '1';
3398 # QEMU returns some flags multiple times, with '_', '.' or '-'
3399 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3400 # We only keep those with underscores, to match /proc/cpuinfo
3401 $prop =~ s/\.|-/_/g;
3402 $flags->{$prop} = 1;
3403 }
3404 };
3405 my $err = $@;
3406
3407 # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3408 vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3409
3410 die $err if $err;
3411
3412 return [ sort keys %$flags ];
3413 };
3414
3415 # We need to query QEMU twice, since KVM and TCG have different supported flags
3416 PVE::QemuConfig->lock_config($fakevmid, sub {
3417 $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3418 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3419
3420 if ($kvm_supported) {
3421 $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3422 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3423 }
3424 });
3425
3426 return $flags;
3427}
3428
3429# Understood CPU flags are written to a file at 'pve-qemu' compile time
3430my $understood_cpu_flag_dir = "/usr/share/kvm";
3431sub query_understood_cpu_flags {
3432 my $arch = get_host_arch();
3433 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3434
3435 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3436 if ! -e $filepath;
3437
3438 my $raw = file_get_contents($filepath);
3439 $raw =~ s/^\s+|\s+$//g;
3440 my @flags = split(/\s+/, $raw);
3441
3442 return \@flags;
3443}
3444
3445# Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3446# anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3447my sub should_disable_smm {
3448 my ($conf, $vga, $machine) = @_;
3449
3450 return if $machine =~ m/^virt/; # there is no smm flag that could be disabled
3451
3452 return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3453 $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3454}
3455
3456my sub print_ovmf_drive_commandlines {
3457 my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3458
3459 my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3460
3461 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3462
3463 my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3464 if ($d) {
3465 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3466 my ($path, $format) = $d->@{'file', 'format'};
3467 if ($storeid) {
3468 $path = PVE::Storage::path($storecfg, $d->{file});
3469 if (!defined($format)) {
3470 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3471 $format = qemu_img_format($scfg, $volname);
3472 }
3473 } elsif (!defined($format)) {
3474 die "efidisk format must be specified\n";
3475 }
3476 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3477 if ($path =~ m/^rbd:/) {
3478 $var_drive_str .= ',cache=writeback';
3479 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3480 }
3481 $var_drive_str .= ",format=$format,file=$path";
3482
3483 $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3484 $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3485 } else {
3486 log_warn("no efidisk configured! Using temporary efivars disk.");
3487 my $path = "/tmp/$vmid-ovmf.fd";
3488 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3489 $var_drive_str .= ",format=raw,file=$path";
3490 $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3491 }
3492
3493 return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3494}
3495
3496sub config_to_command {
3497 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3498 $live_restore_backing) = @_;
3499
3500 my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3501 my $devices = [];
3502 my $bridges = {};
3503 my $ostype = $conf->{ostype};
3504 my $winversion = windows_version($ostype);
3505 my $kvm = $conf->{kvm};
3506 my $nodename = nodename();
3507
3508 my $machine_conf = PVE::QemuServer::Machine::parse_machine($conf->{machine});
3509
3510 my $arch = get_vm_arch($conf);
3511 my $kvm_binary = get_command_for_arch($arch);
3512 my $kvmver = kvm_user_version($kvm_binary);
3513
3514 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3515 $kvmver //= "undefined";
3516 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3517 }
3518
3519 my $add_pve_version = min_version($kvmver, 4, 1);
3520
3521 my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3522 my $machine_version = extract_version($machine_type, $kvmver);
3523 $kvm //= 1 if is_native_arch($arch);
3524
3525 $machine_version =~ m/(\d+)\.(\d+)/;
3526 my ($machine_major, $machine_minor) = ($1, $2);
3527
3528 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3529 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3530 } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3531 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3532 ." please upgrade node '$nodename'\n"
3533 } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3534 my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3535 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3536 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3537 ." node '$nodename'\n";
3538 }
3539
3540 # if a specific +pve version is required for a feature, use $version_guard
3541 # instead of min_version to allow machines to be run with the minimum
3542 # required version
3543 my $required_pve_version = 0;
3544 my $version_guard = sub {
3545 my ($major, $minor, $pve) = @_;
3546 return 0 if !min_version($machine_version, $major, $minor, $pve);
3547 my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3548 return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3549 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3550 return 1;
3551 };
3552
3553 if ($kvm && !defined kvm_version()) {
3554 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3555 ." or enable in BIOS.\n";
3556 }
3557
3558 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3559 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3560 my $use_old_bios_files = undef;
3561 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3562
3563 my $cmd = [];
3564 if ($conf->{affinity}) {
3565 push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3566 }
3567
3568 push @$cmd, $kvm_binary;
3569
3570 push @$cmd, '-id', $vmid;
3571
3572 my $vmname = $conf->{name} || "vm$vmid";
3573
3574 push @$cmd, '-name', "$vmname,debug-threads=on";
3575
3576 push @$cmd, '-no-shutdown';
3577
3578 my $use_virtio = 0;
3579
3580 my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3581 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3582 push @$cmd, '-mon', "chardev=qmp,mode=control";
3583
3584 if (min_version($machine_version, 2, 12)) {
3585 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3586 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3587 }
3588
3589 push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3590
3591 push @$cmd, '-daemonize';
3592
3593 if ($conf->{smbios1}) {
3594 my $smbios_conf = parse_smbios1($conf->{smbios1});
3595 if ($smbios_conf->{base64}) {
3596 # Do not pass base64 flag to qemu
3597 delete $smbios_conf->{base64};
3598 my $smbios_string = "";
3599 foreach my $key (keys %$smbios_conf) {
3600 my $value;
3601 if ($key eq "uuid") {
3602 $value = $smbios_conf->{uuid}
3603 } else {
3604 $value = decode_base64($smbios_conf->{$key});
3605 }
3606 # qemu accepts any binary data, only commas need escaping by double comma
3607 $value =~ s/,/,,/g;
3608 $smbios_string .= "," . $key . "=" . $value if $value;
3609 }
3610 push @$cmd, '-smbios', "type=1" . $smbios_string;
3611 } else {
3612 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3613 }
3614 }
3615
3616 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3617 die "OVMF (UEFI) BIOS is not supported on 32-bit CPU types\n"
3618 if !$forcecpu && get_cpu_bitness($conf->{cpu}, $arch) == 32;
3619
3620 my ($code_drive_str, $var_drive_str) =
3621 print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3622 push $cmd->@*, '-drive', $code_drive_str;
3623 push $cmd->@*, '-drive', $var_drive_str;
3624 }
3625
3626 if ($q35) { # tell QEMU to load q35 config early
3627 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3628 if (min_version($machine_version, 4, 0)) {
3629 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3630 } else {
3631 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3632 }
3633 }
3634
3635 if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3636 push @$cmd, $fixups->@*;
3637 }
3638
3639 if ($conf->{vmgenid}) {
3640 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3641 }
3642
3643 # add usb controllers
3644 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3645 $conf, $bridges, $arch, $machine_type, $machine_version);
3646 push @$devices, @usbcontrollers if @usbcontrollers;
3647 my $vga = parse_vga($conf->{vga});
3648
3649 my $qxlnum = vga_conf_has_spice($conf->{vga});
3650 $vga->{type} = 'qxl' if $qxlnum;
3651
3652 if (!$vga->{type}) {
3653 if ($arch eq 'aarch64') {
3654 $vga->{type} = 'virtio';
3655 } elsif (min_version($machine_version, 2, 9)) {
3656 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3657 } else {
3658 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3659 }
3660 }
3661
3662 # enable absolute mouse coordinates (needed by vnc)
3663 my $tablet = $conf->{tablet};
3664 if (!defined($tablet)) {
3665 $tablet = $defaults->{tablet};
3666 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3667 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3668 }
3669
3670 if ($tablet) {
3671 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3672 my $kbd = print_keyboarddevice_full($conf, $arch);
3673 push @$devices, '-device', $kbd if defined($kbd);
3674 }
3675
3676 my $bootorder = device_bootorder($conf);
3677
3678 # host pci device passthrough
3679 my ($kvm_off, $gpu_passthrough, $legacy_igd, $pci_devices) = PVE::QemuServer::PCI::print_hostpci_devices(
3680 $vmid, $conf, $devices, $vga, $winversion, $bridges, $arch, $machine_type, $bootorder);
3681
3682 # usb devices
3683 my $usb_dev_features = {};
3684 $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3685
3686 my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3687 $conf, $usb_dev_features, $bootorder, $machine_version);
3688 push @$devices, @usbdevices if @usbdevices;
3689
3690 # serial devices
3691 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3692 my $path = $conf->{"serial$i"} or next;
3693 if ($path eq 'socket') {
3694 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3695 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3696 # On aarch64, serial0 is the UART device. QEMU only allows
3697 # connecting UART devices via the '-serial' command line, as
3698 # the device has a fixed slot on the hardware...
3699 if ($arch eq 'aarch64' && $i == 0) {
3700 push @$devices, '-serial', "chardev:serial$i";
3701 } else {
3702 push @$devices, '-device', "isa-serial,chardev=serial$i";
3703 }
3704 } else {
3705 die "no such serial device\n" if ! -c $path;
3706 push @$devices, '-chardev', "serial,id=serial$i,path=$path";
3707 push @$devices, '-device', "isa-serial,chardev=serial$i";
3708 }
3709 }
3710
3711 # parallel devices
3712 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3713 if (my $path = $conf->{"parallel$i"}) {
3714 die "no such parallel device\n" if ! -c $path;
3715 my $devtype = $path =~ m!^/dev/usb/lp! ? 'serial' : 'parallel';
3716 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3717 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3718 }
3719 }
3720
3721 if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3722 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3723 my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3724 push @$devices, @$audio_devs;
3725 }
3726
3727 # Add a TPM only if the VM is not a template,
3728 # to support backing up template VMs even if the TPM disk is write-protected.
3729 add_tpm_device($vmid, $devices, $conf) if (!PVE::QemuConfig->is_template($conf));
3730
3731 my $sockets = 1;
3732 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3733 $sockets = $conf->{sockets} if $conf->{sockets};
3734
3735 my $cores = $conf->{cores} || 1;
3736
3737 my $maxcpus = $sockets * $cores;
3738
3739 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3740
3741 my $allowed_vcpus = $cpuinfo->{cpus};
3742
3743 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3744
3745 if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3746 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3747 for (my $i = 2; $i <= $vcpus; $i++) {
3748 my $cpustr = print_cpu_device($conf, $arch, $i);
3749 push @$cmd, '-device', $cpustr;
3750 }
3751
3752 } else {
3753
3754 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3755 }
3756 push @$cmd, '-nodefaults';
3757
3758 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3759
3760 push $machineFlags->@*, 'acpi=off' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3761
3762 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
3763
3764 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3765 push @$devices, '-device', print_vga_device(
3766 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3767
3768 push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3769
3770 my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3771 push @$cmd, '-vnc', "unix:$socket,password=on";
3772 } else {
3773 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3774 push @$cmd, '-nographic';
3775 }
3776
3777 # time drift fix
3778 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3779 my $useLocaltime = $conf->{localtime};
3780
3781 if ($winversion >= 5) { # windows
3782 $useLocaltime = 1 if !defined($conf->{localtime});
3783
3784 # use time drift fix when acpi is enabled
3785 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3786 $tdf = 1 if !defined($conf->{tdf});
3787 }
3788 }
3789
3790 if ($winversion >= 6) {
3791 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3792 push @$machineFlags, 'hpet=off';
3793 }
3794
3795 push @$rtcFlags, 'driftfix=slew' if $tdf;
3796
3797 if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3798 push @$rtcFlags, "base=$conf->{startdate}";
3799 } elsif ($useLocaltime) {
3800 push @$rtcFlags, 'base=localtime';
3801 }
3802
3803 if ($forcecpu) {
3804 push @$cmd, '-cpu', $forcecpu;
3805 } else {
3806 push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3807 }
3808
3809 PVE::QemuServer::Memory::config(
3810 $conf, $vmid, $sockets, $cores, $hotplug_features->{memory}, $cmd);
3811
3812 push @$cmd, '-S' if $conf->{freeze};
3813
3814 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3815
3816 my $guest_agent = parse_guest_agent($conf);
3817
3818 if ($guest_agent->{enabled}) {
3819 my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3820 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3821
3822 if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3823 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3824 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3825 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3826 } elsif ($guest_agent->{type} eq 'isa') {
3827 push @$devices, '-device', "isa-serial,chardev=qga0";
3828 }
3829 }
3830
3831 my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3832 if ($rng && $version_guard->(4, 1, 2)) {
3833 check_rng_source($rng->{source});
3834
3835 my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3836 my $period = $rng->{period} // $rng_fmt->{period}->{default};
3837 my $limiter_str = "";
3838 if ($max_bytes) {
3839 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3840 }
3841
3842 my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3843 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3844 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3845 }
3846
3847 my $spice_port;
3848
3849 assert_clipboard_config($vga);
3850 my $is_spice = $qxlnum || $vga->{type} =~ /^virtio/;
3851
3852 if ($is_spice || ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc')) {
3853 if ($qxlnum > 1) {
3854 if ($winversion){
3855 for (my $i = 1; $i < $qxlnum; $i++){
3856 push @$devices, '-device', print_vga_device(
3857 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3858 }
3859 } else {
3860 # assume other OS works like Linux
3861 my ($ram, $vram) = ("134217728", "67108864");
3862 if ($vga->{memory}) {
3863 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3864 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3865 }
3866 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3867 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3868 }
3869 }
3870
3871 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3872
3873 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3874 if ($vga->{'clipboard'} && $vga->{'clipboard'} eq 'vnc') {
3875 push @$devices, '-chardev', 'qemu-vdagent,id=vdagent,name=vdagent,clipboard=on';
3876 } else {
3877 push @$devices, '-chardev', 'spicevmc,id=vdagent,name=vdagent';
3878 }
3879 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3880
3881 if ($is_spice) {
3882 my $pfamily = PVE::Tools::get_host_address_family($nodename);
3883 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3884 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3885
3886 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3887 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3888
3889 my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3890 my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3891 if ($spice_enhancement->{foldersharing}) {
3892 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3893 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3894 }
3895
3896 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3897 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3898 if $spice_enhancement->{videostreaming};
3899 push @$devices, '-spice', "$spice_opts";
3900 }
3901 }
3902
3903 # enable balloon by default, unless explicitly disabled
3904 if (!defined($conf->{balloon}) || $conf->{balloon}) {
3905 my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3906 my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3907 $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3908 push @$devices, '-device', $ballooncmd;
3909 }
3910
3911 if ($conf->{watchdog}) {
3912 my $wdopts = parse_watchdog($conf->{watchdog});
3913 my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3914 my $watchdog = $wdopts->{model} || 'i6300esb';
3915 push @$devices, '-device', "$watchdog$pciaddr";
3916 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3917 }
3918
3919 my $vollist = [];
3920 my $scsicontroller = {};
3921 my $ahcicontroller = {};
3922 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3923
3924 # Add iscsi initiator name if available
3925 if (my $initiator = get_initiator_name()) {
3926 push @$devices, '-iscsi', "initiator-name=$initiator";
3927 }
3928
3929 PVE::QemuConfig->foreach_volume($conf, sub {
3930 my ($ds, $drive) = @_;
3931
3932 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3933 check_volume_storage_type($storecfg, $drive->{file});
3934 push @$vollist, $drive->{file};
3935 }
3936
3937 # ignore efidisk here, already added in bios/fw handling code above
3938 return if $drive->{interface} eq 'efidisk';
3939 # similar for TPM
3940 return if $drive->{interface} eq 'tpmstate';
3941
3942 $use_virtio = 1 if $ds =~ m/^virtio/;
3943
3944 $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3945
3946 if ($drive->{interface} eq 'virtio'){
3947 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3948 }
3949
3950 if ($drive->{interface} eq 'scsi') {
3951
3952 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3953
3954 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3955 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3956
3957 my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3958 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3959
3960 my $iothread = '';
3961 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3962 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3963 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3964 } elsif ($drive->{iothread}) {
3965 log_warn(
3966 "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3967 );
3968 }
3969
3970 my $queues = '';
3971 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3972 $queues = ",num_queues=$drive->{queues}";
3973 }
3974
3975 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3976 if !$scsicontroller->{$controller};
3977 $scsicontroller->{$controller}=1;
3978 }
3979
3980 if ($drive->{interface} eq 'sata') {
3981 my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3982 my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3983 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3984 if !$ahcicontroller->{$controller};
3985 $ahcicontroller->{$controller}=1;
3986 }
3987
3988 my $live_restore = $live_restore_backing->{$ds};
3989 my $live_blockdev_name = undef;
3990 if ($live_restore) {
3991 $live_blockdev_name = $live_restore->{name};
3992 push @$devices, '-blockdev', $live_restore->{blockdev};
3993 }
3994
3995 my $drive_cmd = print_drive_commandline_full(
3996 $storecfg, $vmid, $drive, $live_blockdev_name, min_version($kvmver, 6, 0));
3997
3998 # extra protection for templates, but SATA and IDE don't support it..
3999 $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4000
4001 push @$devices, '-drive',$drive_cmd;
4002 push @$devices, '-device', print_drivedevice_full(
4003 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4004 });
4005
4006 for (my $i = 0; $i < $MAX_NETS; $i++) {
4007 my $netname = "net$i";
4008
4009 next if !$conf->{$netname};
4010 my $d = parse_net($conf->{$netname});
4011 next if !$d;
4012 # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4013
4014 $use_virtio = 1 if $d->{model} eq 'virtio';
4015
4016 $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4017
4018 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4019 push @$devices, '-netdev', $netdevfull;
4020
4021 my $netdevicefull = print_netdevice_full(
4022 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4023
4024 push @$devices, '-device', $netdevicefull;
4025 }
4026
4027 if ($conf->{ivshmem}) {
4028 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4029
4030 my $bus;
4031 if ($q35) {
4032 $bus = print_pcie_addr("ivshmem");
4033 } else {
4034 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4035 }
4036
4037 my $ivshmem_name = $ivshmem->{name} // $vmid;
4038 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4039
4040 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4041 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4042 .",size=$ivshmem->{size}M";
4043 }
4044
4045 # pci.4 is nested in pci.1
4046 $bridges->{1} = 1 if $bridges->{4};
4047
4048 if (!$q35) { # add pci bridges
4049 if (min_version($machine_version, 2, 3)) {
4050 $bridges->{1} = 1;
4051 $bridges->{2} = 1;
4052 }
4053 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4054 }
4055
4056 for my $k (sort {$b cmp $a} keys %$bridges) {
4057 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4058
4059 my $k_name = $k;
4060 if ($k == 2 && $legacy_igd) {
4061 $k_name = "$k-igd";
4062 }
4063 my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4064 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4065
4066 if ($q35) { # add after -readconfig pve-q35.cfg
4067 splice @$devices, 2, 0, '-device', $devstr;
4068 } else {
4069 unshift @$devices, '-device', $devstr if $k > 0;
4070 }
4071 }
4072
4073 if (!$kvm) {
4074 push @$machineFlags, 'accel=tcg';
4075 }
4076
4077 push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga, $machine_type);
4078
4079 my $machine_type_min = $machine_type;
4080 if ($add_pve_version) {
4081 $machine_type_min =~ s/\+pve\d+$//;
4082 $machine_type_min .= "+pve$required_pve_version";
4083 }
4084 push @$machineFlags, "type=${machine_type_min}";
4085
4086 PVE::QemuServer::Machine::assert_valid_machine_property($conf, $machine_conf);
4087
4088 if (my $viommu = $machine_conf->{viommu}) {
4089 if ($viommu eq 'intel') {
4090 unshift @$devices, '-device', 'intel-iommu,intremap=on,caching-mode=on';
4091 push @$machineFlags, 'kernel-irqchip=split';
4092 } elsif ($viommu eq 'virtio') {
4093 push @$devices, '-device', 'virtio-iommu-pci';
4094 }
4095 }
4096
4097 push @$cmd, @$devices;
4098 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4099 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4100 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4101
4102 if (my $vmstate = $conf->{vmstate}) {
4103 my $statepath = PVE::Storage::path($storecfg, $vmstate);
4104 push @$vollist, $vmstate;
4105 push @$cmd, '-loadstate', $statepath;
4106 print "activating and using '$vmstate' as vmstate\n";
4107 }
4108
4109 if (PVE::QemuConfig->is_template($conf)) {
4110 # needed to workaround base volumes being read-only
4111 push @$cmd, '-snapshot';
4112 }
4113
4114 # add custom args
4115 if ($conf->{args}) {
4116 my $aa = PVE::Tools::split_args($conf->{args});
4117 push @$cmd, @$aa;
4118 }
4119
4120 return wantarray ? ($cmd, $vollist, $spice_port, $pci_devices) : $cmd;
4121}
4122
4123sub check_rng_source {
4124 my ($source) = @_;
4125
4126 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4127 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4128 if ! -e $source;
4129
4130 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4131 if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4132 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4133 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4134 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4135 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4136 ." to the host.\n";
4137 }
4138}
4139
4140sub spice_port {
4141 my ($vmid) = @_;
4142
4143 my $res = mon_cmd($vmid, 'query-spice');
4144
4145 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4146}
4147
4148sub vm_devices_list {
4149 my ($vmid) = @_;
4150
4151 my $res = mon_cmd($vmid, 'query-pci');
4152 my $devices_to_check = [];
4153 my $devices = {};
4154 foreach my $pcibus (@$res) {
4155 push @$devices_to_check, @{$pcibus->{devices}},
4156 }
4157
4158 while (@$devices_to_check) {
4159 my $to_check = [];
4160 for my $d (@$devices_to_check) {
4161 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4162 next if !$d->{'pci_bridge'} || !$d->{'pci_bridge'}->{devices};
4163
4164 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4165 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4166 }
4167 $devices_to_check = $to_check;
4168 }
4169
4170 my $resblock = mon_cmd($vmid, 'query-block');
4171 foreach my $block (@$resblock) {
4172 if($block->{device} =~ m/^drive-(\S+)/){
4173 $devices->{$1} = 1;
4174 }
4175 }
4176
4177 my $resmice = mon_cmd($vmid, 'query-mice');
4178 foreach my $mice (@$resmice) {
4179 if ($mice->{name} eq 'QEMU HID Tablet') {
4180 $devices->{tablet} = 1;
4181 last;
4182 }
4183 }
4184
4185 # for usb devices there is no query-usb
4186 # but we can iterate over the entries in
4187 # qom-list path=/machine/peripheral
4188 my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4189 foreach my $per (@$resperipheral) {
4190 if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4191 $devices->{$per->{name}} = 1;
4192 }
4193 }
4194
4195 return $devices;
4196}
4197
4198sub vm_deviceplug {
4199 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4200
4201 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4202
4203 my $devices_list = vm_devices_list($vmid);
4204 return 1 if defined($devices_list->{$deviceid});
4205
4206 # add PCI bridge if we need it for the device
4207 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4208
4209 if ($deviceid eq 'tablet') {
4210 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4211 } elsif ($deviceid eq 'keyboard') {
4212 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4213 } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4214 my $id = $1;
4215 qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4216 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4217 } elsif ($deviceid =~ m/^usb(\d+)$/) {
4218 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4219 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4220 qemu_iothread_add($vmid, $deviceid, $device);
4221
4222 qemu_driveadd($storecfg, $vmid, $device);
4223 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4224
4225 qemu_deviceadd($vmid, $devicefull);
4226 eval { qemu_deviceaddverify($vmid, $deviceid); };
4227 if (my $err = $@) {
4228 eval { qemu_drivedel($vmid, $deviceid); };
4229 warn $@ if $@;
4230 die $err;
4231 }
4232 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4233 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4234 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4235 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4236
4237 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4238
4239 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4240 qemu_iothread_add($vmid, $deviceid, $device);
4241 $devicefull .= ",iothread=iothread-$deviceid";
4242 }
4243
4244 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4245 $devicefull .= ",num_queues=$device->{queues}";
4246 }
4247
4248 qemu_deviceadd($vmid, $devicefull);
4249 qemu_deviceaddverify($vmid, $deviceid);
4250 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4251 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4252 qemu_driveadd($storecfg, $vmid, $device);
4253
4254 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4255 eval { qemu_deviceadd($vmid, $devicefull); };
4256 if (my $err = $@) {
4257 eval { qemu_drivedel($vmid, $deviceid); };
4258 warn $@ if $@;
4259 die $err;
4260 }
4261 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4262 return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4263
4264 my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4265 my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4266 my $use_old_bios_files = undef;
4267 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4268
4269 my $netdevicefull = print_netdevice_full(
4270 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4271 qemu_deviceadd($vmid, $netdevicefull);
4272 eval {
4273 qemu_deviceaddverify($vmid, $deviceid);
4274 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4275 };
4276 if (my $err = $@) {
4277 eval { qemu_netdevdel($vmid, $deviceid); };
4278 warn $@ if $@;
4279 die $err;
4280 }
4281 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4282 my $bridgeid = $2;
4283 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4284 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4285
4286 qemu_deviceadd($vmid, $devicefull);
4287 qemu_deviceaddverify($vmid, $deviceid);
4288 } else {
4289 die "can't hotplug device '$deviceid'\n";
4290 }
4291
4292 return 1;
4293}
4294
4295# fixme: this should raise exceptions on error!
4296sub vm_deviceunplug {
4297 my ($vmid, $conf, $deviceid) = @_;
4298
4299 my $devices_list = vm_devices_list($vmid);
4300 return 1 if !defined($devices_list->{$deviceid});
4301
4302 my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4303 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4304
4305 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4306 qemu_devicedel($vmid, $deviceid);
4307 } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4308 qemu_devicedel($vmid, $deviceid);
4309 qemu_devicedelverify($vmid, $deviceid);
4310 } elsif ($deviceid =~ m/^usb\d+$/) {
4311 qemu_devicedel($vmid, $deviceid);
4312 qemu_devicedelverify($vmid, $deviceid);
4313 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4314 my $device = parse_drive($deviceid, $conf->{$deviceid});
4315
4316 qemu_devicedel($vmid, $deviceid);
4317 qemu_devicedelverify($vmid, $deviceid);
4318 qemu_drivedel($vmid, $deviceid);
4319 qemu_iothread_del($vmid, $deviceid, $device);
4320 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4321 qemu_devicedel($vmid, $deviceid);
4322 qemu_devicedelverify($vmid, $deviceid);
4323 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4324 my $device = parse_drive($deviceid, $conf->{$deviceid});
4325
4326 qemu_devicedel($vmid, $deviceid);
4327 qemu_devicedelverify($vmid, $deviceid);
4328 qemu_drivedel($vmid, $deviceid);
4329 qemu_deletescsihw($conf, $vmid, $deviceid);
4330
4331 qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4332 if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4333 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4334 qemu_devicedel($vmid, $deviceid);
4335 qemu_devicedelverify($vmid, $deviceid);
4336 qemu_netdevdel($vmid, $deviceid);
4337 } else {
4338 die "can't unplug device '$deviceid'\n";
4339 }
4340
4341 return 1;
4342}
4343
4344sub qemu_spice_usbredir_chardev_add {
4345 my ($vmid, $id) = @_;
4346
4347 mon_cmd($vmid, "chardev-add" , (
4348 id => $id,
4349 backend => {
4350 type => 'spicevmc',
4351 data => {
4352 type => "usbredir",
4353 },
4354 },
4355 ));
4356}
4357
4358sub qemu_iothread_add {
4359 my ($vmid, $deviceid, $device) = @_;
4360
4361 if ($device->{iothread}) {
4362 my $iothreads = vm_iothreads_list($vmid);
4363 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4364 }
4365}
4366
4367sub qemu_iothread_del {
4368 my ($vmid, $deviceid, $device) = @_;
4369
4370 if ($device->{iothread}) {
4371 my $iothreads = vm_iothreads_list($vmid);
4372 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4373 }
4374}
4375
4376sub qemu_driveadd {
4377 my ($storecfg, $vmid, $device) = @_;
4378
4379 my $kvmver = get_running_qemu_version($vmid);
4380 my $io_uring = min_version($kvmver, 6, 0);
4381 my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4382 $drive =~ s/\\/\\\\/g;
4383 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4384
4385 # If the command succeeds qemu prints: "OK"
4386 return 1 if $ret =~ m/OK/s;
4387
4388 die "adding drive failed: $ret\n";
4389}
4390
4391sub qemu_drivedel {
4392 my ($vmid, $deviceid) = @_;
4393
4394 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4395 $ret =~ s/^\s+//;
4396
4397 return 1 if $ret eq "";
4398
4399 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4400 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4401
4402 die "deleting drive $deviceid failed : $ret\n";
4403}
4404
4405sub qemu_deviceaddverify {
4406 my ($vmid, $deviceid) = @_;
4407
4408 for (my $i = 0; $i <= 5; $i++) {
4409 my $devices_list = vm_devices_list($vmid);
4410 return 1 if defined($devices_list->{$deviceid});
4411 sleep 1;
4412 }
4413
4414 die "error on hotplug device '$deviceid'\n";
4415}
4416
4417
4418sub qemu_devicedelverify {
4419 my ($vmid, $deviceid) = @_;
4420
4421 # need to verify that the device is correctly removed as device_del
4422 # is async and empty return is not reliable
4423
4424 for (my $i = 0; $i <= 5; $i++) {
4425 my $devices_list = vm_devices_list($vmid);
4426 return 1 if !defined($devices_list->{$deviceid});
4427 sleep 1;
4428 }
4429
4430 die "error on hot-unplugging device '$deviceid'\n";
4431}
4432
4433sub qemu_findorcreatescsihw {
4434 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4435
4436 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4437
4438 my $scsihwid="$controller_prefix$controller";
4439 my $devices_list = vm_devices_list($vmid);
4440
4441 if (!defined($devices_list->{$scsihwid})) {
4442 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4443 }
4444
4445 return 1;
4446}
4447
4448sub qemu_deletescsihw {
4449 my ($conf, $vmid, $opt) = @_;
4450
4451 my $device = parse_drive($opt, $conf->{$opt});
4452
4453 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4454 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4455 return 1;
4456 }
4457
4458 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4459
4460 my $devices_list = vm_devices_list($vmid);
4461 foreach my $opt (keys %{$devices_list}) {
4462 if (is_valid_drivename($opt)) {
4463 my $drive = parse_drive($opt, $conf->{$opt});
4464 if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4465 return 1;
4466 }
4467 }
4468 }
4469
4470 my $scsihwid="scsihw$controller";
4471
4472 vm_deviceunplug($vmid, $conf, $scsihwid);
4473
4474 return 1;
4475}
4476
4477sub qemu_add_pci_bridge {
4478 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4479
4480 my $bridges = {};
4481
4482 my $bridgeid;
4483
4484 print_pci_addr($device, $bridges, $arch, $machine_type);
4485
4486 while (my ($k, $v) = each %$bridges) {
4487 $bridgeid = $k;
4488 }
4489 return 1 if !defined($bridgeid) || $bridgeid < 1;
4490
4491 my $bridge = "pci.$bridgeid";
4492 my $devices_list = vm_devices_list($vmid);
4493
4494 if (!defined($devices_list->{$bridge})) {
4495 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4496 }
4497
4498 return 1;
4499}
4500
4501sub qemu_set_link_status {
4502 my ($vmid, $device, $up) = @_;
4503
4504 mon_cmd($vmid, "set_link", name => $device,
4505 up => $up ? JSON::true : JSON::false);
4506}
4507
4508sub qemu_netdevadd {
4509 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4510
4511 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4512 my %options = split(/[=,]/, $netdev);
4513
4514 if (defined(my $vhost = $options{vhost})) {
4515 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4516 }
4517
4518 if (defined(my $queues = $options{queues})) {
4519 $options{queues} = $queues + 0;
4520 }
4521
4522 mon_cmd($vmid, "netdev_add", %options);
4523 return 1;
4524}
4525
4526sub qemu_netdevdel {
4527 my ($vmid, $deviceid) = @_;
4528
4529 mon_cmd($vmid, "netdev_del", id => $deviceid);
4530}
4531
4532sub qemu_usb_hotplug {
4533 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4534
4535 return if !$device;
4536
4537 # remove the old one first
4538 vm_deviceunplug($vmid, $conf, $deviceid);
4539
4540 # check if xhci controller is necessary and available
4541 my $devicelist = vm_devices_list($vmid);
4542
4543 if (!$devicelist->{xhci}) {
4544 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4545 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4546 }
4547
4548 # add the new one
4549 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type);
4550}
4551
4552sub qemu_cpu_hotplug {
4553 my ($vmid, $conf, $vcpus) = @_;
4554
4555 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4556
4557 my $sockets = 1;
4558 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4559 $sockets = $conf->{sockets} if $conf->{sockets};
4560 my $cores = $conf->{cores} || 1;
4561 my $maxcpus = $sockets * $cores;
4562
4563 $vcpus = $maxcpus if !$vcpus;
4564
4565 die "you can't add more vcpus than maxcpus\n"
4566 if $vcpus > $maxcpus;
4567
4568 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4569
4570 if ($vcpus < $currentvcpus) {
4571
4572 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4573
4574 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4575 qemu_devicedel($vmid, "cpu$i");
4576 my $retry = 0;
4577 my $currentrunningvcpus = undef;
4578 while (1) {
4579 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4580 last if scalar(@{$currentrunningvcpus}) == $i-1;
4581 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4582 $retry++;
4583 sleep 1;
4584 }
4585 #update conf after each succesfull cpu unplug
4586 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4587 PVE::QemuConfig->write_config($vmid, $conf);
4588 }
4589 } else {
4590 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4591 }
4592
4593 return;
4594 }
4595
4596 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4597 die "vcpus in running vm does not match its configuration\n"
4598 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4599
4600 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4601 my $arch = get_vm_arch($conf);
4602
4603 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4604 my $cpustr = print_cpu_device($conf, $arch, $i);
4605 qemu_deviceadd($vmid, $cpustr);
4606
4607 my $retry = 0;
4608 my $currentrunningvcpus = undef;
4609 while (1) {
4610 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4611 last if scalar(@{$currentrunningvcpus}) == $i;
4612 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4613 sleep 1;
4614 $retry++;
4615 }
4616 #update conf after each succesfull cpu hotplug
4617 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4618 PVE::QemuConfig->write_config($vmid, $conf);
4619 }
4620 } else {
4621
4622 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4623 mon_cmd($vmid, "cpu-add", id => int($i));
4624 }
4625 }
4626}
4627
4628sub qemu_block_set_io_throttle {
4629 my ($vmid, $deviceid,
4630 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4631 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4632 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4633 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4634
4635 return if !check_running($vmid) ;
4636
4637 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4638 bps => int($bps),
4639 bps_rd => int($bps_rd),
4640 bps_wr => int($bps_wr),
4641 iops => int($iops),
4642 iops_rd => int($iops_rd),
4643 iops_wr => int($iops_wr),
4644 bps_max => int($bps_max),
4645 bps_rd_max => int($bps_rd_max),
4646 bps_wr_max => int($bps_wr_max),
4647 iops_max => int($iops_max),
4648 iops_rd_max => int($iops_rd_max),
4649 iops_wr_max => int($iops_wr_max),
4650 bps_max_length => int($bps_max_length),
4651 bps_rd_max_length => int($bps_rd_max_length),
4652 bps_wr_max_length => int($bps_wr_max_length),
4653 iops_max_length => int($iops_max_length),
4654 iops_rd_max_length => int($iops_rd_max_length),
4655 iops_wr_max_length => int($iops_wr_max_length),
4656 );
4657
4658}
4659
4660sub qemu_block_resize {
4661 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4662
4663 my $running = check_running($vmid);
4664
4665 PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4666
4667 return if !$running;
4668
4669 my $padding = (1024 - $size % 1024) % 1024;
4670 $size = $size + $padding;
4671
4672 mon_cmd(
4673 $vmid,
4674 "block_resize",
4675 device => $deviceid,
4676 size => int($size),
4677 timeout => 60,
4678 );
4679}
4680
4681sub qemu_volume_snapshot {
4682 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4683
4684 my $running = check_running($vmid);
4685
4686 if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4687 mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4688 } else {
4689 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4690 }
4691}
4692
4693sub qemu_volume_snapshot_delete {
4694 my ($vmid, $storecfg, $volid, $snap) = @_;
4695
4696 my $running = check_running($vmid);
4697 my $attached_deviceid;
4698
4699 if ($running) {
4700 my $conf = PVE::QemuConfig->load_config($vmid);
4701 PVE::QemuConfig->foreach_volume($conf, sub {
4702 my ($ds, $drive) = @_;
4703 $attached_deviceid = "drive-$ds" if $drive->{file} eq $volid;
4704 });
4705 }
4706
4707 if ($attached_deviceid && do_snapshots_with_qemu($storecfg, $volid, $attached_deviceid)) {
4708 mon_cmd(
4709 $vmid,
4710 'blockdev-snapshot-delete-internal-sync',
4711 device => $attached_deviceid,
4712 name => $snap,
4713 );
4714 } else {
4715 PVE::Storage::volume_snapshot_delete(
4716 $storecfg, $volid, $snap, $attached_deviceid ? 1 : undef);
4717 }
4718}
4719
4720sub set_migration_caps {
4721 my ($vmid, $savevm) = @_;
4722
4723 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4724
4725 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4726 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4727
4728 my $cap_ref = [];
4729
4730 my $enabled_cap = {
4731 "auto-converge" => 1,
4732 "xbzrle" => 1,
4733 "x-rdma-pin-all" => 0,
4734 "zero-blocks" => 0,
4735 "compress" => 0,
4736 "dirty-bitmaps" => $dirty_bitmaps,
4737 };
4738
4739 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4740
4741 for my $supported_capability (@$supported_capabilities) {
4742 push @$cap_ref, {
4743 capability => $supported_capability->{capability},
4744 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4745 };
4746 }
4747
4748 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4749}
4750
4751sub foreach_volid {
4752 my ($conf, $func, @param) = @_;
4753
4754 my $volhash = {};
4755
4756 my $test_volid = sub {
4757 my ($key, $drive, $snapname, $pending) = @_;
4758
4759 my $volid = $drive->{file};
4760 return if !$volid;
4761
4762 $volhash->{$volid}->{cdrom} //= 1;
4763 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4764
4765 my $replicate = $drive->{replicate} // 1;
4766 $volhash->{$volid}->{replicate} //= 0;
4767 $volhash->{$volid}->{replicate} = 1 if $replicate;
4768
4769 $volhash->{$volid}->{shared} //= 0;
4770 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4771
4772 $volhash->{$volid}->{is_unused} //= 0;
4773 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4774
4775 $volhash->{$volid}->{is_attached} //= 0;
4776 $volhash->{$volid}->{is_attached} = 1
4777 if !$volhash->{$volid}->{is_unused} && !defined($snapname) && !$pending;
4778
4779 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4780 if defined($snapname);
4781
4782 $volhash->{$volid}->{referenced_in_pending} = 1 if $pending;
4783
4784 my $size = $drive->{size};
4785 $volhash->{$volid}->{size} //= $size if $size;
4786
4787 $volhash->{$volid}->{is_vmstate} //= 0;
4788 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4789
4790 $volhash->{$volid}->{is_tpmstate} //= 0;
4791 $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4792
4793 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4794 };
4795
4796 my $include_opts = {
4797 extra_keys => ['vmstate'],
4798 include_unused => 1,
4799 };
4800
4801 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4802
4803 PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $test_volid, undef, 1)
4804 if defined($conf->{pending}) && $conf->{pending}->%*;
4805
4806 foreach my $snapname (keys %{$conf->{snapshots}}) {
4807 my $snap = $conf->{snapshots}->{$snapname};
4808 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4809 }
4810
4811 foreach my $volid (keys %$volhash) {
4812 &$func($volid, $volhash->{$volid}, @param);
4813 }
4814}
4815
4816my $fast_plug_option = {
4817 'description' => 1,
4818 'hookscript' => 1,
4819 'lock' => 1,
4820 'migrate_downtime' => 1,
4821 'migrate_speed' => 1,
4822 'name' => 1,
4823 'onboot' => 1,
4824 'protection' => 1,
4825 'shares' => 1,
4826 'startup' => 1,
4827 'tags' => 1,
4828 'vmstatestorage' => 1,
4829};
4830
4831for my $opt (keys %$confdesc_cloudinit) {
4832 $fast_plug_option->{$opt} = 1;
4833};
4834
4835# hotplug changes in [PENDING]
4836# $selection hash can be used to only apply specified options, for
4837# example: { cores => 1 } (only apply changed 'cores')
4838# $errors ref is used to return error messages
4839sub vmconfig_hotplug_pending {
4840 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4841
4842 my $defaults = load_defaults();
4843 my $arch = get_vm_arch($conf);
4844 my $machine_type = get_vm_machine($conf, undef, $arch);
4845
4846 # commit values which do not have any impact on running VM first
4847 # Note: those option cannot raise errors, we we do not care about
4848 # $selection and always apply them.
4849
4850 my $add_error = sub {
4851 my ($opt, $msg) = @_;
4852 $errors->{$opt} = "hotplug problem - $msg";
4853 };
4854
4855 my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4856
4857 my $cloudinit_record_changed = sub {
4858 my ($conf, $opt, $old, $new) = @_;
4859 return if !$cloudinit_pending_properties->{$opt};
4860
4861 my $ci = ($conf->{cloudinit} //= {});
4862
4863 my $recorded = $ci->{$opt};
4864 my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4865
4866 if (defined($new)) {
4867 if (defined($old)) {
4868 # an existing value is being modified
4869 if (defined($recorded)) {
4870 # the value was already not in sync
4871 if ($new eq $recorded) {
4872 # a value is being reverted to the cloud-init state:
4873 delete $ci->{$opt};
4874 delete $added{$opt};
4875 } else {
4876 # the value was changed multiple times, do nothing
4877 }
4878 } elsif ($added{$opt}) {
4879 # the value had been marked as added and is being changed, do nothing
4880 } else {
4881 # the value is new, record it:
4882 $ci->{$opt} = $old;
4883 }
4884 } else {
4885 # a new value is being added
4886 if (defined($recorded)) {
4887 # it was already not in sync
4888 if ($new eq $recorded) {
4889 # a value is being reverted to the cloud-init state:
4890 delete $ci->{$opt};
4891 delete $added{$opt};
4892 } else {
4893 # the value had temporarily been removed, do nothing
4894 }
4895 } elsif ($added{$opt}) {
4896 # the value had been marked as added already, do nothing
4897 } else {
4898 # the value is new, add it
4899 $added{$opt} = 1;
4900 }
4901 }
4902 } elsif (!defined($old)) {
4903 # a non-existent value is being removed? ignore...
4904 } else {
4905 # a value is being deleted
4906 if (defined($recorded)) {
4907 # a value was already recorded, just keep it
4908 } elsif ($added{$opt}) {
4909 # the value was marked as added, remove it
4910 delete $added{$opt};
4911 } else {
4912 # a previously unrecorded value is being removed, record the old value:
4913 $ci->{$opt} = $old;
4914 }
4915 }
4916
4917 my $added = join(',', sort keys %added);
4918 $ci->{added} = $added if length($added);
4919 };
4920
4921 my $changes = 0;
4922 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4923 if ($fast_plug_option->{$opt}) {
4924 my $new = delete $conf->{pending}->{$opt};
4925 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4926 $conf->{$opt} = $new;
4927 $changes = 1;
4928 }
4929 }
4930
4931 if ($changes) {
4932 PVE::QemuConfig->write_config($vmid, $conf);
4933 }
4934
4935 my $ostype = $conf->{ostype};
4936 my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4937 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4938 my $usb_hotplug = $hotplug_features->{usb}
4939 && min_version($version, 7, 1)
4940 && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4941
4942 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4943 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4944
4945 foreach my $opt (sort keys %$pending_delete_hash) {
4946 next if $selection && !$selection->{$opt};
4947 my $force = $pending_delete_hash->{$opt}->{force};
4948 eval {
4949 if ($opt eq 'hotplug') {
4950 die "skip\n" if ($conf->{hotplug} =~ /(cpu|memory)/);
4951 } elsif ($opt eq 'tablet') {
4952 die "skip\n" if !$hotplug_features->{usb};
4953 if ($defaults->{tablet}) {
4954 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4955 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4956 if $arch eq 'aarch64';
4957 } else {
4958 vm_deviceunplug($vmid, $conf, 'tablet');
4959 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4960 }
4961 } elsif ($opt =~ m/^usb(\d+)$/) {
4962 my $index = $1;
4963 die "skip\n" if !$usb_hotplug;
4964 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4965 vm_deviceunplug($vmid, $conf, $opt);
4966 } elsif ($opt eq 'vcpus') {
4967 die "skip\n" if !$hotplug_features->{cpu};
4968 qemu_cpu_hotplug($vmid, $conf, undef);
4969 } elsif ($opt eq 'balloon') {
4970 # enable balloon device is not hotpluggable
4971 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4972 # here we reset the ballooning value to memory
4973 my $balloon = get_current_memory($conf->{memory});
4974 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4975 } elsif ($fast_plug_option->{$opt}) {
4976 # do nothing
4977 } elsif ($opt =~ m/^net(\d+)$/) {
4978 die "skip\n" if !$hotplug_features->{network};
4979 vm_deviceunplug($vmid, $conf, $opt);
4980 if($have_sdn) {
4981 my $net = PVE::QemuServer::parse_net($conf->{$opt});
4982 PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name});
4983 }
4984 } elsif (is_valid_drivename($opt)) {
4985 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4986 vm_deviceunplug($vmid, $conf, $opt);
4987 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4988 } elsif ($opt =~ m/^memory$/) {
4989 die "skip\n" if !$hotplug_features->{memory};
4990 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf);
4991 } elsif ($opt eq 'cpuunits') {
4992 $cgroup->change_cpu_shares(undef);
4993 } elsif ($opt eq 'cpulimit') {
4994 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4995 } else {
4996 die "skip\n";
4997 }
4998 };
4999 if (my $err = $@) {
5000 &$add_error($opt, $err) if $err ne "skip\n";
5001 } else {
5002 my $old = delete $conf->{$opt};
5003 $cloudinit_record_changed->($conf, $opt, $old, undef);
5004 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5005 }
5006 }
5007
5008 my $cloudinit_opt;
5009 foreach my $opt (keys %{$conf->{pending}}) {
5010 next if $selection && !$selection->{$opt};
5011 my $value = $conf->{pending}->{$opt};
5012 eval {
5013 if ($opt eq 'hotplug') {
5014 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5015 die "skip\n" if ($value =~ /cpu/) || ($value !~ /cpu/ && $conf->{hotplug} =~ /cpu/);
5016 } elsif ($opt eq 'tablet') {
5017 die "skip\n" if !$hotplug_features->{usb};
5018 if ($value == 1) {
5019 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5020 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5021 if $arch eq 'aarch64';
5022 } elsif ($value == 0) {
5023 vm_deviceunplug($vmid, $conf, 'tablet');
5024 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5025 }
5026 } elsif ($opt =~ m/^usb(\d+)$/) {
5027 my $index = $1;
5028 die "skip\n" if !$usb_hotplug;
5029 my $d = eval { parse_property_string('pve-qm-usb', $value) };
5030 my $id = $opt;
5031 if ($d->{host} =~ m/^spice$/i) {
5032 $id = "usbredirdev$index";
5033 }
5034 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5035 } elsif ($opt eq 'vcpus') {
5036 die "skip\n" if !$hotplug_features->{cpu};
5037 qemu_cpu_hotplug($vmid, $conf, $value);
5038 } elsif ($opt eq 'balloon') {
5039 # enable/disable balloning device is not hotpluggable
5040 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
5041 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5042 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5043
5044 # allow manual ballooning if shares is set to zero
5045 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5046 my $memory = get_current_memory($conf->{memory});
5047 my $balloon = $conf->{pending}->{balloon} || $memory;
5048 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5049 }
5050 } elsif ($opt =~ m/^net(\d+)$/) {
5051 # some changes can be done without hotplug
5052 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5053 $vmid, $opt, $value, $arch, $machine_type);
5054 } elsif (is_valid_drivename($opt)) {
5055 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5056 # some changes can be done without hotplug
5057 my $drive = parse_drive($opt, $value);
5058 if (drive_is_cloudinit($drive)) {
5059 $cloudinit_opt = [$opt, $drive];
5060 # apply all the other changes first, then generate the cloudinit disk
5061 die "skip\n";
5062 }
5063 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5064 $vmid, $opt, $value, $arch, $machine_type);
5065 } elsif ($opt =~ m/^memory$/) { #dimms
5066 die "skip\n" if !$hotplug_features->{memory};
5067 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $value);
5068 } elsif ($opt eq 'cpuunits') {
5069 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5070 $cgroup->change_cpu_shares($new_cpuunits);
5071 } elsif ($opt eq 'cpulimit') {
5072 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5073 $cgroup->change_cpu_quota($cpulimit, 100000);
5074 } elsif ($opt eq 'agent') {
5075 vmconfig_update_agent($conf, $opt, $value);
5076 } else {
5077 die "skip\n"; # skip non-hot-pluggable options
5078 }
5079 };
5080 if (my $err = $@) {
5081 &$add_error($opt, $err) if $err ne "skip\n";
5082 } else {
5083 $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5084 $conf->{$opt} = $value;
5085 delete $conf->{pending}->{$opt};
5086 }
5087 }
5088
5089 if (defined($cloudinit_opt)) {
5090 my ($opt, $drive) = @$cloudinit_opt;
5091 my $value = $conf->{pending}->{$opt};
5092 eval {
5093 my $temp = {%$conf, $opt => $value};
5094 PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5095 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5096 $vmid, $opt, $value, $arch, $machine_type);
5097 };
5098 if (my $err = $@) {
5099 &$add_error($opt, $err) if $err ne "skip\n";
5100 } else {
5101 $conf->{$opt} = $value;
5102 delete $conf->{pending}->{$opt};
5103 }
5104 }
5105
5106 # unplug xhci controller if no usb device is left
5107 if ($usb_hotplug) {
5108 my $has_usb = 0;
5109 for (my $i = 0; $i < $PVE::QemuServer::USB::MAX_USB_DEVICES; $i++) {
5110 next if !defined($conf->{"usb$i"});
5111 $has_usb = 1;
5112 last;
5113 }
5114 if (!$has_usb) {
5115 vm_deviceunplug($vmid, $conf, 'xhci');
5116 }
5117 }
5118
5119 PVE::QemuConfig->write_config($vmid, $conf);
5120
5121 if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5122 PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5123 }
5124}
5125
5126sub try_deallocate_drive {
5127 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5128
5129 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5130 my $volid = $drive->{file};
5131 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5132 my $sid = PVE::Storage::parse_volume_id($volid);
5133 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5134
5135 # check if the disk is really unused
5136 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5137 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5138 PVE::Storage::vdisk_free($storecfg, $volid);
5139 return 1;
5140 } else {
5141 # If vm is not owner of this disk remove from config
5142 return 1;
5143 }
5144 }
5145
5146 return;
5147}
5148
5149sub vmconfig_delete_or_detach_drive {
5150 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5151
5152 my $drive = parse_drive($opt, $conf->{$opt});
5153
5154 my $rpcenv = PVE::RPCEnvironment::get();
5155 my $authuser = $rpcenv->get_user();
5156
5157 if ($force) {
5158 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5159 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5160 } else {
5161 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5162 }
5163}
5164
5165
5166
5167sub vmconfig_apply_pending {
5168 my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5169
5170 return if !scalar(keys %{$conf->{pending}});
5171
5172 my $add_apply_error = sub {
5173 my ($opt, $msg) = @_;
5174 my $err_msg = "unable to apply pending change $opt : $msg";
5175 $errors->{$opt} = $err_msg;
5176 warn $err_msg;
5177 };
5178
5179 # cold plug
5180
5181 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5182 foreach my $opt (sort keys %$pending_delete_hash) {
5183 my $force = $pending_delete_hash->{$opt}->{force};
5184 eval {
5185 if ($opt =~ m/^unused/) {
5186 die "internal error";
5187 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5188 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5189 } elsif (defined($conf->{$opt}) && $opt =~ m/^net\d+$/) {
5190 if($have_sdn) {
5191 my $net = PVE::QemuServer::parse_net($conf->{$opt});
5192 eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
5193 warn if $@;
5194 }
5195 }
5196 };
5197 if (my $err = $@) {
5198 $add_apply_error->($opt, $err);
5199 } else {
5200 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5201 delete $conf->{$opt};
5202 }
5203 }
5204
5205 PVE::QemuConfig->cleanup_pending($conf);
5206
5207 my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5208
5209 foreach my $opt (keys %{$conf->{pending}}) { # add/change
5210 next if $opt eq 'delete'; # just to be sure
5211 eval {
5212 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5213 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5214 } elsif (defined($conf->{pending}->{$opt}) && $opt =~ m/^net\d+$/) {
5215 return if !$have_sdn; # return from eval if SDN is not available
5216
5217 my $new_net = PVE::QemuServer::parse_net($conf->{pending}->{$opt});
5218 if ($conf->{$opt}) {
5219 my $old_net = PVE::QemuServer::parse_net($conf->{$opt});
5220
5221 if (defined($old_net->{bridge}) && defined($old_net->{macaddr}) && (
5222 safe_string_ne($old_net->{bridge}, $new_net->{bridge}) ||
5223 safe_string_ne($old_net->{macaddr}, $new_net->{macaddr})
5224 )) {
5225 PVE::Network::SDN::Vnets::del_ips_from_mac($old_net->{bridge}, $old_net->{macaddr}, $conf->{name});
5226 }
5227 }
5228 #fixme: reuse ip if mac change && same bridge
5229 PVE::Network::SDN::Vnets::add_next_free_cidr($new_net->{bridge}, $conf->{name}, $new_net->{macaddr}, $vmid, undef, 1);
5230 }
5231 };
5232 if (my $err = $@) {
5233 $add_apply_error->($opt, $err);
5234 } else {
5235
5236 if (is_valid_drivename($opt)) {
5237 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5238 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5239 }
5240
5241 $conf->{$opt} = delete $conf->{pending}->{$opt};
5242 }
5243 }
5244
5245 # write all changes at once to avoid unnecessary i/o
5246 PVE::QemuConfig->write_config($vmid, $conf);
5247 if ($generate_cloudinit) {
5248 if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5249 # After successful generation and if there were changes to be applied, update the
5250 # config to drop the {cloudinit} entry.
5251 PVE::QemuConfig->write_config($vmid, $conf);
5252 }
5253 }
5254}
5255
5256sub vmconfig_update_net {
5257 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5258
5259 my $newnet = parse_net($value);
5260
5261 if ($conf->{$opt}) {
5262 my $oldnet = parse_net($conf->{$opt});
5263
5264 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5265 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5266 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5267 safe_num_ne($oldnet->{mtu}, $newnet->{mtu}) ||
5268 !($newnet->{bridge} && $oldnet->{bridge})
5269 ) { # bridge/nat mode change
5270
5271 # for non online change, we try to hot-unplug
5272 die "skip\n" if !$hotplug;
5273 vm_deviceunplug($vmid, $conf, $opt);
5274
5275 if ($have_sdn) {
5276 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5277 }
5278
5279 } else {
5280
5281 die "internal error" if $opt !~ m/net(\d+)/;
5282 my $iface = "tap${vmid}i$1";
5283
5284 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5285 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5286 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5287 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})
5288 ) {
5289 PVE::Network::tap_unplug($iface);
5290
5291 #set link_down in guest if bridge or vlan change to notify guest (dhcp renew for example)
5292 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5293 safe_num_ne($oldnet->{tag}, $newnet->{tag})
5294 ) {
5295 qemu_set_link_status($vmid, $opt, 0);
5296 }
5297
5298 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge})) {
5299 if ($have_sdn) {
5300 PVE::Network::SDN::Vnets::del_ips_from_mac($oldnet->{bridge}, $oldnet->{macaddr}, $conf->{name});
5301 PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5302 }
5303 }
5304
5305 if ($have_sdn) {
5306 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5307 } else {
5308 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5309 }
5310
5311 #set link_up in guest if bridge or vlan change to notify guest (dhcp renew for example)
5312 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5313 safe_num_ne($oldnet->{tag}, $newnet->{tag})
5314 ) {
5315 qemu_set_link_status($vmid, $opt, 1);
5316 }
5317
5318 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5319 # Rate can be applied on its own but any change above needs to
5320 # include the rate in tap_plug since OVS resets everything.
5321 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5322 }
5323
5324 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5325 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5326 }
5327
5328 return 1;
5329 }
5330 }
5331
5332 if ($hotplug) {
5333 if ($have_sdn) {
5334 PVE::Network::SDN::Vnets::add_next_free_cidr($newnet->{bridge}, $conf->{name}, $newnet->{macaddr}, $vmid, undef, 1);
5335 PVE::Network::SDN::Vnets::add_dhcp_mapping($newnet->{bridge}, $newnet->{macaddr}, $vmid, $conf->{name});
5336 }
5337 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5338 } else {
5339 die "skip\n";
5340 }
5341}
5342
5343sub vmconfig_update_agent {
5344 my ($conf, $opt, $value) = @_;
5345
5346 die "skip\n" if !$conf->{$opt};
5347
5348 my $hotplug_options = { fstrim_cloned_disks => 1 };
5349
5350 my $old_agent = parse_guest_agent($conf);
5351 my $agent = parse_guest_agent({$opt => $value});
5352
5353 for my $option (keys %$agent) { # added/changed options
5354 next if defined($hotplug_options->{$option});
5355 die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5356 }
5357
5358 for my $option (keys %$old_agent) { # removed options
5359 next if defined($hotplug_options->{$option});
5360 die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5361 }
5362
5363 return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5364}
5365
5366sub vmconfig_update_disk {
5367 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5368
5369 my $drive = parse_drive($opt, $value);
5370
5371 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5372 my $media = $drive->{media} || 'disk';
5373 my $oldmedia = $old_drive->{media} || 'disk';
5374 die "unable to change media type\n" if $media ne $oldmedia;
5375
5376 if (!drive_is_cdrom($old_drive)) {
5377
5378 if ($drive->{file} ne $old_drive->{file}) {
5379
5380 die "skip\n" if !$hotplug;
5381
5382 # unplug and register as unused
5383 vm_deviceunplug($vmid, $conf, $opt);
5384 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5385
5386 } else {
5387 # update existing disk
5388
5389 # skip non hotpluggable value
5390 if (safe_string_ne($drive->{aio}, $old_drive->{aio}) ||
5391 safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5392 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5393 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5394 safe_string_ne($drive->{product}, $old_drive->{product}) ||
5395 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5396 safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5397 safe_string_ne($drive->{vendor}, $old_drive->{vendor}) ||
5398 safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5399 die "skip\n";
5400 }
5401
5402 # apply throttle
5403 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5404 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5405 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5406 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5407 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5408 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5409 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5410 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5411 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5412 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5413 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5414 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5415 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5416 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5417 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5418 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5419 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5420 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5421
5422 qemu_block_set_io_throttle(
5423 $vmid,"drive-$opt",
5424 ($drive->{mbps} || 0)*1024*1024,
5425 ($drive->{mbps_rd} || 0)*1024*1024,
5426 ($drive->{mbps_wr} || 0)*1024*1024,
5427 $drive->{iops} || 0,
5428 $drive->{iops_rd} || 0,
5429 $drive->{iops_wr} || 0,
5430 ($drive->{mbps_max} || 0)*1024*1024,
5431 ($drive->{mbps_rd_max} || 0)*1024*1024,
5432 ($drive->{mbps_wr_max} || 0)*1024*1024,
5433 $drive->{iops_max} || 0,
5434 $drive->{iops_rd_max} || 0,
5435 $drive->{iops_wr_max} || 0,
5436 $drive->{bps_max_length} || 1,
5437 $drive->{bps_rd_max_length} || 1,
5438 $drive->{bps_wr_max_length} || 1,
5439 $drive->{iops_max_length} || 1,
5440 $drive->{iops_rd_max_length} || 1,
5441 $drive->{iops_wr_max_length} || 1,
5442 );
5443
5444 }
5445
5446 return 1;
5447 }
5448
5449 } else { # cdrom
5450
5451 if ($drive->{file} eq 'none') {
5452 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5453 if (drive_is_cloudinit($old_drive)) {
5454 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5455 }
5456 } else {
5457 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5458
5459 # force eject if locked
5460 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5461
5462 if ($path) {
5463 mon_cmd($vmid, "blockdev-change-medium",
5464 id => "$opt", filename => "$path");
5465 }
5466 }
5467
5468 return 1;
5469 }
5470 }
5471
5472 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5473 # hotplug new disks
5474 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5475 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5476}
5477
5478sub vmconfig_update_cloudinit_drive {
5479 my ($storecfg, $conf, $vmid) = @_;
5480
5481 my $cloudinit_ds = undef;
5482 my $cloudinit_drive = undef;
5483
5484 PVE::QemuConfig->foreach_volume($conf, sub {
5485 my ($ds, $drive) = @_;
5486 if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5487 $cloudinit_ds = $ds;
5488 $cloudinit_drive = $drive;
5489 }
5490 });
5491
5492 return if !$cloudinit_drive;
5493
5494 if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5495 PVE::QemuConfig->write_config($vmid, $conf);
5496 }
5497
5498 my $running = PVE::QemuServer::check_running($vmid);
5499
5500 if ($running) {
5501 my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5502 if ($path) {
5503 mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5504 mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5505 }
5506 }
5507}
5508
5509# called in locked context by incoming migration
5510sub vm_migrate_get_nbd_disks {
5511 my ($storecfg, $conf, $replicated_volumes) = @_;
5512
5513 my $local_volumes = {};
5514 PVE::QemuConfig->foreach_volume($conf, sub {
5515 my ($ds, $drive) = @_;
5516
5517 return if drive_is_cdrom($drive);
5518 return if $ds eq 'tpmstate0';
5519
5520 my $volid = $drive->{file};
5521
5522 return if !$volid;
5523
5524 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5525
5526 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5527 return if $scfg->{shared};
5528
5529 my $format = qemu_img_format($scfg, $volname);
5530
5531 # replicated disks re-use existing state via bitmap
5532 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5533 $local_volumes->{$ds} = [$volid, $storeid, $drive, $use_existing, $format];
5534 });
5535 return $local_volumes;
5536}
5537
5538# called in locked context by incoming migration
5539sub vm_migrate_alloc_nbd_disks {
5540 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5541
5542 my $nbd = {};
5543 foreach my $opt (sort keys %$source_volumes) {
5544 my ($volid, $storeid, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5545
5546 if ($use_existing) {
5547 $nbd->{$opt}->{drivestr} = print_drive($drive);
5548 $nbd->{$opt}->{volid} = $volid;
5549 $nbd->{$opt}->{replicated} = 1;
5550 next;
5551 }
5552
5553 $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5554
5555 # order of precedence, filtered by whether storage supports it:
5556 # 1. explicit requested format
5557 # 2. default format of storage
5558 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5559 $format = $defFormat if !$format || !grep { $format eq $_ } $validFormats->@*;
5560
5561 my $size = $drive->{size} / 1024;
5562 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5563 my $newdrive = $drive;
5564 $newdrive->{format} = $format;
5565 $newdrive->{file} = $newvolid;
5566 my $drivestr = print_drive($newdrive);
5567 $nbd->{$opt}->{drivestr} = $drivestr;
5568 $nbd->{$opt}->{volid} = $newvolid;
5569 }
5570
5571 return $nbd;
5572}
5573
5574# see vm_start_nolock for parameters, additionally:
5575# migrate_opts:
5576# storagemap = parsed storage map for allocating NBD disks
5577sub vm_start {
5578 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5579
5580 return PVE::QemuConfig->lock_config($vmid, sub {
5581 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5582
5583 die "you can't start a vm if it's a template\n"
5584 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5585
5586 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5587 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5588
5589 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5590
5591 if ($has_backup_lock && $running) {
5592 # a backup is currently running, attempt to start the guest in the
5593 # existing QEMU instance
5594 return vm_resume($vmid);
5595 }
5596
5597 PVE::QemuConfig->check_lock($conf)
5598 if !($params->{skiplock} || $has_suspended_lock);
5599
5600 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5601
5602 die "VM $vmid already running\n" if $running;
5603
5604 if (my $storagemap = $migrate_opts->{storagemap}) {
5605 my $replicated = $migrate_opts->{replicated_volumes};
5606 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5607 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5608
5609 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5610 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5611 }
5612 }
5613
5614 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5615 });
5616}
5617
5618
5619# params:
5620# statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5621# skiplock => 0/1, skip checking for config lock
5622# skiptemplate => 0/1, skip checking whether VM is template
5623# forcemachine => to force QEMU machine (rollback/migration)
5624# forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5625# timeout => in seconds
5626# paused => start VM in paused state (backup)
5627# resume => resume from hibernation
5628# live-restore-backing => {
5629# sata0 => {
5630# name => blockdev-name,
5631# blockdev => "arg to the -blockdev command instantiating device named 'name'",
5632# },
5633# virtio2 => ...
5634# }
5635# migrate_opts:
5636# nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5637# migratedfrom => source node
5638# spice_ticket => used for spice migration, passed via tunnel/stdin
5639# network => CIDR of migration network
5640# type => secure/insecure - tunnel over encrypted connection or plain-text
5641# nbd_proto_version => int, 0 for TCP, 1 for UNIX
5642# replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5643# offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5644# contained in config
5645sub vm_start_nolock {
5646 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5647
5648 my $statefile = $params->{statefile};
5649 my $resume = $params->{resume};
5650
5651 my $migratedfrom = $migrate_opts->{migratedfrom};
5652 my $migration_type = $migrate_opts->{type};
5653
5654 my $res = {};
5655
5656 # clean up leftover reboot request files
5657 eval { clear_reboot_request($vmid); };
5658 warn $@ if $@;
5659
5660 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5661 vmconfig_apply_pending($vmid, $conf, $storecfg);
5662 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5663 }
5664
5665 # don't regenerate the ISO if the VM is started as part of a live migration
5666 # this way we can reuse the old ISO with the correct config
5667 if (!$migratedfrom) {
5668 if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5669 # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5670 # $conf->{cloudinit}, so we could just not do this?
5671 # But we do it above, so for now let's be consistent.
5672 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5673 }
5674 }
5675
5676 # override offline migrated volumes, conf is out of date still
5677 if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5678 for my $key (sort keys $offline_volumes->%*) {
5679 my $parsed = parse_drive($key, $conf->{$key});
5680 $parsed->{file} = $offline_volumes->{$key};
5681 $conf->{$key} = print_drive($parsed);
5682 }
5683 }
5684
5685 my $defaults = load_defaults();
5686
5687 # set environment variable useful inside network script
5688 # for remote migration the config is available on the target node!
5689 if (!$migrate_opts->{remote_node}) {
5690 $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5691 }
5692
5693 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5694
5695 my $forcemachine = $params->{forcemachine};
5696 my $forcecpu = $params->{forcecpu};
5697 if ($resume) {
5698 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5699 $forcemachine = $conf->{runningmachine};
5700 $forcecpu = $conf->{runningcpu};
5701 print "Resuming suspended VM\n";
5702 }
5703
5704 my ($cmd, $vollist, $spice_port, $pci_devices) = config_to_command($storecfg, $vmid,
5705 $conf, $defaults, $forcemachine, $forcecpu, $params->{'live-restore-backing'});
5706
5707 my $migration_ip;
5708 my $get_migration_ip = sub {
5709 my ($nodename) = @_;
5710
5711 return $migration_ip if defined($migration_ip);
5712
5713 my $cidr = $migrate_opts->{network};
5714
5715 if (!defined($cidr)) {
5716 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5717 $cidr = $dc_conf->{migration}->{network};
5718 }
5719
5720 if (defined($cidr)) {
5721 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5722
5723 die "could not get IP: no address configured on local " .
5724 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5725
5726 die "could not get IP: multiple addresses configured on local " .
5727 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5728
5729 $migration_ip = @$ips[0];
5730 }
5731
5732 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5733 if !defined($migration_ip);
5734
5735 return $migration_ip;
5736 };
5737
5738 if ($statefile) {
5739 if ($statefile eq 'tcp') {
5740 my $migrate = $res->{migrate} = { proto => 'tcp' };
5741 $migrate->{addr} = "localhost";
5742 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5743 my $nodename = nodename();
5744
5745 if (!defined($migration_type)) {
5746 if (defined($datacenterconf->{migration}->{type})) {
5747 $migration_type = $datacenterconf->{migration}->{type};
5748 } else {
5749 $migration_type = 'secure';
5750 }
5751 }
5752
5753 if ($migration_type eq 'insecure') {
5754 $migrate->{addr} = $get_migration_ip->($nodename);
5755 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5756 }
5757
5758 # see #4501: port reservation should be done close to usage - tell QEMU where to listen
5759 # via QMP later
5760 push @$cmd, '-incoming', 'defer';
5761 push @$cmd, '-S';
5762
5763 } elsif ($statefile eq 'unix') {
5764 # should be default for secure migrations as a ssh TCP forward
5765 # tunnel is not deterministic reliable ready and fails regurarly
5766 # to set up in time, so use UNIX socket forwards
5767 my $migrate = $res->{migrate} = { proto => 'unix' };
5768 $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5769 unlink $migrate->{addr};
5770
5771 $migrate->{uri} = "unix:$migrate->{addr}";
5772 push @$cmd, '-incoming', $migrate->{uri};
5773 push @$cmd, '-S';
5774
5775 } elsif (-e $statefile) {
5776 push @$cmd, '-loadstate', $statefile;
5777 } else {
5778 my $statepath = PVE::Storage::path($storecfg, $statefile);
5779 push @$vollist, $statefile;
5780 push @$cmd, '-loadstate', $statepath;
5781 }
5782 } elsif ($params->{paused}) {
5783 push @$cmd, '-S';
5784 }
5785
5786 my $memory = get_current_memory($conf->{memory});
5787 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $memory, $resume);
5788
5789 my $pci_reserve_list = [];
5790 for my $device (values $pci_devices->%*) {
5791 next if $device->{mdev}; # we don't reserve for mdev devices
5792 push $pci_reserve_list->@*, map { $_->{id} } $device->{ids}->@*;
5793 }
5794
5795 # reserve all PCI IDs before actually doing anything with them
5796 PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, $start_timeout);
5797
5798 eval {
5799 my $uuid;
5800 for my $id (sort keys %$pci_devices) {
5801 my $d = $pci_devices->{$id};
5802 my ($index) = ($id =~ m/^hostpci(\d+)$/);
5803
5804 my $chosen_mdev;
5805 for my $dev ($d->{ids}->@*) {
5806 my $info = eval { PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $index, $d->{mdev}) };
5807 if ($d->{mdev}) {
5808 warn $@ if $@;
5809 $chosen_mdev = $info;
5810 last if $chosen_mdev; # if successful, we're done
5811 } else {
5812 die $@ if $@;
5813 }
5814 }
5815
5816 next if !$d->{mdev};
5817 die "could not create mediated device\n" if !defined($chosen_mdev);
5818
5819 # nvidia grid needs the uuid of the mdev as qemu parameter
5820 if (!defined($uuid) && $chosen_mdev->{vendor} =~ m/^(0x)?10de$/) {
5821 if (defined($conf->{smbios1})) {
5822 my $smbios_conf = parse_smbios1($conf->{smbios1});
5823 $uuid = $smbios_conf->{uuid} if defined($smbios_conf->{uuid});
5824 }
5825 $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $index) if !defined($uuid);
5826 }
5827 }
5828 push @$cmd, '-uuid', $uuid if defined($uuid);
5829 };
5830 if (my $err = $@) {
5831 eval { cleanup_pci_devices($vmid, $conf) };
5832 warn $@ if $@;
5833 die $err;
5834 }
5835
5836 PVE::Storage::activate_volumes($storecfg, $vollist);
5837
5838
5839 my %silence_std_outs = (outfunc => sub {}, errfunc => sub {});
5840 eval { run_command(['/bin/systemctl', 'reset-failed', "$vmid.scope"], %silence_std_outs) };
5841 eval { run_command(['/bin/systemctl', 'stop', "$vmid.scope"], %silence_std_outs) };
5842 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5843 # timeout should be more than enough here...
5844 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5845
5846 my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5847
5848 my %run_params = (
5849 timeout => $statefile ? undef : $start_timeout,
5850 umask => 0077,
5851 noerr => 1,
5852 );
5853
5854 # when migrating, prefix QEMU output so other side can pick up any
5855 # errors that might occur and show the user
5856 if ($migratedfrom) {
5857 $run_params{quiet} = 1;
5858 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5859 }
5860
5861 my %systemd_properties = (
5862 Slice => 'qemu.slice',
5863 KillMode => 'process',
5864 SendSIGKILL => 0,
5865 TimeoutStopUSec => ULONG_MAX, # infinity
5866 );
5867
5868 if (PVE::CGroup::cgroup_mode() == 2) {
5869 $systemd_properties{CPUWeight} = $cpuunits;
5870 } else {
5871 $systemd_properties{CPUShares} = $cpuunits;
5872 }
5873
5874 if (my $cpulimit = $conf->{cpulimit}) {
5875 $systemd_properties{CPUQuota} = int($cpulimit * 100);
5876 }
5877 $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5878
5879 my $run_qemu = sub {
5880 PVE::Tools::run_fork sub {
5881 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5882
5883 my $tpmpid;
5884 if ((my $tpm = $conf->{tpmstate0}) && !PVE::QemuConfig->is_template($conf)) {
5885 # start the TPM emulator so QEMU can connect on start
5886 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5887 }
5888
5889 my $exitcode = run_command($cmd, %run_params);
5890 if ($exitcode) {
5891 if ($tpmpid) {
5892 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5893 kill 'TERM', $tpmpid;
5894 }
5895 die "QEMU exited with code $exitcode\n";
5896 }
5897 };
5898 };
5899
5900 if ($conf->{hugepages}) {
5901
5902 my $code = sub {
5903 my $hotplug_features =
5904 parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
5905 my $hugepages_topology =
5906 PVE::QemuServer::Memory::hugepages_topology($conf, $hotplug_features->{memory});
5907
5908 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5909
5910 PVE::QemuServer::Memory::hugepages_mount();
5911 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5912
5913 eval { $run_qemu->() };
5914 if (my $err = $@) {
5915 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5916 if !$conf->{keephugepages};
5917 die $err;
5918 }
5919
5920 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5921 if !$conf->{keephugepages};
5922 };
5923 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5924
5925 } else {
5926 eval { $run_qemu->() };
5927 }
5928
5929 if (my $err = $@) {
5930 # deactivate volumes if start fails
5931 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5932 warn $@ if $@;
5933 eval { cleanup_pci_devices($vmid, $conf) };
5934 warn $@ if $@;
5935
5936 die "start failed: $err";
5937 }
5938
5939 # re-reserve all PCI IDs now that we can know the actual VM PID
5940 my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5941 eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_reserve_list, $vmid, undef, $pid) };
5942 warn $@ if $@;
5943
5944 if (defined(my $migrate = $res->{migrate})) {
5945 if ($migrate->{proto} eq 'tcp') {
5946 my $nodename = nodename();
5947 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5948 $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5949 $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5950 mon_cmd($vmid, "migrate-incoming", uri => $migrate->{uri});
5951 }
5952 print "migration listens on $migrate->{uri}\n";
5953 } elsif ($statefile) {
5954 eval { mon_cmd($vmid, "cont"); };
5955 warn $@ if $@;
5956 }
5957
5958 #start nbd server for storage migration
5959 if (my $nbd = $migrate_opts->{nbd}) {
5960 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5961
5962 my $migrate_storage_uri;
5963 # nbd_protocol_version > 0 for unix socket support
5964 if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5965 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5966 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5967 $migrate_storage_uri = "nbd:unix:$socket_path";
5968 $res->{migrate}->{unix_sockets} = [$socket_path];
5969 } else {
5970 my $nodename = nodename();
5971 my $localip = $get_migration_ip->($nodename);
5972 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5973 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5974
5975 mon_cmd($vmid, "nbd-server-start", addr => {
5976 type => 'inet',
5977 data => {
5978 host => "${localip}",
5979 port => "${storage_migrate_port}",
5980 },
5981 });
5982 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5983 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5984 }
5985
5986 my $block_info = mon_cmd($vmid, "query-block");
5987 $block_info = { map { $_->{device} => $_ } $block_info->@* };
5988
5989 foreach my $opt (sort keys %$nbd) {
5990 my $drivestr = $nbd->{$opt}->{drivestr};
5991 my $volid = $nbd->{$opt}->{volid};
5992
5993 my $block_node = $block_info->{"drive-$opt"}->{inserted}->{'node-name'};
5994
5995 mon_cmd(
5996 $vmid,
5997 "block-export-add",
5998 id => "drive-$opt",
5999 'node-name' => $block_node,
6000 writable => JSON::true,
6001 type => "nbd",
6002 name => "drive-$opt", # NBD export name
6003 );
6004
6005 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
6006 print "storage migration listens on $nbd_uri volume:$drivestr\n";
6007 print "re-using replicated volume: $opt - $volid\n"
6008 if $nbd->{$opt}->{replicated};
6009
6010 $res->{drives}->{$opt} = $nbd->{$opt};
6011 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
6012 }
6013 }
6014
6015 if ($migratedfrom) {
6016 eval {
6017 set_migration_caps($vmid);
6018 };
6019 warn $@ if $@;
6020
6021 if ($spice_port) {
6022 print "spice listens on port $spice_port\n";
6023 $res->{spice_port} = $spice_port;
6024 if ($migrate_opts->{spice_ticket}) {
6025 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
6026 $migrate_opts->{spice_ticket});
6027 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
6028 }
6029 }
6030
6031 } else {
6032 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6033 if !$statefile && $conf->{balloon};
6034
6035 foreach my $opt (keys %$conf) {
6036 next if $opt !~ m/^net\d+$/;
6037 my $nicconf = parse_net($conf->{$opt});
6038 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6039 }
6040 add_nets_bridge_fdb($conf, $vmid);
6041 }
6042
6043 if (!defined($conf->{balloon}) || $conf->{balloon}) {
6044 eval {
6045 mon_cmd(
6046 $vmid,
6047 'qom-set',
6048 path => "machine/peripheral/balloon0",
6049 property => "guest-stats-polling-interval",
6050 value => 2
6051 );
6052 };
6053 log_warn("could not set polling interval for ballooning - $@") if $@;
6054 }
6055
6056 if ($resume) {
6057 print "Resumed VM, removing state\n";
6058 if (my $vmstate = $conf->{vmstate}) {
6059 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6060 PVE::Storage::vdisk_free($storecfg, $vmstate);
6061 }
6062 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6063 PVE::QemuConfig->write_config($vmid, $conf);
6064 }
6065
6066 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6067
6068 my ($current_machine, $is_deprecated) =
6069 PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
6070 if ($is_deprecated) {
6071 log_warn(
6072 "current machine version '$current_machine' is deprecated - see the documentation and ".
6073 "change to a newer one",
6074 );
6075 }
6076
6077 return $res;
6078}
6079
6080sub vm_commandline {
6081 my ($storecfg, $vmid, $snapname) = @_;
6082
6083 my $conf = PVE::QemuConfig->load_config($vmid);
6084
6085 my ($forcemachine, $forcecpu);
6086 if ($snapname) {
6087 my $snapshot = $conf->{snapshots}->{$snapname};
6088 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6089
6090 # check for machine or CPU overrides in snapshot
6091 $forcemachine = $snapshot->{runningmachine};
6092 $forcecpu = $snapshot->{runningcpu};
6093
6094 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6095
6096 $conf = $snapshot;
6097 }
6098
6099 my $defaults = load_defaults();
6100
6101 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6102
6103 return PVE::Tools::cmd2string($cmd);
6104}
6105
6106sub vm_reset {
6107 my ($vmid, $skiplock) = @_;
6108
6109 PVE::QemuConfig->lock_config($vmid, sub {
6110
6111 my $conf = PVE::QemuConfig->load_config($vmid);
6112
6113 PVE::QemuConfig->check_lock($conf) if !$skiplock;
6114
6115 mon_cmd($vmid, "system_reset");
6116 });
6117}
6118
6119sub get_vm_volumes {
6120 my ($conf) = @_;
6121
6122 my $vollist = [];
6123 foreach_volid($conf, sub {
6124 my ($volid, $attr) = @_;
6125
6126 return if $volid =~ m|^/|;
6127
6128 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6129 return if !$sid;
6130
6131 push @$vollist, $volid;
6132 });
6133
6134 return $vollist;
6135}
6136
6137sub cleanup_pci_devices {
6138 my ($vmid, $conf) = @_;
6139
6140 foreach my $key (keys %$conf) {
6141 next if $key !~ m/^hostpci(\d+)$/;
6142 my $hostpciindex = $1;
6143 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6144 my $d = parse_hostpci($conf->{$key});
6145 if ($d->{mdev}) {
6146 # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6147 # don't want to break ABI just for this two liner
6148 my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6149
6150 # some nvidia vgpu driver versions want to clean the mdevs up themselves, and error
6151 # out when we do it first. so wait for up to 10 seconds and then try it manually
6152 if ($d->{ids}->[0]->[0]->{vendor} =~ m/^(0x)?10de$/ && -e $dev_sysfs_dir) {
6153 my $count = 0;
6154 while (-e $dev_sysfs_dir && $count < 10) {
6155 sleep 1;
6156 $count++;
6157 }
6158 print "waited $count seconds for mediated device driver finishing clean up\n";
6159 }
6160
6161 if (-e $dev_sysfs_dir) {
6162 print "actively clean up mediated device with UUID $uuid\n";
6163 PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1");
6164 }
6165 }
6166 }
6167 PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6168}
6169
6170sub vm_stop_cleanup {
6171 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6172
6173 eval {
6174
6175 if (!$keepActive) {
6176 my $vollist = get_vm_volumes($conf);
6177 PVE::Storage::deactivate_volumes($storecfg, $vollist);
6178
6179 if (my $tpmdrive = $conf->{tpmstate0}) {
6180 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6181 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6182 if ($storeid) {
6183 PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6184 }
6185 }
6186 }
6187
6188 foreach my $ext (qw(mon qmp pid vnc qga)) {
6189 unlink "/var/run/qemu-server/${vmid}.$ext";
6190 }
6191
6192 if ($conf->{ivshmem}) {
6193 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6194 # just delete it for now, VMs which have this already open do not
6195 # are affected, but new VMs will get a separated one. If this
6196 # becomes an issue we either add some sort of ref-counting or just
6197 # add a "don't delete on stop" flag to the ivshmem format.
6198 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6199 }
6200
6201 cleanup_pci_devices($vmid, $conf);
6202
6203 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6204 };
6205 warn $@ if $@; # avoid errors - just warn
6206}
6207
6208# call only in locked context
6209sub _do_vm_stop {
6210 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6211
6212 my $pid = check_running($vmid, $nocheck);
6213 return if !$pid;
6214
6215 my $conf;
6216 if (!$nocheck) {
6217 $conf = PVE::QemuConfig->load_config($vmid);
6218 PVE::QemuConfig->check_lock($conf) if !$skiplock;
6219 if (!defined($timeout) && $shutdown && $conf->{startup}) {
6220 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6221 $timeout = $opts->{down} if $opts->{down};
6222 }
6223 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6224 }
6225
6226 eval {
6227 if ($shutdown) {
6228 if (defined($conf) && get_qga_key($conf, 'enabled')) {
6229 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6230 } else {
6231 mon_cmd($vmid, "system_powerdown");
6232 }
6233 } else {
6234 mon_cmd($vmid, "quit");
6235 }
6236 };
6237 my $err = $@;
6238
6239 if (!$err) {
6240 $timeout = 60 if !defined($timeout);
6241
6242 my $count = 0;
6243 while (($count < $timeout) && check_running($vmid, $nocheck)) {
6244 $count++;
6245 sleep 1;
6246 }
6247
6248 if ($count >= $timeout) {
6249 if ($force) {
6250 warn "VM still running - terminating now with SIGTERM\n";
6251 kill 15, $pid;
6252 } else {
6253 die "VM quit/powerdown failed - got timeout\n";
6254 }
6255 } else {
6256 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6257 return;
6258 }
6259 } else {
6260 if (!check_running($vmid, $nocheck)) {
6261 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6262 return;
6263 }
6264 if ($force) {
6265 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6266 kill 15, $pid;
6267 } else {
6268 die "VM quit/powerdown failed\n";
6269 }
6270 }
6271
6272 # wait again
6273 $timeout = 10;
6274
6275 my $count = 0;
6276 while (($count < $timeout) && check_running($vmid, $nocheck)) {
6277 $count++;
6278 sleep 1;
6279 }
6280
6281 if ($count >= $timeout) {
6282 warn "VM still running - terminating now with SIGKILL\n";
6283 kill 9, $pid;
6284 sleep 1;
6285 }
6286
6287 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6288}
6289
6290# Note: use $nocheck to skip tests if VM configuration file exists.
6291# We need that when migration VMs to other nodes (files already moved)
6292# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6293sub vm_stop {
6294 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6295
6296 $force = 1 if !defined($force) && !$shutdown;
6297
6298 if ($migratedfrom){
6299 my $pid = check_running($vmid, $nocheck, $migratedfrom);
6300 kill 15, $pid if $pid;
6301 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6302 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6303 return;
6304 }
6305
6306 PVE::QemuConfig->lock_config($vmid, sub {
6307 _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6308 });
6309}
6310
6311sub vm_reboot {
6312 my ($vmid, $timeout) = @_;
6313
6314 PVE::QemuConfig->lock_config($vmid, sub {
6315 eval {
6316
6317 # only reboot if running, as qmeventd starts it again on a stop event
6318 return if !check_running($vmid);
6319
6320 create_reboot_request($vmid);
6321
6322 my $storecfg = PVE::Storage::config();
6323 _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6324
6325 };
6326 if (my $err = $@) {
6327 # avoid that the next normal shutdown will be confused for a reboot
6328 clear_reboot_request($vmid);
6329 die $err;
6330 }
6331 });
6332}
6333
6334# note: if using the statestorage parameter, the caller has to check privileges
6335sub vm_suspend {
6336 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6337
6338 my $conf;
6339 my $path;
6340 my $storecfg;
6341 my $vmstate;
6342
6343 PVE::QemuConfig->lock_config($vmid, sub {
6344
6345 $conf = PVE::QemuConfig->load_config($vmid);
6346
6347 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6348 PVE::QemuConfig->check_lock($conf)
6349 if !($skiplock || $is_backing_up);
6350
6351 die "cannot suspend to disk during backup\n"
6352 if $is_backing_up && $includestate;
6353
6354 if ($includestate) {
6355 $conf->{lock} = 'suspending';
6356 my $date = strftime("%Y-%m-%d", localtime(time()));
6357 $storecfg = PVE::Storage::config();
6358 if (!$statestorage) {
6359 $statestorage = find_vmstate_storage($conf, $storecfg);
6360 # check permissions for the storage
6361 my $rpcenv = PVE::RPCEnvironment::get();
6362 if ($rpcenv->{type} ne 'cli') {
6363 my $authuser = $rpcenv->get_user();
6364 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6365 }
6366 }
6367
6368
6369 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6370 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6371 $path = PVE::Storage::path($storecfg, $vmstate);
6372 PVE::QemuConfig->write_config($vmid, $conf);
6373 } else {
6374 mon_cmd($vmid, "stop");
6375 }
6376 });
6377
6378 if ($includestate) {
6379 # save vm state
6380 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6381
6382 eval {
6383 set_migration_caps($vmid, 1);
6384 mon_cmd($vmid, "savevm-start", statefile => $path);
6385 for(;;) {
6386 my $state = mon_cmd($vmid, "query-savevm");
6387 if (!$state->{status}) {
6388 die "savevm not active\n";
6389 } elsif ($state->{status} eq 'active') {
6390 sleep(1);
6391 next;
6392 } elsif ($state->{status} eq 'completed') {
6393 print "State saved, quitting\n";
6394 last;
6395 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6396 die "query-savevm failed with error '$state->{error}'\n"
6397 } else {
6398 die "query-savevm returned status '$state->{status}'\n";
6399 }
6400 }
6401 };
6402 my $err = $@;
6403
6404 PVE::QemuConfig->lock_config($vmid, sub {
6405 $conf = PVE::QemuConfig->load_config($vmid);
6406 if ($err) {
6407 # cleanup, but leave suspending lock, to indicate something went wrong
6408 eval {
6409 eval { mon_cmd($vmid, "savevm-end"); };
6410 warn $@ if $@;
6411 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6412 PVE::Storage::vdisk_free($storecfg, $vmstate);
6413 delete $conf->@{qw(vmstate runningmachine runningcpu)};
6414 PVE::QemuConfig->write_config($vmid, $conf);
6415 };
6416 warn $@ if $@;
6417 die $err;
6418 }
6419
6420 die "lock changed unexpectedly\n"
6421 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6422
6423 mon_cmd($vmid, "quit");
6424 $conf->{lock} = 'suspended';
6425 PVE::QemuConfig->write_config($vmid, $conf);
6426 });
6427 }
6428}
6429
6430# $nocheck is set when called as part of a migration - in this context the
6431# location of the config file (source or target node) is not deterministic,
6432# since migration cannot wait for pmxcfs to process the rename
6433sub vm_resume {
6434 my ($vmid, $skiplock, $nocheck) = @_;
6435
6436 PVE::QemuConfig->lock_config($vmid, sub {
6437 my $res = mon_cmd($vmid, 'query-status');
6438 my $resume_cmd = 'cont';
6439 my $reset = 0;
6440 my $conf;
6441 if ($nocheck) {
6442 $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6443 if ($@) {
6444 my $vmlist = PVE::Cluster::get_vmlist();
6445 if (exists($vmlist->{ids}->{$vmid})) {
6446 my $node = $vmlist->{ids}->{$vmid}->{node};
6447 $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6448 }
6449 if (!$conf) {
6450 PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6451 $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6452 }
6453 }
6454 } else {
6455 $conf = PVE::QemuConfig->load_config($vmid);
6456 }
6457
6458 if ($res->{status}) {
6459 return if $res->{status} eq 'running'; # job done, go home
6460 $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6461 $reset = 1 if $res->{status} eq 'shutdown';
6462 }
6463
6464 if (!$nocheck) {
6465 PVE::QemuConfig->check_lock($conf)
6466 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6467 }
6468
6469 if ($reset) {
6470 # required if a VM shuts down during a backup and we get a resume
6471 # request before the backup finishes for example
6472 mon_cmd($vmid, "system_reset");
6473 }
6474
6475 add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6476
6477 mon_cmd($vmid, $resume_cmd);
6478 });
6479}
6480
6481sub vm_sendkey {
6482 my ($vmid, $skiplock, $key) = @_;
6483
6484 PVE::QemuConfig->lock_config($vmid, sub {
6485
6486 my $conf = PVE::QemuConfig->load_config($vmid);
6487
6488 # there is no qmp command, so we use the human monitor command
6489 my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6490 die $res if $res ne '';
6491 });
6492}
6493
6494sub check_bridge_access {
6495 my ($rpcenv, $authuser, $conf) = @_;
6496
6497 return 1 if $authuser eq 'root@pam';
6498
6499 for my $opt (sort keys $conf->%*) {
6500 next if $opt !~ m/^net\d+$/;
6501 my $net = parse_net($conf->{$opt});
6502 my ($bridge, $tag, $trunks) = $net->@{'bridge', 'tag', 'trunks'};
6503 PVE::GuestHelpers::check_vnet_access($rpcenv, $authuser, $bridge, $tag, $trunks);
6504 }
6505 return 1;
6506};
6507
6508sub check_mapping_access {
6509 my ($rpcenv, $user, $conf) = @_;
6510
6511 for my $opt (keys $conf->%*) {
6512 if ($opt =~ m/^usb\d+$/) {
6513 my $device = PVE::JSONSchema::parse_property_string('pve-qm-usb', $conf->{$opt});
6514 if (my $host = $device->{host}) {
6515 die "only root can set '$opt' config for real devices\n"
6516 if $host !~ m/^spice$/i && $user ne 'root@pam';
6517 } elsif ($device->{mapping}) {
6518 $rpcenv->check_full($user, "/mapping/usb/$device->{mapping}", ['Mapping.Use']);
6519 } else {
6520 die "either 'host' or 'mapping' must be set.\n";
6521 }
6522 } elsif ($opt =~ m/^hostpci\d+$/) {
6523 my $device = PVE::JSONSchema::parse_property_string('pve-qm-hostpci', $conf->{$opt});
6524 if ($device->{host}) {
6525 die "only root can set '$opt' config for non-mapped devices\n" if $user ne 'root@pam';
6526 } elsif ($device->{mapping}) {
6527 $rpcenv->check_full($user, "/mapping/pci/$device->{mapping}", ['Mapping.Use']);
6528 } else {
6529 die "either 'host' or 'mapping' must be set.\n";
6530 }
6531 }
6532 }
6533};
6534
6535sub check_restore_permissions {
6536 my ($rpcenv, $user, $conf) = @_;
6537
6538 check_bridge_access($rpcenv, $user, $conf);
6539 check_mapping_access($rpcenv, $user, $conf);
6540}
6541# vzdump restore implementaion
6542
6543sub tar_archive_read_firstfile {
6544 my $archive = shift;
6545
6546 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6547
6548 # try to detect archive type first
6549 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6550 die "unable to open file '$archive'\n";
6551 my $firstfile = <$fh>;
6552 kill 15, $pid;
6553 close $fh;
6554
6555 die "ERROR: archive contaions no data\n" if !$firstfile;
6556 chomp $firstfile;
6557
6558 return $firstfile;
6559}
6560
6561sub tar_restore_cleanup {
6562 my ($storecfg, $statfile) = @_;
6563
6564 print STDERR "starting cleanup\n";
6565
6566 if (my $fd = IO::File->new($statfile, "r")) {
6567 while (defined(my $line = <$fd>)) {
6568 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6569 my $volid = $2;
6570 eval {
6571 if ($volid =~ m|^/|) {
6572 unlink $volid || die 'unlink failed\n';
6573 } else {
6574 PVE::Storage::vdisk_free($storecfg, $volid);
6575 }
6576 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6577 };
6578 print STDERR "unable to cleanup '$volid' - $@" if $@;
6579 } else {
6580 print STDERR "unable to parse line in statfile - $line";
6581 }
6582 }
6583 $fd->close();
6584 }
6585}
6586
6587sub restore_file_archive {
6588 my ($archive, $vmid, $user, $opts) = @_;
6589
6590 return restore_vma_archive($archive, $vmid, $user, $opts)
6591 if $archive eq '-';
6592
6593 my $info = PVE::Storage::archive_info($archive);
6594 my $format = $opts->{format} // $info->{format};
6595 my $comp = $info->{compression};
6596
6597 # try to detect archive format
6598 if ($format eq 'tar') {
6599 return restore_tar_archive($archive, $vmid, $user, $opts);
6600 } else {
6601 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6602 }
6603}
6604
6605# hepler to remove disks that will not be used after restore
6606my $restore_cleanup_oldconf = sub {
6607 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6608
6609 my $kept_disks = {};
6610
6611 PVE::QemuConfig->foreach_volume($oldconf, sub {
6612 my ($ds, $drive) = @_;
6613
6614 return if drive_is_cdrom($drive, 1);
6615
6616 my $volid = $drive->{file};
6617 return if !$volid || $volid =~ m|^/|;
6618
6619 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6620 return if !$path || !$owner || ($owner != $vmid);
6621
6622 # Note: only delete disk we want to restore
6623 # other volumes will become unused
6624 if ($virtdev_hash->{$ds}) {
6625 eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6626 if (my $err = $@) {
6627 warn $err;
6628 }
6629 } else {
6630 $kept_disks->{$volid} = 1;
6631 }
6632 });
6633
6634 # after the restore we have no snapshots anymore
6635 for my $snapname (keys $oldconf->{snapshots}->%*) {
6636 my $snap = $oldconf->{snapshots}->{$snapname};
6637 if ($snap->{vmstate}) {
6638 eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6639 if (my $err = $@) {
6640 warn $err;
6641 }
6642 }
6643
6644 for my $volid (keys $kept_disks->%*) {
6645 eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6646 warn $@ if $@;
6647 }
6648 }
6649};
6650
6651# Helper to parse vzdump backup device hints
6652#
6653# $rpcenv: Environment, used to ckeck storage permissions
6654# $user: User ID, to check storage permissions
6655# $storecfg: Storage configuration
6656# $fh: the file handle for reading the configuration
6657# $devinfo: should contain device sizes for all backu-up'ed devices
6658# $options: backup options (pool, default storage)
6659#
6660# Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6661my $parse_backup_hints = sub {
6662 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6663
6664 my $check_storage = sub { # assert if an image can be allocate
6665 my ($storeid, $scfg) = @_;
6666 die "Content type 'images' is not available on storage '$storeid'\n"
6667 if !$scfg->{content}->{images};
6668 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6669 if $user ne 'root@pam';
6670 };
6671
6672 my $virtdev_hash = {};
6673 while (defined(my $line = <$fh>)) {
6674 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6675 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6676 die "archive does not contain data for drive '$virtdev'\n"
6677 if !$devinfo->{$devname};
6678
6679 if (defined($options->{storage})) {
6680 $storeid = $options->{storage} || 'local';
6681 } elsif (!$storeid) {
6682 $storeid = 'local';
6683 }
6684 $format = 'raw' if !$format;
6685 $devinfo->{$devname}->{devname} = $devname;
6686 $devinfo->{$devname}->{virtdev} = $virtdev;
6687 $devinfo->{$devname}->{format} = $format;
6688 $devinfo->{$devname}->{storeid} = $storeid;
6689
6690 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6691 $check_storage->($storeid, $scfg); # permission and content type check
6692
6693 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6694 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6695 my $virtdev = $1;
6696 my $drive = parse_drive($virtdev, $2);
6697
6698 if (drive_is_cloudinit($drive)) {
6699 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6700 $storeid = $options->{storage} if defined ($options->{storage});
6701 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6702 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6703
6704 $check_storage->($storeid, $scfg); # permission and content type check
6705
6706 $virtdev_hash->{$virtdev} = {
6707 format => $format,
6708 storeid => $storeid,
6709 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6710 is_cloudinit => 1,
6711 };
6712 }
6713 }
6714 }
6715
6716 return $virtdev_hash;
6717};
6718
6719# Helper to allocate and activate all volumes required for a restore
6720#
6721# $storecfg: Storage configuration
6722# $virtdev_hash: as returned by parse_backup_hints()
6723#
6724# Returns: { $virtdev => $volid }
6725my $restore_allocate_devices = sub {
6726 my ($storecfg, $virtdev_hash, $vmid) = @_;
6727
6728 my $map = {};
6729 foreach my $virtdev (sort keys %$virtdev_hash) {
6730 my $d = $virtdev_hash->{$virtdev};
6731 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6732 my $storeid = $d->{storeid};
6733 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6734
6735 # test if requested format is supported
6736 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6737 my $supported = grep { $_ eq $d->{format} } @$validFormats;
6738 $d->{format} = $defFormat if !$supported;
6739
6740 my $name;
6741 if ($d->{is_cloudinit}) {
6742 $name = "vm-$vmid-cloudinit";
6743 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6744 if ($scfg->{path}) {
6745 $name .= ".$d->{format}";
6746 }
6747 }
6748
6749 my $volid = PVE::Storage::vdisk_alloc(
6750 $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6751
6752 print STDERR "new volume ID is '$volid'\n";
6753 $d->{volid} = $volid;
6754
6755 PVE::Storage::activate_volumes($storecfg, [$volid]);
6756
6757 $map->{$virtdev} = $volid;
6758 }
6759
6760 return $map;
6761};
6762
6763sub restore_update_config_line {
6764 my ($cookie, $map, $line, $unique) = @_;
6765
6766 return '' if $line =~ m/^\#qmdump\#/;
6767 return '' if $line =~ m/^\#vzdump\#/;
6768 return '' if $line =~ m/^lock:/;
6769 return '' if $line =~ m/^unused\d+:/;
6770 return '' if $line =~ m/^parent:/;
6771
6772 my $res = '';
6773
6774 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6775 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6776 # try to convert old 1.X settings
6777 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6778 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6779 my ($model, $macaddr) = split(/\=/, $devconfig);
6780 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6781 my $net = {
6782 model => $model,
6783 bridge => "vmbr$ind",
6784 macaddr => $macaddr,
6785 };
6786 my $netstr = print_net($net);
6787
6788 $res .= "net$cookie->{netcount}: $netstr\n";
6789 $cookie->{netcount}++;
6790 }
6791 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6792 my ($id, $netstr) = ($1, $2);
6793 my $net = parse_net($netstr);
6794 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6795 $netstr = print_net($net);
6796 $res .= "$id: $netstr\n";
6797 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6798 my $virtdev = $1;
6799 my $value = $3;
6800 my $di = parse_drive($virtdev, $value);
6801 if (defined($di->{backup}) && !$di->{backup}) {
6802 $res .= "#$line";
6803 } elsif ($map->{$virtdev}) {
6804 delete $di->{format}; # format can change on restore
6805 $di->{file} = $map->{$virtdev};
6806 $value = print_drive($di);
6807 $res .= "$virtdev: $value\n";
6808 } else {
6809 $res .= $line;
6810 }
6811 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6812 my $vmgenid = $1;
6813 if ($vmgenid ne '0') {
6814 # always generate a new vmgenid if there was a valid one setup
6815 $vmgenid = generate_uuid();
6816 }
6817 $res .= "vmgenid: $vmgenid\n";
6818 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6819 my ($uuid, $uuid_str);
6820 UUID::generate($uuid);
6821 UUID::unparse($uuid, $uuid_str);
6822 my $smbios1 = parse_smbios1($2);
6823 $smbios1->{uuid} = $uuid_str;
6824 $res .= $1.print_smbios1($smbios1)."\n";
6825 } else {
6826 $res .= $line;
6827 }
6828
6829 return $res;
6830}
6831
6832my $restore_deactivate_volumes = sub {
6833 my ($storecfg, $virtdev_hash) = @_;
6834
6835 my $vollist = [];
6836 for my $dev (values $virtdev_hash->%*) {
6837 push $vollist->@*, $dev->{volid} if $dev->{volid};
6838 }
6839
6840 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6841 print STDERR $@ if $@;
6842};
6843
6844my $restore_destroy_volumes = sub {
6845 my ($storecfg, $virtdev_hash) = @_;
6846
6847 for my $dev (values $virtdev_hash->%*) {
6848 my $volid = $dev->{volid} or next;
6849 eval {
6850 PVE::Storage::vdisk_free($storecfg, $volid);
6851 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6852 };
6853 print STDERR "unable to cleanup '$volid' - $@" if $@;
6854 }
6855};
6856
6857sub restore_merge_config {
6858 my ($filename, $backup_conf_raw, $override_conf) = @_;
6859
6860 my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6861 for my $key (keys $override_conf->%*) {
6862 $backup_conf->{$key} = $override_conf->{$key};
6863 }
6864
6865 return $backup_conf;
6866}
6867
6868sub scan_volids {
6869 my ($cfg, $vmid) = @_;
6870
6871 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6872
6873 my $volid_hash = {};
6874 foreach my $storeid (keys %$info) {
6875 foreach my $item (@{$info->{$storeid}}) {
6876 next if !($item->{volid} && $item->{size});
6877 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6878 $volid_hash->{$item->{volid}} = $item;
6879 }
6880 }
6881
6882 return $volid_hash;
6883}
6884
6885sub update_disk_config {
6886 my ($vmid, $conf, $volid_hash) = @_;
6887
6888 my $changes;
6889 my $prefix = "VM $vmid";
6890
6891 # used and unused disks
6892 my $referenced = {};
6893
6894 # Note: it is allowed to define multiple storages with same path (alias), so
6895 # we need to check both 'volid' and real 'path' (two different volid can point
6896 # to the same path).
6897
6898 my $referencedpath = {};
6899
6900 # update size info
6901 PVE::QemuConfig->foreach_volume($conf, sub {
6902 my ($opt, $drive) = @_;
6903
6904 my $volid = $drive->{file};
6905 return if !$volid;
6906 my $volume = $volid_hash->{$volid};
6907
6908 # mark volid as "in-use" for next step
6909 $referenced->{$volid} = 1;
6910 if ($volume && (my $path = $volume->{path})) {
6911 $referencedpath->{$path} = 1;
6912 }
6913
6914 return if drive_is_cdrom($drive);
6915 return if !$volume;
6916
6917 my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6918 if (defined($updated)) {
6919 $changes = 1;
6920 $conf->{$opt} = print_drive($updated);
6921 print "$prefix ($opt): $msg\n";
6922 }
6923 });
6924
6925 # remove 'unusedX' entry if volume is used
6926 PVE::QemuConfig->foreach_unused_volume($conf, sub {
6927 my ($opt, $drive) = @_;
6928
6929 my $volid = $drive->{file};
6930 return if !$volid;
6931
6932 my $path;
6933 $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6934 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6935 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6936 $changes = 1;
6937 delete $conf->{$opt};
6938 }
6939
6940 $referenced->{$volid} = 1;
6941 $referencedpath->{$path} = 1 if $path;
6942 });
6943
6944 foreach my $volid (sort keys %$volid_hash) {
6945 next if $volid =~ m/vm-$vmid-state-/;
6946 next if $referenced->{$volid};
6947 my $path = $volid_hash->{$volid}->{path};
6948 next if !$path; # just to be sure
6949 next if $referencedpath->{$path};
6950 $changes = 1;
6951 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6952 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6953 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6954 }
6955
6956 return $changes;
6957}
6958
6959sub rescan {
6960 my ($vmid, $nolock, $dryrun) = @_;
6961
6962 my $cfg = PVE::Storage::config();
6963
6964 print "rescan volumes...\n";
6965 my $volid_hash = scan_volids($cfg, $vmid);
6966
6967 my $updatefn = sub {
6968 my ($vmid) = @_;
6969
6970 my $conf = PVE::QemuConfig->load_config($vmid);
6971
6972 PVE::QemuConfig->check_lock($conf);
6973
6974 my $vm_volids = {};
6975 foreach my $volid (keys %$volid_hash) {
6976 my $info = $volid_hash->{$volid};
6977 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6978 }
6979
6980 my $changes = update_disk_config($vmid, $conf, $vm_volids);
6981
6982 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6983 };
6984
6985 if (defined($vmid)) {
6986 if ($nolock) {
6987 &$updatefn($vmid);
6988 } else {
6989 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6990 }
6991 } else {
6992 my $vmlist = config_list();
6993 foreach my $vmid (keys %$vmlist) {
6994 if ($nolock) {
6995 &$updatefn($vmid);
6996 } else {
6997 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6998 }
6999 }
7000 }
7001}
7002
7003sub restore_proxmox_backup_archive {
7004 my ($archive, $vmid, $user, $options) = @_;
7005
7006 my $storecfg = PVE::Storage::config();
7007
7008 my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
7009 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7010
7011 my $fingerprint = $scfg->{fingerprint};
7012 my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
7013
7014 my $repo = PVE::PBSClient::get_repository($scfg);
7015 my $namespace = $scfg->{namespace};
7016
7017 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
7018 my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
7019 local $ENV{PBS_PASSWORD} = $password;
7020 local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
7021
7022 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
7023 PVE::Storage::parse_volname($storecfg, $archive);
7024
7025 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
7026
7027 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
7028
7029 my $tmpdir = "/var/tmp/vzdumptmp$$";
7030 rmtree $tmpdir;
7031 mkpath $tmpdir;
7032
7033 my $conffile = PVE::QemuConfig->config_file($vmid);
7034 # disable interrupts (always do cleanups)
7035 local $SIG{INT} =
7036 local $SIG{TERM} =
7037 local $SIG{QUIT} =
7038 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7039
7040 # Note: $oldconf is undef if VM does not exists
7041 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7042 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7043 my $new_conf_raw = '';
7044
7045 my $rpcenv = PVE::RPCEnvironment::get();
7046 my $devinfo = {}; # info about drives included in backup
7047 my $virtdev_hash = {}; # info about allocated drives
7048
7049 eval {
7050 # enable interrupts
7051 local $SIG{INT} =
7052 local $SIG{TERM} =
7053 local $SIG{QUIT} =
7054 local $SIG{HUP} =
7055 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7056
7057 my $cfgfn = "$tmpdir/qemu-server.conf";
7058 my $firewall_config_fn = "$tmpdir/fw.conf";
7059 my $index_fn = "$tmpdir/index.json";
7060
7061 my $cmd = "restore";
7062
7063 my $param = [$pbs_backup_name, "index.json", $index_fn];
7064 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7065 my $index = PVE::Tools::file_get_contents($index_fn);
7066 $index = decode_json($index);
7067
7068 foreach my $info (@{$index->{files}}) {
7069 if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
7070 my $devname = $1;
7071 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
7072 $devinfo->{$devname}->{size} = $1;
7073 } else {
7074 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
7075 }
7076 }
7077 }
7078
7079 my $is_qemu_server_backup = scalar(
7080 grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
7081 );
7082 if (!$is_qemu_server_backup) {
7083 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
7084 }
7085 my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
7086
7087 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
7088 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7089
7090 if ($has_firewall_config) {
7091 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
7092 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
7093
7094 my $pve_firewall_dir = '/etc/pve/firewall';
7095 mkdir $pve_firewall_dir; # make sure the dir exists
7096 PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
7097 }
7098
7099 my $fh = IO::File->new($cfgfn, "r") ||
7100 die "unable to read qemu-server.conf - $!\n";
7101
7102 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
7103
7104 # fixme: rate limit?
7105
7106 # create empty/temp config
7107 PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
7108
7109 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
7110
7111 # allocate volumes
7112 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7113
7114 foreach my $virtdev (sort keys %$virtdev_hash) {
7115 my $d = $virtdev_hash->{$virtdev};
7116 next if $d->{is_cloudinit}; # no need to restore cloudinit
7117
7118 # this fails if storage is unavailable
7119 my $volid = $d->{volid};
7120 my $path = PVE::Storage::path($storecfg, $volid);
7121
7122 # for live-restore we only want to preload the efidisk and TPM state
7123 next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7124
7125 my @ns_arg;
7126 if (defined(my $ns = $scfg->{namespace})) {
7127 @ns_arg = ('--ns', $ns);
7128 }
7129
7130 my $pbs_restore_cmd = [
7131 '/usr/bin/pbs-restore',
7132 '--repository', $repo,
7133 @ns_arg,
7134 $pbs_backup_name,
7135 "$d->{devname}.img.fidx",
7136 $path,
7137 '--verbose',
7138 ];
7139
7140 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7141 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7142
7143 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7144 push @$pbs_restore_cmd, '--skip-zero';
7145 }
7146
7147 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7148 print "restore proxmox backup image: $dbg_cmdstring\n";
7149 run_command($pbs_restore_cmd);
7150 }
7151
7152 $fh->seek(0, 0) || die "seek failed - $!\n";
7153
7154 my $cookie = { netcount => 0 };
7155 while (defined(my $line = <$fh>)) {
7156 $new_conf_raw .= restore_update_config_line(
7157 $cookie,
7158 $map,
7159 $line,
7160 $options->{unique},
7161 );
7162 }
7163
7164 $fh->close();
7165 };
7166 my $err = $@;
7167
7168 if ($err || !$options->{live}) {
7169 $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7170 }
7171
7172 rmtree $tmpdir;
7173
7174 if ($err) {
7175 $restore_destroy_volumes->($storecfg, $virtdev_hash);
7176 die $err;
7177 }
7178
7179 if ($options->{live}) {
7180 # keep lock during live-restore
7181 $new_conf_raw .= "\nlock: create";
7182 }
7183
7184 my $new_conf = restore_merge_config($conffile, $new_conf_raw, $options->{override_conf});
7185 check_restore_permissions($rpcenv, $user, $new_conf);
7186 PVE::QemuConfig->write_config($vmid, $new_conf);
7187
7188 eval { rescan($vmid, 1); };
7189 warn $@ if $@;
7190
7191 PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7192
7193 if ($options->{live}) {
7194 # enable interrupts
7195 local $SIG{INT} =
7196 local $SIG{TERM} =
7197 local $SIG{QUIT} =
7198 local $SIG{HUP} =
7199 local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7200
7201 my $conf = PVE::QemuConfig->load_config($vmid);
7202 die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7203
7204 # these special drives are already restored before start
7205 delete $devinfo->{'drive-efidisk0'};
7206 delete $devinfo->{'drive-tpmstate0-backup'};
7207
7208 my $pbs_opts = {
7209 repo => $repo,
7210 keyfile => $keyfile,
7211 snapshot => $pbs_backup_name,
7212 namespace => $namespace,
7213 };
7214 pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7215
7216 PVE::QemuConfig->remove_lock($vmid, "create");
7217 }
7218}
7219
7220sub pbs_live_restore {
7221 my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7222
7223 print "starting VM for live-restore\n";
7224 print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7225
7226 my $live_restore_backing = {};
7227 for my $ds (keys %$restored_disks) {
7228 $ds =~ m/^drive-(.*)$/;
7229 my $confname = $1;
7230 my $pbs_conf = {};
7231 $pbs_conf = {
7232 repository => $opts->{repo},
7233 snapshot => $opts->{snapshot},
7234 archive => "$ds.img.fidx",
7235 };
7236 $pbs_conf->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7237 $pbs_conf->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7238
7239 my $drive = parse_drive($confname, $conf->{$confname});
7240 print "restoring '$ds' to '$drive->{file}'\n";
7241
7242 my $pbs_name = "drive-${confname}-pbs";
7243 $live_restore_backing->{$confname} = {
7244 name => $pbs_name,
7245 blockdev => print_pbs_blockdev($pbs_conf, $pbs_name),
7246 };
7247 }
7248
7249 my $drives_streamed = 0;
7250 eval {
7251 # make sure HA doesn't interrupt our restore by stopping the VM
7252 if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7253 run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7254 }
7255
7256 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7257 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7258 vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'live-restore-backing' => $live_restore_backing}, {});
7259
7260 my $qmeventd_fd = register_qmeventd_handle($vmid);
7261
7262 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7263 # this will effectively collapse the backing image chain consisting of
7264 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7265 # removes itself once all backing images vanish with 'auto-remove=on')
7266 my $jobs = {};
7267 for my $ds (sort keys %$restored_disks) {
7268 my $job_id = "restore-$ds";
7269 mon_cmd($vmid, 'block-stream',
7270 'job-id' => $job_id,
7271 device => "$ds",
7272 );
7273 $jobs->{$job_id} = {};
7274 }
7275
7276 mon_cmd($vmid, 'cont');
7277 qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7278
7279 print "restore-drive jobs finished successfully, removing all tracking block devices"
7280 ." to disconnect from Proxmox Backup Server\n";
7281
7282 for my $ds (sort keys %$restored_disks) {
7283 mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7284 }
7285
7286 close($qmeventd_fd);
7287 };
7288
7289 my $err = $@;
7290
7291 if ($err) {
7292 warn "An error occurred during live-restore: $err\n";
7293 _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7294 die "live-restore failed\n";
7295 }
7296}
7297
7298# Inspired by pbs live-restore, this restores with the disks being available as files.
7299# Theoretically this can also be used to quick-start a full-clone vm if the
7300# disks are all available as files.
7301#
7302# The mapping should provide a path by config entry, such as
7303# `{ scsi0 => { format => <qcow2|raw|...>, path => "/path/to/file", sata1 => ... } }`
7304#
7305# This is used when doing a `create` call with the `--live-import` parameter,
7306# where the disks get an `import-from=` property. The non-live part is
7307# therefore already handled in the `$create_disks()` call happening in the
7308# `create` api call
7309sub live_import_from_files {
7310 my ($mapping, $vmid, $conf, $restore_options) = @_;
7311
7312 my $live_restore_backing = {};
7313 for my $dev (keys %$mapping) {
7314 die "disk not support for live-restoring: '$dev'\n"
7315 if !is_valid_drivename($dev) || $dev =~ /^(?:efidisk|tpmstate)/;
7316
7317 die "mapping contains disk '$dev' which does not exist in the config\n"
7318 if !exists($conf->{$dev});
7319
7320 my $info = $mapping->{$dev};
7321 my ($format, $path) = $info->@{qw(format path)};
7322 die "missing path for '$dev' mapping\n" if !$path;
7323 die "missing format for '$dev' mapping\n" if !$format;
7324 die "invalid format '$format' for '$dev' mapping\n"
7325 if !grep { $format eq $_ } qw(raw qcow2 vmdk);
7326
7327 $live_restore_backing->{$dev} = {
7328 name => "drive-$dev-restore",
7329 blockdev => "driver=$format,node-name=drive-$dev-restore"
7330 . ",read-only=on"
7331 . ",file.driver=file,file.filename=$path"
7332 };
7333 };
7334
7335 my $storecfg = PVE::Storage::config();
7336 eval {
7337
7338 # make sure HA doesn't interrupt our restore by stopping the VM
7339 if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7340 run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
7341 }
7342
7343 vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'live-restore-backing' => $live_restore_backing}, {});
7344
7345 # prevent shutdowns from qmeventd when the VM powers off from the inside
7346 my $qmeventd_fd = register_qmeventd_handle($vmid);
7347
7348 # begin streaming, i.e. data copy from PBS to target disk for every vol,
7349 # this will effectively collapse the backing image chain consisting of
7350 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7351 # removes itself once all backing images vanish with 'auto-remove=on')
7352 my $jobs = {};
7353 for my $ds (sort keys %$live_restore_backing) {
7354 my $job_id = "restore-$ds";
7355 mon_cmd($vmid, 'block-stream',
7356 'job-id' => $job_id,
7357 device => "drive-$ds",
7358 );
7359 $jobs->{$job_id} = {};
7360 }
7361
7362 mon_cmd($vmid, 'cont');
7363 qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7364
7365 print "restore-drive jobs finished successfully, removing all tracking block devices\n";
7366
7367 for my $ds (sort keys %$live_restore_backing) {
7368 mon_cmd($vmid, 'blockdev-del', 'node-name' => "drive-$ds-restore");
7369 }
7370
7371 close($qmeventd_fd);
7372 };
7373
7374 my $err = $@;
7375
7376 if ($err) {
7377 warn "An error occurred during live-restore: $err\n";
7378 _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7379 die "live-restore failed\n";
7380 }
7381
7382 PVE::QemuConfig->remove_lock($vmid, "import");
7383}
7384
7385sub restore_vma_archive {
7386 my ($archive, $vmid, $user, $opts, $comp) = @_;
7387
7388 my $readfrom = $archive;
7389
7390 my $cfg = PVE::Storage::config();
7391 my $commands = [];
7392 my $bwlimit = $opts->{bwlimit};
7393
7394 my $dbg_cmdstring = '';
7395 my $add_pipe = sub {
7396 my ($cmd) = @_;
7397 push @$commands, $cmd;
7398 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7399 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7400 $readfrom = '-';
7401 };
7402
7403 my $input = undef;
7404 if ($archive eq '-') {
7405 $input = '<&STDIN';
7406 } else {
7407 # If we use a backup from a PVE defined storage we also consider that
7408 # storage's rate limit:
7409 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7410 if (defined($volid)) {
7411 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7412 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7413 if ($readlimit) {
7414 print STDERR "applying read rate limit: $readlimit\n";
7415 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7416 $add_pipe->($cstream);
7417 }
7418 }
7419 }
7420
7421 if ($comp) {
7422 my $info = PVE::Storage::decompressor_info('vma', $comp);
7423 my $cmd = $info->{decompressor};
7424 push @$cmd, $readfrom;
7425 $add_pipe->($cmd);
7426 }
7427
7428 my $tmpdir = "/var/tmp/vzdumptmp$$";
7429 rmtree $tmpdir;
7430
7431 # disable interrupts (always do cleanups)
7432 local $SIG{INT} =
7433 local $SIG{TERM} =
7434 local $SIG{QUIT} =
7435 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7436
7437 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7438 POSIX::mkfifo($mapfifo, 0600);
7439 my $fifofh;
7440 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7441
7442 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7443
7444 my $devinfo = {}; # info about drives included in backup
7445 my $virtdev_hash = {}; # info about allocated drives
7446
7447 my $rpcenv = PVE::RPCEnvironment::get();
7448
7449 my $conffile = PVE::QemuConfig->config_file($vmid);
7450
7451 # Note: $oldconf is undef if VM does not exist
7452 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7453 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7454 my $new_conf_raw = '';
7455
7456 my %storage_limits;
7457
7458 my $print_devmap = sub {
7459 my $cfgfn = "$tmpdir/qemu-server.conf";
7460
7461 # we can read the config - that is already extracted
7462 my $fh = IO::File->new($cfgfn, "r") ||
7463 die "unable to read qemu-server.conf - $!\n";
7464
7465 my $fwcfgfn = "$tmpdir/qemu-server.fw";
7466 if (-f $fwcfgfn) {
7467 my $pve_firewall_dir = '/etc/pve/firewall';
7468 mkdir $pve_firewall_dir; # make sure the dir exists
7469 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7470 }
7471
7472 $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7473
7474 foreach my $info (values %{$virtdev_hash}) {
7475 my $storeid = $info->{storeid};
7476 next if defined($storage_limits{$storeid});
7477
7478 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7479 print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7480 $storage_limits{$storeid} = $limit * 1024;
7481 }
7482
7483 foreach my $devname (keys %$devinfo) {
7484 die "found no device mapping information for device '$devname'\n"
7485 if !$devinfo->{$devname}->{virtdev};
7486 }
7487
7488 # create empty/temp config
7489 if ($oldconf) {
7490 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7491 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7492 }
7493
7494 # allocate volumes
7495 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7496
7497 # print restore information to $fifofh
7498 foreach my $virtdev (sort keys %$virtdev_hash) {
7499 my $d = $virtdev_hash->{$virtdev};
7500 next if $d->{is_cloudinit}; # no need to restore cloudinit
7501
7502 my $storeid = $d->{storeid};
7503 my $volid = $d->{volid};
7504
7505 my $map_opts = '';
7506 if (my $limit = $storage_limits{$storeid}) {
7507 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7508 }
7509
7510 my $write_zeros = 1;
7511 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7512 $write_zeros = 0;
7513 }
7514
7515 my $path = PVE::Storage::path($cfg, $volid);
7516
7517 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7518
7519 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7520 }
7521
7522 $fh->seek(0, 0) || die "seek failed - $!\n";
7523
7524 my $cookie = { netcount => 0 };
7525 while (defined(my $line = <$fh>)) {
7526 $new_conf_raw .= restore_update_config_line(
7527 $cookie,
7528 $map,
7529 $line,
7530 $opts->{unique},
7531 );
7532 }
7533
7534 $fh->close();
7535 };
7536
7537 my $oldtimeout;
7538
7539 eval {
7540 # enable interrupts
7541 local $SIG{INT} =
7542 local $SIG{TERM} =
7543 local $SIG{QUIT} =
7544 local $SIG{HUP} =
7545 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7546 local $SIG{ALRM} = sub { die "got timeout\n"; };
7547
7548 $oldtimeout = alarm(5); # for reading the VMA header - might hang with a corrupted one
7549
7550 my $parser = sub {
7551 my $line = shift;
7552
7553 print "$line\n";
7554
7555 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7556 my ($dev_id, $size, $devname) = ($1, $2, $3);
7557 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7558 } elsif ($line =~ m/^CTIME: /) {
7559 # we correctly received the vma config, so we can disable
7560 # the timeout now for disk allocation
7561 alarm($oldtimeout || 0);
7562 $oldtimeout = undef;
7563 &$print_devmap();
7564 print $fifofh "done\n";
7565 close($fifofh);
7566 $fifofh = undef;
7567 }
7568 };
7569
7570 print "restore vma archive: $dbg_cmdstring\n";
7571 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7572 };
7573 my $err = $@;
7574
7575 alarm($oldtimeout) if $oldtimeout;
7576
7577 $restore_deactivate_volumes->($cfg, $virtdev_hash);
7578
7579 close($fifofh) if $fifofh;
7580 unlink $mapfifo;
7581 rmtree $tmpdir;
7582
7583 if ($err) {
7584 $restore_destroy_volumes->($cfg, $virtdev_hash);
7585 die $err;
7586 }
7587
7588 my $new_conf = restore_merge_config($conffile, $new_conf_raw, $opts->{override_conf});
7589 check_restore_permissions($rpcenv, $user, $new_conf);
7590 PVE::QemuConfig->write_config($vmid, $new_conf);
7591
7592 eval { rescan($vmid, 1); };
7593 warn $@ if $@;
7594
7595 PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7596}
7597
7598sub restore_tar_archive {
7599 my ($archive, $vmid, $user, $opts) = @_;
7600
7601 if (scalar(keys $opts->{override_conf}->%*) > 0) {
7602 my $keystring = join(' ', keys $opts->{override_conf}->%*);
7603 die "cannot pass along options ($keystring) when restoring from tar archive\n";
7604 }
7605
7606 if ($archive ne '-') {
7607 my $firstfile = tar_archive_read_firstfile($archive);
7608 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7609 if $firstfile ne 'qemu-server.conf';
7610 }
7611
7612 my $storecfg = PVE::Storage::config();
7613
7614 # avoid zombie disks when restoring over an existing VM -> cleanup first
7615 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7616 # skiplock=1 because qmrestore has set the 'create' lock itself already
7617 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7618 destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7619
7620 my $tocmd = "/usr/lib/qemu-server/qmextract";
7621
7622 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7623 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7624 $tocmd .= ' --prealloc' if $opts->{prealloc};
7625 $tocmd .= ' --info' if $opts->{info};
7626
7627 # tar option "xf" does not autodetect compression when read from STDIN,
7628 # so we pipe to zcat
7629 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7630 PVE::Tools::shellquote("--to-command=$tocmd");
7631
7632 my $tmpdir = "/var/tmp/vzdumptmp$$";
7633 mkpath $tmpdir;
7634
7635 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7636 local $ENV{VZDUMP_VMID} = $vmid;
7637 local $ENV{VZDUMP_USER} = $user;
7638
7639 my $conffile = PVE::QemuConfig->config_file($vmid);
7640 my $new_conf_raw = '';
7641
7642 # disable interrupts (always do cleanups)
7643 local $SIG{INT} =
7644 local $SIG{TERM} =
7645 local $SIG{QUIT} =
7646 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7647
7648 eval {
7649 # enable interrupts
7650 local $SIG{INT} =
7651 local $SIG{TERM} =
7652 local $SIG{QUIT} =
7653 local $SIG{HUP} =
7654 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7655
7656 if ($archive eq '-') {
7657 print "extracting archive from STDIN\n";
7658 run_command($cmd, input => "<&STDIN");
7659 } else {
7660 print "extracting archive '$archive'\n";
7661 run_command($cmd);
7662 }
7663
7664 return if $opts->{info};
7665
7666 # read new mapping
7667 my $map = {};
7668 my $statfile = "$tmpdir/qmrestore.stat";
7669 if (my $fd = IO::File->new($statfile, "r")) {
7670 while (defined (my $line = <$fd>)) {
7671 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7672 $map->{$1} = $2 if $1;
7673 } else {
7674 print STDERR "unable to parse line in statfile - $line\n";
7675 }
7676 }
7677 $fd->close();
7678 }
7679
7680 my $confsrc = "$tmpdir/qemu-server.conf";
7681
7682 my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7683
7684 my $cookie = { netcount => 0 };
7685 while (defined (my $line = <$srcfd>)) {
7686 $new_conf_raw .= restore_update_config_line(
7687 $cookie,
7688 $map,
7689 $line,
7690 $opts->{unique},
7691 );
7692 }
7693
7694 $srcfd->close();
7695 };
7696 if (my $err = $@) {
7697 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7698 die $err;
7699 }
7700
7701 rmtree $tmpdir;
7702
7703 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7704
7705 PVE::Cluster::cfs_update(); # make sure we read new file
7706
7707 eval { rescan($vmid, 1); };
7708 warn $@ if $@;
7709};
7710
7711sub foreach_storage_used_by_vm {
7712 my ($conf, $func) = @_;
7713
7714 my $sidhash = {};
7715
7716 PVE::QemuConfig->foreach_volume($conf, sub {
7717 my ($ds, $drive) = @_;
7718 return if drive_is_cdrom($drive);
7719
7720 my $volid = $drive->{file};
7721
7722 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7723 $sidhash->{$sid} = $sid if $sid;
7724 });
7725
7726 foreach my $sid (sort keys %$sidhash) {
7727 &$func($sid);
7728 }
7729}
7730
7731my $qemu_snap_storage = {
7732 rbd => 1,
7733};
7734sub do_snapshots_with_qemu {
7735 my ($storecfg, $volid, $deviceid) = @_;
7736
7737 return if $deviceid =~ m/tpmstate0/;
7738
7739 my $storage_name = PVE::Storage::parse_volume_id($volid);
7740 my $scfg = $storecfg->{ids}->{$storage_name};
7741 die "could not find storage '$storage_name'\n" if !defined($scfg);
7742
7743 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7744 return 1;
7745 }
7746
7747 if ($volid =~ m/\.(qcow2|qed)$/){
7748 return 1;
7749 }
7750
7751 return;
7752}
7753
7754sub qga_check_running {
7755 my ($vmid, $nowarn) = @_;
7756
7757 eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7758 if ($@) {
7759 warn "QEMU Guest Agent is not running - $@" if !$nowarn;
7760 return 0;
7761 }
7762 return 1;
7763}
7764
7765sub template_create {
7766 my ($vmid, $conf, $disk) = @_;
7767
7768 my $storecfg = PVE::Storage::config();
7769
7770 PVE::QemuConfig->foreach_volume($conf, sub {
7771 my ($ds, $drive) = @_;
7772
7773 return if drive_is_cdrom($drive);
7774 return if $disk && $ds ne $disk;
7775
7776 my $volid = $drive->{file};
7777 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7778
7779 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7780 $drive->{file} = $voliddst;
7781 $conf->{$ds} = print_drive($drive);
7782 PVE::QemuConfig->write_config($vmid, $conf);
7783 });
7784}
7785
7786sub convert_iscsi_path {
7787 my ($path) = @_;
7788
7789 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7790 my $portal = $1;
7791 my $target = $2;
7792 my $lun = $3;
7793
7794 my $initiator_name = get_initiator_name();
7795
7796 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7797 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7798 }
7799
7800 die "cannot convert iscsi path '$path', unkown format\n";
7801}
7802
7803sub qemu_img_convert {
7804 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized, $bwlimit) = @_;
7805
7806 my $storecfg = PVE::Storage::config();
7807 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7808 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7809
7810 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7811
7812 my $cachemode;
7813 my $src_path;
7814 my $src_is_iscsi = 0;
7815 my $src_format;
7816
7817 if ($src_storeid) {
7818 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7819 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7820 $src_format = qemu_img_format($src_scfg, $src_volname);
7821 $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7822 $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7823 $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7824 } elsif (-f $src_volid || -b $src_volid) {
7825 $src_path = $src_volid;
7826 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7827 $src_format = $1;
7828 }
7829 }
7830
7831 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7832
7833 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7834 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7835 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7836 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7837
7838 my $cmd = [];
7839 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7840 push @$cmd, '-l', "snapshot.name=$snapname"
7841 if $snapname && $src_format && $src_format eq "qcow2";
7842 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7843 push @$cmd, '-T', $cachemode if defined($cachemode);
7844 push @$cmd, '-r', "${bwlimit}K" if defined($bwlimit);
7845
7846 if ($src_is_iscsi) {
7847 push @$cmd, '--image-opts';
7848 $src_path = convert_iscsi_path($src_path);
7849 } elsif ($src_format) {
7850 push @$cmd, '-f', $src_format;
7851 }
7852
7853 if ($dst_is_iscsi) {
7854 push @$cmd, '--target-image-opts';
7855 $dst_path = convert_iscsi_path($dst_path);
7856 } else {
7857 push @$cmd, '-O', $dst_format;
7858 }
7859
7860 push @$cmd, $src_path;
7861
7862 if (!$dst_is_iscsi && $is_zero_initialized) {
7863 push @$cmd, "zeroinit:$dst_path";
7864 } else {
7865 push @$cmd, $dst_path;
7866 }
7867
7868 my $parser = sub {
7869 my $line = shift;
7870 if($line =~ m/\((\S+)\/100\%\)/){
7871 my $percent = $1;
7872 my $transferred = int($size * $percent / 100);
7873 my $total_h = render_bytes($size, 1);
7874 my $transferred_h = render_bytes($transferred, 1);
7875
7876 print "transferred $transferred_h of $total_h ($percent%)\n";
7877 }
7878
7879 };
7880
7881 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
7882 my $err = $@;
7883 die "copy failed: $err" if $err;
7884}
7885
7886sub qemu_img_format {
7887 my ($scfg, $volname) = @_;
7888
7889 # FIXME: this entire function is kind of weird given that `parse_volname`
7890 # also already gives us a format?
7891 my $is_path_storage = $scfg->{path} || $scfg->{type} eq 'esxi';
7892
7893 if ($is_path_storage && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7894 return $1;
7895 } else {
7896 return "raw";
7897 }
7898}
7899
7900sub qemu_drive_mirror {
7901 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7902
7903 $jobs = {} if !$jobs;
7904
7905 my $qemu_target;
7906 my $format;
7907 $jobs->{"drive-$drive"} = {};
7908
7909 if ($dst_volid =~ /^nbd:/) {
7910 $qemu_target = $dst_volid;
7911 $format = "nbd";
7912 } else {
7913 my $storecfg = PVE::Storage::config();
7914 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7915
7916 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7917
7918 $format = qemu_img_format($dst_scfg, $dst_volname);
7919
7920 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7921
7922 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7923 }
7924
7925 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7926 $opts->{format} = $format if $format;
7927
7928 if (defined($src_bitmap)) {
7929 $opts->{sync} = 'incremental';
7930 $opts->{bitmap} = $src_bitmap;
7931 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7932 }
7933
7934 if (defined($bwlimit)) {
7935 $opts->{speed} = $bwlimit * 1024;
7936 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7937 } else {
7938 print "drive mirror is starting for drive-$drive\n";
7939 }
7940
7941 # if a job already runs for this device we get an error, catch it for cleanup
7942 eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7943 if (my $err = $@) {
7944 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7945 warn "$@\n" if $@;
7946 die "mirroring error: $err\n";
7947 }
7948
7949 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7950}
7951
7952# $completion can be either
7953# 'complete': wait until all jobs are ready, block-job-complete them (default)
7954# 'cancel': wait until all jobs are ready, block-job-cancel them
7955# 'skip': wait until all jobs are ready, return with block jobs in ready state
7956# 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7957sub qemu_drive_mirror_monitor {
7958 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7959
7960 $completion //= 'complete';
7961 $op //= "mirror";
7962
7963 eval {
7964 my $err_complete = 0;
7965
7966 my $starttime = time ();
7967 while (1) {
7968 die "block job ('$op') timed out\n" if $err_complete > 300;
7969
7970 my $stats = mon_cmd($vmid, "query-block-jobs");
7971 my $ctime = time();
7972
7973 my $running_jobs = {};
7974 for my $stat (@$stats) {
7975 next if $stat->{type} ne $op;
7976 $running_jobs->{$stat->{device}} = $stat;
7977 }
7978
7979 my $readycounter = 0;
7980
7981 for my $job_id (sort keys %$jobs) {
7982 my $job = $running_jobs->{$job_id};
7983
7984 my $vanished = !defined($job);
7985 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7986 if($complete || ($vanished && $completion eq 'auto')) {
7987 print "$job_id: $op-job finished\n";
7988 delete $jobs->{$job_id};
7989 next;
7990 }
7991
7992 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7993
7994 my $busy = $job->{busy};
7995 my $ready = $job->{ready};
7996 if (my $total = $job->{len}) {
7997 my $transferred = $job->{offset} || 0;
7998 my $remaining = $total - $transferred;
7999 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
8000
8001 my $duration = $ctime - $starttime;
8002 my $total_h = render_bytes($total, 1);
8003 my $transferred_h = render_bytes($transferred, 1);
8004
8005 my $status = sprintf(
8006 "transferred $transferred_h of $total_h ($percent%%) in %s",
8007 render_duration($duration),
8008 );
8009
8010 if ($ready) {
8011 if ($busy) {
8012 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
8013 } else {
8014 $status .= ", ready";
8015 }
8016 }
8017 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
8018 $jobs->{$job_id}->{ready} = $ready;
8019 }
8020
8021 $readycounter++ if $job->{ready};
8022 }
8023
8024 last if scalar(keys %$jobs) == 0;
8025
8026 if ($readycounter == scalar(keys %$jobs)) {
8027 print "all '$op' jobs are ready\n";
8028
8029 # do the complete later (or has already been done)
8030 last if $completion eq 'skip' || $completion eq 'auto';
8031
8032 if ($vmiddst && $vmiddst != $vmid) {
8033 my $agent_running = $qga && qga_check_running($vmid);
8034 if ($agent_running) {
8035 print "freeze filesystem\n";
8036 eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
8037 warn $@ if $@;
8038 } else {
8039 print "suspend vm\n";
8040 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
8041 warn $@ if $@;
8042 }
8043
8044 # if we clone a disk for a new target vm, we don't switch the disk
8045 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
8046
8047 if ($agent_running) {
8048 print "unfreeze filesystem\n";
8049 eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
8050 warn $@ if $@;
8051 } else {
8052 print "resume vm\n";
8053 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
8054 warn $@ if $@;
8055 }
8056
8057 last;
8058 } else {
8059
8060 for my $job_id (sort keys %$jobs) {
8061 # try to switch the disk if source and destination are on the same guest
8062 print "$job_id: Completing block job_id...\n";
8063
8064 my $op;
8065 if ($completion eq 'complete') {
8066 $op = 'block-job-complete';
8067 } elsif ($completion eq 'cancel') {
8068 $op = 'block-job-cancel';
8069 } else {
8070 die "invalid completion value: $completion\n";
8071 }
8072 eval { mon_cmd($vmid, $op, device => $job_id) };
8073 if ($@ =~ m/cannot be completed/) {
8074 print "$job_id: block job cannot be completed, trying again.\n";
8075 $err_complete++;
8076 }else {
8077 print "$job_id: Completed successfully.\n";
8078 $jobs->{$job_id}->{complete} = 1;
8079 }
8080 }
8081 }
8082 }
8083 sleep 1;
8084 }
8085 };
8086 my $err = $@;
8087
8088 if ($err) {
8089 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
8090 die "block job ($op) error: $err";
8091 }
8092}
8093
8094sub qemu_blockjobs_cancel {
8095 my ($vmid, $jobs) = @_;
8096
8097 foreach my $job (keys %$jobs) {
8098 print "$job: Cancelling block job\n";
8099 eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
8100 $jobs->{$job}->{cancel} = 1;
8101 }
8102
8103 while (1) {
8104 my $stats = mon_cmd($vmid, "query-block-jobs");
8105
8106 my $running_jobs = {};
8107 foreach my $stat (@$stats) {
8108 $running_jobs->{$stat->{device}} = $stat;
8109 }
8110
8111 foreach my $job (keys %$jobs) {
8112
8113 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
8114 print "$job: Done.\n";
8115 delete $jobs->{$job};
8116 }
8117 }
8118
8119 last if scalar(keys %$jobs) == 0;
8120
8121 sleep 1;
8122 }
8123}
8124
8125# Check for bug #4525: drive-mirror will open the target drive with the same aio setting as the
8126# source, but some storages have problems with io_uring, sometimes even leading to crashes.
8127my sub clone_disk_check_io_uring {
8128 my ($src_drive, $storecfg, $src_storeid, $dst_storeid, $use_drive_mirror) = @_;
8129
8130 return if !$use_drive_mirror;
8131
8132 # Don't complain when not changing storage.
8133 # Assume if it works for the source, it'll work for the target too.
8134 return if $src_storeid eq $dst_storeid;
8135
8136 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
8137 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
8138
8139 my $cache_direct = drive_uses_cache_direct($src_drive);
8140
8141 my $src_uses_io_uring;
8142 if ($src_drive->{aio}) {
8143 $src_uses_io_uring = $src_drive->{aio} eq 'io_uring';
8144 } else {
8145 $src_uses_io_uring = storage_allows_io_uring_default($src_scfg, $cache_direct);
8146 }
8147
8148 die "target storage is known to cause issues with aio=io_uring (used by current drive)\n"
8149 if $src_uses_io_uring && !storage_allows_io_uring_default($dst_scfg, $cache_direct);
8150}
8151
8152sub clone_disk {
8153 my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
8154
8155 my ($vmid, $running) = $source->@{qw(vmid running)};
8156 my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
8157
8158 my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
8159 my ($storage, $format) = $dest->@{qw(storage format)};
8160
8161 my $unused = defined($src_drivename) && $src_drivename =~ /^unused/;
8162 my $use_drive_mirror = $full && $running && $src_drivename && !$snapname && !$unused;
8163
8164 if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
8165 die "cloning from/to EFI disk requires EFI disk\n"
8166 if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
8167 die "cloning from/to TPM state requires TPM state\n"
8168 if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
8169
8170 # This would lead to two device nodes in QEMU pointing to the same backing image!
8171 die "cannot change drive name when cloning disk from/to the same VM\n"
8172 if $use_drive_mirror && $vmid == $newvmid;
8173 }
8174
8175 die "cannot move TPM state while VM is running\n"
8176 if $use_drive_mirror && $src_drivename eq 'tpmstate0';
8177
8178 my $newvolid;
8179
8180 print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
8181 print "$src_drivename " if $src_drivename;
8182 print "($drive->{file})\n";
8183
8184 if (!$full) {
8185 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
8186 push @$newvollist, $newvolid;
8187 } else {
8188 my ($src_storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
8189 my $storeid = $storage || $src_storeid;
8190
8191 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
8192
8193 my $name = undef;
8194 my $size = undef;
8195 if (drive_is_cloudinit($drive)) {
8196 $name = "vm-$newvmid-cloudinit";
8197 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8198 if ($scfg->{path}) {
8199 $name .= ".$dst_format";
8200 }
8201 $snapname = undef;
8202 $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
8203 } elsif ($dst_drivename eq 'efidisk0') {
8204 $size = $efisize or die "internal error - need to specify EFI disk size\n";
8205 } elsif ($dst_drivename eq 'tpmstate0') {
8206 $dst_format = 'raw';
8207 $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8208 } else {
8209 clone_disk_check_io_uring($drive, $storecfg, $src_storeid, $storeid, $use_drive_mirror);
8210
8211 $size = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
8212 }
8213 $newvolid = PVE::Storage::vdisk_alloc(
8214 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
8215 );
8216 push @$newvollist, $newvolid;
8217
8218 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
8219
8220 if (drive_is_cloudinit($drive)) {
8221 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
8222 # if this is the case, we have to complete any block-jobs still there from
8223 # previous drive-mirrors
8224 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
8225 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
8226 }
8227 goto no_data_clone;
8228 }
8229
8230 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
8231 if ($use_drive_mirror) {
8232 qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
8233 $completion, $qga, $bwlimit);
8234 } else {
8235 if ($dst_drivename eq 'efidisk0') {
8236 # the relevant data on the efidisk may be smaller than the source
8237 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8238 # that is given by the OVMF_VARS.fd
8239 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8240 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8241
8242 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8243
8244 # better for Ceph if block size is not too small, see bug #3324
8245 my $bs = 1024*1024;
8246
8247 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8248
8249 if ($src_format eq 'qcow2' && $snapname) {
8250 die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8251 if !min_version(kvm_user_version(), 6, 2);
8252 push $cmd->@*, '-l', $snapname;
8253 }
8254 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8255 run_command($cmd);
8256 } else {
8257 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit, $bwlimit);
8258 }
8259 }
8260 }
8261
8262no_data_clone:
8263 my $size = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8264
8265 my $disk = dclone($drive);
8266 delete $disk->{format};
8267 $disk->{file} = $newvolid;
8268 $disk->{size} = $size if defined($size) && !$unused;
8269
8270 return $disk;
8271}
8272
8273sub get_running_qemu_version {
8274 my ($vmid) = @_;
8275 my $res = mon_cmd($vmid, "query-version");
8276 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8277}
8278
8279sub qemu_use_old_bios_files {
8280 my ($machine_type) = @_;
8281
8282 return if !$machine_type;
8283
8284 my $use_old_bios_files = undef;
8285
8286 if ($machine_type =~ m/^(\S+)\.pxe$/) {
8287 $machine_type = $1;
8288 $use_old_bios_files = 1;
8289 } else {
8290 my $version = extract_version($machine_type, kvm_user_version());
8291 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8292 # load new efi bios files on migration. So this hack is required to allow
8293 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8294 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8295 $use_old_bios_files = !min_version($version, 2, 4);
8296 }
8297
8298 return ($use_old_bios_files, $machine_type);
8299}
8300
8301sub get_efivars_size {
8302 my ($conf, $efidisk) = @_;
8303
8304 my $arch = get_vm_arch($conf);
8305 $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8306 my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8307 my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8308 return -s $ovmf_vars;
8309}
8310
8311sub update_efidisk_size {
8312 my ($conf) = @_;
8313
8314 return if !defined($conf->{efidisk0});
8315
8316 my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8317 $disk->{size} = get_efivars_size($conf);
8318 $conf->{efidisk0} = print_drive($disk);
8319
8320 return;
8321}
8322
8323sub update_tpmstate_size {
8324 my ($conf) = @_;
8325
8326 my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8327 $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8328 $conf->{tpmstate0} = print_drive($disk);
8329}
8330
8331sub create_efidisk($$$$$$$) {
8332 my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8333
8334 my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8335
8336 my $vars_size_b = -s $ovmf_vars;
8337 my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8338 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8339 PVE::Storage::activate_volumes($storecfg, [$volid]);
8340
8341 qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8342 my $size = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8343
8344 return ($volid, $size/1024);
8345}
8346
8347sub vm_iothreads_list {
8348 my ($vmid) = @_;
8349
8350 my $res = mon_cmd($vmid, 'query-iothreads');
8351
8352 my $iothreads = {};
8353 foreach my $iothread (@$res) {
8354 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8355 }
8356
8357 return $iothreads;
8358}
8359
8360sub scsihw_infos {
8361 my ($conf, $drive) = @_;
8362
8363 my $maxdev = 0;
8364
8365 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8366 $maxdev = 7;
8367 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8368 $maxdev = 1;
8369 } else {
8370 $maxdev = 256;
8371 }
8372
8373 my $controller = int($drive->{index} / $maxdev);
8374 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8375 ? "virtioscsi"
8376 : "scsihw";
8377
8378 return ($maxdev, $controller, $controller_prefix);
8379}
8380
8381sub resolve_dst_disk_format {
8382 my ($storecfg, $storeid, $src_volname, $format) = @_;
8383 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8384
8385 if (!$format) {
8386 # if no target format is specified, use the source disk format as hint
8387 if ($src_volname) {
8388 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8389 $format = qemu_img_format($scfg, $src_volname);
8390 } else {
8391 return $defFormat;
8392 }
8393 }
8394
8395 # test if requested format is supported - else use default
8396 my $supported = grep { $_ eq $format } @$validFormats;
8397 $format = $defFormat if !$supported;
8398 return $format;
8399}
8400
8401# NOTE: if this logic changes, please update docs & possibly gui logic
8402sub find_vmstate_storage {
8403 my ($conf, $storecfg) = @_;
8404
8405 # first, return storage from conf if set
8406 return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8407
8408 my ($target, $shared, $local);
8409
8410 foreach_storage_used_by_vm($conf, sub {
8411 my ($sid) = @_;
8412 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8413 my $dst = $scfg->{shared} ? \$shared : \$local;
8414 $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8415 });
8416
8417 # second, use shared storage where VM has at least one disk
8418 # third, use local storage where VM has at least one disk
8419 # fall back to local storage
8420 $target = $shared // $local // 'local';
8421
8422 return $target;
8423}
8424
8425sub generate_uuid {
8426 my ($uuid, $uuid_str);
8427 UUID::generate($uuid);
8428 UUID::unparse($uuid, $uuid_str);
8429 return $uuid_str;
8430}
8431
8432sub generate_smbios1_uuid {
8433 return "uuid=".generate_uuid();
8434}
8435
8436sub nbd_stop {
8437 my ($vmid) = @_;
8438
8439 mon_cmd($vmid, 'nbd-server-stop', timeout => 25);
8440}
8441
8442sub create_reboot_request {
8443 my ($vmid) = @_;
8444 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8445 or die "failed to create reboot trigger file: $!\n";
8446 close($fh);
8447}
8448
8449sub clear_reboot_request {
8450 my ($vmid) = @_;
8451 my $path = "/run/qemu-server/$vmid.reboot";
8452 my $res = 0;
8453
8454 $res = unlink($path);
8455 die "could not remove reboot request for $vmid: $!"
8456 if !$res && $! != POSIX::ENOENT;
8457
8458 return $res;
8459}
8460
8461sub bootorder_from_legacy {
8462 my ($conf, $bootcfg) = @_;
8463
8464 my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8465 my $bootindex_hash = {};
8466 my $i = 1;
8467 foreach my $o (split(//, $boot)) {
8468 $bootindex_hash->{$o} = $i*100;
8469 $i++;
8470 }
8471
8472 my $bootorder = {};
8473
8474 PVE::QemuConfig->foreach_volume($conf, sub {
8475 my ($ds, $drive) = @_;
8476
8477 if (drive_is_cdrom ($drive, 1)) {
8478 if ($bootindex_hash->{d}) {
8479 $bootorder->{$ds} = $bootindex_hash->{d};
8480 $bootindex_hash->{d} += 1;
8481 }
8482 } elsif ($bootindex_hash->{c}) {
8483 $bootorder->{$ds} = $bootindex_hash->{c}
8484 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8485 $bootindex_hash->{c} += 1;
8486 }
8487 });
8488
8489 if ($bootindex_hash->{n}) {
8490 for (my $i = 0; $i < $MAX_NETS; $i++) {
8491 my $netname = "net$i";
8492 next if !$conf->{$netname};
8493 $bootorder->{$netname} = $bootindex_hash->{n};
8494 $bootindex_hash->{n} += 1;
8495 }
8496 }
8497
8498 return $bootorder;
8499}
8500
8501# Generate default device list for 'boot: order=' property. Matches legacy
8502# default boot order, but with explicit device names. This is important, since
8503# the fallback for when neither 'order' nor the old format is specified relies
8504# on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8505sub get_default_bootdevices {
8506 my ($conf) = @_;
8507
8508 my @ret = ();
8509
8510 # harddisk
8511 my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8512 push @ret, $first if $first;
8513
8514 # cdrom
8515 $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8516 push @ret, $first if $first;
8517
8518 # network
8519 for (my $i = 0; $i < $MAX_NETS; $i++) {
8520 my $netname = "net$i";
8521 next if !$conf->{$netname};
8522 push @ret, $netname;
8523 last;
8524 }
8525
8526 return \@ret;
8527}
8528
8529sub device_bootorder {
8530 my ($conf) = @_;
8531
8532 return bootorder_from_legacy($conf) if !defined($conf->{boot});
8533
8534 my $boot = parse_property_string($boot_fmt, $conf->{boot});
8535
8536 my $bootorder = {};
8537 if (!defined($boot) || $boot->{legacy}) {
8538 $bootorder = bootorder_from_legacy($conf, $boot);
8539 } elsif ($boot->{order}) {
8540 my $i = 100; # start at 100 to allow user to insert devices before us with -args
8541 for my $dev (PVE::Tools::split_list($boot->{order})) {
8542 $bootorder->{$dev} = $i++;
8543 }
8544 }
8545
8546 return $bootorder;
8547}
8548
8549sub register_qmeventd_handle {
8550 my ($vmid) = @_;
8551
8552 my $fh;
8553 my $peer = "/var/run/qmeventd.sock";
8554 my $count = 0;
8555
8556 for (;;) {
8557 $count++;
8558 $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8559 last if $fh;
8560 if ($! != EINTR && $! != EAGAIN) {
8561 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8562 }
8563 if ($count > 4) {
8564 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8565 . "after $count retries\n";
8566 }
8567 usleep(25000);
8568 }
8569
8570 # send handshake to mark VM as backing up
8571 print $fh to_json({vzdump => {vmid => "$vmid"}});
8572
8573 # return handle to be closed later when inhibit is no longer required
8574 return $fh;
8575}
8576
8577# bash completion helper
8578
8579sub complete_backup_archives {
8580 my ($cmdname, $pname, $cvalue) = @_;
8581
8582 my $cfg = PVE::Storage::config();
8583
8584 my $storeid;
8585
8586 if ($cvalue =~ m/^([^:]+):/) {
8587 $storeid = $1;
8588 }
8589
8590 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8591
8592 my $res = [];
8593 foreach my $id (keys %$data) {
8594 foreach my $item (@{$data->{$id}}) {
8595 next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8596 push @$res, $item->{volid} if defined($item->{volid});
8597 }
8598 }
8599
8600 return $res;
8601}
8602
8603my $complete_vmid_full = sub {
8604 my ($running) = @_;
8605
8606 my $idlist = vmstatus();
8607
8608 my $res = [];
8609
8610 foreach my $id (keys %$idlist) {
8611 my $d = $idlist->{$id};
8612 if (defined($running)) {
8613 next if $d->{template};
8614 next if $running && $d->{status} ne 'running';
8615 next if !$running && $d->{status} eq 'running';
8616 }
8617 push @$res, $id;
8618
8619 }
8620 return $res;
8621};
8622
8623sub complete_vmid {
8624 return &$complete_vmid_full();
8625}
8626
8627sub complete_vmid_stopped {
8628 return &$complete_vmid_full(0);
8629}
8630
8631sub complete_vmid_running {
8632 return &$complete_vmid_full(1);
8633}
8634
8635sub complete_storage {
8636
8637 my $cfg = PVE::Storage::config();
8638 my $ids = $cfg->{ids};
8639
8640 my $res = [];
8641 foreach my $sid (keys %$ids) {
8642 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8643 next if !$ids->{$sid}->{content}->{images};
8644 push @$res, $sid;
8645 }
8646
8647 return $res;
8648}
8649
8650sub complete_migration_storage {
8651 my ($cmd, $param, $current_value, $all_args) = @_;
8652
8653 my $targetnode = @$all_args[1];
8654
8655 my $cfg = PVE::Storage::config();
8656 my $ids = $cfg->{ids};
8657
8658 my $res = [];
8659 foreach my $sid (keys %$ids) {
8660 next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8661 next if !$ids->{$sid}->{content}->{images};
8662 push @$res, $sid;
8663 }
8664
8665 return $res;
8666}
8667
8668sub vm_is_paused {
8669 my ($vmid, $include_suspended) = @_;
8670 my $qmpstatus = eval {
8671 PVE::QemuConfig::assert_config_exists_on_node($vmid);
8672 mon_cmd($vmid, "query-status");
8673 };
8674 warn "$@\n" if $@;
8675 return $qmpstatus && (
8676 $qmpstatus->{status} eq "paused" ||
8677 $qmpstatus->{status} eq "prelaunch" ||
8678 ($include_suspended && $qmpstatus->{status} eq "suspended")
8679 );
8680}
8681
8682sub check_volume_storage_type {
8683 my ($storecfg, $vol) = @_;
8684
8685 my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8686 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8687 my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8688
8689 die "storage '$storeid' does not support content-type '$vtype'\n"
8690 if !$scfg->{content}->{$vtype};
8691
8692 return 1;
8693}
8694
8695sub add_nets_bridge_fdb {
8696 my ($conf, $vmid) = @_;
8697
8698 for my $opt (keys %$conf) {
8699 next if $opt !~ m/^net(\d+)$/;
8700 my $iface = "tap${vmid}i$1";
8701 # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8702 my $net = parse_net($conf->{$opt}, 1) or next;
8703
8704 my $mac = $net->{macaddr};
8705 if (!$mac) {
8706 log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8707 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8708 next;
8709 }
8710
8711 my $bridge = $net->{bridge};
8712 if (!$bridge) {
8713 log_warn("Interface '$iface' not attached to any bridge.");
8714 next;
8715 }
8716 if ($have_sdn) {
8717 PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge);
8718 } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8719 PVE::Network::add_bridge_fdb($iface, $mac);
8720 }
8721 }
8722}
8723
8724sub del_nets_bridge_fdb {
8725 my ($conf, $vmid) = @_;
8726
8727 for my $opt (keys %$conf) {
8728 next if $opt !~ m/^net(\d+)$/;
8729 my $iface = "tap${vmid}i$1";
8730
8731 my $net = parse_net($conf->{$opt}) or next;
8732 my $mac = $net->{macaddr} or next;
8733
8734 my $bridge = $net->{bridge};
8735 if ($have_sdn) {
8736 PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge);
8737 } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8738 PVE::Network::del_bridge_fdb($iface, $mac);
8739 }
8740 }
8741}
8742
8743sub create_ifaces_ipams_ips {
8744 my ($conf, $vmid) = @_;
8745
8746 return if !$have_sdn;
8747
8748 foreach my $opt (keys %$conf) {
8749 if ($opt =~ m/^net(\d+)$/) {
8750 my $value = $conf->{$opt};
8751 my $net = PVE::QemuServer::parse_net($value);
8752 eval { PVE::Network::SDN::Vnets::add_next_free_cidr($net->{bridge}, $conf->{name}, $net->{macaddr}, $vmid, undef, 1) };
8753 warn $@ if $@;
8754 }
8755 }
8756}
8757
8758sub delete_ifaces_ipams_ips {
8759 my ($conf, $vmid) = @_;
8760
8761 return if !$have_sdn;
8762
8763 foreach my $opt (keys %$conf) {
8764 if ($opt =~ m/^net(\d+)$/) {
8765 my $net = PVE::QemuServer::parse_net($conf->{$opt});
8766 eval { PVE::Network::SDN::Vnets::del_ips_from_mac($net->{bridge}, $net->{macaddr}, $conf->{name}) };
8767 warn $@ if $@;
8768 }
8769 }
8770}
8771
87721;
Virtual Machine Manager