]> git.proxmox.com Git - qemu-server.git/blame_incremental - PVE/QemuServer.pm
projects / qemu-server.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
bump version to 7.0-9
[qemu-server.git] / PVE / QemuServer.pm
... / ...
CommitLineData
1package PVE::QemuServer;
2
3use strict;
4use warnings;
5
6use Cwd 'abs_path';
7use Digest::SHA;
8use Fcntl ':flock';
9use Fcntl;
10use File::Basename;
11use File::Copy qw(copy);
12use File::Path;
13use File::stat;
14use Getopt::Long;
15use IO::Dir;
16use IO::File;
17use IO::Handle;
18use IO::Select;
19use IO::Socket::UNIX;
20use IPC::Open3;
21use JSON;
22use MIME::Base64;
23use POSIX;
24use Storable qw(dclone);
25use Time::HiRes qw(gettimeofday);
26use URI::Escape;
27use UUID;
28
29use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
30use PVE::CGroup;
31use PVE::DataCenterConfig;
32use PVE::Exception qw(raise raise_param_exc);
33use PVE::Format qw(render_duration render_bytes);
34use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
35use PVE::INotify;
36use PVE::JSONSchema qw(get_standard_option parse_property_string);
37use PVE::ProcFSTools;
38use PVE::PBSClient;
39use PVE::RPCEnvironment;
40use PVE::Storage;
41use PVE::SysFSTools;
42use PVE::Systemd;
43use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44
45use PVE::QMPClient;
46use PVE::QemuConfig;
47use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
48use PVE::QemuServer::Cloudinit;
49use PVE::QemuServer::CGroup;
50use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
51use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
52use PVE::QemuServer::Machine;
53use PVE::QemuServer::Memory;
54use PVE::QemuServer::Monitor qw(mon_cmd);
55use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56use PVE::QemuServer::USB qw(parse_usb_device);
57
58my $have_sdn;
59eval {
60 require PVE::Network::SDN::Zones;
61 $have_sdn = 1;
62};
63
64my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
65my $OVMF = {
66 x86_64 => [
67 "$EDK2_FW_BASE/OVMF_CODE.fd",
68 "$EDK2_FW_BASE/OVMF_VARS.fd"
69 ],
70 aarch64 => [
71 "$EDK2_FW_BASE/AAVMF_CODE.fd",
72 "$EDK2_FW_BASE/AAVMF_VARS.fd"
73 ],
74};
75
76my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
77
78# Note about locking: we use flock on the config file protect
79# against concurent actions.
80# Aditionaly, we have a 'lock' setting in the config file. This
81# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
82# allowed when such lock is set. But you can ignore this kind of
83# lock with the --skiplock flag.
84
85cfs_register_file('/qemu-server/',
86 \&parse_vm_config,
87 \&write_vm_config);
88
89PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
90 description => "Some command save/restore state from this location.",
91 type => 'string',
92 maxLength => 128,
93 optional => 1,
94});
95
96PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
97 description => "Specifies the Qemu machine type.",
98 type => 'string',
99 pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
100 maxLength => 40,
101 optional => 1,
102});
103
104
105sub map_storage {
106 my ($map, $source) = @_;
107
108 return $source if !defined($map);
109
110 return $map->{entries}->{$source}
111 if $map->{entries} && defined($map->{entries}->{$source});
112
113 return $map->{default} if $map->{default};
114
115 # identity (fallback)
116 return $source;
117}
118
119PVE::JSONSchema::register_standard_option('pve-targetstorage', {
120 description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
121 type => 'string',
122 format => 'storagepair-list',
123 optional => 1,
124});
125
126#no warnings 'redefine';
127
128my $nodename_cache;
129sub nodename {
130 $nodename_cache //= PVE::INotify::nodename();
131 return $nodename_cache;
132}
133
134my $watchdog_fmt = {
135 model => {
136 default_key => 1,
137 type => 'string',
138 enum => [qw(i6300esb ib700)],
139 description => "Watchdog type to emulate.",
140 default => 'i6300esb',
141 optional => 1,
142 },
143 action => {
144 type => 'string',
145 enum => [qw(reset shutdown poweroff pause debug none)],
146 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
147 optional => 1,
148 },
149};
150PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
151
152my $agent_fmt = {
153 enabled => {
154 description => "Enable/disable Qemu GuestAgent.",
155 type => 'boolean',
156 default => 0,
157 default_key => 1,
158 },
159 fstrim_cloned_disks => {
160 description => "Run fstrim after moving a disk or migrating the VM.",
161 type => 'boolean',
162 optional => 1,
163 default => 0
164 },
165 type => {
166 description => "Select the agent type",
167 type => 'string',
168 default => 'virtio',
169 optional => 1,
170 enum => [qw(virtio isa)],
171 },
172};
173
174my $vga_fmt = {
175 type => {
176 description => "Select the VGA type.",
177 type => 'string',
178 default => 'std',
179 optional => 1,
180 default_key => 1,
181 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
182 },
183 memory => {
184 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
185 type => 'integer',
186 optional => 1,
187 minimum => 4,
188 maximum => 512,
189 },
190};
191
192my $ivshmem_fmt = {
193 size => {
194 type => 'integer',
195 minimum => 1,
196 description => "The size of the file in MB.",
197 },
198 name => {
199 type => 'string',
200 pattern => '[a-zA-Z0-9\-]+',
201 optional => 1,
202 format_description => 'string',
203 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
204 },
205};
206
207my $audio_fmt = {
208 device => {
209 type => 'string',
210 enum => [qw(ich9-intel-hda intel-hda AC97)],
211 description => "Configure an audio device."
212 },
213 driver => {
214 type => 'string',
215 enum => ['spice', 'none'],
216 default => 'spice',
217 optional => 1,
218 description => "Driver backend for the audio device."
219 },
220};
221
222my $spice_enhancements_fmt = {
223 foldersharing => {
224 type => 'boolean',
225 optional => 1,
226 default => '0',
227 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
228 },
229 videostreaming => {
230 type => 'string',
231 enum => ['off', 'all', 'filter'],
232 default => 'off',
233 optional => 1,
234 description => "Enable video streaming. Uses compression for detected video streams."
235 },
236};
237
238my $rng_fmt = {
239 source => {
240 type => 'string',
241 enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
242 default_key => 1,
243 description => "The file on the host to gather entropy from. In most"
244 . " cases /dev/urandom should be preferred over /dev/random"
245 . " to avoid entropy-starvation issues on the host. Using"
246 . " urandom does *not* decrease security in any meaningful"
247 . " way, as it's still seeded from real entropy, and the"
248 . " bytes provided will most likely be mixed with real"
249 . " entropy on the guest as well. /dev/hwrng can be used"
250 . " to pass through a hardware RNG from the host.",
251 },
252 max_bytes => {
253 type => 'integer',
254 description => "Maximum bytes of entropy injected into the guest every"
255 . " 'period' milliseconds. Prefer a lower value when using"
256 . " /dev/random as source. Use 0 to disable limiting"
257 . " (potentially dangerous!).",
258 optional => 1,
259
260 # default is 1 KiB/s, provides enough entropy to the guest to avoid
261 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
262 # of overwhelming the host, provided we're reading from /dev/urandom
263 default => 1024,
264 },
265 period => {
266 type => 'integer',
267 description => "Every 'period' milliseconds the entropy-injection quota"
268 . " is reset, allowing the guest to retrieve another"
269 . " 'max_bytes' of entropy.",
270 optional => 1,
271 default => 1000,
272 },
273};
274
275my $confdesc = {
276 onboot => {
277 optional => 1,
278 type => 'boolean',
279 description => "Specifies whether a VM will be started during system bootup.",
280 default => 0,
281 },
282 autostart => {
283 optional => 1,
284 type => 'boolean',
285 description => "Automatic restart after crash (currently ignored).",
286 default => 0,
287 },
288 hotplug => {
289 optional => 1,
290 type => 'string', format => 'pve-hotplug-features',
291 description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
292 default => 'network,disk,usb',
293 },
294 reboot => {
295 optional => 1,
296 type => 'boolean',
297 description => "Allow reboot. If set to '0' the VM exit on reboot.",
298 default => 1,
299 },
300 lock => {
301 optional => 1,
302 type => 'string',
303 description => "Lock/unlock the VM.",
304 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
305 },
306 cpulimit => {
307 optional => 1,
308 type => 'number',
309 description => "Limit of CPU usage.",
310 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
311 minimum => 0,
312 maximum => 128,
313 default => 0,
314 },
315 cpuunits => {
316 optional => 1,
317 type => 'integer',
318 description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
319 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
320 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
321 ." weights of all the other running VMs.",
322 minimum => 2,
323 maximum => 262144,
324 default => 'cgroup v1: 1024, cgroup v2: 100',
325 },
326 memory => {
327 optional => 1,
328 type => 'integer',
329 description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
330 minimum => 16,
331 default => 512,
332 },
333 balloon => {
334 optional => 1,
335 type => 'integer',
336 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
337 minimum => 0,
338 },
339 shares => {
340 optional => 1,
341 type => 'integer',
342 description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
343 minimum => 0,
344 maximum => 50000,
345 default => 1000,
346 },
347 keyboard => {
348 optional => 1,
349 type => 'string',
350 description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
351 "It should not be necessary to set it.",
352 enum => PVE::Tools::kvmkeymaplist(),
353 default => undef,
354 },
355 name => {
356 optional => 1,
357 type => 'string', format => 'dns-name',
358 description => "Set a name for the VM. Only used on the configuration web interface.",
359 },
360 scsihw => {
361 optional => 1,
362 type => 'string',
363 description => "SCSI controller model",
364 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
365 default => 'lsi',
366 },
367 description => {
368 optional => 1,
369 type => 'string',
370 description => "Description for the VM. Shown in the web-interface VM's summary."
371 ." This is saved as comment inside the configuration file.",
372 maxLength => 1024 * 8,
373 },
374 ostype => {
375 optional => 1,
376 type => 'string',
377 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
378 description => "Specify guest operating system.",
379 verbose_description => <<EODESC,
380Specify guest operating system. This is used to enable special
381optimization/features for specific operating systems:
382
383[horizontal]
384other;; unspecified OS
385wxp;; Microsoft Windows XP
386w2k;; Microsoft Windows 2000
387w2k3;; Microsoft Windows 2003
388w2k8;; Microsoft Windows 2008
389wvista;; Microsoft Windows Vista
390win7;; Microsoft Windows 7
391win8;; Microsoft Windows 8/2012/2012r2
392win10;; Microsoft Windows 10/2016/2019
393l24;; Linux 2.4 Kernel
394l26;; Linux 2.6 - 5.X Kernel
395solaris;; Solaris/OpenSolaris/OpenIndiania kernel
396EODESC
397 },
398 boot => {
399 optional => 1,
400 type => 'string', format => 'pve-qm-boot',
401 description => "Specify guest boot order. Use with 'order=', usage with"
402 . " no key or 'legacy=' is deprecated.",
403 },
404 bootdisk => {
405 optional => 1,
406 type => 'string', format => 'pve-qm-bootdisk',
407 description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
408 pattern => '(ide|sata|scsi|virtio)\d+',
409 },
410 smp => {
411 optional => 1,
412 type => 'integer',
413 description => "The number of CPUs. Please use option -sockets instead.",
414 minimum => 1,
415 default => 1,
416 },
417 sockets => {
418 optional => 1,
419 type => 'integer',
420 description => "The number of CPU sockets.",
421 minimum => 1,
422 default => 1,
423 },
424 cores => {
425 optional => 1,
426 type => 'integer',
427 description => "The number of cores per socket.",
428 minimum => 1,
429 default => 1,
430 },
431 numa => {
432 optional => 1,
433 type => 'boolean',
434 description => "Enable/disable NUMA.",
435 default => 0,
436 },
437 hugepages => {
438 optional => 1,
439 type => 'string',
440 description => "Enable/disable hugepages memory.",
441 enum => [qw(any 2 1024)],
442 },
443 keephugepages => {
444 optional => 1,
445 type => 'boolean',
446 default => 0,
447 description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
448 ." after VM shutdown and can be used for subsequent starts.",
449 },
450 vcpus => {
451 optional => 1,
452 type => 'integer',
453 description => "Number of hotplugged vcpus.",
454 minimum => 1,
455 default => 0,
456 },
457 acpi => {
458 optional => 1,
459 type => 'boolean',
460 description => "Enable/disable ACPI.",
461 default => 1,
462 },
463 agent => {
464 optional => 1,
465 description => "Enable/disable Qemu GuestAgent and its properties.",
466 type => 'string',
467 format => $agent_fmt,
468 },
469 kvm => {
470 optional => 1,
471 type => 'boolean',
472 description => "Enable/disable KVM hardware virtualization.",
473 default => 1,
474 },
475 tdf => {
476 optional => 1,
477 type => 'boolean',
478 description => "Enable/disable time drift fix.",
479 default => 0,
480 },
481 localtime => {
482 optional => 1,
483 type => 'boolean',
484 description => "Set the real time clock to local time. This is enabled by default if ostype"
485 ." indicates a Microsoft OS.",
486 },
487 freeze => {
488 optional => 1,
489 type => 'boolean',
490 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
491 },
492 vga => {
493 optional => 1,
494 type => 'string', format => $vga_fmt,
495 description => "Configure the VGA hardware.",
496 verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
497 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
498 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
499 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
500 ." display server. For win* OS you can select how many independent displays you want,"
501 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
502 ." using a serial device as terminal.",
503 },
504 watchdog => {
505 optional => 1,
506 type => 'string', format => 'pve-qm-watchdog',
507 description => "Create a virtual hardware watchdog device.",
508 verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
509 ." action), the watchdog must be periodically polled by an agent inside the guest or"
510 ." else the watchdog will reset the guest (or execute the respective action specified)",
511 },
512 startdate => {
513 optional => 1,
514 type => 'string',
515 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
516 description => "Set the initial date of the real time clock. Valid format for date are:"
517 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
518 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
519 default => 'now',
520 },
521 startup => get_standard_option('pve-startup-order'),
522 template => {
523 optional => 1,
524 type => 'boolean',
525 description => "Enable/disable Template.",
526 default => 0,
527 },
528 args => {
529 optional => 1,
530 type => 'string',
531 description => "Arbitrary arguments passed to kvm.",
532 verbose_description => <<EODESCR,
533Arbitrary arguments passed to kvm, for example:
534
535args: -no-reboot -no-hpet
536
537NOTE: this option is for experts only.
538EODESCR
539 },
540 tablet => {
541 optional => 1,
542 type => 'boolean',
543 default => 1,
544 description => "Enable/disable the USB tablet device.",
545 verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
546 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
547 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
548 ." may consider disabling this to save some context switches. This is turned off by"
549 ." default if you use spice (`qm set <vmid> --vga qxl`).",
550 },
551 migrate_speed => {
552 optional => 1,
553 type => 'integer',
554 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
555 minimum => 0,
556 default => 0,
557 },
558 migrate_downtime => {
559 optional => 1,
560 type => 'number',
561 description => "Set maximum tolerated downtime (in seconds) for migrations.",
562 minimum => 0,
563 default => 0.1,
564 },
565 cdrom => {
566 optional => 1,
567 type => 'string', format => 'pve-qm-ide',
568 typetext => '<volume>',
569 description => "This is an alias for option -ide2",
570 },
571 cpu => {
572 optional => 1,
573 description => "Emulated CPU type.",
574 type => 'string',
575 format => 'pve-vm-cpu-conf',
576 },
577 parent => get_standard_option('pve-snapshot-name', {
578 optional => 1,
579 description => "Parent snapshot name. This is used internally, and should not be modified.",
580 }),
581 snaptime => {
582 optional => 1,
583 description => "Timestamp for snapshots.",
584 type => 'integer',
585 minimum => 0,
586 },
587 vmstate => {
588 optional => 1,
589 type => 'string', format => 'pve-volume-id',
590 description => "Reference to a volume which stores the VM state. This is used internally"
591 ." for snapshots.",
592 },
593 vmstatestorage => get_standard_option('pve-storage-id', {
594 description => "Default storage for VM state volumes/files.",
595 optional => 1,
596 }),
597 runningmachine => get_standard_option('pve-qemu-machine', {
598 description => "Specifies the QEMU machine type of the running vm. This is used internally"
599 ." for snapshots.",
600 }),
601 runningcpu => {
602 description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
603 ." internally for snapshots.",
604 optional => 1,
605 type => 'string',
606 pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
607 format_description => 'QEMU -cpu parameter'
608 },
609 machine => get_standard_option('pve-qemu-machine'),
610 arch => {
611 description => "Virtual processor architecture. Defaults to the host.",
612 optional => 1,
613 type => 'string',
614 enum => [qw(x86_64 aarch64)],
615 },
616 smbios1 => {
617 description => "Specify SMBIOS type 1 fields.",
618 type => 'string', format => 'pve-qm-smbios1',
619 maxLength => 512,
620 optional => 1,
621 },
622 protection => {
623 optional => 1,
624 type => 'boolean',
625 description => "Sets the protection flag of the VM. This will disable the remove VM and"
626 ." remove disk operations.",
627 default => 0,
628 },
629 bios => {
630 optional => 1,
631 type => 'string',
632 enum => [ qw(seabios ovmf) ],
633 description => "Select BIOS implementation.",
634 default => 'seabios',
635 },
636 vmgenid => {
637 type => 'string',
638 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
639 format_description => 'UUID',
640 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
641 ." to disable explicitly.",
642 verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
643 ." value identifier to the guest OS. This allows to notify the guest operating system"
644 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
645 ." execution or creation from a template). The guest operating system notices the"
646 ." change, and is then able to react as appropriate by marking its copies of"
647 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
648 ."Note that auto-creation only works when done through API/CLI create or update methods"
649 .", but not when manually editing the config file.",
650 default => "1 (autogenerated)",
651 optional => 1,
652 },
653 hookscript => {
654 type => 'string',
655 format => 'pve-volume-id',
656 optional => 1,
657 description => "Script that will be executed during various steps in the vms lifetime.",
658 },
659 ivshmem => {
660 type => 'string',
661 format => $ivshmem_fmt,
662 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
663 ." the host.",
664 optional => 1,
665 },
666 audio0 => {
667 type => 'string',
668 format => $audio_fmt,
669 description => "Configure a audio device, useful in combination with QXL/Spice.",
670 optional => 1
671 },
672 spice_enhancements => {
673 type => 'string',
674 format => $spice_enhancements_fmt,
675 description => "Configure additional enhancements for SPICE.",
676 optional => 1
677 },
678 tags => {
679 type => 'string', format => 'pve-tag-list',
680 description => 'Tags of the VM. This is only meta information.',
681 optional => 1,
682 },
683 rng0 => {
684 type => 'string',
685 format => $rng_fmt,
686 description => "Configure a VirtIO-based Random Number Generator.",
687 optional => 1,
688 },
689};
690
691my $cicustom_fmt = {
692 meta => {
693 type => 'string',
694 optional => 1,
695 description => 'Specify a custom file containing all meta data passed to the VM via"
696 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
697 format => 'pve-volume-id',
698 format_description => 'volume',
699 },
700 network => {
701 type => 'string',
702 optional => 1,
703 description => 'Specify a custom file containing all network data passed to the VM via'
704 .' cloud-init.',
705 format => 'pve-volume-id',
706 format_description => 'volume',
707 },
708 user => {
709 type => 'string',
710 optional => 1,
711 description => 'Specify a custom file containing all user data passed to the VM via'
712 .' cloud-init.',
713 format => 'pve-volume-id',
714 format_description => 'volume',
715 },
716};
717PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
718
719my $confdesc_cloudinit = {
720 citype => {
721 optional => 1,
722 type => 'string',
723 description => 'Specifies the cloud-init configuration format. The default depends on the'
724 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
725 .' and `configdrive2` for windows.',
726 enum => ['configdrive2', 'nocloud', 'opennebula'],
727 },
728 ciuser => {
729 optional => 1,
730 type => 'string',
731 description => "cloud-init: User name to change ssh keys and password for instead of the"
732 ." image's configured default user.",
733 },
734 cipassword => {
735 optional => 1,
736 type => 'string',
737 description => 'cloud-init: Password to assign the user. Using this is generally not'
738 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
739 .' support hashed passwords.',
740 },
741 cicustom => {
742 optional => 1,
743 type => 'string',
744 description => 'cloud-init: Specify custom files to replace the automatically generated'
745 .' ones at start.',
746 format => 'pve-qm-cicustom',
747 },
748 searchdomain => {
749 optional => 1,
750 type => 'string',
751 description => "cloud-init: Sets DNS search domains for a container. Create will'
752 .' automatically use the setting from the host if neither searchdomain nor nameserver'
753 .' are set.",
754 },
755 nameserver => {
756 optional => 1,
757 type => 'string', format => 'address-list',
758 description => "cloud-init: Sets DNS server IP address for a container. Create will'
759 .' automatically use the setting from the host if neither searchdomain nor nameserver'
760 .' are set.",
761 },
762 sshkeys => {
763 optional => 1,
764 type => 'string',
765 format => 'urlencoded',
766 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
767 },
768};
769
770# what about other qemu settings ?
771#cpu => 'string',
772#machine => 'string',
773#fda => 'file',
774#fdb => 'file',
775#mtdblock => 'file',
776#sd => 'file',
777#pflash => 'file',
778#snapshot => 'bool',
779#bootp => 'file',
780##tftp => 'dir',
781##smb => 'dir',
782#kernel => 'file',
783#append => 'string',
784#initrd => 'file',
785##soundhw => 'string',
786
787while (my ($k, $v) = each %$confdesc) {
788 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
789}
790
791my $MAX_USB_DEVICES = 5;
792my $MAX_NETS = 32;
793my $MAX_SERIAL_PORTS = 4;
794my $MAX_PARALLEL_PORTS = 3;
795my $MAX_NUMA = 8;
796
797my $numa_fmt = {
798 cpus => {
799 type => "string",
800 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
801 description => "CPUs accessing this NUMA node.",
802 format_description => "id[-id];...",
803 },
804 memory => {
805 type => "number",
806 description => "Amount of memory this NUMA node provides.",
807 optional => 1,
808 },
809 hostnodes => {
810 type => "string",
811 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
812 description => "Host NUMA nodes to use.",
813 format_description => "id[-id];...",
814 optional => 1,
815 },
816 policy => {
817 type => 'string',
818 enum => [qw(preferred bind interleave)],
819 description => "NUMA allocation policy.",
820 optional => 1,
821 },
822};
823PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
824my $numadesc = {
825 optional => 1,
826 type => 'string', format => $numa_fmt,
827 description => "NUMA topology.",
828};
829PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
830
831for (my $i = 0; $i < $MAX_NUMA; $i++) {
832 $confdesc->{"numa$i"} = $numadesc;
833}
834
835my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
836 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
837 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
838my $nic_model_list_txt = join(' ', sort @$nic_model_list);
839
840my $net_fmt_bridge_descr = <<__EOD__;
841Bridge to attach the network device to. The Proxmox VE standard bridge
842is called 'vmbr0'.
843
844If you do not specify a bridge, we create a kvm user (NATed) network
845device, which provides DHCP and DNS services. The following addresses
846are used:
847
848 10.0.2.2 Gateway
849 10.0.2.3 DNS Server
850 10.0.2.4 SMB Server
851
852The DHCP server assign addresses to the guest starting from 10.0.2.15.
853__EOD__
854
855my $net_fmt = {
856 macaddr => get_standard_option('mac-addr', {
857 description => "MAC address. That address must be unique withing your network. This is"
858 ." automatically generated if not specified.",
859 }),
860 model => {
861 type => 'string',
862 description => "Network Card Model. The 'virtio' model provides the best performance with"
863 ." very low CPU overhead. If your guest does not support this driver, it is usually"
864 ." best to use 'e1000'.",
865 enum => $nic_model_list,
866 default_key => 1,
867 },
868 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
869 bridge => {
870 type => 'string',
871 description => $net_fmt_bridge_descr,
872 format_description => 'bridge',
873 pattern => '[-_.\w\d]+',
874 optional => 1,
875 },
876 queues => {
877 type => 'integer',
878 minimum => 0, maximum => 16,
879 description => 'Number of packet queues to be used on the device.',
880 optional => 1,
881 },
882 rate => {
883 type => 'number',
884 minimum => 0,
885 description => "Rate limit in mbps (megabytes per second) as floating point number.",
886 optional => 1,
887 },
888 tag => {
889 type => 'integer',
890 minimum => 1, maximum => 4094,
891 description => 'VLAN tag to apply to packets on this interface.',
892 optional => 1,
893 },
894 trunks => {
895 type => 'string',
896 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
897 description => 'VLAN trunks to pass through this interface.',
898 format_description => 'vlanid[;vlanid...]',
899 optional => 1,
900 },
901 firewall => {
902 type => 'boolean',
903 description => 'Whether this interface should be protected by the firewall.',
904 optional => 1,
905 },
906 link_down => {
907 type => 'boolean',
908 description => 'Whether this interface should be disconnected (like pulling the plug).',
909 optional => 1,
910 },
911 mtu => {
912 type => 'integer',
913 minimum => 1, maximum => 65520,
914 description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
915 optional => 1,
916 },
917};
918
919my $netdesc = {
920 optional => 1,
921 type => 'string', format => $net_fmt,
922 description => "Specify network devices.",
923};
924
925PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
926
927my $ipconfig_fmt = {
928 ip => {
929 type => 'string',
930 format => 'pve-ipv4-config',
931 format_description => 'IPv4Format/CIDR',
932 description => 'IPv4 address in CIDR format.',
933 optional => 1,
934 default => 'dhcp',
935 },
936 gw => {
937 type => 'string',
938 format => 'ipv4',
939 format_description => 'GatewayIPv4',
940 description => 'Default gateway for IPv4 traffic.',
941 optional => 1,
942 requires => 'ip',
943 },
944 ip6 => {
945 type => 'string',
946 format => 'pve-ipv6-config',
947 format_description => 'IPv6Format/CIDR',
948 description => 'IPv6 address in CIDR format.',
949 optional => 1,
950 default => 'dhcp',
951 },
952 gw6 => {
953 type => 'string',
954 format => 'ipv6',
955 format_description => 'GatewayIPv6',
956 description => 'Default gateway for IPv6 traffic.',
957 optional => 1,
958 requires => 'ip6',
959 },
960};
961PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
962my $ipconfigdesc = {
963 optional => 1,
964 type => 'string', format => 'pve-qm-ipconfig',
965 description => <<'EODESCR',
966cloud-init: Specify IP addresses and gateways for the corresponding interface.
967
968IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
969
970The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
971gateway should be provided.
972For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
973cloud-init 19.4 or newer.
974
975If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
976dhcp on IPv4.
977EODESCR
978};
979PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
980
981for (my $i = 0; $i < $MAX_NETS; $i++) {
982 $confdesc->{"net$i"} = $netdesc;
983 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
984}
985
986foreach my $key (keys %$confdesc_cloudinit) {
987 $confdesc->{$key} = $confdesc_cloudinit->{$key};
988}
989
990PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
991sub verify_volume_id_or_qm_path {
992 my ($volid, $noerr) = @_;
993
994 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
995 return $volid;
996 }
997
998 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
999 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1000 if ($@) {
1001 return if $noerr;
1002 die $@;
1003 }
1004 return $volid;
1005}
1006
1007my $usb_fmt = {
1008 host => {
1009 default_key => 1,
1010 type => 'string', format => 'pve-qm-usb-device',
1011 format_description => 'HOSTUSBDEVICE|spice',
1012 description => <<EODESCR,
1013The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1014
1015 'bus-port(.port)*' (decimal numbers) or
1016 'vendor_id:product_id' (hexadeciaml numbers) or
1017 'spice'
1018
1019You can use the 'lsusb -t' command to list existing usb devices.
1020
1021NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1022machines - use with special care.
1023
1024The value 'spice' can be used to add a usb redirection devices for spice.
1025EODESCR
1026 },
1027 usb3 => {
1028 optional => 1,
1029 type => 'boolean',
1030 description => "Specifies whether if given host option is a USB3 device or port.",
1031 default => 0,
1032 },
1033};
1034
1035my $usbdesc = {
1036 optional => 1,
1037 type => 'string', format => $usb_fmt,
1038 description => "Configure an USB device (n is 0 to 4).",
1039};
1040PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1041
1042my $serialdesc = {
1043 optional => 1,
1044 type => 'string',
1045 pattern => '(/dev/.+|socket)',
1046 description => "Create a serial device inside the VM (n is 0 to 3)",
1047 verbose_description => <<EODESCR,
1048Create a serial device inside the VM (n is 0 to 3), and pass through a
1049host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1050host side (use 'qm terminal' to open a terminal connection).
1051
1052NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1053use with special care.
1054
1055CAUTION: Experimental! User reported problems with this option.
1056EODESCR
1057};
1058
1059my $paralleldesc= {
1060 optional => 1,
1061 type => 'string',
1062 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1063 description => "Map host parallel devices (n is 0 to 2).",
1064 verbose_description => <<EODESCR,
1065Map host parallel devices (n is 0 to 2).
1066
1067NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1068machines - use with special care.
1069
1070CAUTION: Experimental! User reported problems with this option.
1071EODESCR
1072};
1073
1074for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1075 $confdesc->{"parallel$i"} = $paralleldesc;
1076}
1077
1078for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1079 $confdesc->{"serial$i"} = $serialdesc;
1080}
1081
1082for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
1083 $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1084}
1085
1086for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1087 $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1088}
1089
1090for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1091 $confdesc->{"usb$i"} = $usbdesc;
1092}
1093
1094my $boot_fmt = {
1095 legacy => {
1096 optional => 1,
1097 default_key => 1,
1098 type => 'string',
1099 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1100 . " Deprecated, use 'order=' instead.",
1101 pattern => '[acdn]{1,4}',
1102 format_description => "[acdn]{1,4}",
1103
1104 # note: this is also the fallback if boot: is not given at all
1105 default => 'cdn',
1106 },
1107 order => {
1108 optional => 1,
1109 type => 'string',
1110 format => 'pve-qm-bootdev-list',
1111 format_description => "device[;device...]",
1112 description => <<EODESC,
1113The guest will attempt to boot from devices in the order they appear here.
1114
1115Disks, optical drives and passed-through storage USB devices will be directly
1116booted from, NICs will load PXE, and PCIe devices will either behave like disks
1117(e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1118
1119Note that only devices in this list will be marked as bootable and thus loaded
1120by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1121(e.g. software-raid), you need to specify all of them here.
1122
1123Overrides the deprecated 'legacy=[acdn]*' value when given.
1124EODESC
1125 },
1126};
1127PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1128
1129PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1130sub verify_bootdev {
1131 my ($dev, $noerr) = @_;
1132
1133 return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1134
1135 my $check = sub {
1136 my ($base) = @_;
1137 return 0 if $dev !~ m/^$base\d+$/;
1138 return 0 if !$confdesc->{$dev};
1139 return 1;
1140 };
1141
1142 return $dev if $check->("net");
1143 return $dev if $check->("usb");
1144 return $dev if $check->("hostpci");
1145
1146 return if $noerr;
1147 die "invalid boot device '$dev'\n";
1148}
1149
1150sub print_bootorder {
1151 my ($devs) = @_;
1152 return "" if !@$devs;
1153 my $data = { order => join(';', @$devs) };
1154 return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1155}
1156
1157my $kvm_api_version = 0;
1158
1159sub kvm_version {
1160 return $kvm_api_version if $kvm_api_version;
1161
1162 open my $fh, '<', '/dev/kvm' or return;
1163
1164 # 0xae00 => KVM_GET_API_VERSION
1165 $kvm_api_version = ioctl($fh, 0xae00, 0);
1166 close($fh);
1167
1168 return $kvm_api_version;
1169}
1170
1171my $kvm_user_version = {};
1172my $kvm_mtime = {};
1173
1174sub kvm_user_version {
1175 my ($binary) = @_;
1176
1177 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1178 my $st = stat($binary);
1179
1180 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1181 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1182 $cachedmtime == $st->mtime;
1183
1184 $kvm_user_version->{$binary} = 'unknown';
1185 $kvm_mtime->{$binary} = $st->mtime;
1186
1187 my $code = sub {
1188 my $line = shift;
1189 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1190 $kvm_user_version->{$binary} = $2;
1191 }
1192 };
1193
1194 eval { run_command([$binary, '--version'], outfunc => $code); };
1195 warn $@ if $@;
1196
1197 return $kvm_user_version->{$binary};
1198
1199}
1200my sub extract_version {
1201 my ($machine_type, $version) = @_;
1202 $version = kvm_user_version() if !defined($version);
1203 PVE::QemuServer::Machine::extract_version($machine_type, $version)
1204}
1205
1206sub kernel_has_vhost_net {
1207 return -c '/dev/vhost-net';
1208}
1209
1210sub option_exists {
1211 my $key = shift;
1212 return defined($confdesc->{$key});
1213}
1214
1215my $cdrom_path;
1216sub get_cdrom_path {
1217
1218 return $cdrom_path if $cdrom_path;
1219
1220 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1221 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1222 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1223}
1224
1225sub get_iso_path {
1226 my ($storecfg, $vmid, $cdrom) = @_;
1227
1228 if ($cdrom eq 'cdrom') {
1229 return get_cdrom_path();
1230 } elsif ($cdrom eq 'none') {
1231 return '';
1232 } elsif ($cdrom =~ m|^/|) {
1233 return $cdrom;
1234 } else {
1235 return PVE::Storage::path($storecfg, $cdrom);
1236 }
1237}
1238
1239# try to convert old style file names to volume IDs
1240sub filename_to_volume_id {
1241 my ($vmid, $file, $media) = @_;
1242
1243 if (!($file eq 'none' || $file eq 'cdrom' ||
1244 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1245
1246 return if $file =~ m|/|;
1247
1248 if ($media && $media eq 'cdrom') {
1249 $file = "local:iso/$file";
1250 } else {
1251 $file = "local:$vmid/$file";
1252 }
1253 }
1254
1255 return $file;
1256}
1257
1258sub verify_media_type {
1259 my ($opt, $vtype, $media) = @_;
1260
1261 return if !$media;
1262
1263 my $etype;
1264 if ($media eq 'disk') {
1265 $etype = 'images';
1266 } elsif ($media eq 'cdrom') {
1267 $etype = 'iso';
1268 } else {
1269 die "internal error";
1270 }
1271
1272 return if ($vtype eq $etype);
1273
1274 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1275}
1276
1277sub cleanup_drive_path {
1278 my ($opt, $storecfg, $drive) = @_;
1279
1280 # try to convert filesystem paths to volume IDs
1281
1282 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1283 ($drive->{file} !~ m|^/dev/.+|) &&
1284 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1285 ($drive->{file} !~ m/^\d+$/)) {
1286 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1287 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1288 if !$vtype;
1289 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1290 verify_media_type($opt, $vtype, $drive->{media});
1291 $drive->{file} = $volid;
1292 }
1293
1294 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1295}
1296
1297sub parse_hotplug_features {
1298 my ($data) = @_;
1299
1300 my $res = {};
1301
1302 return $res if $data eq '0';
1303
1304 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1305
1306 foreach my $feature (PVE::Tools::split_list($data)) {
1307 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1308 $res->{$1} = 1;
1309 } else {
1310 die "invalid hotplug feature '$feature'\n";
1311 }
1312 }
1313 return $res;
1314}
1315
1316PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1317sub pve_verify_hotplug_features {
1318 my ($value, $noerr) = @_;
1319
1320 return $value if parse_hotplug_features($value);
1321
1322 return if $noerr;
1323
1324 die "unable to parse hotplug option\n";
1325}
1326
1327sub scsi_inquiry {
1328 my($fh, $noerr) = @_;
1329
1330 my $SG_IO = 0x2285;
1331 my $SG_GET_VERSION_NUM = 0x2282;
1332
1333 my $versionbuf = "\x00" x 8;
1334 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1335 if (!$ret) {
1336 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1337 return;
1338 }
1339 my $version = unpack("I", $versionbuf);
1340 if ($version < 30000) {
1341 die "scsi generic interface too old\n" if !$noerr;
1342 return;
1343 }
1344
1345 my $buf = "\x00" x 36;
1346 my $sensebuf = "\x00" x 8;
1347 my $cmd = pack("C x3 C x1", 0x12, 36);
1348
1349 # see /usr/include/scsi/sg.h
1350 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1351
1352 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1353 length($sensebuf), 0, length($buf), $buf,
1354 $cmd, $sensebuf, 6000);
1355
1356 $ret = ioctl($fh, $SG_IO, $packet);
1357 if (!$ret) {
1358 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1359 return;
1360 }
1361
1362 my @res = unpack($sg_io_hdr_t, $packet);
1363 if ($res[17] || $res[18]) {
1364 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1365 return;
1366 }
1367
1368 my $res = {};
1369 (my $byte0, my $byte1, $res->{vendor},
1370 $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1371
1372 $res->{removable} = $byte1 & 128 ? 1 : 0;
1373 $res->{type} = $byte0 & 31;
1374
1375 return $res;
1376}
1377
1378sub path_is_scsi {
1379 my ($path) = @_;
1380
1381 my $fh = IO::File->new("+<$path") || return;
1382 my $res = scsi_inquiry($fh, 1);
1383 close($fh);
1384
1385 return $res;
1386}
1387
1388sub print_tabletdevice_full {
1389 my ($conf, $arch) = @_;
1390
1391 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1392
1393 # we use uhci for old VMs because tablet driver was buggy in older qemu
1394 my $usbbus;
1395 if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1396 $usbbus = 'ehci';
1397 } else {
1398 $usbbus = 'uhci';
1399 }
1400
1401 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1402}
1403
1404sub print_keyboarddevice_full {
1405 my ($conf, $arch, $machine) = @_;
1406
1407 return if $arch ne 'aarch64';
1408
1409 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1410}
1411
1412my sub get_drive_id {
1413 my ($drive) = @_;
1414 return "$drive->{interface}$drive->{index}";
1415}
1416
1417sub print_drivedevice_full {
1418 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1419
1420 my $device = '';
1421 my $maxdev = 0;
1422
1423 my $drive_id = get_drive_id($drive);
1424 if ($drive->{interface} eq 'virtio') {
1425 my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1426 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1427 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1428 } elsif ($drive->{interface} eq 'scsi') {
1429
1430 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1431 my $unit = $drive->{index} % $maxdev;
1432 my $devicetype = 'hd';
1433 my $path = '';
1434 if (drive_is_cdrom($drive)) {
1435 $devicetype = 'cd';
1436 } else {
1437 if ($drive->{file} =~ m|^/|) {
1438 $path = $drive->{file};
1439 if (my $info = path_is_scsi($path)) {
1440 if ($info->{type} == 0 && $drive->{scsiblock}) {
1441 $devicetype = 'block';
1442 } elsif ($info->{type} == 1) { # tape
1443 $devicetype = 'generic';
1444 }
1445 }
1446 } else {
1447 $path = PVE::Storage::path($storecfg, $drive->{file});
1448 }
1449
1450 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1451 my $version = extract_version($machine_type, kvm_user_version());
1452 if ($path =~ m/^iscsi\:\/\// &&
1453 !min_version($version, 4, 1)) {
1454 $devicetype = 'generic';
1455 }
1456 }
1457
1458 if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1459 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1460 } else {
1461 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1462 .",lun=$drive->{index}";
1463 }
1464 $device .= ",drive=drive-$drive_id,id=$drive_id";
1465
1466 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1467 $device .= ",rotation_rate=1";
1468 }
1469 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1470
1471 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1472 my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1473 my $controller = int($drive->{index} / $maxdev);
1474 my $unit = $drive->{index} % $maxdev;
1475 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1476
1477 $device = "ide-$devicetype";
1478 if ($drive->{interface} eq 'ide') {
1479 $device .= ",bus=ide.$controller,unit=$unit";
1480 } else {
1481 $device .= ",bus=ahci$controller.$unit";
1482 }
1483 $device .= ",drive=drive-$drive_id,id=$drive_id";
1484
1485 if ($devicetype eq 'hd') {
1486 if (my $model = $drive->{model}) {
1487 $model = URI::Escape::uri_unescape($model);
1488 $device .= ",model=$model";
1489 }
1490 if ($drive->{ssd}) {
1491 $device .= ",rotation_rate=1";
1492 }
1493 }
1494 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1495 } elsif ($drive->{interface} eq 'usb') {
1496 die "implement me";
1497 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1498 } else {
1499 die "unsupported interface type";
1500 }
1501
1502 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1503
1504 if (my $serial = $drive->{serial}) {
1505 $serial = URI::Escape::uri_unescape($serial);
1506 $device .= ",serial=$serial";
1507 }
1508
1509
1510 return $device;
1511}
1512
1513sub get_initiator_name {
1514 my $initiator;
1515
1516 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1517 while (defined(my $line = <$fh>)) {
1518 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1519 $initiator = $1;
1520 last;
1521 }
1522 $fh->close();
1523
1524 return $initiator;
1525}
1526
1527sub print_drive_commandline_full {
1528 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1529
1530 my $path;
1531 my $volid = $drive->{file};
1532 my $format = $drive->{format};
1533 my $drive_id = get_drive_id($drive);
1534
1535 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1536 my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1537
1538 if (drive_is_cdrom($drive)) {
1539 $path = get_iso_path($storecfg, $vmid, $volid);
1540 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1541 } else {
1542 if ($storeid) {
1543 $path = PVE::Storage::path($storecfg, $volid);
1544 $format //= qemu_img_format($scfg, $volname);
1545 } else {
1546 $path = $volid;
1547 $format //= "raw";
1548 }
1549 }
1550
1551 my $is_rbd = $path =~ m/^rbd:/;
1552
1553 my $opts = '';
1554 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1555 foreach my $o (@qemu_drive_options) {
1556 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1557 }
1558
1559 # snapshot only accepts on|off
1560 if (defined($drive->{snapshot})) {
1561 my $v = $drive->{snapshot} ? 'on' : 'off';
1562 $opts .= ",snapshot=$v";
1563 }
1564
1565 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1566 my ($dir, $qmpname) = @$type;
1567 if (my $v = $drive->{"mbps$dir"}) {
1568 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1569 }
1570 if (my $v = $drive->{"mbps${dir}_max"}) {
1571 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1572 }
1573 if (my $v = $drive->{"bps${dir}_max_length"}) {
1574 $opts .= ",throttling.bps$qmpname-max-length=$v";
1575 }
1576 if (my $v = $drive->{"iops${dir}"}) {
1577 $opts .= ",throttling.iops$qmpname=$v";
1578 }
1579 if (my $v = $drive->{"iops${dir}_max"}) {
1580 $opts .= ",throttling.iops$qmpname-max=$v";
1581 }
1582 if (my $v = $drive->{"iops${dir}_max_length"}) {
1583 $opts .= ",throttling.iops$qmpname-max-length=$v";
1584 }
1585 }
1586
1587 if ($pbs_name) {
1588 $format = "rbd" if $is_rbd;
1589 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1590 if !$format;
1591 $opts .= ",format=alloc-track,file.driver=$format";
1592 } elsif ($format) {
1593 $opts .= ",format=$format";
1594 }
1595
1596 my $cache_direct = 0;
1597
1598 if (my $cache = $drive->{cache}) {
1599 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1600 } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1601 $opts .= ",cache=none";
1602 $cache_direct = 1;
1603 }
1604
1605 # io_uring with cache mode writeback or writethrough on krbd will hang...
1606 my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1607
1608 if (!$drive->{aio}) {
1609 if ($io_uring && !$rbd_no_io_uring) {
1610 # io_uring supports all cache modes
1611 $opts .= ",aio=io_uring";
1612 } else {
1613 # aio native works only with O_DIRECT
1614 if($cache_direct) {
1615 $opts .= ",aio=native";
1616 } else {
1617 $opts .= ",aio=threads";
1618 }
1619 }
1620 }
1621
1622 if (!drive_is_cdrom($drive)) {
1623 my $detectzeroes;
1624 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1625 $detectzeroes = 'off';
1626 } elsif ($drive->{discard}) {
1627 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1628 } else {
1629 # This used to be our default with discard not being specified:
1630 $detectzeroes = 'on';
1631 }
1632
1633 # note: 'detect-zeroes' works per blockdev and we want it to persist
1634 # after the alloc-track is removed, so put it on 'file' directly
1635 my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1636 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1637 }
1638
1639 if ($pbs_name) {
1640 $opts .= ",backing=$pbs_name";
1641 $opts .= ",auto-remove=on";
1642 }
1643
1644 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1645 my $file_param = "file";
1646 if ($pbs_name) {
1647 # non-rbd drivers require the underlying file to be a seperate block
1648 # node, so add a second .file indirection
1649 $file_param .= ".file" if !$is_rbd;
1650 $file_param .= ".filename";
1651 }
1652 my $pathinfo = $path ? "$file_param=$path," : '';
1653
1654 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1655}
1656
1657sub print_pbs_blockdev {
1658 my ($pbs_conf, $pbs_name) = @_;
1659 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1660 $blockdev .= ",repository=$pbs_conf->{repository}";
1661 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1662 $blockdev .= ",archive=$pbs_conf->{archive}";
1663 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1664 return $blockdev;
1665}
1666
1667sub print_netdevice_full {
1668 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1669
1670 my $device = $net->{model};
1671 if ($net->{model} eq 'virtio') {
1672 $device = 'virtio-net-pci';
1673 };
1674
1675 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1676 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1677 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1678 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1679 # and out of each queue plus one config interrupt and control vector queue
1680 my $vectors = $net->{queues} * 2 + 2;
1681 $tmpstr .= ",vectors=$vectors,mq=on";
1682 }
1683 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1684
1685 if (my $mtu = $net->{mtu}) {
1686 if ($net->{model} eq 'virtio' && $net->{bridge}) {
1687 my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1688 if ($mtu == 1) {
1689 $mtu = $bridge_mtu;
1690 } elsif ($mtu < 576) {
1691 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1692 } elsif ($mtu > $bridge_mtu) {
1693 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1694 }
1695 $tmpstr .= ",host_mtu=$mtu";
1696 } else {
1697 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1698 }
1699 }
1700
1701 if ($use_old_bios_files) {
1702 my $romfile;
1703 if ($device eq 'virtio-net-pci') {
1704 $romfile = 'pxe-virtio.rom';
1705 } elsif ($device eq 'e1000') {
1706 $romfile = 'pxe-e1000.rom';
1707 } elsif ($device eq 'ne2k') {
1708 $romfile = 'pxe-ne2k_pci.rom';
1709 } elsif ($device eq 'pcnet') {
1710 $romfile = 'pxe-pcnet.rom';
1711 } elsif ($device eq 'rtl8139') {
1712 $romfile = 'pxe-rtl8139.rom';
1713 }
1714 $tmpstr .= ",romfile=$romfile" if $romfile;
1715 }
1716
1717 return $tmpstr;
1718}
1719
1720sub print_netdev_full {
1721 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1722
1723 my $i = '';
1724 if ($netid =~ m/^net(\d+)$/) {
1725 $i = int($1);
1726 }
1727
1728 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1729
1730 my $ifname = "tap${vmid}i$i";
1731
1732 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1733 die "interface name '$ifname' is too long (max 15 character)\n"
1734 if length($ifname) >= 16;
1735
1736 my $vhostparam = '';
1737 if (is_native($arch)) {
1738 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1739 }
1740
1741 my $vmname = $conf->{name} || "vm$vmid";
1742
1743 my $netdev = "";
1744 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1745
1746 if ($net->{bridge}) {
1747 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1748 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1749 } else {
1750 $netdev = "type=user,id=$netid,hostname=$vmname";
1751 }
1752
1753 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1754
1755 return $netdev;
1756}
1757
1758my $vga_map = {
1759 'cirrus' => 'cirrus-vga',
1760 'std' => 'VGA',
1761 'vmware' => 'vmware-svga',
1762 'virtio' => 'virtio-vga',
1763};
1764
1765sub print_vga_device {
1766 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1767
1768 my $type = $vga_map->{$vga->{type}};
1769 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1770 $type = 'virtio-gpu';
1771 }
1772 my $vgamem_mb = $vga->{memory};
1773
1774 my $max_outputs = '';
1775 if ($qxlnum) {
1776 $type = $id ? 'qxl' : 'qxl-vga';
1777
1778 if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1779 # set max outputs so linux can have up to 4 qxl displays with one device
1780 if (min_version($machine_version, 4, 1)) {
1781 $max_outputs = ",max_outputs=4";
1782 }
1783 }
1784 }
1785
1786 die "no devicetype for $vga->{type}\n" if !$type;
1787
1788 my $memory = "";
1789 if ($vgamem_mb) {
1790 if ($vga->{type} eq 'virtio') {
1791 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1792 $memory = ",max_hostmem=$bytes";
1793 } elsif ($qxlnum) {
1794 # from https://www.spice-space.org/multiple-monitors.html
1795 $memory = ",vgamem_mb=$vga->{memory}";
1796 my $ram = $vgamem_mb * 4;
1797 my $vram = $vgamem_mb * 2;
1798 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1799 } else {
1800 $memory = ",vgamem_mb=$vga->{memory}";
1801 }
1802 } elsif ($qxlnum && $id) {
1803 $memory = ",ram_size=67108864,vram_size=33554432";
1804 }
1805
1806 my $edidoff = "";
1807 if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1808 $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1809 }
1810
1811 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1812 my $vgaid = "vga" . ($id // '');
1813 my $pciaddr;
1814
1815 if ($q35 && $vgaid eq 'vga') {
1816 # the first display uses pcie.0 bus on q35 machines
1817 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1818 } else {
1819 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1820 }
1821
1822 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1823}
1824
1825sub parse_number_sets {
1826 my ($set) = @_;
1827 my $res = [];
1828 foreach my $part (split(/;/, $set)) {
1829 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1830 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1831 push @$res, [ $1, $2 ];
1832 } else {
1833 die "invalid range: $part\n";
1834 }
1835 }
1836 return $res;
1837}
1838
1839sub parse_numa {
1840 my ($data) = @_;
1841
1842 my $res = parse_property_string($numa_fmt, $data);
1843 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1844 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1845 return $res;
1846}
1847
1848# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1849sub parse_net {
1850 my ($data) = @_;
1851
1852 my $res = eval { parse_property_string($net_fmt, $data) };
1853 if ($@) {
1854 warn $@;
1855 return;
1856 }
1857 if (!defined($res->{macaddr})) {
1858 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1859 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1860 }
1861 return $res;
1862}
1863
1864# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1865sub parse_ipconfig {
1866 my ($data) = @_;
1867
1868 my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1869 if ($@) {
1870 warn $@;
1871 return;
1872 }
1873
1874 if ($res->{gw} && !$res->{ip}) {
1875 warn 'gateway specified without specifying an IP address';
1876 return;
1877 }
1878 if ($res->{gw6} && !$res->{ip6}) {
1879 warn 'IPv6 gateway specified without specifying an IPv6 address';
1880 return;
1881 }
1882 if ($res->{gw} && $res->{ip} eq 'dhcp') {
1883 warn 'gateway specified together with DHCP';
1884 return;
1885 }
1886 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1887 # gw6 + auto/dhcp
1888 warn "IPv6 gateway specified together with $res->{ip6} address";
1889 return;
1890 }
1891
1892 if (!$res->{ip} && !$res->{ip6}) {
1893 return { ip => 'dhcp', ip6 => 'dhcp' };
1894 }
1895
1896 return $res;
1897}
1898
1899sub print_net {
1900 my $net = shift;
1901
1902 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1903}
1904
1905sub add_random_macs {
1906 my ($settings) = @_;
1907
1908 foreach my $opt (keys %$settings) {
1909 next if $opt !~ m/^net(\d+)$/;
1910 my $net = parse_net($settings->{$opt});
1911 next if !$net;
1912 $settings->{$opt} = print_net($net);
1913 }
1914}
1915
1916sub vm_is_volid_owner {
1917 my ($storecfg, $vmid, $volid) = @_;
1918
1919 if ($volid !~ m|^/|) {
1920 my ($path, $owner);
1921 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1922 if ($owner && ($owner == $vmid)) {
1923 return 1;
1924 }
1925 }
1926
1927 return;
1928}
1929
1930sub vmconfig_register_unused_drive {
1931 my ($storecfg, $vmid, $conf, $drive) = @_;
1932
1933 if (drive_is_cloudinit($drive)) {
1934 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1935 warn $@ if $@;
1936 } elsif (!drive_is_cdrom($drive)) {
1937 my $volid = $drive->{file};
1938 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1939 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1940 }
1941 }
1942}
1943
1944# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1945my $smbios1_fmt = {
1946 uuid => {
1947 type => 'string',
1948 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1949 format_description => 'UUID',
1950 description => "Set SMBIOS1 UUID.",
1951 optional => 1,
1952 },
1953 version => {
1954 type => 'string',
1955 pattern => '[A-Za-z0-9+\/]+={0,2}',
1956 format_description => 'Base64 encoded string',
1957 description => "Set SMBIOS1 version.",
1958 optional => 1,
1959 },
1960 serial => {
1961 type => 'string',
1962 pattern => '[A-Za-z0-9+\/]+={0,2}',
1963 format_description => 'Base64 encoded string',
1964 description => "Set SMBIOS1 serial number.",
1965 optional => 1,
1966 },
1967 manufacturer => {
1968 type => 'string',
1969 pattern => '[A-Za-z0-9+\/]+={0,2}',
1970 format_description => 'Base64 encoded string',
1971 description => "Set SMBIOS1 manufacturer.",
1972 optional => 1,
1973 },
1974 product => {
1975 type => 'string',
1976 pattern => '[A-Za-z0-9+\/]+={0,2}',
1977 format_description => 'Base64 encoded string',
1978 description => "Set SMBIOS1 product ID.",
1979 optional => 1,
1980 },
1981 sku => {
1982 type => 'string',
1983 pattern => '[A-Za-z0-9+\/]+={0,2}',
1984 format_description => 'Base64 encoded string',
1985 description => "Set SMBIOS1 SKU string.",
1986 optional => 1,
1987 },
1988 family => {
1989 type => 'string',
1990 pattern => '[A-Za-z0-9+\/]+={0,2}',
1991 format_description => 'Base64 encoded string',
1992 description => "Set SMBIOS1 family string.",
1993 optional => 1,
1994 },
1995 base64 => {
1996 type => 'boolean',
1997 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1998 optional => 1,
1999 },
2000};
2001
2002sub parse_smbios1 {
2003 my ($data) = @_;
2004
2005 my $res = eval { parse_property_string($smbios1_fmt, $data) };
2006 warn $@ if $@;
2007 return $res;
2008}
2009
2010sub print_smbios1 {
2011 my ($smbios1) = @_;
2012 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2013}
2014
2015PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2016
2017sub parse_watchdog {
2018 my ($value) = @_;
2019
2020 return if !$value;
2021
2022 my $res = eval { parse_property_string($watchdog_fmt, $value) };
2023 warn $@ if $@;
2024 return $res;
2025}
2026
2027sub parse_guest_agent {
2028 my ($conf) = @_;
2029
2030 return {} if !defined($conf->{agent});
2031
2032 my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2033 warn $@ if $@;
2034
2035 # if the agent is disabled ignore the other potentially set properties
2036 return {} if !$res->{enabled};
2037 return $res;
2038}
2039
2040sub get_qga_key {
2041 my ($conf, $key) = @_;
2042 return undef if !defined($conf->{agent});
2043
2044 my $agent = parse_guest_agent($conf);
2045 return $agent->{$key};
2046}
2047
2048sub parse_vga {
2049 my ($value) = @_;
2050
2051 return {} if !$value;
2052 my $res = eval { parse_property_string($vga_fmt, $value) };
2053 warn $@ if $@;
2054 return $res;
2055}
2056
2057sub parse_rng {
2058 my ($value) = @_;
2059
2060 return if !$value;
2061
2062 my $res = eval { parse_property_string($rng_fmt, $value) };
2063 warn $@ if $@;
2064 return $res;
2065}
2066
2067PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2068sub verify_usb_device {
2069 my ($value, $noerr) = @_;
2070
2071 return $value if parse_usb_device($value);
2072
2073 return if $noerr;
2074
2075 die "unable to parse usb device\n";
2076}
2077
2078# add JSON properties for create and set function
2079sub json_config_properties {
2080 my $prop = shift;
2081
2082 foreach my $opt (keys %$confdesc) {
2083 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2084 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2085 $prop->{$opt} = $confdesc->{$opt};
2086 }
2087
2088 return $prop;
2089}
2090
2091# return copy of $confdesc_cloudinit to generate documentation
2092sub cloudinit_config_properties {
2093
2094 return dclone($confdesc_cloudinit);
2095}
2096
2097sub check_type {
2098 my ($key, $value) = @_;
2099
2100 die "unknown setting '$key'\n" if !$confdesc->{$key};
2101
2102 my $type = $confdesc->{$key}->{type};
2103
2104 if (!defined($value)) {
2105 die "got undefined value\n";
2106 }
2107
2108 if ($value =~ m/[\n\r]/) {
2109 die "property contains a line feed\n";
2110 }
2111
2112 if ($type eq 'boolean') {
2113 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2114 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2115 die "type check ('boolean') failed - got '$value'\n";
2116 } elsif ($type eq 'integer') {
2117 return int($1) if $value =~ m/^(\d+)$/;
2118 die "type check ('integer') failed - got '$value'\n";
2119 } elsif ($type eq 'number') {
2120 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2121 die "type check ('number') failed - got '$value'\n";
2122 } elsif ($type eq 'string') {
2123 if (my $fmt = $confdesc->{$key}->{format}) {
2124 PVE::JSONSchema::check_format($fmt, $value);
2125 return $value;
2126 }
2127 $value =~ s/^\"(.*)\"$/$1/;
2128 return $value;
2129 } else {
2130 die "internal error"
2131 }
2132}
2133
2134sub destroy_vm {
2135 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2136
2137 my $conf = PVE::QemuConfig->load_config($vmid);
2138
2139 PVE::QemuConfig->check_lock($conf) if !$skiplock;
2140
2141 if ($conf->{template}) {
2142 # check if any base image is still used by a linked clone
2143 PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2144 my ($ds, $drive) = @_;
2145 return if drive_is_cdrom($drive);
2146
2147 my $volid = $drive->{file};
2148 return if !$volid || $volid =~ m|^/|;
2149
2150 die "base volume '$volid' is still in use by linked cloned\n"
2151 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2152
2153 });
2154 }
2155
2156 my $remove_owned_drive = sub {
2157 my ($ds, $drive) = @_;
2158 return if drive_is_cdrom($drive, 1);
2159
2160 my $volid = $drive->{file};
2161 return if !$volid || $volid =~ m|^/|;
2162
2163 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2164 return if !$path || !$owner || ($owner != $vmid);
2165
2166 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2167 warn "Could not remove disk '$volid', check manually: $@" if $@;
2168 };
2169
2170 # only remove disks owned by this VM (referenced in the config)
2171 my $include_opts = {
2172 include_unused => 1,
2173 extra_keys => ['vmstate'],
2174 };
2175 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2176
2177 for my $snap (values %{$conf->{snapshots}}) {
2178 next if !defined($snap->{vmstate});
2179 my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2180 next if !defined($drive);
2181 $remove_owned_drive->('vmstate', $drive);
2182 }
2183
2184 if ($purge_unreferenced) { # also remove unreferenced disk
2185 my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2186 PVE::Storage::foreach_volid($vmdisks, sub {
2187 my ($volid, $sid, $volname, $d) = @_;
2188 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2189 warn $@ if $@;
2190 });
2191 }
2192
2193 if (defined $replacement_conf) {
2194 PVE::QemuConfig->write_config($vmid, $replacement_conf);
2195 } else {
2196 PVE::QemuConfig->destroy_config($vmid);
2197 }
2198}
2199
2200sub parse_vm_config {
2201 my ($filename, $raw) = @_;
2202
2203 return if !defined($raw);
2204
2205 my $res = {
2206 digest => Digest::SHA::sha1_hex($raw),
2207 snapshots => {},
2208 pending => {},
2209 };
2210
2211 $filename =~ m|/qemu-server/(\d+)\.conf$|
2212 || die "got strange filename '$filename'";
2213
2214 my $vmid = $1;
2215
2216 my $conf = $res;
2217 my $descr;
2218 my $section = '';
2219
2220 my @lines = split(/\n/, $raw);
2221 foreach my $line (@lines) {
2222 next if $line =~ m/^\s*$/;
2223
2224 if ($line =~ m/^\[PENDING\]\s*$/i) {
2225 $section = 'pending';
2226 if (defined($descr)) {
2227 $descr =~ s/\s+$//;
2228 $conf->{description} = $descr;
2229 }
2230 $descr = undef;
2231 $conf = $res->{$section} = {};
2232 next;
2233
2234 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2235 $section = $1;
2236 if (defined($descr)) {
2237 $descr =~ s/\s+$//;
2238 $conf->{description} = $descr;
2239 }
2240 $descr = undef;
2241 $conf = $res->{snapshots}->{$section} = {};
2242 next;
2243 }
2244
2245 if ($line =~ m/^\#(.*)\s*$/) {
2246 $descr = '' if !defined($descr);
2247 $descr .= PVE::Tools::decode_text($1) . "\n";
2248 next;
2249 }
2250
2251 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2252 $descr = '' if !defined($descr);
2253 $descr .= PVE::Tools::decode_text($2);
2254 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2255 $conf->{snapstate} = $1;
2256 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2257 my $key = $1;
2258 my $value = $2;
2259 $conf->{$key} = $value;
2260 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2261 my $value = $1;
2262 if ($section eq 'pending') {
2263 $conf->{delete} = $value; # we parse this later
2264 } else {
2265 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2266 }
2267 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2268 my $key = $1;
2269 my $value = $2;
2270 eval { $value = check_type($key, $value); };
2271 if ($@) {
2272 warn "vm $vmid - unable to parse value of '$key' - $@";
2273 } else {
2274 $key = 'ide2' if $key eq 'cdrom';
2275 my $fmt = $confdesc->{$key}->{format};
2276 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2277 my $v = parse_drive($key, $value);
2278 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2279 $v->{file} = $volid;
2280 $value = print_drive($v);
2281 } else {
2282 warn "vm $vmid - unable to parse value of '$key'\n";
2283 next;
2284 }
2285 }
2286
2287 $conf->{$key} = $value;
2288 }
2289 } else {
2290 warn "vm $vmid - unable to parse config: $line\n";
2291 }
2292 }
2293
2294 if (defined($descr)) {
2295 $descr =~ s/\s+$//;
2296 $conf->{description} = $descr;
2297 }
2298 delete $res->{snapstate}; # just to be sure
2299
2300 return $res;
2301}
2302
2303sub write_vm_config {
2304 my ($filename, $conf) = @_;
2305
2306 delete $conf->{snapstate}; # just to be sure
2307
2308 if ($conf->{cdrom}) {
2309 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2310 $conf->{ide2} = $conf->{cdrom};
2311 delete $conf->{cdrom};
2312 }
2313
2314 # we do not use 'smp' any longer
2315 if ($conf->{sockets}) {
2316 delete $conf->{smp};
2317 } elsif ($conf->{smp}) {
2318 $conf->{sockets} = $conf->{smp};
2319 delete $conf->{cores};
2320 delete $conf->{smp};
2321 }
2322
2323 my $used_volids = {};
2324
2325 my $cleanup_config = sub {
2326 my ($cref, $pending, $snapname) = @_;
2327
2328 foreach my $key (keys %$cref) {
2329 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2330 $key eq 'snapstate' || $key eq 'pending';
2331 my $value = $cref->{$key};
2332 if ($key eq 'delete') {
2333 die "propertry 'delete' is only allowed in [PENDING]\n"
2334 if !$pending;
2335 # fixme: check syntax?
2336 next;
2337 }
2338 eval { $value = check_type($key, $value); };
2339 die "unable to parse value of '$key' - $@" if $@;
2340
2341 $cref->{$key} = $value;
2342
2343 if (!$snapname && is_valid_drivename($key)) {
2344 my $drive = parse_drive($key, $value);
2345 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2346 }
2347 }
2348 };
2349
2350 &$cleanup_config($conf);
2351
2352 &$cleanup_config($conf->{pending}, 1);
2353
2354 foreach my $snapname (keys %{$conf->{snapshots}}) {
2355 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2356 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2357 }
2358
2359 # remove 'unusedX' settings if we re-add a volume
2360 foreach my $key (keys %$conf) {
2361 my $value = $conf->{$key};
2362 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2363 delete $conf->{$key};
2364 }
2365 }
2366
2367 my $generate_raw_config = sub {
2368 my ($conf, $pending) = @_;
2369
2370 my $raw = '';
2371
2372 # add description as comment to top of file
2373 if (defined(my $descr = $conf->{description})) {
2374 if ($descr) {
2375 foreach my $cl (split(/\n/, $descr)) {
2376 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2377 }
2378 } else {
2379 $raw .= "#\n" if $pending;
2380 }
2381 }
2382
2383 foreach my $key (sort keys %$conf) {
2384 next if $key =~ /^(digest|description|pending|snapshots)$/;
2385 $raw .= "$key: $conf->{$key}\n";
2386 }
2387 return $raw;
2388 };
2389
2390 my $raw = &$generate_raw_config($conf);
2391
2392 if (scalar(keys %{$conf->{pending}})){
2393 $raw .= "\n[PENDING]\n";
2394 $raw .= &$generate_raw_config($conf->{pending}, 1);
2395 }
2396
2397 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2398 $raw .= "\n[$snapname]\n";
2399 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2400 }
2401
2402 return $raw;
2403}
2404
2405sub load_defaults {
2406
2407 my $res = {};
2408
2409 # we use static defaults from our JSON schema configuration
2410 foreach my $key (keys %$confdesc) {
2411 if (defined(my $default = $confdesc->{$key}->{default})) {
2412 $res->{$key} = $default;
2413 }
2414 }
2415
2416 return $res;
2417}
2418
2419sub config_list {
2420 my $vmlist = PVE::Cluster::get_vmlist();
2421 my $res = {};
2422 return $res if !$vmlist || !$vmlist->{ids};
2423 my $ids = $vmlist->{ids};
2424 my $nodename = nodename();
2425
2426 foreach my $vmid (keys %$ids) {
2427 my $d = $ids->{$vmid};
2428 next if !$d->{node} || $d->{node} ne $nodename;
2429 next if !$d->{type} || $d->{type} ne 'qemu';
2430 $res->{$vmid}->{exists} = 1;
2431 }
2432 return $res;
2433}
2434
2435# test if VM uses local resources (to prevent migration)
2436sub check_local_resources {
2437 my ($conf, $noerr) = @_;
2438
2439 my @loc_res = ();
2440
2441 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2442 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2443
2444 push @loc_res, "ivshmem" if $conf->{ivshmem};
2445
2446 foreach my $k (keys %$conf) {
2447 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2448 # sockets are safe: they will recreated be on the target side post-migrate
2449 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2450 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2451 }
2452
2453 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2454
2455 return \@loc_res;
2456}
2457
2458# check if used storages are available on all nodes (use by migrate)
2459sub check_storage_availability {
2460 my ($storecfg, $conf, $node) = @_;
2461
2462 PVE::QemuConfig->foreach_volume($conf, sub {
2463 my ($ds, $drive) = @_;
2464
2465 my $volid = $drive->{file};
2466 return if !$volid;
2467
2468 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2469 return if !$sid;
2470
2471 # check if storage is available on both nodes
2472 my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2473 PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2474
2475 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2476
2477 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2478 if !$scfg->{content}->{$vtype};
2479 });
2480}
2481
2482# list nodes where all VM images are available (used by has_feature API)
2483sub shared_nodes {
2484 my ($conf, $storecfg) = @_;
2485
2486 my $nodelist = PVE::Cluster::get_nodelist();
2487 my $nodehash = { map { $_ => 1 } @$nodelist };
2488 my $nodename = nodename();
2489
2490 PVE::QemuConfig->foreach_volume($conf, sub {
2491 my ($ds, $drive) = @_;
2492
2493 my $volid = $drive->{file};
2494 return if !$volid;
2495
2496 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2497 if ($storeid) {
2498 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2499 if ($scfg->{disable}) {
2500 $nodehash = {};
2501 } elsif (my $avail = $scfg->{nodes}) {
2502 foreach my $node (keys %$nodehash) {
2503 delete $nodehash->{$node} if !$avail->{$node};
2504 }
2505 } elsif (!$scfg->{shared}) {
2506 foreach my $node (keys %$nodehash) {
2507 delete $nodehash->{$node} if $node ne $nodename
2508 }
2509 }
2510 }
2511 });
2512
2513 return $nodehash
2514}
2515
2516sub check_local_storage_availability {
2517 my ($conf, $storecfg) = @_;
2518
2519 my $nodelist = PVE::Cluster::get_nodelist();
2520 my $nodehash = { map { $_ => {} } @$nodelist };
2521
2522 PVE::QemuConfig->foreach_volume($conf, sub {
2523 my ($ds, $drive) = @_;
2524
2525 my $volid = $drive->{file};
2526 return if !$volid;
2527
2528 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2529 if ($storeid) {
2530 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2531
2532 if ($scfg->{disable}) {
2533 foreach my $node (keys %$nodehash) {
2534 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2535 }
2536 } elsif (my $avail = $scfg->{nodes}) {
2537 foreach my $node (keys %$nodehash) {
2538 if (!$avail->{$node}) {
2539 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2540 }
2541 }
2542 }
2543 }
2544 });
2545
2546 foreach my $node (values %$nodehash) {
2547 if (my $unavail = $node->{unavailable_storages}) {
2548 $node->{unavailable_storages} = [ sort keys %$unavail ];
2549 }
2550 }
2551
2552 return $nodehash
2553}
2554
2555# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2556sub check_running {
2557 my ($vmid, $nocheck, $node) = @_;
2558
2559 PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2560 return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2561}
2562
2563sub vzlist {
2564
2565 my $vzlist = config_list();
2566
2567 my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2568
2569 while (defined(my $de = $fd->read)) {
2570 next if $de !~ m/^(\d+)\.pid$/;
2571 my $vmid = $1;
2572 next if !defined($vzlist->{$vmid});
2573 if (my $pid = check_running($vmid)) {
2574 $vzlist->{$vmid}->{pid} = $pid;
2575 }
2576 }
2577
2578 return $vzlist;
2579}
2580
2581our $vmstatus_return_properties = {
2582 vmid => get_standard_option('pve-vmid'),
2583 status => {
2584 description => "Qemu process status.",
2585 type => 'string',
2586 enum => ['stopped', 'running'],
2587 },
2588 maxmem => {
2589 description => "Maximum memory in bytes.",
2590 type => 'integer',
2591 optional => 1,
2592 renderer => 'bytes',
2593 },
2594 maxdisk => {
2595 description => "Root disk size in bytes.",
2596 type => 'integer',
2597 optional => 1,
2598 renderer => 'bytes',
2599 },
2600 name => {
2601 description => "VM name.",
2602 type => 'string',
2603 optional => 1,
2604 },
2605 qmpstatus => {
2606 description => "Qemu QMP agent status.",
2607 type => 'string',
2608 optional => 1,
2609 },
2610 pid => {
2611 description => "PID of running qemu process.",
2612 type => 'integer',
2613 optional => 1,
2614 },
2615 uptime => {
2616 description => "Uptime.",
2617 type => 'integer',
2618 optional => 1,
2619 renderer => 'duration',
2620 },
2621 cpus => {
2622 description => "Maximum usable CPUs.",
2623 type => 'number',
2624 optional => 1,
2625 },
2626 lock => {
2627 description => "The current config lock, if any.",
2628 type => 'string',
2629 optional => 1,
2630 },
2631 tags => {
2632 description => "The current configured tags, if any",
2633 type => 'string',
2634 optional => 1,
2635 },
2636 'running-machine' => {
2637 description => "The currently running machine type (if running).",
2638 type => 'string',
2639 optional => 1,
2640 },
2641 'running-qemu' => {
2642 description => "The currently running QEMU version (if running).",
2643 type => 'string',
2644 optional => 1,
2645 },
2646};
2647
2648my $last_proc_pid_stat;
2649
2650# get VM status information
2651# This must be fast and should not block ($full == false)
2652# We only query KVM using QMP if $full == true (this can be slow)
2653sub vmstatus {
2654 my ($opt_vmid, $full) = @_;
2655
2656 my $res = {};
2657
2658 my $storecfg = PVE::Storage::config();
2659
2660 my $list = vzlist();
2661 my $defaults = load_defaults();
2662
2663 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2664
2665 my $cpucount = $cpuinfo->{cpus} || 1;
2666
2667 foreach my $vmid (keys %$list) {
2668 next if $opt_vmid && ($vmid ne $opt_vmid);
2669
2670 my $conf = PVE::QemuConfig->load_config($vmid);
2671
2672 my $d = { vmid => int($vmid) };
2673 $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2674
2675 # fixme: better status?
2676 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2677
2678 my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2679 if (defined($size)) {
2680 $d->{disk} = 0; # no info available
2681 $d->{maxdisk} = $size;
2682 } else {
2683 $d->{disk} = 0;
2684 $d->{maxdisk} = 0;
2685 }
2686
2687 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2688 * ($conf->{cores} || $defaults->{cores});
2689 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2690 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2691
2692 $d->{name} = $conf->{name} || "VM $vmid";
2693 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2694 : $defaults->{memory}*(1024*1024);
2695
2696 if ($conf->{balloon}) {
2697 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2698 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2699 : $defaults->{shares};
2700 }
2701
2702 $d->{uptime} = 0;
2703 $d->{cpu} = 0;
2704 $d->{mem} = 0;
2705
2706 $d->{netout} = 0;
2707 $d->{netin} = 0;
2708
2709 $d->{diskread} = 0;
2710 $d->{diskwrite} = 0;
2711
2712 $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2713
2714 $d->{serial} = 1 if conf_has_serial($conf);
2715 $d->{lock} = $conf->{lock} if $conf->{lock};
2716 $d->{tags} = $conf->{tags} if defined($conf->{tags});
2717
2718 $res->{$vmid} = $d;
2719 }
2720
2721 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2722 foreach my $dev (keys %$netdev) {
2723 next if $dev !~ m/^tap([1-9]\d*)i/;
2724 my $vmid = $1;
2725 my $d = $res->{$vmid};
2726 next if !$d;
2727
2728 $d->{netout} += $netdev->{$dev}->{receive};
2729 $d->{netin} += $netdev->{$dev}->{transmit};
2730
2731 if ($full) {
2732 $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2733 $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2734 }
2735
2736 }
2737
2738 my $ctime = gettimeofday;
2739
2740 foreach my $vmid (keys %$list) {
2741
2742 my $d = $res->{$vmid};
2743 my $pid = $d->{pid};
2744 next if !$pid;
2745
2746 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2747 next if !$pstat; # not running
2748
2749 my $used = $pstat->{utime} + $pstat->{stime};
2750
2751 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2752
2753 if ($pstat->{vsize}) {
2754 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2755 }
2756
2757 my $old = $last_proc_pid_stat->{$pid};
2758 if (!$old) {
2759 $last_proc_pid_stat->{$pid} = {
2760 time => $ctime,
2761 used => $used,
2762 cpu => 0,
2763 };
2764 next;
2765 }
2766
2767 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2768
2769 if ($dtime > 1000) {
2770 my $dutime = $used - $old->{used};
2771
2772 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2773 $last_proc_pid_stat->{$pid} = {
2774 time => $ctime,
2775 used => $used,
2776 cpu => $d->{cpu},
2777 };
2778 } else {
2779 $d->{cpu} = $old->{cpu};
2780 }
2781 }
2782
2783 return $res if !$full;
2784
2785 my $qmpclient = PVE::QMPClient->new();
2786
2787 my $ballooncb = sub {
2788 my ($vmid, $resp) = @_;
2789
2790 my $info = $resp->{'return'};
2791 return if !$info->{max_mem};
2792
2793 my $d = $res->{$vmid};
2794
2795 # use memory assigned to VM
2796 $d->{maxmem} = $info->{max_mem};
2797 $d->{balloon} = $info->{actual};
2798
2799 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2800 $d->{mem} = $info->{total_mem} - $info->{free_mem};
2801 $d->{freemem} = $info->{free_mem};
2802 }
2803
2804 $d->{ballooninfo} = $info;
2805 };
2806
2807 my $blockstatscb = sub {
2808 my ($vmid, $resp) = @_;
2809 my $data = $resp->{'return'} || [];
2810 my $totalrdbytes = 0;
2811 my $totalwrbytes = 0;
2812
2813 for my $blockstat (@$data) {
2814 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2815 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2816
2817 $blockstat->{device} =~ s/drive-//;
2818 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2819 }
2820 $res->{$vmid}->{diskread} = $totalrdbytes;
2821 $res->{$vmid}->{diskwrite} = $totalwrbytes;
2822 };
2823
2824 my $machinecb = sub {
2825 my ($vmid, $resp) = @_;
2826 my $data = $resp->{'return'} || [];
2827
2828 $res->{$vmid}->{'running-machine'} =
2829 PVE::QemuServer::Machine::current_from_query_machines($data);
2830 };
2831
2832 my $versioncb = sub {
2833 my ($vmid, $resp) = @_;
2834 my $data = $resp->{'return'} // {};
2835 my $version = 'unknown';
2836
2837 if (my $v = $data->{qemu}) {
2838 $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2839 }
2840
2841 $res->{$vmid}->{'running-qemu'} = $version;
2842 };
2843
2844 my $statuscb = sub {
2845 my ($vmid, $resp) = @_;
2846
2847 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2848 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2849 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2850 # this fails if ballon driver is not loaded, so this must be
2851 # the last commnand (following command are aborted if this fails).
2852 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2853
2854 my $status = 'unknown';
2855 if (!defined($status = $resp->{'return'}->{status})) {
2856 warn "unable to get VM status\n";
2857 return;
2858 }
2859
2860 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2861 };
2862
2863 foreach my $vmid (keys %$list) {
2864 next if $opt_vmid && ($vmid ne $opt_vmid);
2865 next if !$res->{$vmid}->{pid}; # not running
2866 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2867 }
2868
2869 $qmpclient->queue_execute(undef, 2);
2870
2871 foreach my $vmid (keys %$list) {
2872 next if $opt_vmid && ($vmid ne $opt_vmid);
2873 next if !$res->{$vmid}->{pid}; #not running
2874
2875 # we can't use the $qmpclient since it might have already aborted on
2876 # 'query-balloon', but this might also fail for older versions...
2877 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2878 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2879 }
2880
2881 foreach my $vmid (keys %$list) {
2882 next if $opt_vmid && ($vmid ne $opt_vmid);
2883 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2884 }
2885
2886 return $res;
2887}
2888
2889sub conf_has_serial {
2890 my ($conf) = @_;
2891
2892 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2893 if ($conf->{"serial$i"}) {
2894 return 1;
2895 }
2896 }
2897
2898 return 0;
2899}
2900
2901sub conf_has_audio {
2902 my ($conf, $id) = @_;
2903
2904 $id //= 0;
2905 my $audio = $conf->{"audio$id"};
2906 return if !defined($audio);
2907
2908 my $audioproperties = parse_property_string($audio_fmt, $audio);
2909 my $audiodriver = $audioproperties->{driver} // 'spice';
2910
2911 return {
2912 dev => $audioproperties->{device},
2913 dev_id => "audiodev$id",
2914 backend => $audiodriver,
2915 backend_id => "$audiodriver-backend${id}",
2916 };
2917}
2918
2919sub audio_devs {
2920 my ($audio, $audiopciaddr, $machine_version) = @_;
2921
2922 my $devs = [];
2923
2924 my $id = $audio->{dev_id};
2925 my $audiodev = "";
2926 if (min_version($machine_version, 4, 2)) {
2927 $audiodev = ",audiodev=$audio->{backend_id}";
2928 }
2929
2930 if ($audio->{dev} eq 'AC97') {
2931 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2932 } elsif ($audio->{dev} =~ /intel\-hda$/) {
2933 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2934 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2935 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2936 } else {
2937 die "unkown audio device '$audio->{dev}', implement me!";
2938 }
2939
2940 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2941
2942 return $devs;
2943}
2944
2945sub vga_conf_has_spice {
2946 my ($vga) = @_;
2947
2948 my $vgaconf = parse_vga($vga);
2949 my $vgatype = $vgaconf->{type};
2950 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2951
2952 return $1 || 1;
2953}
2954
2955sub is_native($) {
2956 my ($arch) = @_;
2957 return get_host_arch() eq $arch;
2958}
2959
2960sub get_vm_arch {
2961 my ($conf) = @_;
2962 return $conf->{arch} // get_host_arch();
2963}
2964
2965my $default_machines = {
2966 x86_64 => 'pc',
2967 aarch64 => 'virt',
2968};
2969
2970sub get_installed_machine_version {
2971 my ($kvmversion) = @_;
2972 $kvmversion = kvm_user_version() if !defined($kvmversion);
2973 $kvmversion =~ m/^(\d+\.\d+)/;
2974 return $1;
2975}
2976
2977sub windows_get_pinned_machine_version {
2978 my ($machine, $base_version, $kvmversion) = @_;
2979
2980 my $pin_version = $base_version;
2981 if (!defined($base_version) ||
2982 !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2983 ) {
2984 $pin_version = get_installed_machine_version($kvmversion);
2985 }
2986 if (!$machine || $machine eq 'pc') {
2987 $machine = "pc-i440fx-$pin_version";
2988 } elsif ($machine eq 'q35') {
2989 $machine = "pc-q35-$pin_version";
2990 } elsif ($machine eq 'virt') {
2991 $machine = "virt-$pin_version";
2992 } else {
2993 warn "unknown machine type '$machine', not touching that!\n";
2994 }
2995
2996 return $machine;
2997}
2998
2999sub get_vm_machine {
3000 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3001
3002 my $machine = $forcemachine || $conf->{machine};
3003
3004 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3005 $kvmversion //= kvm_user_version();
3006 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3007 # layout which confuses windows quite a bit and may result in various regressions..
3008 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3009 if (windows_version($conf->{ostype})) {
3010 $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3011 }
3012 $arch //= 'x86_64';
3013 $machine ||= $default_machines->{$arch};
3014 if ($add_pve_version) {
3015 my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3016 $machine .= "+pve$pvever";
3017 }
3018 }
3019
3020 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3021 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3022 $machine = $1 if $is_pxe;
3023
3024 # for version-pinned machines that do not include a pve-version (e.g.
3025 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3026 $machine .= '+pve0';
3027
3028 $machine .= '.pxe' if $is_pxe;
3029 }
3030
3031 return $machine;
3032}
3033
3034sub get_ovmf_files($) {
3035 my ($arch) = @_;
3036
3037 my $ovmf = $OVMF->{$arch}
3038 or die "no OVMF images known for architecture '$arch'\n";
3039
3040 return @$ovmf;
3041}
3042
3043my $Arch2Qemu = {
3044 aarch64 => '/usr/bin/qemu-system-aarch64',
3045 x86_64 => '/usr/bin/qemu-system-x86_64',
3046};
3047sub get_command_for_arch($) {
3048 my ($arch) = @_;
3049 return '/usr/bin/kvm' if is_native($arch);
3050
3051 my $cmd = $Arch2Qemu->{$arch}
3052 or die "don't know how to emulate architecture '$arch'\n";
3053 return $cmd;
3054}
3055
3056# To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3057# to use in a QEMU command line (-cpu element), first array_intersect the result
3058# of query_supported_ with query_understood_. This is necessary because:
3059#
3060# a) query_understood_ returns flags the host cannot use and
3061# b) query_supported_ (rather the QMP call) doesn't actually return CPU
3062# flags, but CPU settings - with most of them being flags. Those settings
3063# (and some flags, curiously) cannot be specified as a "-cpu" argument.
3064#
3065# query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3066# expensive. If you need the value returned from this, you can get it much
3067# cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3068# $accel being 'kvm' or 'tcg'.
3069#
3070# pvestatd calls this function on startup and whenever the QEMU/KVM version
3071# changes, automatically populating pmxcfs.
3072#
3073# Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3074# since kvm and tcg machines support different flags
3075#
3076sub query_supported_cpu_flags {
3077 my ($arch) = @_;
3078
3079 $arch //= get_host_arch();
3080 my $default_machine = $default_machines->{$arch};
3081
3082 my $flags = {};
3083
3084 # FIXME: Once this is merged, the code below should work for ARM as well:
3085 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3086 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3087 $arch eq "aarch64";
3088
3089 my $kvm_supported = defined(kvm_version());
3090 my $qemu_cmd = get_command_for_arch($arch);
3091 my $fakevmid = -1;
3092 my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3093
3094 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3095 my $query_supported_run_qemu = sub {
3096 my ($kvm) = @_;
3097
3098 my $flags = {};
3099 my $cmd = [
3100 $qemu_cmd,
3101 '-machine', $default_machine,
3102 '-display', 'none',
3103 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3104 '-mon', 'chardev=qmp,mode=control',
3105 '-pidfile', $pidfile,
3106 '-S', '-daemonize'
3107 ];
3108
3109 if (!$kvm) {
3110 push @$cmd, '-accel', 'tcg';
3111 }
3112
3113 my $rc = run_command($cmd, noerr => 1, quiet => 0);
3114 die "QEMU flag querying VM exited with code " . $rc if $rc;
3115
3116 eval {
3117 my $cmd_result = mon_cmd(
3118 $fakevmid,
3119 'query-cpu-model-expansion',
3120 type => 'full',
3121 model => { name => 'host' }
3122 );
3123
3124 my $props = $cmd_result->{model}->{props};
3125 foreach my $prop (keys %$props) {
3126 next if $props->{$prop} ne '1';
3127 # QEMU returns some flags multiple times, with '_', '.' or '-'
3128 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3129 # We only keep those with underscores, to match /proc/cpuinfo
3130 $prop =~ s/\.|-/_/g;
3131 $flags->{$prop} = 1;
3132 }
3133 };
3134 my $err = $@;
3135
3136 # force stop with 10 sec timeout and 'nocheck'
3137 # always stop, even if QMP failed
3138 vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3139
3140 die $err if $err;
3141
3142 return [ sort keys %$flags ];
3143 };
3144
3145 # We need to query QEMU twice, since KVM and TCG have different supported flags
3146 PVE::QemuConfig->lock_config($fakevmid, sub {
3147 $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3148 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3149
3150 if ($kvm_supported) {
3151 $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3152 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3153 }
3154 });
3155
3156 return $flags;
3157}
3158
3159# Understood CPU flags are written to a file at 'pve-qemu' compile time
3160my $understood_cpu_flag_dir = "/usr/share/kvm";
3161sub query_understood_cpu_flags {
3162 my $arch = get_host_arch();
3163 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3164
3165 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3166 if ! -e $filepath;
3167
3168 my $raw = file_get_contents($filepath);
3169 $raw =~ s/^\s+|\s+$//g;
3170 my @flags = split(/\s+/, $raw);
3171
3172 return \@flags;
3173}
3174
3175my sub get_cpuunits {
3176 my ($conf) = @_;
3177 return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3178}
3179sub config_to_command {
3180 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3181 $pbs_backing) = @_;
3182
3183 my $cmd = [];
3184 my $globalFlags = [];
3185 my $machineFlags = [];
3186 my $rtcFlags = [];
3187 my $devices = [];
3188 my $pciaddr = '';
3189 my $bridges = {};
3190 my $ostype = $conf->{ostype};
3191 my $winversion = windows_version($ostype);
3192 my $kvm = $conf->{kvm};
3193 my $nodename = nodename();
3194
3195 my $arch = get_vm_arch($conf);
3196 my $kvm_binary = get_command_for_arch($arch);
3197 my $kvmver = kvm_user_version($kvm_binary);
3198
3199 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3200 $kvmver //= "undefined";
3201 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3202 }
3203
3204 my $add_pve_version = min_version($kvmver, 4, 1);
3205
3206 my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3207 my $machine_version = extract_version($machine_type, $kvmver);
3208 $kvm //= 1 if is_native($arch);
3209
3210 $machine_version =~ m/(\d+)\.(\d+)/;
3211 my ($machine_major, $machine_minor) = ($1, $2);
3212
3213 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3214 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3215 } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3216 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3217 ." please upgrade node '$nodename'\n"
3218 } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3219 my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3220 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3221 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3222 ." node '$nodename'\n";
3223 }
3224
3225 # if a specific +pve version is required for a feature, use $version_guard
3226 # instead of min_version to allow machines to be run with the minimum
3227 # required version
3228 my $required_pve_version = 0;
3229 my $version_guard = sub {
3230 my ($major, $minor, $pve) = @_;
3231 return 0 if !min_version($machine_version, $major, $minor, $pve);
3232 my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3233 return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3234 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3235 return 1;
3236 };
3237
3238 if ($kvm && !defined kvm_version()) {
3239 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3240 ." or enable in BIOS.\n";
3241 }
3242
3243 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3244 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3245 my $use_old_bios_files = undef;
3246 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3247
3248 my $cpuunits = get_cpuunits($conf);
3249
3250 push @$cmd, $kvm_binary;
3251
3252 push @$cmd, '-id', $vmid;
3253
3254 my $vmname = $conf->{name} || "vm$vmid";
3255
3256 push @$cmd, '-name', $vmname;
3257
3258 push @$cmd, '-no-shutdown';
3259
3260 my $use_virtio = 0;
3261
3262 my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3263 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3264 push @$cmd, '-mon', "chardev=qmp,mode=control";
3265
3266 if (min_version($machine_version, 2, 12)) {
3267 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3268 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3269 }
3270
3271 push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3272
3273 push @$cmd, '-daemonize';
3274
3275 if ($conf->{smbios1}) {
3276 my $smbios_conf = parse_smbios1($conf->{smbios1});
3277 if ($smbios_conf->{base64}) {
3278 # Do not pass base64 flag to qemu
3279 delete $smbios_conf->{base64};
3280 my $smbios_string = "";
3281 foreach my $key (keys %$smbios_conf) {
3282 my $value;
3283 if ($key eq "uuid") {
3284 $value = $smbios_conf->{uuid}
3285 } else {
3286 $value = decode_base64($smbios_conf->{$key});
3287 }
3288 # qemu accepts any binary data, only commas need escaping by double comma
3289 $value =~ s/,/,,/g;
3290 $smbios_string .= "," . $key . "=" . $value if $value;
3291 }
3292 push @$cmd, '-smbios', "type=1" . $smbios_string;
3293 } else {
3294 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3295 }
3296 }
3297
3298 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3299 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3300 die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3301
3302 my ($path, $format);
3303 my $read_only_str = '';
3304 if (my $efidisk = $conf->{efidisk0}) {
3305 my $d = parse_drive('efidisk0', $efidisk);
3306 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3307 $format = $d->{format};
3308 if ($storeid) {
3309 $path = PVE::Storage::path($storecfg, $d->{file});
3310 if (!defined($format)) {
3311 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3312 $format = qemu_img_format($scfg, $volname);
3313 }
3314 } else {
3315 $path = $d->{file};
3316 die "efidisk format must be specified\n"
3317 if !defined($format);
3318 }
3319
3320 $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3321 } else {
3322 warn "no efidisk configured! Using temporary efivars disk.\n";
3323 $path = "/tmp/$vmid-ovmf.fd";
3324 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3325 $format = 'raw';
3326 }
3327
3328 my $size_str = "";
3329
3330 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3331 $size_str = ",size=" . (-s $ovmf_vars);
3332 }
3333
3334 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3335 my $cache = "";
3336 if ($path =~ m/^rbd:/) {
3337 $cache = ',cache=writeback';
3338 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3339 }
3340
3341 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3342 push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3343 }
3344
3345 # load q35 config
3346 if ($q35) {
3347 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3348 if (min_version($machine_version, 4, 0)) {
3349 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3350 } else {
3351 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3352 }
3353 }
3354
3355 if ($conf->{vmgenid}) {
3356 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3357 }
3358
3359 # add usb controllers
3360 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3361 $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3362 push @$devices, @usbcontrollers if @usbcontrollers;
3363 my $vga = parse_vga($conf->{vga});
3364
3365 my $qxlnum = vga_conf_has_spice($conf->{vga});
3366 $vga->{type} = 'qxl' if $qxlnum;
3367
3368 if (!$vga->{type}) {
3369 if ($arch eq 'aarch64') {
3370 $vga->{type} = 'virtio';
3371 } elsif (min_version($machine_version, 2, 9)) {
3372 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3373 } else {
3374 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3375 }
3376 }
3377
3378 # enable absolute mouse coordinates (needed by vnc)
3379 my $tablet;
3380 if (defined($conf->{tablet})) {
3381 $tablet = $conf->{tablet};
3382 } else {
3383 $tablet = $defaults->{tablet};
3384 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3385 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3386 }
3387
3388 if ($tablet) {
3389 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3390 my $kbd = print_keyboarddevice_full($conf, $arch);
3391 push @$devices, '-device', $kbd if defined($kbd);
3392 }
3393
3394 my $bootorder = device_bootorder($conf);
3395
3396 # host pci device passthrough
3397 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3398 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3399
3400 # usb devices
3401 my $usb_dev_features = {};
3402 $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3403
3404 my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3405 $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3406 push @$devices, @usbdevices if @usbdevices;
3407
3408 # serial devices
3409 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3410 if (my $path = $conf->{"serial$i"}) {
3411 if ($path eq 'socket') {
3412 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3413 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3414 # On aarch64, serial0 is the UART device. Qemu only allows
3415 # connecting UART devices via the '-serial' command line, as
3416 # the device has a fixed slot on the hardware...
3417 if ($arch eq 'aarch64' && $i == 0) {
3418 push @$devices, '-serial', "chardev:serial$i";
3419 } else {
3420 push @$devices, '-device', "isa-serial,chardev=serial$i";
3421 }
3422 } else {
3423 die "no such serial device\n" if ! -c $path;
3424 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3425 push @$devices, '-device', "isa-serial,chardev=serial$i";
3426 }
3427 }
3428 }
3429
3430 # parallel devices
3431 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3432 if (my $path = $conf->{"parallel$i"}) {
3433 die "no such parallel device\n" if ! -c $path;
3434 my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3435 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3436 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3437 }
3438 }
3439
3440 if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3441 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3442 my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3443 push @$devices, @$audio_devs;
3444 }
3445
3446 my $sockets = 1;
3447 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3448 $sockets = $conf->{sockets} if $conf->{sockets};
3449
3450 my $cores = $conf->{cores} || 1;
3451
3452 my $maxcpus = $sockets * $cores;
3453
3454 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3455
3456 my $allowed_vcpus = $cpuinfo->{cpus};
3457
3458 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3459 if ($allowed_vcpus < $maxcpus);
3460
3461 if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3462
3463 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3464 for (my $i = 2; $i <= $vcpus; $i++) {
3465 my $cpustr = print_cpu_device($conf,$i);
3466 push @$cmd, '-device', $cpustr;
3467 }
3468
3469 } else {
3470
3471 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3472 }
3473 push @$cmd, '-nodefaults';
3474
3475 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3476
3477 push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3478
3479 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
3480
3481 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3482 push @$devices, '-device', print_vga_device(
3483 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3484 my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3485 push @$cmd, '-vnc', "unix:$socket,password=on";
3486 } else {
3487 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3488 push @$cmd, '-nographic';
3489 }
3490
3491 # time drift fix
3492 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3493 my $useLocaltime = $conf->{localtime};
3494
3495 if ($winversion >= 5) { # windows
3496 $useLocaltime = 1 if !defined($conf->{localtime});
3497
3498 # use time drift fix when acpi is enabled
3499 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3500 $tdf = 1 if !defined($conf->{tdf});
3501 }
3502 }
3503
3504 if ($winversion >= 6) {
3505 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3506 push @$cmd, '-no-hpet';
3507 }
3508
3509 push @$rtcFlags, 'driftfix=slew' if $tdf;
3510
3511 if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3512 push @$rtcFlags, "base=$conf->{startdate}";
3513 } elsif ($useLocaltime) {
3514 push @$rtcFlags, 'base=localtime';
3515 }
3516
3517 if ($forcecpu) {
3518 push @$cmd, '-cpu', $forcecpu;
3519 } else {
3520 push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3521 }
3522
3523 PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3524
3525 push @$cmd, '-S' if $conf->{freeze};
3526
3527 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3528
3529 my $guest_agent = parse_guest_agent($conf);
3530
3531 if ($guest_agent->{enabled}) {
3532 my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3533 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3534
3535 if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3536 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3537 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3538 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3539 } elsif ($guest_agent->{type} eq 'isa') {
3540 push @$devices, '-device', "isa-serial,chardev=qga0";
3541 }
3542 }
3543
3544 my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3545 if ($rng && $version_guard->(4, 1, 2)) {
3546 check_rng_source($rng->{source});
3547
3548 my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3549 my $period = $rng->{period} // $rng_fmt->{period}->{default};
3550 my $limiter_str = "";
3551 if ($max_bytes) {
3552 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3553 }
3554
3555 my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3556 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3557 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3558 }
3559
3560 my $spice_port;
3561
3562 if ($qxlnum) {
3563 if ($qxlnum > 1) {
3564 if ($winversion){
3565 for (my $i = 1; $i < $qxlnum; $i++){
3566 push @$devices, '-device', print_vga_device(
3567 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3568 }
3569 } else {
3570 # assume other OS works like Linux
3571 my ($ram, $vram) = ("134217728", "67108864");
3572 if ($vga->{memory}) {
3573 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3574 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3575 }
3576 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3577 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3578 }
3579 }
3580
3581 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3582
3583 my $pfamily = PVE::Tools::get_host_address_family($nodename);
3584 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3585 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3586
3587 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3588 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3589 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3590
3591 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3592 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3593
3594 my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3595 my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3596 if ($spice_enhancement->{foldersharing}) {
3597 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3598 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3599 }
3600
3601 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3602 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3603 if $spice_enhancement->{videostreaming};
3604
3605 push @$devices, '-spice', "$spice_opts";
3606 }
3607
3608 # enable balloon by default, unless explicitly disabled
3609 if (!defined($conf->{balloon}) || $conf->{balloon}) {
3610 $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3611 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3612 }
3613
3614 if ($conf->{watchdog}) {
3615 my $wdopts = parse_watchdog($conf->{watchdog});
3616 $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3617 my $watchdog = $wdopts->{model} || 'i6300esb';
3618 push @$devices, '-device', "$watchdog$pciaddr";
3619 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3620 }
3621
3622 my $vollist = [];
3623 my $scsicontroller = {};
3624 my $ahcicontroller = {};
3625 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3626
3627 # Add iscsi initiator name if available
3628 if (my $initiator = get_initiator_name()) {
3629 push @$devices, '-iscsi', "initiator-name=$initiator";
3630 }
3631
3632 PVE::QemuConfig->foreach_volume($conf, sub {
3633 my ($ds, $drive) = @_;
3634
3635 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3636 check_volume_storage_type($storecfg, $drive->{file});
3637 push @$vollist, $drive->{file};
3638 }
3639
3640 # ignore efidisk here, already added in bios/fw handling code above
3641 return if $drive->{interface} eq 'efidisk';
3642
3643 $use_virtio = 1 if $ds =~ m/^virtio/;
3644
3645 $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3646
3647 if ($drive->{interface} eq 'virtio'){
3648 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3649 }
3650
3651 if ($drive->{interface} eq 'scsi') {
3652
3653 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3654
3655 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3656 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3657
3658 $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3659 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3660
3661 my $iothread = '';
3662 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3663 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3664 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3665 } elsif ($drive->{iothread}) {
3666 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3667 }
3668
3669 my $queues = '';
3670 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3671 $queues = ",num_queues=$drive->{queues}";
3672 }
3673
3674 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3675 if !$scsicontroller->{$controller};
3676 $scsicontroller->{$controller}=1;
3677 }
3678
3679 if ($drive->{interface} eq 'sata') {
3680 my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3681 $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3682 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3683 if !$ahcicontroller->{$controller};
3684 $ahcicontroller->{$controller}=1;
3685 }
3686
3687 my $pbs_conf = $pbs_backing->{$ds};
3688 my $pbs_name = undef;
3689 if ($pbs_conf) {
3690 $pbs_name = "drive-$ds-pbs";
3691 push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3692 }
3693
3694 my $drive_cmd = print_drive_commandline_full(
3695 $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3696
3697 # extra protection for templates, but SATA and IDE don't support it..
3698 $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3699
3700 push @$devices, '-drive',$drive_cmd;
3701 push @$devices, '-device', print_drivedevice_full(
3702 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3703 });
3704
3705 for (my $i = 0; $i < $MAX_NETS; $i++) {
3706 my $netname = "net$i";
3707
3708 next if !$conf->{$netname};
3709 my $d = parse_net($conf->{$netname});
3710 next if !$d;
3711
3712 $use_virtio = 1 if $d->{model} eq 'virtio';
3713
3714 $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3715
3716 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3717 push @$devices, '-netdev', $netdevfull;
3718
3719 my $netdevicefull = print_netdevice_full(
3720 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3721
3722 push @$devices, '-device', $netdevicefull;
3723 }
3724
3725 if ($conf->{ivshmem}) {
3726 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3727
3728 my $bus;
3729 if ($q35) {
3730 $bus = print_pcie_addr("ivshmem");
3731 } else {
3732 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3733 }
3734
3735 my $ivshmem_name = $ivshmem->{name} // $vmid;
3736 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3737
3738 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3739 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3740 .",size=$ivshmem->{size}M";
3741 }
3742
3743 # pci.4 is nested in pci.1
3744 $bridges->{1} = 1 if $bridges->{4};
3745
3746 if (!$q35) {
3747 # add pci bridges
3748 if (min_version($machine_version, 2, 3)) {
3749 $bridges->{1} = 1;
3750 $bridges->{2} = 1;
3751 }
3752
3753 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3754
3755 }
3756
3757 for my $k (sort {$b cmp $a} keys %$bridges) {
3758 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3759
3760 my $k_name = $k;
3761 if ($k == 2 && $legacy_igd) {
3762 $k_name = "$k-igd";
3763 }
3764 $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3765
3766 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3767 if ($q35) {
3768 # add after -readconfig pve-q35.cfg
3769 splice @$devices, 2, 0, '-device', $devstr;
3770 } else {
3771 unshift @$devices, '-device', $devstr if $k > 0;
3772 }
3773 }
3774
3775 if (!$kvm) {
3776 push @$machineFlags, 'accel=tcg';
3777 }
3778
3779 my $machine_type_min = $machine_type;
3780 if ($add_pve_version) {
3781 $machine_type_min =~ s/\+pve\d+$//;
3782 $machine_type_min .= "+pve$required_pve_version";
3783 }
3784 push @$machineFlags, "type=${machine_type_min}";
3785
3786 push @$cmd, @$devices;
3787 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3788 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3789 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3790
3791 if (my $vmstate = $conf->{vmstate}) {
3792 my $statepath = PVE::Storage::path($storecfg, $vmstate);
3793 push @$vollist, $vmstate;
3794 push @$cmd, '-loadstate', $statepath;
3795 print "activating and using '$vmstate' as vmstate\n";
3796 }
3797
3798 if (PVE::QemuConfig->is_template($conf)) {
3799 # needed to workaround base volumes being read-only
3800 push @$cmd, '-snapshot';
3801 }
3802
3803 # add custom args
3804 if ($conf->{args}) {
3805 my $aa = PVE::Tools::split_args($conf->{args});
3806 push @$cmd, @$aa;
3807 }
3808
3809 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3810}
3811
3812sub check_rng_source {
3813 my ($source) = @_;
3814
3815 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3816 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3817 if ! -e $source;
3818
3819 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3820 if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3821 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3822 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3823 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3824 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3825 ." to the host.\n";
3826 }
3827}
3828
3829sub spice_port {
3830 my ($vmid) = @_;
3831
3832 my $res = mon_cmd($vmid, 'query-spice');
3833
3834 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3835}
3836
3837sub vm_devices_list {
3838 my ($vmid) = @_;
3839
3840 my $res = mon_cmd($vmid, 'query-pci');
3841 my $devices_to_check = [];
3842 my $devices = {};
3843 foreach my $pcibus (@$res) {
3844 push @$devices_to_check, @{$pcibus->{devices}},
3845 }
3846
3847 while (@$devices_to_check) {
3848 my $to_check = [];
3849 for my $d (@$devices_to_check) {
3850 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3851 next if !$d->{'pci_bridge'};
3852
3853 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3854 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3855 }
3856 $devices_to_check = $to_check;
3857 }
3858
3859 my $resblock = mon_cmd($vmid, 'query-block');
3860 foreach my $block (@$resblock) {
3861 if($block->{device} =~ m/^drive-(\S+)/){
3862 $devices->{$1} = 1;
3863 }
3864 }
3865
3866 my $resmice = mon_cmd($vmid, 'query-mice');
3867 foreach my $mice (@$resmice) {
3868 if ($mice->{name} eq 'QEMU HID Tablet') {
3869 $devices->{tablet} = 1;
3870 last;
3871 }
3872 }
3873
3874 # for usb devices there is no query-usb
3875 # but we can iterate over the entries in
3876 # qom-list path=/machine/peripheral
3877 my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3878 foreach my $per (@$resperipheral) {
3879 if ($per->{name} =~ m/^usb\d+$/) {
3880 $devices->{$per->{name}} = 1;
3881 }
3882 }
3883
3884 return $devices;
3885}
3886
3887sub vm_deviceplug {
3888 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3889
3890 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3891
3892 my $devices_list = vm_devices_list($vmid);
3893 return 1 if defined($devices_list->{$deviceid});
3894
3895 # add PCI bridge if we need it for the device
3896 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3897
3898 if ($deviceid eq 'tablet') {
3899
3900 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3901
3902 } elsif ($deviceid eq 'keyboard') {
3903
3904 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3905
3906 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3907
3908 die "usb hotplug currently not reliable\n";
3909 # since we can't reliably hot unplug all added usb devices and usb
3910 # passthrough breaks live migration we disable usb hotplugging for now
3911 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3912
3913 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3914
3915 qemu_iothread_add($vmid, $deviceid, $device);
3916
3917 qemu_driveadd($storecfg, $vmid, $device);
3918 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3919
3920 qemu_deviceadd($vmid, $devicefull);
3921 eval { qemu_deviceaddverify($vmid, $deviceid); };
3922 if (my $err = $@) {
3923 eval { qemu_drivedel($vmid, $deviceid); };
3924 warn $@ if $@;
3925 die $err;
3926 }
3927
3928 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3929
3930
3931 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3932 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3933 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3934
3935 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3936
3937 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3938 qemu_iothread_add($vmid, $deviceid, $device);
3939 $devicefull .= ",iothread=iothread-$deviceid";
3940 }
3941
3942 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3943 $devicefull .= ",num_queues=$device->{queues}";
3944 }
3945
3946 qemu_deviceadd($vmid, $devicefull);
3947 qemu_deviceaddverify($vmid, $deviceid);
3948
3949 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3950
3951 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3952 qemu_driveadd($storecfg, $vmid, $device);
3953
3954 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3955 eval { qemu_deviceadd($vmid, $devicefull); };
3956 if (my $err = $@) {
3957 eval { qemu_drivedel($vmid, $deviceid); };
3958 warn $@ if $@;
3959 die $err;
3960 }
3961
3962 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3963
3964 return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3965
3966 my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3967 my $use_old_bios_files = undef;
3968 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3969
3970 my $netdevicefull = print_netdevice_full(
3971 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3972 qemu_deviceadd($vmid, $netdevicefull);
3973 eval {
3974 qemu_deviceaddverify($vmid, $deviceid);
3975 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3976 };
3977 if (my $err = $@) {
3978 eval { qemu_netdevdel($vmid, $deviceid); };
3979 warn $@ if $@;
3980 die $err;
3981 }
3982
3983 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3984
3985 my $bridgeid = $2;
3986 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3987 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3988
3989 qemu_deviceadd($vmid, $devicefull);
3990 qemu_deviceaddverify($vmid, $deviceid);
3991
3992 } else {
3993 die "can't hotplug device '$deviceid'\n";
3994 }
3995
3996 return 1;
3997}
3998
3999# fixme: this should raise exceptions on error!
4000sub vm_deviceunplug {
4001 my ($vmid, $conf, $deviceid) = @_;
4002
4003 my $devices_list = vm_devices_list($vmid);
4004 return 1 if !defined($devices_list->{$deviceid});
4005
4006 my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4007 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4008
4009 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4010
4011 qemu_devicedel($vmid, $deviceid);
4012
4013 } elsif ($deviceid =~ m/^usb\d+$/) {
4014
4015 die "usb hotplug currently not reliable\n";
4016 # when unplugging usb devices this way, there may be remaining usb
4017 # controllers/hubs so we disable it for now
4018 #qemu_devicedel($vmid, $deviceid);
4019 #qemu_devicedelverify($vmid, $deviceid);
4020
4021 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4022
4023 qemu_devicedel($vmid, $deviceid);
4024 qemu_devicedelverify($vmid, $deviceid);
4025 qemu_drivedel($vmid, $deviceid);
4026 qemu_iothread_del($conf, $vmid, $deviceid);
4027
4028 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4029
4030 qemu_devicedel($vmid, $deviceid);
4031 qemu_devicedelverify($vmid, $deviceid);
4032 qemu_iothread_del($conf, $vmid, $deviceid);
4033
4034 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4035
4036 qemu_devicedel($vmid, $deviceid);
4037 qemu_drivedel($vmid, $deviceid);
4038 qemu_deletescsihw($conf, $vmid, $deviceid);
4039
4040 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4041
4042 qemu_devicedel($vmid, $deviceid);
4043 qemu_devicedelverify($vmid, $deviceid);
4044 qemu_netdevdel($vmid, $deviceid);
4045
4046 } else {
4047 die "can't unplug device '$deviceid'\n";
4048 }
4049
4050 return 1;
4051}
4052
4053sub qemu_deviceadd {
4054 my ($vmid, $devicefull) = @_;
4055
4056 $devicefull = "driver=".$devicefull;
4057 my %options = split(/[=,]/, $devicefull);
4058
4059 mon_cmd($vmid, "device_add" , %options);
4060}
4061
4062sub qemu_devicedel {
4063 my ($vmid, $deviceid) = @_;
4064
4065 my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4066}
4067
4068sub qemu_iothread_add {
4069 my($vmid, $deviceid, $device) = @_;
4070
4071 if ($device->{iothread}) {
4072 my $iothreads = vm_iothreads_list($vmid);
4073 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4074 }
4075}
4076
4077sub qemu_iothread_del {
4078 my($conf, $vmid, $deviceid) = @_;
4079
4080 my $confid = $deviceid;
4081 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4082 $confid = 'scsi' . $1;
4083 }
4084 my $device = parse_drive($confid, $conf->{$confid});
4085 if ($device->{iothread}) {
4086 my $iothreads = vm_iothreads_list($vmid);
4087 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4088 }
4089}
4090
4091sub qemu_objectadd {
4092 my($vmid, $objectid, $qomtype) = @_;
4093
4094 mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4095
4096 return 1;
4097}
4098
4099sub qemu_objectdel {
4100 my($vmid, $objectid) = @_;
4101
4102 mon_cmd($vmid, "object-del", id => $objectid);
4103
4104 return 1;
4105}
4106
4107sub qemu_driveadd {
4108 my ($storecfg, $vmid, $device) = @_;
4109
4110 my $kvmver = get_running_qemu_version($vmid);
4111 my $io_uring = min_version($kvmver, 6, 0);
4112 my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4113 $drive =~ s/\\/\\\\/g;
4114 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4115
4116 # If the command succeeds qemu prints: "OK"
4117 return 1 if $ret =~ m/OK/s;
4118
4119 die "adding drive failed: $ret\n";
4120}
4121
4122sub qemu_drivedel {
4123 my($vmid, $deviceid) = @_;
4124
4125 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4126 $ret =~ s/^\s+//;
4127
4128 return 1 if $ret eq "";
4129
4130 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4131 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4132
4133 die "deleting drive $deviceid failed : $ret\n";
4134}
4135
4136sub qemu_deviceaddverify {
4137 my ($vmid, $deviceid) = @_;
4138
4139 for (my $i = 0; $i <= 5; $i++) {
4140 my $devices_list = vm_devices_list($vmid);
4141 return 1 if defined($devices_list->{$deviceid});
4142 sleep 1;
4143 }
4144
4145 die "error on hotplug device '$deviceid'\n";
4146}
4147
4148
4149sub qemu_devicedelverify {
4150 my ($vmid, $deviceid) = @_;
4151
4152 # need to verify that the device is correctly removed as device_del
4153 # is async and empty return is not reliable
4154
4155 for (my $i = 0; $i <= 5; $i++) {
4156 my $devices_list = vm_devices_list($vmid);
4157 return 1 if !defined($devices_list->{$deviceid});
4158 sleep 1;
4159 }
4160
4161 die "error on hot-unplugging device '$deviceid'\n";
4162}
4163
4164sub qemu_findorcreatescsihw {
4165 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4166
4167 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4168
4169 my $scsihwid="$controller_prefix$controller";
4170 my $devices_list = vm_devices_list($vmid);
4171
4172 if(!defined($devices_list->{$scsihwid})) {
4173 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4174 }
4175
4176 return 1;
4177}
4178
4179sub qemu_deletescsihw {
4180 my ($conf, $vmid, $opt) = @_;
4181
4182 my $device = parse_drive($opt, $conf->{$opt});
4183
4184 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4185 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4186 return 1;
4187 }
4188
4189 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4190
4191 my $devices_list = vm_devices_list($vmid);
4192 foreach my $opt (keys %{$devices_list}) {
4193 if (is_valid_drivename($opt)) {
4194 my $drive = parse_drive($opt, $conf->{$opt});
4195 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4196 return 1;
4197 }
4198 }
4199 }
4200
4201 my $scsihwid="scsihw$controller";
4202
4203 vm_deviceunplug($vmid, $conf, $scsihwid);
4204
4205 return 1;
4206}
4207
4208sub qemu_add_pci_bridge {
4209 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4210
4211 my $bridges = {};
4212
4213 my $bridgeid;
4214
4215 print_pci_addr($device, $bridges, $arch, $machine_type);
4216
4217 while (my ($k, $v) = each %$bridges) {
4218 $bridgeid = $k;
4219 }
4220 return 1 if !defined($bridgeid) || $bridgeid < 1;
4221
4222 my $bridge = "pci.$bridgeid";
4223 my $devices_list = vm_devices_list($vmid);
4224
4225 if (!defined($devices_list->{$bridge})) {
4226 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4227 }
4228
4229 return 1;
4230}
4231
4232sub qemu_set_link_status {
4233 my ($vmid, $device, $up) = @_;
4234
4235 mon_cmd($vmid, "set_link", name => $device,
4236 up => $up ? JSON::true : JSON::false);
4237}
4238
4239sub qemu_netdevadd {
4240 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4241
4242 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4243 my %options = split(/[=,]/, $netdev);
4244
4245 if (defined(my $vhost = $options{vhost})) {
4246 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4247 }
4248
4249 if (defined(my $queues = $options{queues})) {
4250 $options{queues} = $queues + 0;
4251 }
4252
4253 mon_cmd($vmid, "netdev_add", %options);
4254 return 1;
4255}
4256
4257sub qemu_netdevdel {
4258 my ($vmid, $deviceid) = @_;
4259
4260 mon_cmd($vmid, "netdev_del", id => $deviceid);
4261}
4262
4263sub qemu_usb_hotplug {
4264 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4265
4266 return if !$device;
4267
4268 # remove the old one first
4269 vm_deviceunplug($vmid, $conf, $deviceid);
4270
4271 # check if xhci controller is necessary and available
4272 if ($device->{usb3}) {
4273
4274 my $devicelist = vm_devices_list($vmid);
4275
4276 if (!$devicelist->{xhci}) {
4277 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4278 qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4279 }
4280 }
4281 my $d = parse_usb_device($device->{host});
4282 $d->{usb3} = $device->{usb3};
4283
4284 # add the new one
4285 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4286}
4287
4288sub qemu_cpu_hotplug {
4289 my ($vmid, $conf, $vcpus) = @_;
4290
4291 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4292
4293 my $sockets = 1;
4294 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4295 $sockets = $conf->{sockets} if $conf->{sockets};
4296 my $cores = $conf->{cores} || 1;
4297 my $maxcpus = $sockets * $cores;
4298
4299 $vcpus = $maxcpus if !$vcpus;
4300
4301 die "you can't add more vcpus than maxcpus\n"
4302 if $vcpus > $maxcpus;
4303
4304 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4305
4306 if ($vcpus < $currentvcpus) {
4307
4308 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4309
4310 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4311 qemu_devicedel($vmid, "cpu$i");
4312 my $retry = 0;
4313 my $currentrunningvcpus = undef;
4314 while (1) {
4315 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4316 last if scalar(@{$currentrunningvcpus}) == $i-1;
4317 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4318 $retry++;
4319 sleep 1;
4320 }
4321 #update conf after each succesfull cpu unplug
4322 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4323 PVE::QemuConfig->write_config($vmid, $conf);
4324 }
4325 } else {
4326 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4327 }
4328
4329 return;
4330 }
4331
4332 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4333 die "vcpus in running vm does not match its configuration\n"
4334 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4335
4336 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4337
4338 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4339 my $cpustr = print_cpu_device($conf, $i);
4340 qemu_deviceadd($vmid, $cpustr);
4341
4342 my $retry = 0;
4343 my $currentrunningvcpus = undef;
4344 while (1) {
4345 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4346 last if scalar(@{$currentrunningvcpus}) == $i;
4347 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4348 sleep 1;
4349 $retry++;
4350 }
4351 #update conf after each succesfull cpu hotplug
4352 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4353 PVE::QemuConfig->write_config($vmid, $conf);
4354 }
4355 } else {
4356
4357 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4358 mon_cmd($vmid, "cpu-add", id => int($i));
4359 }
4360 }
4361}
4362
4363sub qemu_block_set_io_throttle {
4364 my ($vmid, $deviceid,
4365 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4366 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4367 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4368 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4369
4370 return if !check_running($vmid) ;
4371
4372 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4373 bps => int($bps),
4374 bps_rd => int($bps_rd),
4375 bps_wr => int($bps_wr),
4376 iops => int($iops),
4377 iops_rd => int($iops_rd),
4378 iops_wr => int($iops_wr),
4379 bps_max => int($bps_max),
4380 bps_rd_max => int($bps_rd_max),
4381 bps_wr_max => int($bps_wr_max),
4382 iops_max => int($iops_max),
4383 iops_rd_max => int($iops_rd_max),
4384 iops_wr_max => int($iops_wr_max),
4385 bps_max_length => int($bps_max_length),
4386 bps_rd_max_length => int($bps_rd_max_length),
4387 bps_wr_max_length => int($bps_wr_max_length),
4388 iops_max_length => int($iops_max_length),
4389 iops_rd_max_length => int($iops_rd_max_length),
4390 iops_wr_max_length => int($iops_wr_max_length),
4391 );
4392
4393}
4394
4395sub qemu_block_resize {
4396 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4397
4398 my $running = check_running($vmid);
4399
4400 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4401
4402 return if !$running;
4403
4404 my $padding = (1024 - $size % 1024) % 1024;
4405 $size = $size + $padding;
4406
4407 mon_cmd(
4408 $vmid,
4409 "block_resize",
4410 device => $deviceid,
4411 size => int($size),
4412 timeout => 60,
4413 );
4414}
4415
4416sub qemu_volume_snapshot {
4417 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4418
4419 my $running = check_running($vmid);
4420
4421 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4422 mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4423 } else {
4424 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4425 }
4426}
4427
4428sub qemu_volume_snapshot_delete {
4429 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4430
4431 my $running = check_running($vmid);
4432
4433 if($running) {
4434
4435 $running = undef;
4436 my $conf = PVE::QemuConfig->load_config($vmid);
4437 PVE::QemuConfig->foreach_volume($conf, sub {
4438 my ($ds, $drive) = @_;
4439 $running = 1 if $drive->{file} eq $volid;
4440 });
4441 }
4442
4443 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4444 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4445 } else {
4446 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4447 }
4448}
4449
4450sub set_migration_caps {
4451 my ($vmid, $savevm) = @_;
4452
4453 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4454
4455 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4456 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4457
4458 my $cap_ref = [];
4459
4460 my $enabled_cap = {
4461 "auto-converge" => 1,
4462 "xbzrle" => 1,
4463 "x-rdma-pin-all" => 0,
4464 "zero-blocks" => 0,
4465 "compress" => 0,
4466 "dirty-bitmaps" => $dirty_bitmaps,
4467 };
4468
4469 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4470
4471 for my $supported_capability (@$supported_capabilities) {
4472 push @$cap_ref, {
4473 capability => $supported_capability->{capability},
4474 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4475 };
4476 }
4477
4478 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4479}
4480
4481sub foreach_volid {
4482 my ($conf, $func, @param) = @_;
4483
4484 my $volhash = {};
4485
4486 my $test_volid = sub {
4487 my ($key, $drive, $snapname) = @_;
4488
4489 my $volid = $drive->{file};
4490 return if !$volid;
4491
4492 $volhash->{$volid}->{cdrom} //= 1;
4493 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4494
4495 my $replicate = $drive->{replicate} // 1;
4496 $volhash->{$volid}->{replicate} //= 0;
4497 $volhash->{$volid}->{replicate} = 1 if $replicate;
4498
4499 $volhash->{$volid}->{shared} //= 0;
4500 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4501
4502 $volhash->{$volid}->{referenced_in_config} //= 0;
4503 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4504
4505 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4506 if defined($snapname);
4507
4508 my $size = $drive->{size};
4509 $volhash->{$volid}->{size} //= $size if $size;
4510
4511 $volhash->{$volid}->{is_vmstate} //= 0;
4512 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4513
4514 $volhash->{$volid}->{is_unused} //= 0;
4515 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4516
4517 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4518 };
4519
4520 my $include_opts = {
4521 extra_keys => ['vmstate'],
4522 include_unused => 1,
4523 };
4524
4525 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4526 foreach my $snapname (keys %{$conf->{snapshots}}) {
4527 my $snap = $conf->{snapshots}->{$snapname};
4528 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4529 }
4530
4531 foreach my $volid (keys %$volhash) {
4532 &$func($volid, $volhash->{$volid}, @param);
4533 }
4534}
4535
4536my $fast_plug_option = {
4537 'lock' => 1,
4538 'name' => 1,
4539 'onboot' => 1,
4540 'shares' => 1,
4541 'startup' => 1,
4542 'description' => 1,
4543 'protection' => 1,
4544 'vmstatestorage' => 1,
4545 'hookscript' => 1,
4546 'tags' => 1,
4547};
4548
4549# hotplug changes in [PENDING]
4550# $selection hash can be used to only apply specified options, for
4551# example: { cores => 1 } (only apply changed 'cores')
4552# $errors ref is used to return error messages
4553sub vmconfig_hotplug_pending {
4554 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4555
4556 my $defaults = load_defaults();
4557 my $arch = get_vm_arch($conf);
4558 my $machine_type = get_vm_machine($conf, undef, $arch);
4559
4560 # commit values which do not have any impact on running VM first
4561 # Note: those option cannot raise errors, we we do not care about
4562 # $selection and always apply them.
4563
4564 my $add_error = sub {
4565 my ($opt, $msg) = @_;
4566 $errors->{$opt} = "hotplug problem - $msg";
4567 };
4568
4569 my $changes = 0;
4570 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4571 if ($fast_plug_option->{$opt}) {
4572 $conf->{$opt} = $conf->{pending}->{$opt};
4573 delete $conf->{pending}->{$opt};
4574 $changes = 1;
4575 }
4576 }
4577
4578 if ($changes) {
4579 PVE::QemuConfig->write_config($vmid, $conf);
4580 }
4581
4582 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4583
4584 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4585 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4586 foreach my $opt (sort keys %$pending_delete_hash) {
4587 next if $selection && !$selection->{$opt};
4588 my $force = $pending_delete_hash->{$opt}->{force};
4589 eval {
4590 if ($opt eq 'hotplug') {
4591 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4592 } elsif ($opt eq 'tablet') {
4593 die "skip\n" if !$hotplug_features->{usb};
4594 if ($defaults->{tablet}) {
4595 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4596 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4597 if $arch eq 'aarch64';
4598 } else {
4599 vm_deviceunplug($vmid, $conf, 'tablet');
4600 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4601 }
4602 } elsif ($opt =~ m/^usb\d+/) {
4603 die "skip\n";
4604 # since we cannot reliably hot unplug usb devices we are disabling it
4605 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4606 #vm_deviceunplug($vmid, $conf, $opt);
4607 } elsif ($opt eq 'vcpus') {
4608 die "skip\n" if !$hotplug_features->{cpu};
4609 qemu_cpu_hotplug($vmid, $conf, undef);
4610 } elsif ($opt eq 'balloon') {
4611 # enable balloon device is not hotpluggable
4612 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4613 # here we reset the ballooning value to memory
4614 my $balloon = $conf->{memory} || $defaults->{memory};
4615 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4616 } elsif ($fast_plug_option->{$opt}) {
4617 # do nothing
4618 } elsif ($opt =~ m/^net(\d+)$/) {
4619 die "skip\n" if !$hotplug_features->{network};
4620 vm_deviceunplug($vmid, $conf, $opt);
4621 } elsif (is_valid_drivename($opt)) {
4622 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4623 vm_deviceunplug($vmid, $conf, $opt);
4624 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4625 } elsif ($opt =~ m/^memory$/) {
4626 die "skip\n" if !$hotplug_features->{memory};
4627 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4628 } elsif ($opt eq 'cpuunits') {
4629 $cgroup->change_cpu_shares(undef, 1024);
4630 } elsif ($opt eq 'cpulimit') {
4631 $cgroup->change_cpu_quota(-1, 100000);
4632 } else {
4633 die "skip\n";
4634 }
4635 };
4636 if (my $err = $@) {
4637 &$add_error($opt, $err) if $err ne "skip\n";
4638 } else {
4639 delete $conf->{$opt};
4640 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4641 }
4642 }
4643
4644 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4645 $apply_pending_cloudinit = sub {
4646 return if $apply_pending_cloudinit_done; # once is enough
4647 $apply_pending_cloudinit_done = 1; # once is enough
4648
4649 my ($key, $value) = @_;
4650
4651 my @cloudinit_opts = keys %$confdesc_cloudinit;
4652 foreach my $opt (keys %{$conf->{pending}}) {
4653 next if !grep { $_ eq $opt } @cloudinit_opts;
4654 $conf->{$opt} = delete $conf->{pending}->{$opt};
4655 }
4656
4657 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4658 foreach my $opt (sort keys %$pending_delete_hash) {
4659 next if !grep { $_ eq $opt } @cloudinit_opts;
4660 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4661 delete $conf->{$opt};
4662 }
4663
4664 my $new_conf = { %$conf };
4665 $new_conf->{$key} = $value;
4666 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4667 };
4668
4669 foreach my $opt (keys %{$conf->{pending}}) {
4670 next if $selection && !$selection->{$opt};
4671 my $value = $conf->{pending}->{$opt};
4672 eval {
4673 if ($opt eq 'hotplug') {
4674 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4675 } elsif ($opt eq 'tablet') {
4676 die "skip\n" if !$hotplug_features->{usb};
4677 if ($value == 1) {
4678 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4679 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4680 if $arch eq 'aarch64';
4681 } elsif ($value == 0) {
4682 vm_deviceunplug($vmid, $conf, 'tablet');
4683 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4684 }
4685 } elsif ($opt =~ m/^usb\d+$/) {
4686 die "skip\n";
4687 # since we cannot reliably hot unplug usb devices we disable it for now
4688 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4689 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4690 #die "skip\n" if !$d;
4691 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4692 } elsif ($opt eq 'vcpus') {
4693 die "skip\n" if !$hotplug_features->{cpu};
4694 qemu_cpu_hotplug($vmid, $conf, $value);
4695 } elsif ($opt eq 'balloon') {
4696 # enable/disable balloning device is not hotpluggable
4697 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4698 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4699 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4700
4701 # allow manual ballooning if shares is set to zero
4702 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4703 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4704 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4705 }
4706 } elsif ($opt =~ m/^net(\d+)$/) {
4707 # some changes can be done without hotplug
4708 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4709 $vmid, $opt, $value, $arch, $machine_type);
4710 } elsif (is_valid_drivename($opt)) {
4711 die "skip\n" if $opt eq 'efidisk0';
4712 # some changes can be done without hotplug
4713 my $drive = parse_drive($opt, $value);
4714 if (drive_is_cloudinit($drive)) {
4715 &$apply_pending_cloudinit($opt, $value);
4716 }
4717 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4718 $vmid, $opt, $value, $arch, $machine_type);
4719 } elsif ($opt =~ m/^memory$/) { #dimms
4720 die "skip\n" if !$hotplug_features->{memory};
4721 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4722 } elsif ($opt eq 'cpuunits') {
4723 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4724 } elsif ($opt eq 'cpulimit') {
4725 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4726 $cgroup->change_cpu_quota($cpulimit, 100000);
4727 } else {
4728 die "skip\n"; # skip non-hot-pluggable options
4729 }
4730 };
4731 if (my $err = $@) {
4732 &$add_error($opt, $err) if $err ne "skip\n";
4733 } else {
4734 $conf->{$opt} = $value;
4735 delete $conf->{pending}->{$opt};
4736 }
4737 }
4738
4739 PVE::QemuConfig->write_config($vmid, $conf);
4740}
4741
4742sub try_deallocate_drive {
4743 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4744
4745 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4746 my $volid = $drive->{file};
4747 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4748 my $sid = PVE::Storage::parse_volume_id($volid);
4749 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4750
4751 # check if the disk is really unused
4752 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4753 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4754 PVE::Storage::vdisk_free($storecfg, $volid);
4755 return 1;
4756 } else {
4757 # If vm is not owner of this disk remove from config
4758 return 1;
4759 }
4760 }
4761
4762 return;
4763}
4764
4765sub vmconfig_delete_or_detach_drive {
4766 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4767
4768 my $drive = parse_drive($opt, $conf->{$opt});
4769
4770 my $rpcenv = PVE::RPCEnvironment::get();
4771 my $authuser = $rpcenv->get_user();
4772
4773 if ($force) {
4774 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4775 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4776 } else {
4777 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4778 }
4779}
4780
4781
4782
4783sub vmconfig_apply_pending {
4784 my ($vmid, $conf, $storecfg, $errors) = @_;
4785
4786 my $add_apply_error = sub {
4787 my ($opt, $msg) = @_;
4788 my $err_msg = "unable to apply pending change $opt : $msg";
4789 $errors->{$opt} = $err_msg;
4790 warn $err_msg;
4791 };
4792
4793 # cold plug
4794
4795 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4796 foreach my $opt (sort keys %$pending_delete_hash) {
4797 my $force = $pending_delete_hash->{$opt}->{force};
4798 eval {
4799 if ($opt =~ m/^unused/) {
4800 die "internal error";
4801 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4802 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4803 }
4804 };
4805 if (my $err = $@) {
4806 $add_apply_error->($opt, $err);
4807 } else {
4808 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4809 delete $conf->{$opt};
4810 }
4811 }
4812
4813 PVE::QemuConfig->cleanup_pending($conf);
4814
4815 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4816 next if $opt eq 'delete'; # just to be sure
4817 eval {
4818 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4819 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4820 }
4821 };
4822 if (my $err = $@) {
4823 $add_apply_error->($opt, $err);
4824 } else {
4825 $conf->{$opt} = delete $conf->{pending}->{$opt};
4826 }
4827 }
4828
4829 # write all changes at once to avoid unnecessary i/o
4830 PVE::QemuConfig->write_config($vmid, $conf);
4831}
4832
4833sub vmconfig_update_net {
4834 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4835
4836 my $newnet = parse_net($value);
4837
4838 if ($conf->{$opt}) {
4839 my $oldnet = parse_net($conf->{$opt});
4840
4841 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4842 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4843 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4844 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4845
4846 # for non online change, we try to hot-unplug
4847 die "skip\n" if !$hotplug;
4848 vm_deviceunplug($vmid, $conf, $opt);
4849 } else {
4850
4851 die "internal error" if $opt !~ m/net(\d+)/;
4852 my $iface = "tap${vmid}i$1";
4853
4854 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4855 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4856 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4857 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4858 PVE::Network::tap_unplug($iface);
4859
4860 if ($have_sdn) {
4861 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4862 } else {
4863 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4864 }
4865 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4866 # Rate can be applied on its own but any change above needs to
4867 # include the rate in tap_plug since OVS resets everything.
4868 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4869 }
4870
4871 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4872 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4873 }
4874
4875 return 1;
4876 }
4877 }
4878
4879 if ($hotplug) {
4880 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4881 } else {
4882 die "skip\n";
4883 }
4884}
4885
4886sub vmconfig_update_disk {
4887 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4888
4889 my $drive = parse_drive($opt, $value);
4890
4891 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4892 my $media = $drive->{media} || 'disk';
4893 my $oldmedia = $old_drive->{media} || 'disk';
4894 die "unable to change media type\n" if $media ne $oldmedia;
4895
4896 if (!drive_is_cdrom($old_drive)) {
4897
4898 if ($drive->{file} ne $old_drive->{file}) {
4899
4900 die "skip\n" if !$hotplug;
4901
4902 # unplug and register as unused
4903 vm_deviceunplug($vmid, $conf, $opt);
4904 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4905
4906 } else {
4907 # update existing disk
4908
4909 # skip non hotpluggable value
4910 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4911 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4912 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4913 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4914 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4915 die "skip\n";
4916 }
4917
4918 # apply throttle
4919 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4920 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4921 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4922 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4923 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4924 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4925 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4926 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4927 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4928 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4929 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4930 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4931 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4932 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4933 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4934 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4935 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4936 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4937
4938 qemu_block_set_io_throttle(
4939 $vmid,"drive-$opt",
4940 ($drive->{mbps} || 0)*1024*1024,
4941 ($drive->{mbps_rd} || 0)*1024*1024,
4942 ($drive->{mbps_wr} || 0)*1024*1024,
4943 $drive->{iops} || 0,
4944 $drive->{iops_rd} || 0,
4945 $drive->{iops_wr} || 0,
4946 ($drive->{mbps_max} || 0)*1024*1024,
4947 ($drive->{mbps_rd_max} || 0)*1024*1024,
4948 ($drive->{mbps_wr_max} || 0)*1024*1024,
4949 $drive->{iops_max} || 0,
4950 $drive->{iops_rd_max} || 0,
4951 $drive->{iops_wr_max} || 0,
4952 $drive->{bps_max_length} || 1,
4953 $drive->{bps_rd_max_length} || 1,
4954 $drive->{bps_wr_max_length} || 1,
4955 $drive->{iops_max_length} || 1,
4956 $drive->{iops_rd_max_length} || 1,
4957 $drive->{iops_wr_max_length} || 1,
4958 );
4959
4960 }
4961
4962 return 1;
4963 }
4964
4965 } else { # cdrom
4966
4967 if ($drive->{file} eq 'none') {
4968 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4969 if (drive_is_cloudinit($old_drive)) {
4970 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4971 }
4972 } else {
4973 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4974
4975 # force eject if locked
4976 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4977
4978 if ($path) {
4979 mon_cmd($vmid, "blockdev-change-medium",
4980 id => "$opt", filename => "$path");
4981 }
4982 }
4983
4984 return 1;
4985 }
4986 }
4987
4988 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4989 # hotplug new disks
4990 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4991 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4992}
4993
4994# called in locked context by incoming migration
4995sub vm_migrate_get_nbd_disks {
4996 my ($storecfg, $conf, $replicated_volumes) = @_;
4997
4998 my $local_volumes = {};
4999 PVE::QemuConfig->foreach_volume($conf, sub {
5000 my ($ds, $drive) = @_;
5001
5002 return if drive_is_cdrom($drive);
5003
5004 my $volid = $drive->{file};
5005
5006 return if !$volid;
5007
5008 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5009
5010 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5011 return if $scfg->{shared};
5012
5013 # replicated disks re-use existing state via bitmap
5014 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5015 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5016 });
5017 return $local_volumes;
5018}
5019
5020# called in locked context by incoming migration
5021sub vm_migrate_alloc_nbd_disks {
5022 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5023
5024 my $format = undef;
5025
5026 my $nbd = {};
5027 foreach my $opt (sort keys %$source_volumes) {
5028 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5029
5030 if ($use_existing) {
5031 $nbd->{$opt}->{drivestr} = print_drive($drive);
5032 $nbd->{$opt}->{volid} = $volid;
5033 $nbd->{$opt}->{replicated} = 1;
5034 next;
5035 }
5036
5037 # If a remote storage is specified and the format of the original
5038 # volume is not available there, fall back to the default format.
5039 # Otherwise use the same format as the original.
5040 if (!$storagemap->{identity}) {
5041 $storeid = map_storage($storagemap, $storeid);
5042 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5043 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5044 my $fileFormat = qemu_img_format($scfg, $volname);
5045 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5046 } else {
5047 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5048 $format = qemu_img_format($scfg, $volname);
5049 }
5050
5051 my $size = $drive->{size} / 1024;
5052 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5053 my $newdrive = $drive;
5054 $newdrive->{format} = $format;
5055 $newdrive->{file} = $newvolid;
5056 my $drivestr = print_drive($newdrive);
5057 $nbd->{$opt}->{drivestr} = $drivestr;
5058 $nbd->{$opt}->{volid} = $newvolid;
5059 }
5060
5061 return $nbd;
5062}
5063
5064# see vm_start_nolock for parameters, additionally:
5065# migrate_opts:
5066# storagemap = parsed storage map for allocating NBD disks
5067sub vm_start {
5068 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5069
5070 return PVE::QemuConfig->lock_config($vmid, sub {
5071 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5072
5073 die "you can't start a vm if it's a template\n"
5074 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5075
5076 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5077 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5078
5079 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5080
5081 if ($has_backup_lock && $running) {
5082 # a backup is currently running, attempt to start the guest in the
5083 # existing QEMU instance
5084 return vm_resume($vmid);
5085 }
5086
5087 PVE::QemuConfig->check_lock($conf)
5088 if !($params->{skiplock} || $has_suspended_lock);
5089
5090 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5091
5092 die "VM $vmid already running\n" if $running;
5093
5094 if (my $storagemap = $migrate_opts->{storagemap}) {
5095 my $replicated = $migrate_opts->{replicated_volumes};
5096 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5097 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5098
5099 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5100 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5101 }
5102 }
5103
5104 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5105 });
5106}
5107
5108
5109# params:
5110# statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5111# skiplock => 0/1, skip checking for config lock
5112# skiptemplate => 0/1, skip checking whether VM is template
5113# forcemachine => to force Qemu machine (rollback/migration)
5114# forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5115# timeout => in seconds
5116# paused => start VM in paused state (backup)
5117# resume => resume from hibernation
5118# pbs-backing => {
5119# sata0 => {
5120# repository
5121# snapshot
5122# keyfile
5123# archive
5124# },
5125# virtio2 => ...
5126# }
5127# migrate_opts:
5128# nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5129# migratedfrom => source node
5130# spice_ticket => used for spice migration, passed via tunnel/stdin
5131# network => CIDR of migration network
5132# type => secure/insecure - tunnel over encrypted connection or plain-text
5133# nbd_proto_version => int, 0 for TCP, 1 for UNIX
5134# replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5135sub vm_start_nolock {
5136 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5137
5138 my $statefile = $params->{statefile};
5139 my $resume = $params->{resume};
5140
5141 my $migratedfrom = $migrate_opts->{migratedfrom};
5142 my $migration_type = $migrate_opts->{type};
5143
5144 my $res = {};
5145
5146 # clean up leftover reboot request files
5147 eval { clear_reboot_request($vmid); };
5148 warn $@ if $@;
5149
5150 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5151 vmconfig_apply_pending($vmid, $conf, $storecfg);
5152 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5153 }
5154
5155 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5156
5157 my $defaults = load_defaults();
5158
5159 # set environment variable useful inside network script
5160 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5161
5162 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5163
5164 my $forcemachine = $params->{forcemachine};
5165 my $forcecpu = $params->{forcecpu};
5166 if ($resume) {
5167 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5168 $forcemachine = $conf->{runningmachine};
5169 $forcecpu = $conf->{runningcpu};
5170 print "Resuming suspended VM\n";
5171 }
5172
5173 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5174 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5175
5176 my $migration_ip;
5177 my $get_migration_ip = sub {
5178 my ($nodename) = @_;
5179
5180 return $migration_ip if defined($migration_ip);
5181
5182 my $cidr = $migrate_opts->{network};
5183
5184 if (!defined($cidr)) {
5185 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5186 $cidr = $dc_conf->{migration}->{network};
5187 }
5188
5189 if (defined($cidr)) {
5190 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5191
5192 die "could not get IP: no address configured on local " .
5193 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5194
5195 die "could not get IP: multiple addresses configured on local " .
5196 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5197
5198 $migration_ip = @$ips[0];
5199 }
5200
5201 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5202 if !defined($migration_ip);
5203
5204 return $migration_ip;
5205 };
5206
5207 my $migrate_uri;
5208 if ($statefile) {
5209 if ($statefile eq 'tcp') {
5210 my $localip = "localhost";
5211 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5212 my $nodename = nodename();
5213
5214 if (!defined($migration_type)) {
5215 if (defined($datacenterconf->{migration}->{type})) {
5216 $migration_type = $datacenterconf->{migration}->{type};
5217 } else {
5218 $migration_type = 'secure';
5219 }
5220 }
5221
5222 if ($migration_type eq 'insecure') {
5223 $localip = $get_migration_ip->($nodename);
5224 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5225 }
5226
5227 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5228 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5229 $migrate_uri = "tcp:${localip}:${migrate_port}";
5230 push @$cmd, '-incoming', $migrate_uri;
5231 push @$cmd, '-S';
5232
5233 } elsif ($statefile eq 'unix') {
5234 # should be default for secure migrations as a ssh TCP forward
5235 # tunnel is not deterministic reliable ready and fails regurarly
5236 # to set up in time, so use UNIX socket forwards
5237 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5238 unlink $socket_addr;
5239
5240 $migrate_uri = "unix:$socket_addr";
5241
5242 push @$cmd, '-incoming', $migrate_uri;
5243 push @$cmd, '-S';
5244
5245 } elsif (-e $statefile) {
5246 push @$cmd, '-loadstate', $statefile;
5247 } else {
5248 my $statepath = PVE::Storage::path($storecfg, $statefile);
5249 push @$vollist, $statefile;
5250 push @$cmd, '-loadstate', $statepath;
5251 }
5252 } elsif ($params->{paused}) {
5253 push @$cmd, '-S';
5254 }
5255
5256 # host pci devices
5257 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5258 my $d = parse_hostpci($conf->{"hostpci$i"});
5259 next if !$d;
5260 my $pcidevices = $d->{pciid};
5261 foreach my $pcidevice (@$pcidevices) {
5262 my $pciid = $pcidevice->{id};
5263
5264 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5265 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5266 die "no pci device info for device '$pciid'\n" if !$info;
5267
5268 if ($d->{mdev}) {
5269 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5270 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5271 } else {
5272 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5273 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5274 die "can't reset PCI device '$pciid'\n"
5275 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5276 }
5277 }
5278 }
5279
5280 PVE::Storage::activate_volumes($storecfg, $vollist);
5281
5282 eval {
5283 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5284 outfunc => sub {}, errfunc => sub {});
5285 };
5286 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5287 # timeout should be more than enough here...
5288 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5289
5290 my $cpuunits = get_cpuunits($conf);
5291
5292 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5293 my %run_params = (
5294 timeout => $statefile ? undef : $start_timeout,
5295 umask => 0077,
5296 noerr => 1,
5297 );
5298
5299 # when migrating, prefix QEMU output so other side can pick up any
5300 # errors that might occur and show the user
5301 if ($migratedfrom) {
5302 $run_params{quiet} = 1;
5303 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5304 }
5305
5306 my %properties = (
5307 Slice => 'qemu.slice',
5308 KillMode => 'process',
5309 SendSIGKILL => 0,
5310 TimeoutStopUSec => ULONG_MAX, # infinity
5311 );
5312
5313 if (PVE::CGroup::cgroup_mode() == 2) {
5314 $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5315 $properties{CPUWeight} = $cpuunits;
5316 } else {
5317 $properties{CPUShares} = $cpuunits;
5318 }
5319
5320 if (my $cpulimit = $conf->{cpulimit}) {
5321 $properties{CPUQuota} = int($cpulimit * 100);
5322 }
5323 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5324
5325 my $run_qemu = sub {
5326 PVE::Tools::run_fork sub {
5327 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5328
5329 my $exitcode = run_command($cmd, %run_params);
5330 die "QEMU exited with code $exitcode\n" if $exitcode;
5331 };
5332 };
5333
5334 if ($conf->{hugepages}) {
5335
5336 my $code = sub {
5337 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5338 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5339
5340 PVE::QemuServer::Memory::hugepages_mount();
5341 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5342
5343 eval { $run_qemu->() };
5344 if (my $err = $@) {
5345 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5346 if !$conf->{keephugepages};
5347 die $err;
5348 }
5349
5350 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5351 if !$conf->{keephugepages};
5352 };
5353 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5354
5355 } else {
5356 eval { $run_qemu->() };
5357 }
5358
5359 if (my $err = $@) {
5360 # deactivate volumes if start fails
5361 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5362 die "start failed: $err";
5363 }
5364
5365 print "migration listens on $migrate_uri\n" if $migrate_uri;
5366 $res->{migrate_uri} = $migrate_uri;
5367
5368 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5369 eval { mon_cmd($vmid, "cont"); };
5370 warn $@ if $@;
5371 }
5372
5373 #start nbd server for storage migration
5374 if (my $nbd = $migrate_opts->{nbd}) {
5375 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5376
5377 my $migrate_storage_uri;
5378 # nbd_protocol_version > 0 for unix socket support
5379 if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5380 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5381 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5382 $migrate_storage_uri = "nbd:unix:$socket_path";
5383 } else {
5384 my $nodename = nodename();
5385 my $localip = $get_migration_ip->($nodename);
5386 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5387 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5388
5389 mon_cmd($vmid, "nbd-server-start", addr => {
5390 type => 'inet',
5391 data => {
5392 host => "${localip}",
5393 port => "${storage_migrate_port}",
5394 },
5395 });
5396 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5397 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5398 }
5399
5400 $res->{migrate_storage_uri} = $migrate_storage_uri;
5401
5402 foreach my $opt (sort keys %$nbd) {
5403 my $drivestr = $nbd->{$opt}->{drivestr};
5404 my $volid = $nbd->{$opt}->{volid};
5405 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5406 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5407 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5408 print "re-using replicated volume: $opt - $volid\n"
5409 if $nbd->{$opt}->{replicated};
5410
5411 $res->{drives}->{$opt} = $nbd->{$opt};
5412 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5413 }
5414 }
5415
5416 if ($migratedfrom) {
5417 eval {
5418 set_migration_caps($vmid);
5419 };
5420 warn $@ if $@;
5421
5422 if ($spice_port) {
5423 print "spice listens on port $spice_port\n";
5424 $res->{spice_port} = $spice_port;
5425 if ($migrate_opts->{spice_ticket}) {
5426 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5427 $migrate_opts->{spice_ticket});
5428 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5429 }
5430 }
5431
5432 } else {
5433 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5434 if !$statefile && $conf->{balloon};
5435
5436 foreach my $opt (keys %$conf) {
5437 next if $opt !~ m/^net\d+$/;
5438 my $nicconf = parse_net($conf->{$opt});
5439 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5440 }
5441 }
5442
5443 mon_cmd($vmid, 'qom-set',
5444 path => "machine/peripheral/balloon0",
5445 property => "guest-stats-polling-interval",
5446 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5447
5448 if ($resume) {
5449 print "Resumed VM, removing state\n";
5450 if (my $vmstate = $conf->{vmstate}) {
5451 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5452 PVE::Storage::vdisk_free($storecfg, $vmstate);
5453 }
5454 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5455 PVE::QemuConfig->write_config($vmid, $conf);
5456 }
5457
5458 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5459
5460 return $res;
5461}
5462
5463sub vm_commandline {
5464 my ($storecfg, $vmid, $snapname) = @_;
5465
5466 my $conf = PVE::QemuConfig->load_config($vmid);
5467 my $forcemachine;
5468 my $forcecpu;
5469
5470 if ($snapname) {
5471 my $snapshot = $conf->{snapshots}->{$snapname};
5472 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5473
5474 # check for machine or CPU overrides in snapshot
5475 $forcemachine = $snapshot->{runningmachine};
5476 $forcecpu = $snapshot->{runningcpu};
5477
5478 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5479
5480 $conf = $snapshot;
5481 }
5482
5483 my $defaults = load_defaults();
5484
5485 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5486 $forcemachine, $forcecpu);
5487
5488 return PVE::Tools::cmd2string($cmd);
5489}
5490
5491sub vm_reset {
5492 my ($vmid, $skiplock) = @_;
5493
5494 PVE::QemuConfig->lock_config($vmid, sub {
5495
5496 my $conf = PVE::QemuConfig->load_config($vmid);
5497
5498 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5499
5500 mon_cmd($vmid, "system_reset");
5501 });
5502}
5503
5504sub get_vm_volumes {
5505 my ($conf) = @_;
5506
5507 my $vollist = [];
5508 foreach_volid($conf, sub {
5509 my ($volid, $attr) = @_;
5510
5511 return if $volid =~ m|^/|;
5512
5513 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5514 return if !$sid;
5515
5516 push @$vollist, $volid;
5517 });
5518
5519 return $vollist;
5520}
5521
5522sub vm_stop_cleanup {
5523 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5524
5525 eval {
5526
5527 if (!$keepActive) {
5528 my $vollist = get_vm_volumes($conf);
5529 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5530 }
5531
5532 foreach my $ext (qw(mon qmp pid vnc qga)) {
5533 unlink "/var/run/qemu-server/${vmid}.$ext";
5534 }
5535
5536 if ($conf->{ivshmem}) {
5537 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5538 # just delete it for now, VMs which have this already open do not
5539 # are affected, but new VMs will get a separated one. If this
5540 # becomes an issue we either add some sort of ref-counting or just
5541 # add a "don't delete on stop" flag to the ivshmem format.
5542 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5543 }
5544
5545 foreach my $key (keys %$conf) {
5546 next if $key !~ m/^hostpci(\d+)$/;
5547 my $hostpciindex = $1;
5548 my $d = parse_hostpci($conf->{$key});
5549 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5550
5551 foreach my $pci (@{$d->{pciid}}) {
5552 my $pciid = $pci->{id};
5553 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5554 }
5555 }
5556
5557 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5558 };
5559 warn $@ if $@; # avoid errors - just warn
5560}
5561
5562# call only in locked context
5563sub _do_vm_stop {
5564 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5565
5566 my $pid = check_running($vmid, $nocheck);
5567 return if !$pid;
5568
5569 my $conf;
5570 if (!$nocheck) {
5571 $conf = PVE::QemuConfig->load_config($vmid);
5572 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5573 if (!defined($timeout) && $shutdown && $conf->{startup}) {
5574 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5575 $timeout = $opts->{down} if $opts->{down};
5576 }
5577 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5578 }
5579
5580 eval {
5581 if ($shutdown) {
5582 if (defined($conf) && get_qga_key($conf, 'enabled')) {
5583 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5584 } else {
5585 mon_cmd($vmid, "system_powerdown");
5586 }
5587 } else {
5588 mon_cmd($vmid, "quit");
5589 }
5590 };
5591 my $err = $@;
5592
5593 if (!$err) {
5594 $timeout = 60 if !defined($timeout);
5595
5596 my $count = 0;
5597 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5598 $count++;
5599 sleep 1;
5600 }
5601
5602 if ($count >= $timeout) {
5603 if ($force) {
5604 warn "VM still running - terminating now with SIGTERM\n";
5605 kill 15, $pid;
5606 } else {
5607 die "VM quit/powerdown failed - got timeout\n";
5608 }
5609 } else {
5610 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5611 return;
5612 }
5613 } else {
5614 if (!check_running($vmid, $nocheck)) {
5615 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5616 return;
5617 }
5618 if ($force) {
5619 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5620 kill 15, $pid;
5621 } else {
5622 die "VM quit/powerdown failed\n";
5623 }
5624 }
5625
5626 # wait again
5627 $timeout = 10;
5628
5629 my $count = 0;
5630 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5631 $count++;
5632 sleep 1;
5633 }
5634
5635 if ($count >= $timeout) {
5636 warn "VM still running - terminating now with SIGKILL\n";
5637 kill 9, $pid;
5638 sleep 1;
5639 }
5640
5641 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5642}
5643
5644# Note: use $nocheck to skip tests if VM configuration file exists.
5645# We need that when migration VMs to other nodes (files already moved)
5646# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5647sub vm_stop {
5648 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5649
5650 $force = 1 if !defined($force) && !$shutdown;
5651
5652 if ($migratedfrom){
5653 my $pid = check_running($vmid, $nocheck, $migratedfrom);
5654 kill 15, $pid if $pid;
5655 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5656 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5657 return;
5658 }
5659
5660 PVE::QemuConfig->lock_config($vmid, sub {
5661 _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5662 });
5663}
5664
5665sub vm_reboot {
5666 my ($vmid, $timeout) = @_;
5667
5668 PVE::QemuConfig->lock_config($vmid, sub {
5669 eval {
5670
5671 # only reboot if running, as qmeventd starts it again on a stop event
5672 return if !check_running($vmid);
5673
5674 create_reboot_request($vmid);
5675
5676 my $storecfg = PVE::Storage::config();
5677 _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5678
5679 };
5680 if (my $err = $@) {
5681 # avoid that the next normal shutdown will be confused for a reboot
5682 clear_reboot_request($vmid);
5683 die $err;
5684 }
5685 });
5686}
5687
5688# note: if using the statestorage parameter, the caller has to check privileges
5689sub vm_suspend {
5690 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5691
5692 my $conf;
5693 my $path;
5694 my $storecfg;
5695 my $vmstate;
5696
5697 PVE::QemuConfig->lock_config($vmid, sub {
5698
5699 $conf = PVE::QemuConfig->load_config($vmid);
5700
5701 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5702 PVE::QemuConfig->check_lock($conf)
5703 if !($skiplock || $is_backing_up);
5704
5705 die "cannot suspend to disk during backup\n"
5706 if $is_backing_up && $includestate;
5707
5708 if ($includestate) {
5709 $conf->{lock} = 'suspending';
5710 my $date = strftime("%Y-%m-%d", localtime(time()));
5711 $storecfg = PVE::Storage::config();
5712 if (!$statestorage) {
5713 $statestorage = find_vmstate_storage($conf, $storecfg);
5714 # check permissions for the storage
5715 my $rpcenv = PVE::RPCEnvironment::get();
5716 if ($rpcenv->{type} ne 'cli') {
5717 my $authuser = $rpcenv->get_user();
5718 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5719 }
5720 }
5721
5722
5723 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5724 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5725 $path = PVE::Storage::path($storecfg, $vmstate);
5726 PVE::QemuConfig->write_config($vmid, $conf);
5727 } else {
5728 mon_cmd($vmid, "stop");
5729 }
5730 });
5731
5732 if ($includestate) {
5733 # save vm state
5734 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5735
5736 eval {
5737 set_migration_caps($vmid, 1);
5738 mon_cmd($vmid, "savevm-start", statefile => $path);
5739 for(;;) {
5740 my $state = mon_cmd($vmid, "query-savevm");
5741 if (!$state->{status}) {
5742 die "savevm not active\n";
5743 } elsif ($state->{status} eq 'active') {
5744 sleep(1);
5745 next;
5746 } elsif ($state->{status} eq 'completed') {
5747 print "State saved, quitting\n";
5748 last;
5749 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5750 die "query-savevm failed with error '$state->{error}'\n"
5751 } else {
5752 die "query-savevm returned status '$state->{status}'\n";
5753 }
5754 }
5755 };
5756 my $err = $@;
5757
5758 PVE::QemuConfig->lock_config($vmid, sub {
5759 $conf = PVE::QemuConfig->load_config($vmid);
5760 if ($err) {
5761 # cleanup, but leave suspending lock, to indicate something went wrong
5762 eval {
5763 mon_cmd($vmid, "savevm-end");
5764 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5765 PVE::Storage::vdisk_free($storecfg, $vmstate);
5766 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5767 PVE::QemuConfig->write_config($vmid, $conf);
5768 };
5769 warn $@ if $@;
5770 die $err;
5771 }
5772
5773 die "lock changed unexpectedly\n"
5774 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5775
5776 mon_cmd($vmid, "quit");
5777 $conf->{lock} = 'suspended';
5778 PVE::QemuConfig->write_config($vmid, $conf);
5779 });
5780 }
5781}
5782
5783sub vm_resume {
5784 my ($vmid, $skiplock, $nocheck) = @_;
5785
5786 PVE::QemuConfig->lock_config($vmid, sub {
5787 my $res = mon_cmd($vmid, 'query-status');
5788 my $resume_cmd = 'cont';
5789 my $reset = 0;
5790
5791 if ($res->{status}) {
5792 return if $res->{status} eq 'running'; # job done, go home
5793 $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5794 $reset = 1 if $res->{status} eq 'shutdown';
5795 }
5796
5797 if (!$nocheck) {
5798
5799 my $conf = PVE::QemuConfig->load_config($vmid);
5800
5801 PVE::QemuConfig->check_lock($conf)
5802 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5803 }
5804
5805 if ($reset) {
5806 # required if a VM shuts down during a backup and we get a resume
5807 # request before the backup finishes for example
5808 mon_cmd($vmid, "system_reset");
5809 }
5810 mon_cmd($vmid, $resume_cmd);
5811 });
5812}
5813
5814sub vm_sendkey {
5815 my ($vmid, $skiplock, $key) = @_;
5816
5817 PVE::QemuConfig->lock_config($vmid, sub {
5818
5819 my $conf = PVE::QemuConfig->load_config($vmid);
5820
5821 # there is no qmp command, so we use the human monitor command
5822 my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5823 die $res if $res ne '';
5824 });
5825}
5826
5827# vzdump restore implementaion
5828
5829sub tar_archive_read_firstfile {
5830 my $archive = shift;
5831
5832 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5833
5834 # try to detect archive type first
5835 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5836 die "unable to open file '$archive'\n";
5837 my $firstfile = <$fh>;
5838 kill 15, $pid;
5839 close $fh;
5840
5841 die "ERROR: archive contaions no data\n" if !$firstfile;
5842 chomp $firstfile;
5843
5844 return $firstfile;
5845}
5846
5847sub tar_restore_cleanup {
5848 my ($storecfg, $statfile) = @_;
5849
5850 print STDERR "starting cleanup\n";
5851
5852 if (my $fd = IO::File->new($statfile, "r")) {
5853 while (defined(my $line = <$fd>)) {
5854 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5855 my $volid = $2;
5856 eval {
5857 if ($volid =~ m|^/|) {
5858 unlink $volid || die 'unlink failed\n';
5859 } else {
5860 PVE::Storage::vdisk_free($storecfg, $volid);
5861 }
5862 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5863 };
5864 print STDERR "unable to cleanup '$volid' - $@" if $@;
5865 } else {
5866 print STDERR "unable to parse line in statfile - $line";
5867 }
5868 }
5869 $fd->close();
5870 }
5871}
5872
5873sub restore_file_archive {
5874 my ($archive, $vmid, $user, $opts) = @_;
5875
5876 return restore_vma_archive($archive, $vmid, $user, $opts)
5877 if $archive eq '-';
5878
5879 my $info = PVE::Storage::archive_info($archive);
5880 my $format = $opts->{format} // $info->{format};
5881 my $comp = $info->{compression};
5882
5883 # try to detect archive format
5884 if ($format eq 'tar') {
5885 return restore_tar_archive($archive, $vmid, $user, $opts);
5886 } else {
5887 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5888 }
5889}
5890
5891# hepler to remove disks that will not be used after restore
5892my $restore_cleanup_oldconf = sub {
5893 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5894
5895 PVE::QemuConfig->foreach_volume($oldconf, sub {
5896 my ($ds, $drive) = @_;
5897
5898 return if drive_is_cdrom($drive, 1);
5899
5900 my $volid = $drive->{file};
5901 return if !$volid || $volid =~ m|^/|;
5902
5903 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5904 return if !$path || !$owner || ($owner != $vmid);
5905
5906 # Note: only delete disk we want to restore
5907 # other volumes will become unused
5908 if ($virtdev_hash->{$ds}) {
5909 eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5910 if (my $err = $@) {
5911 warn $err;
5912 }
5913 }
5914 });
5915
5916 # delete vmstate files, after the restore we have no snapshots anymore
5917 foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5918 my $snap = $oldconf->{snapshots}->{$snapname};
5919 if ($snap->{vmstate}) {
5920 eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5921 if (my $err = $@) {
5922 warn $err;
5923 }
5924 }
5925 }
5926};
5927
5928# Helper to parse vzdump backup device hints
5929#
5930# $rpcenv: Environment, used to ckeck storage permissions
5931# $user: User ID, to check storage permissions
5932# $storecfg: Storage configuration
5933# $fh: the file handle for reading the configuration
5934# $devinfo: should contain device sizes for all backu-up'ed devices
5935# $options: backup options (pool, default storage)
5936#
5937# Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5938my $parse_backup_hints = sub {
5939 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5940
5941 my $virtdev_hash = {};
5942
5943 while (defined(my $line = <$fh>)) {
5944 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5945 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5946 die "archive does not contain data for drive '$virtdev'\n"
5947 if !$devinfo->{$devname};
5948
5949 if (defined($options->{storage})) {
5950 $storeid = $options->{storage} || 'local';
5951 } elsif (!$storeid) {
5952 $storeid = 'local';
5953 }
5954 $format = 'raw' if !$format;
5955 $devinfo->{$devname}->{devname} = $devname;
5956 $devinfo->{$devname}->{virtdev} = $virtdev;
5957 $devinfo->{$devname}->{format} = $format;
5958 $devinfo->{$devname}->{storeid} = $storeid;
5959
5960 # check permission on storage
5961 my $pool = $options->{pool}; # todo: do we need that?
5962 if ($user ne 'root@pam') {
5963 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5964 }
5965
5966 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5967 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5968 my $virtdev = $1;
5969 my $drive = parse_drive($virtdev, $2);
5970 if (drive_is_cloudinit($drive)) {
5971 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5972 $storeid = $options->{storage} if defined ($options->{storage});
5973 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5974 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5975
5976 $virtdev_hash->{$virtdev} = {
5977 format => $format,
5978 storeid => $storeid,
5979 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5980 is_cloudinit => 1,
5981 };
5982 }
5983 }
5984 }
5985
5986 return $virtdev_hash;
5987};
5988
5989# Helper to allocate and activate all volumes required for a restore
5990#
5991# $storecfg: Storage configuration
5992# $virtdev_hash: as returned by parse_backup_hints()
5993#
5994# Returns: { $virtdev => $volid }
5995my $restore_allocate_devices = sub {
5996 my ($storecfg, $virtdev_hash, $vmid) = @_;
5997
5998 my $map = {};
5999 foreach my $virtdev (sort keys %$virtdev_hash) {
6000 my $d = $virtdev_hash->{$virtdev};
6001 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6002 my $storeid = $d->{storeid};
6003 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6004
6005 # test if requested format is supported
6006 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6007 my $supported = grep { $_ eq $d->{format} } @$validFormats;
6008 $d->{format} = $defFormat if !$supported;
6009
6010 my $name;
6011 if ($d->{is_cloudinit}) {
6012 $name = "vm-$vmid-cloudinit";
6013 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6014 if ($scfg->{path}) {
6015 $name .= ".$d->{format}";
6016 }
6017 }
6018
6019 my $volid = PVE::Storage::vdisk_alloc(
6020 $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6021
6022 print STDERR "new volume ID is '$volid'\n";
6023 $d->{volid} = $volid;
6024
6025 PVE::Storage::activate_volumes($storecfg, [$volid]);
6026
6027 $map->{$virtdev} = $volid;
6028 }
6029
6030 return $map;
6031};
6032
6033sub restore_update_config_line {
6034 my ($cookie, $map, $line, $unique) = @_;
6035
6036 return '' if $line =~ m/^\#qmdump\#/;
6037 return '' if $line =~ m/^\#vzdump\#/;
6038 return '' if $line =~ m/^lock:/;
6039 return '' if $line =~ m/^unused\d+:/;
6040 return '' if $line =~ m/^parent:/;
6041
6042 my $res = '';
6043
6044 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6045 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6046 # try to convert old 1.X settings
6047 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6048 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6049 my ($model, $macaddr) = split(/\=/, $devconfig);
6050 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6051 my $net = {
6052 model => $model,
6053 bridge => "vmbr$ind",
6054 macaddr => $macaddr,
6055 };
6056 my $netstr = print_net($net);
6057
6058 $res .= "net$cookie->{netcount}: $netstr\n";
6059 $cookie->{netcount}++;
6060 }
6061 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6062 my ($id, $netstr) = ($1, $2);
6063 my $net = parse_net($netstr);
6064 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6065 $netstr = print_net($net);
6066 $res .= "$id: $netstr\n";
6067 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6068 my $virtdev = $1;
6069 my $value = $3;
6070 my $di = parse_drive($virtdev, $value);
6071 if (defined($di->{backup}) && !$di->{backup}) {
6072 $res .= "#$line";
6073 } elsif ($map->{$virtdev}) {
6074 delete $di->{format}; # format can change on restore
6075 $di->{file} = $map->{$virtdev};
6076 $value = print_drive($di);
6077 $res .= "$virtdev: $value\n";
6078 } else {
6079 $res .= $line;
6080 }
6081 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6082 my $vmgenid = $1;
6083 if ($vmgenid ne '0') {
6084 # always generate a new vmgenid if there was a valid one setup
6085 $vmgenid = generate_uuid();
6086 }
6087 $res .= "vmgenid: $vmgenid\n";
6088 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6089 my ($uuid, $uuid_str);
6090 UUID::generate($uuid);
6091 UUID::unparse($uuid, $uuid_str);
6092 my $smbios1 = parse_smbios1($2);
6093 $smbios1->{uuid} = $uuid_str;
6094 $res .= $1.print_smbios1($smbios1)."\n";
6095 } else {
6096 $res .= $line;
6097 }
6098
6099 return $res;
6100}
6101
6102my $restore_deactivate_volumes = sub {
6103 my ($storecfg, $devinfo) = @_;
6104
6105 my $vollist = [];
6106 foreach my $devname (keys %$devinfo) {
6107 my $volid = $devinfo->{$devname}->{volid};
6108 push @$vollist, $volid if $volid;
6109 }
6110
6111 PVE::Storage::deactivate_volumes($storecfg, $vollist);
6112};
6113
6114my $restore_destroy_volumes = sub {
6115 my ($storecfg, $devinfo) = @_;
6116
6117 foreach my $devname (keys %$devinfo) {
6118 my $volid = $devinfo->{$devname}->{volid};
6119 next if !$volid;
6120 eval {
6121 if ($volid =~ m|^/|) {
6122 unlink $volid || die 'unlink failed\n';
6123 } else {
6124 PVE::Storage::vdisk_free($storecfg, $volid);
6125 }
6126 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6127 };
6128 print STDERR "unable to cleanup '$volid' - $@" if $@;
6129 }
6130};
6131
6132sub scan_volids {
6133 my ($cfg, $vmid) = @_;
6134
6135 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6136
6137 my $volid_hash = {};
6138 foreach my $storeid (keys %$info) {
6139 foreach my $item (@{$info->{$storeid}}) {
6140 next if !($item->{volid} && $item->{size});
6141 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6142 $volid_hash->{$item->{volid}} = $item;
6143 }
6144 }
6145
6146 return $volid_hash;
6147}
6148
6149sub update_disk_config {
6150 my ($vmid, $conf, $volid_hash) = @_;
6151
6152 my $changes;
6153 my $prefix = "VM $vmid";
6154
6155 # used and unused disks
6156 my $referenced = {};
6157
6158 # Note: it is allowed to define multiple storages with same path (alias), so
6159 # we need to check both 'volid' and real 'path' (two different volid can point
6160 # to the same path).
6161
6162 my $referencedpath = {};
6163
6164 # update size info
6165 PVE::QemuConfig->foreach_volume($conf, sub {
6166 my ($opt, $drive) = @_;
6167
6168 my $volid = $drive->{file};
6169 return if !$volid;
6170 my $volume = $volid_hash->{$volid};
6171
6172 # mark volid as "in-use" for next step
6173 $referenced->{$volid} = 1;
6174 if ($volume && (my $path = $volume->{path})) {
6175 $referencedpath->{$path} = 1;
6176 }
6177
6178 return if drive_is_cdrom($drive);
6179 return if !$volume;
6180
6181 my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6182 if (defined($updated)) {
6183 $changes = 1;
6184 $conf->{$opt} = print_drive($updated);
6185 print "$prefix ($opt): $msg\n";
6186 }
6187 });
6188
6189 # remove 'unusedX' entry if volume is used
6190 PVE::QemuConfig->foreach_unused_volume($conf, sub {
6191 my ($opt, $drive) = @_;
6192
6193 my $volid = $drive->{file};
6194 return if !$volid;
6195
6196 my $path;
6197 $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6198 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6199 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6200 $changes = 1;
6201 delete $conf->{$opt};
6202 }
6203
6204 $referenced->{$volid} = 1;
6205 $referencedpath->{$path} = 1 if $path;
6206 });
6207
6208 foreach my $volid (sort keys %$volid_hash) {
6209 next if $volid =~ m/vm-$vmid-state-/;
6210 next if $referenced->{$volid};
6211 my $path = $volid_hash->{$volid}->{path};
6212 next if !$path; # just to be sure
6213 next if $referencedpath->{$path};
6214 $changes = 1;
6215 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6216 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6217 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6218 }
6219
6220 return $changes;
6221}
6222
6223sub rescan {
6224 my ($vmid, $nolock, $dryrun) = @_;
6225
6226 my $cfg = PVE::Storage::config();
6227
6228 print "rescan volumes...\n";
6229 my $volid_hash = scan_volids($cfg, $vmid);
6230
6231 my $updatefn = sub {
6232 my ($vmid) = @_;
6233
6234 my $conf = PVE::QemuConfig->load_config($vmid);
6235
6236 PVE::QemuConfig->check_lock($conf);
6237
6238 my $vm_volids = {};
6239 foreach my $volid (keys %$volid_hash) {
6240 my $info = $volid_hash->{$volid};
6241 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6242 }
6243
6244 my $changes = update_disk_config($vmid, $conf, $vm_volids);
6245
6246 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6247 };
6248
6249 if (defined($vmid)) {
6250 if ($nolock) {
6251 &$updatefn($vmid);
6252 } else {
6253 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6254 }
6255 } else {
6256 my $vmlist = config_list();
6257 foreach my $vmid (keys %$vmlist) {
6258 if ($nolock) {
6259 &$updatefn($vmid);
6260 } else {
6261 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6262 }
6263 }
6264 }
6265}
6266
6267sub restore_proxmox_backup_archive {
6268 my ($archive, $vmid, $user, $options) = @_;
6269
6270 my $storecfg = PVE::Storage::config();
6271
6272 my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6273 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6274
6275 my $fingerprint = $scfg->{fingerprint};
6276 my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6277
6278 my $repo = PVE::PBSClient::get_repository($scfg);
6279
6280 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6281 my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6282 local $ENV{PBS_PASSWORD} = $password;
6283 local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6284
6285 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6286 PVE::Storage::parse_volname($storecfg, $archive);
6287
6288 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6289
6290 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6291
6292 my $tmpdir = "/var/tmp/vzdumptmp$$";
6293 rmtree $tmpdir;
6294 mkpath $tmpdir;
6295
6296 my $conffile = PVE::QemuConfig->config_file($vmid);
6297 # disable interrupts (always do cleanups)
6298 local $SIG{INT} =
6299 local $SIG{TERM} =
6300 local $SIG{QUIT} =
6301 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6302
6303 # Note: $oldconf is undef if VM does not exists
6304 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6305 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6306 my $new_conf_raw = '';
6307
6308 my $rpcenv = PVE::RPCEnvironment::get();
6309 my $devinfo = {};
6310
6311 eval {
6312 # enable interrupts
6313 local $SIG{INT} =
6314 local $SIG{TERM} =
6315 local $SIG{QUIT} =
6316 local $SIG{HUP} =
6317 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6318
6319 my $cfgfn = "$tmpdir/qemu-server.conf";
6320 my $firewall_config_fn = "$tmpdir/fw.conf";
6321 my $index_fn = "$tmpdir/index.json";
6322
6323 my $cmd = "restore";
6324
6325 my $param = [$pbs_backup_name, "index.json", $index_fn];
6326 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6327 my $index = PVE::Tools::file_get_contents($index_fn);
6328 $index = decode_json($index);
6329
6330 # print Dumper($index);
6331 foreach my $info (@{$index->{files}}) {
6332 if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6333 my $devname = $1;
6334 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6335 $devinfo->{$devname}->{size} = $1;
6336 } else {
6337 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6338 }
6339 }
6340 }
6341
6342 my $is_qemu_server_backup = scalar(
6343 grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6344 );
6345 if (!$is_qemu_server_backup) {
6346 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6347 }
6348 my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6349
6350 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6351 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6352
6353 if ($has_firewall_config) {
6354 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6355 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6356
6357 my $pve_firewall_dir = '/etc/pve/firewall';
6358 mkdir $pve_firewall_dir; # make sure the dir exists
6359 PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6360 }
6361
6362 my $fh = IO::File->new($cfgfn, "r") ||
6363 die "unable to read qemu-server.conf - $!\n";
6364
6365 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6366
6367 # fixme: rate limit?
6368
6369 # create empty/temp config
6370 PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6371
6372 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6373
6374 # allocate volumes
6375 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6376
6377 foreach my $virtdev (sort keys %$virtdev_hash) {
6378 my $d = $virtdev_hash->{$virtdev};
6379 next if $d->{is_cloudinit}; # no need to restore cloudinit
6380
6381 # for live-restore we only want to preload the efidisk
6382 next if $options->{live} && $virtdev ne 'efidisk0';
6383
6384 my $volid = $d->{volid};
6385
6386 my $path = PVE::Storage::path($storecfg, $volid);
6387
6388 my $pbs_restore_cmd = [
6389 '/usr/bin/pbs-restore',
6390 '--repository', $repo,
6391 $pbs_backup_name,
6392 "$d->{devname}.img.fidx",
6393 $path,
6394 '--verbose',
6395 ];
6396
6397 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6398 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6399
6400 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6401 push @$pbs_restore_cmd, '--skip-zero';
6402 }
6403
6404 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6405 print "restore proxmox backup image: $dbg_cmdstring\n";
6406 run_command($pbs_restore_cmd);
6407 }
6408
6409 $fh->seek(0, 0) || die "seek failed - $!\n";
6410
6411 my $cookie = { netcount => 0 };
6412 while (defined(my $line = <$fh>)) {
6413 $new_conf_raw .= restore_update_config_line(
6414 $cookie,
6415 $map,
6416 $line,
6417 $options->{unique},
6418 );
6419 }
6420
6421 $fh->close();
6422 };
6423 my $err = $@;
6424
6425 if ($err || !$options->{live}) {
6426 $restore_deactivate_volumes->($storecfg, $devinfo);
6427 }
6428
6429 rmtree $tmpdir;
6430
6431 if ($err) {
6432 $restore_destroy_volumes->($storecfg, $devinfo);
6433 die $err;
6434 }
6435
6436 if ($options->{live}) {
6437 # keep lock during live-restore
6438 $new_conf_raw .= "\nlock: create";
6439 }
6440
6441 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6442
6443 PVE::Cluster::cfs_update(); # make sure we read new file
6444
6445 eval { rescan($vmid, 1); };
6446 warn $@ if $@;
6447
6448 PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6449
6450 if ($options->{live}) {
6451 # enable interrupts
6452 local $SIG{INT} =
6453 local $SIG{TERM} =
6454 local $SIG{QUIT} =
6455 local $SIG{HUP} =
6456 local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6457
6458 my $conf = PVE::QemuConfig->load_config($vmid);
6459 die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6460
6461 delete $devinfo->{'drive-efidisk0'}; # this special drive is already restored before start
6462 pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6463
6464 PVE::QemuConfig->remove_lock($vmid, "create");
6465 }
6466}
6467
6468sub pbs_live_restore {
6469 my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6470
6471 print "starting VM for live-restore\n";
6472 print "repository: '$repo', snapshot: '$snap'\n";
6473
6474 my $pbs_backing = {};
6475 for my $ds (keys %$restored_disks) {
6476 $ds =~ m/^drive-(.*)$/;
6477 my $confname = $1;
6478 $pbs_backing->{$confname} = {
6479 repository => $repo,
6480 snapshot => $snap,
6481 archive => "$ds.img.fidx",
6482 };
6483 $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6484
6485 my $drive = parse_drive($confname, $conf->{$confname});
6486 print "restoring '$ds' to '$drive->{file}'\n";
6487 }
6488
6489 my $drives_streamed = 0;
6490 eval {
6491 # make sure HA doesn't interrupt our restore by stopping the VM
6492 if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6493 run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
6494 }
6495
6496 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6497 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6498 vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6499
6500 my $qmeventd_fd = register_qmeventd_handle($vmid);
6501
6502 # begin streaming, i.e. data copy from PBS to target disk for every vol,
6503 # this will effectively collapse the backing image chain consisting of
6504 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6505 # removes itself once all backing images vanish with 'auto-remove=on')
6506 my $jobs = {};
6507 for my $ds (sort keys %$restored_disks) {
6508 my $job_id = "restore-$ds";
6509 mon_cmd($vmid, 'block-stream',
6510 'job-id' => $job_id,
6511 device => "$ds",
6512 );
6513 $jobs->{$job_id} = {};
6514 }
6515
6516 mon_cmd($vmid, 'cont');
6517 qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6518
6519 print "restore-drive jobs finished successfully, removing all tracking block devices"
6520 ." to disconnect from Proxmox Backup Server\n";
6521
6522 for my $ds (sort keys %$restored_disks) {
6523 mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6524 }
6525
6526 close($qmeventd_fd);
6527 };
6528
6529 my $err = $@;
6530
6531 if ($err) {
6532 warn "An error occured during live-restore: $err\n";
6533 _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6534 die "live-restore failed\n";
6535 }
6536}
6537
6538sub restore_vma_archive {
6539 my ($archive, $vmid, $user, $opts, $comp) = @_;
6540
6541 my $readfrom = $archive;
6542
6543 my $cfg = PVE::Storage::config();
6544 my $commands = [];
6545 my $bwlimit = $opts->{bwlimit};
6546
6547 my $dbg_cmdstring = '';
6548 my $add_pipe = sub {
6549 my ($cmd) = @_;
6550 push @$commands, $cmd;
6551 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6552 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6553 $readfrom = '-';
6554 };
6555
6556 my $input = undef;
6557 if ($archive eq '-') {
6558 $input = '<&STDIN';
6559 } else {
6560 # If we use a backup from a PVE defined storage we also consider that
6561 # storage's rate limit:
6562 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6563 if (defined($volid)) {
6564 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6565 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6566 if ($readlimit) {
6567 print STDERR "applying read rate limit: $readlimit\n";
6568 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6569 $add_pipe->($cstream);
6570 }
6571 }
6572 }
6573
6574 if ($comp) {
6575 my $info = PVE::Storage::decompressor_info('vma', $comp);
6576 my $cmd = $info->{decompressor};
6577 push @$cmd, $readfrom;
6578 $add_pipe->($cmd);
6579 }
6580
6581 my $tmpdir = "/var/tmp/vzdumptmp$$";
6582 rmtree $tmpdir;
6583
6584 # disable interrupts (always do cleanups)
6585 local $SIG{INT} =
6586 local $SIG{TERM} =
6587 local $SIG{QUIT} =
6588 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6589
6590 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6591 POSIX::mkfifo($mapfifo, 0600);
6592 my $fifofh;
6593 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6594
6595 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6596
6597 my $oldtimeout;
6598 my $timeout = 5;
6599
6600 my $devinfo = {};
6601
6602 my $rpcenv = PVE::RPCEnvironment::get();
6603
6604 my $conffile = PVE::QemuConfig->config_file($vmid);
6605
6606 # Note: $oldconf is undef if VM does not exist
6607 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6608 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6609 my $new_conf_raw = '';
6610
6611 my %storage_limits;
6612
6613 my $print_devmap = sub {
6614 my $cfgfn = "$tmpdir/qemu-server.conf";
6615
6616 # we can read the config - that is already extracted
6617 my $fh = IO::File->new($cfgfn, "r") ||
6618 die "unable to read qemu-server.conf - $!\n";
6619
6620 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6621 if (-f $fwcfgfn) {
6622 my $pve_firewall_dir = '/etc/pve/firewall';
6623 mkdir $pve_firewall_dir; # make sure the dir exists
6624 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6625 }
6626
6627 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6628
6629 foreach my $info (values %{$virtdev_hash}) {
6630 my $storeid = $info->{storeid};
6631 next if defined($storage_limits{$storeid});
6632
6633 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6634 print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6635 $storage_limits{$storeid} = $limit * 1024;
6636 }
6637
6638 foreach my $devname (keys %$devinfo) {
6639 die "found no device mapping information for device '$devname'\n"
6640 if !$devinfo->{$devname}->{virtdev};
6641 }
6642
6643 # create empty/temp config
6644 if ($oldconf) {
6645 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6646 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6647 }
6648
6649 # allocate volumes
6650 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6651
6652 # print restore information to $fifofh
6653 foreach my $virtdev (sort keys %$virtdev_hash) {
6654 my $d = $virtdev_hash->{$virtdev};
6655 next if $d->{is_cloudinit}; # no need to restore cloudinit
6656
6657 my $storeid = $d->{storeid};
6658 my $volid = $d->{volid};
6659
6660 my $map_opts = '';
6661 if (my $limit = $storage_limits{$storeid}) {
6662 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6663 }
6664
6665 my $write_zeros = 1;
6666 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6667 $write_zeros = 0;
6668 }
6669
6670 my $path = PVE::Storage::path($cfg, $volid);
6671
6672 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6673
6674 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6675 }
6676
6677 $fh->seek(0, 0) || die "seek failed - $!\n";
6678
6679 my $cookie = { netcount => 0 };
6680 while (defined(my $line = <$fh>)) {
6681 $new_conf_raw .= restore_update_config_line(
6682 $cookie,
6683 $map,
6684 $line,
6685 $opts->{unique},
6686 );
6687 }
6688
6689 $fh->close();
6690 };
6691
6692 eval {
6693 # enable interrupts
6694 local $SIG{INT} =
6695 local $SIG{TERM} =
6696 local $SIG{QUIT} =
6697 local $SIG{HUP} =
6698 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6699 local $SIG{ALRM} = sub { die "got timeout\n"; };
6700
6701 $oldtimeout = alarm($timeout);
6702
6703 my $parser = sub {
6704 my $line = shift;
6705
6706 print "$line\n";
6707
6708 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6709 my ($dev_id, $size, $devname) = ($1, $2, $3);
6710 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6711 } elsif ($line =~ m/^CTIME: /) {
6712 # we correctly received the vma config, so we can disable
6713 # the timeout now for disk allocation (set to 10 minutes, so
6714 # that we always timeout if something goes wrong)
6715 alarm(600);
6716 &$print_devmap();
6717 print $fifofh "done\n";
6718 my $tmp = $oldtimeout || 0;
6719 $oldtimeout = undef;
6720 alarm($tmp);
6721 close($fifofh);
6722 $fifofh = undef;
6723 }
6724 };
6725
6726 print "restore vma archive: $dbg_cmdstring\n";
6727 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6728 };
6729 my $err = $@;
6730
6731 alarm($oldtimeout) if $oldtimeout;
6732
6733 $restore_deactivate_volumes->($cfg, $devinfo);
6734
6735 close($fifofh) if $fifofh;
6736 unlink $mapfifo;
6737 rmtree $tmpdir;
6738
6739 if ($err) {
6740 $restore_destroy_volumes->($cfg, $devinfo);
6741 die $err;
6742 }
6743
6744 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6745
6746 PVE::Cluster::cfs_update(); # make sure we read new file
6747
6748 eval { rescan($vmid, 1); };
6749 warn $@ if $@;
6750
6751 PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6752}
6753
6754sub restore_tar_archive {
6755 my ($archive, $vmid, $user, $opts) = @_;
6756
6757 if ($archive ne '-') {
6758 my $firstfile = tar_archive_read_firstfile($archive);
6759 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6760 if $firstfile ne 'qemu-server.conf';
6761 }
6762
6763 my $storecfg = PVE::Storage::config();
6764
6765 # avoid zombie disks when restoring over an existing VM -> cleanup first
6766 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6767 # skiplock=1 because qmrestore has set the 'create' lock itself already
6768 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6769 destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6770
6771 my $tocmd = "/usr/lib/qemu-server/qmextract";
6772
6773 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6774 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6775 $tocmd .= ' --prealloc' if $opts->{prealloc};
6776 $tocmd .= ' --info' if $opts->{info};
6777
6778 # tar option "xf" does not autodetect compression when read from STDIN,
6779 # so we pipe to zcat
6780 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6781 PVE::Tools::shellquote("--to-command=$tocmd");
6782
6783 my $tmpdir = "/var/tmp/vzdumptmp$$";
6784 mkpath $tmpdir;
6785
6786 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6787 local $ENV{VZDUMP_VMID} = $vmid;
6788 local $ENV{VZDUMP_USER} = $user;
6789
6790 my $conffile = PVE::QemuConfig->config_file($vmid);
6791 my $new_conf_raw = '';
6792
6793 # disable interrupts (always do cleanups)
6794 local $SIG{INT} =
6795 local $SIG{TERM} =
6796 local $SIG{QUIT} =
6797 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6798
6799 eval {
6800 # enable interrupts
6801 local $SIG{INT} =
6802 local $SIG{TERM} =
6803 local $SIG{QUIT} =
6804 local $SIG{HUP} =
6805 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6806
6807 if ($archive eq '-') {
6808 print "extracting archive from STDIN\n";
6809 run_command($cmd, input => "<&STDIN");
6810 } else {
6811 print "extracting archive '$archive'\n";
6812 run_command($cmd);
6813 }
6814
6815 return if $opts->{info};
6816
6817 # read new mapping
6818 my $map = {};
6819 my $statfile = "$tmpdir/qmrestore.stat";
6820 if (my $fd = IO::File->new($statfile, "r")) {
6821 while (defined (my $line = <$fd>)) {
6822 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6823 $map->{$1} = $2 if $1;
6824 } else {
6825 print STDERR "unable to parse line in statfile - $line\n";
6826 }
6827 }
6828 $fd->close();
6829 }
6830
6831 my $confsrc = "$tmpdir/qemu-server.conf";
6832
6833 my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6834
6835 my $cookie = { netcount => 0 };
6836 while (defined (my $line = <$srcfd>)) {
6837 $new_conf_raw .= restore_update_config_line(
6838 $cookie,
6839 $map,
6840 $line,
6841 $opts->{unique},
6842 );
6843 }
6844
6845 $srcfd->close();
6846 };
6847 if (my $err = $@) {
6848 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6849 die $err;
6850 }
6851
6852 rmtree $tmpdir;
6853
6854 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6855
6856 PVE::Cluster::cfs_update(); # make sure we read new file
6857
6858 eval { rescan($vmid, 1); };
6859 warn $@ if $@;
6860};
6861
6862sub foreach_storage_used_by_vm {
6863 my ($conf, $func) = @_;
6864
6865 my $sidhash = {};
6866
6867 PVE::QemuConfig->foreach_volume($conf, sub {
6868 my ($ds, $drive) = @_;
6869 return if drive_is_cdrom($drive);
6870
6871 my $volid = $drive->{file};
6872
6873 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6874 $sidhash->{$sid} = $sid if $sid;
6875 });
6876
6877 foreach my $sid (sort keys %$sidhash) {
6878 &$func($sid);
6879 }
6880}
6881
6882my $qemu_snap_storage = {
6883 rbd => 1,
6884};
6885sub do_snapshots_with_qemu {
6886 my ($storecfg, $volid) = @_;
6887
6888 my $storage_name = PVE::Storage::parse_volume_id($volid);
6889 my $scfg = $storecfg->{ids}->{$storage_name};
6890 die "could not find storage '$storage_name'\n" if !defined($scfg);
6891
6892 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6893 return 1;
6894 }
6895
6896 if ($volid =~ m/\.(qcow2|qed)$/){
6897 return 1;
6898 }
6899
6900 return;
6901}
6902
6903sub qga_check_running {
6904 my ($vmid, $nowarn) = @_;
6905
6906 eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6907 if ($@) {
6908 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6909 return 0;
6910 }
6911 return 1;
6912}
6913
6914sub template_create {
6915 my ($vmid, $conf, $disk) = @_;
6916
6917 my $storecfg = PVE::Storage::config();
6918
6919 PVE::QemuConfig->foreach_volume($conf, sub {
6920 my ($ds, $drive) = @_;
6921
6922 return if drive_is_cdrom($drive);
6923 return if $disk && $ds ne $disk;
6924
6925 my $volid = $drive->{file};
6926 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6927
6928 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6929 $drive->{file} = $voliddst;
6930 $conf->{$ds} = print_drive($drive);
6931 PVE::QemuConfig->write_config($vmid, $conf);
6932 });
6933}
6934
6935sub convert_iscsi_path {
6936 my ($path) = @_;
6937
6938 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6939 my $portal = $1;
6940 my $target = $2;
6941 my $lun = $3;
6942
6943 my $initiator_name = get_initiator_name();
6944
6945 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6946 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6947 }
6948
6949 die "cannot convert iscsi path '$path', unkown format\n";
6950}
6951
6952sub qemu_img_convert {
6953 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6954
6955 my $storecfg = PVE::Storage::config();
6956 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6957 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6958
6959 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6960
6961 my $cachemode;
6962 my $src_path;
6963 my $src_is_iscsi = 0;
6964 my $src_format;
6965
6966 if ($src_storeid) {
6967 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6968 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6969 $src_format = qemu_img_format($src_scfg, $src_volname);
6970 $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6971 $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6972 $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6973 } elsif (-f $src_volid) {
6974 $src_path = $src_volid;
6975 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6976 $src_format = $1;
6977 }
6978 }
6979
6980 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6981
6982 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6983 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6984 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6985 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6986
6987 my $cmd = [];
6988 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6989 push @$cmd, '-l', "snapshot.name=$snapname"
6990 if $snapname && $src_format && $src_format eq "qcow2";
6991 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6992 push @$cmd, '-T', $cachemode if defined($cachemode);
6993
6994 if ($src_is_iscsi) {
6995 push @$cmd, '--image-opts';
6996 $src_path = convert_iscsi_path($src_path);
6997 } elsif ($src_format) {
6998 push @$cmd, '-f', $src_format;
6999 }
7000
7001 if ($dst_is_iscsi) {
7002 push @$cmd, '--target-image-opts';
7003 $dst_path = convert_iscsi_path($dst_path);
7004 } else {
7005 push @$cmd, '-O', $dst_format;
7006 }
7007
7008 push @$cmd, $src_path;
7009
7010 if (!$dst_is_iscsi && $is_zero_initialized) {
7011 push @$cmd, "zeroinit:$dst_path";
7012 } else {
7013 push @$cmd, $dst_path;
7014 }
7015
7016 my $parser = sub {
7017 my $line = shift;
7018 if($line =~ m/\((\S+)\/100\%\)/){
7019 my $percent = $1;
7020 my $transferred = int($size * $percent / 100);
7021 my $total_h = render_bytes($size, 1);
7022 my $transferred_h = render_bytes($transferred, 1);
7023
7024 print "transferred $transferred_h of $total_h ($percent%)\n";
7025 }
7026
7027 };
7028
7029 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
7030 my $err = $@;
7031 die "copy failed: $err" if $err;
7032}
7033
7034sub qemu_img_format {
7035 my ($scfg, $volname) = @_;
7036
7037 if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7038 return $1;
7039 } else {
7040 return "raw";
7041 }
7042}
7043
7044sub qemu_drive_mirror {
7045 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7046
7047 $jobs = {} if !$jobs;
7048
7049 my $qemu_target;
7050 my $format;
7051 $jobs->{"drive-$drive"} = {};
7052
7053 if ($dst_volid =~ /^nbd:/) {
7054 $qemu_target = $dst_volid;
7055 $format = "nbd";
7056 } else {
7057 my $storecfg = PVE::Storage::config();
7058 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7059
7060 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7061
7062 $format = qemu_img_format($dst_scfg, $dst_volname);
7063
7064 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7065
7066 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7067 }
7068
7069 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7070 $opts->{format} = $format if $format;
7071
7072 if (defined($src_bitmap)) {
7073 $opts->{sync} = 'incremental';
7074 $opts->{bitmap} = $src_bitmap;
7075 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7076 }
7077
7078 if (defined($bwlimit)) {
7079 $opts->{speed} = $bwlimit * 1024;
7080 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7081 } else {
7082 print "drive mirror is starting for drive-$drive\n";
7083 }
7084
7085 # if a job already runs for this device we get an error, catch it for cleanup
7086 eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7087 if (my $err = $@) {
7088 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7089 warn "$@\n" if $@;
7090 die "mirroring error: $err\n";
7091 }
7092
7093 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7094}
7095
7096# $completion can be either
7097# 'complete': wait until all jobs are ready, block-job-complete them (default)
7098# 'cancel': wait until all jobs are ready, block-job-cancel them
7099# 'skip': wait until all jobs are ready, return with block jobs in ready state
7100# 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7101sub qemu_drive_mirror_monitor {
7102 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7103
7104 $completion //= 'complete';
7105 $op //= "mirror";
7106
7107 eval {
7108 my $err_complete = 0;
7109
7110 my $starttime = time ();
7111 while (1) {
7112 die "block job ('$op') timed out\n" if $err_complete > 300;
7113
7114 my $stats = mon_cmd($vmid, "query-block-jobs");
7115 my $ctime = time();
7116
7117 my $running_jobs = {};
7118 for my $stat (@$stats) {
7119 next if $stat->{type} ne $op;
7120 $running_jobs->{$stat->{device}} = $stat;
7121 }
7122
7123 my $readycounter = 0;
7124
7125 for my $job_id (sort keys %$jobs) {
7126 my $job = $running_jobs->{$job_id};
7127
7128 my $vanished = !defined($job);
7129 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7130 if($complete || ($vanished && $completion eq 'auto')) {
7131 print "$job_id: $op-job finished\n";
7132 delete $jobs->{$job_id};
7133 next;
7134 }
7135
7136 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7137
7138 my $busy = $job->{busy};
7139 my $ready = $job->{ready};
7140 if (my $total = $job->{len}) {
7141 my $transferred = $job->{offset} || 0;
7142 my $remaining = $total - $transferred;
7143 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7144
7145 my $duration = $ctime - $starttime;
7146 my $total_h = render_bytes($total, 1);
7147 my $transferred_h = render_bytes($transferred, 1);
7148
7149 my $status = sprintf(
7150 "transferred $transferred_h of $total_h ($percent%%) in %s",
7151 render_duration($duration),
7152 );
7153
7154 if ($ready) {
7155 if ($busy) {
7156 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7157 } else {
7158 $status .= ", ready";
7159 }
7160 }
7161 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7162 $jobs->{$job_id}->{ready} = $ready;
7163 }
7164
7165 $readycounter++ if $job->{ready};
7166 }
7167
7168 last if scalar(keys %$jobs) == 0;
7169
7170 if ($readycounter == scalar(keys %$jobs)) {
7171 print "all '$op' jobs are ready\n";
7172
7173 # do the complete later (or has already been done)
7174 last if $completion eq 'skip' || $completion eq 'auto';
7175
7176 if ($vmiddst && $vmiddst != $vmid) {
7177 my $agent_running = $qga && qga_check_running($vmid);
7178 if ($agent_running) {
7179 print "freeze filesystem\n";
7180 eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7181 } else {
7182 print "suspend vm\n";
7183 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7184 }
7185
7186 # if we clone a disk for a new target vm, we don't switch the disk
7187 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7188
7189 if ($agent_running) {
7190 print "unfreeze filesystem\n";
7191 eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7192 } else {
7193 print "resume vm\n";
7194 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7195 }
7196
7197 last;
7198 } else {
7199
7200 for my $job_id (sort keys %$jobs) {
7201 # try to switch the disk if source and destination are on the same guest
7202 print "$job_id: Completing block job_id...\n";
7203
7204 my $op;
7205 if ($completion eq 'complete') {
7206 $op = 'block-job-complete';
7207 } elsif ($completion eq 'cancel') {
7208 $op = 'block-job-cancel';
7209 } else {
7210 die "invalid completion value: $completion\n";
7211 }
7212 eval { mon_cmd($vmid, $op, device => $job_id) };
7213 if ($@ =~ m/cannot be completed/) {
7214 print "$job_id: block job cannot be completed, trying again.\n";
7215 $err_complete++;
7216 }else {
7217 print "$job_id: Completed successfully.\n";
7218 $jobs->{$job_id}->{complete} = 1;
7219 }
7220 }
7221 }
7222 }
7223 sleep 1;
7224 }
7225 };
7226 my $err = $@;
7227
7228 if ($err) {
7229 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7230 die "block job ($op) error: $err";
7231 }
7232}
7233
7234sub qemu_blockjobs_cancel {
7235 my ($vmid, $jobs) = @_;
7236
7237 foreach my $job (keys %$jobs) {
7238 print "$job: Cancelling block job\n";
7239 eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7240 $jobs->{$job}->{cancel} = 1;
7241 }
7242
7243 while (1) {
7244 my $stats = mon_cmd($vmid, "query-block-jobs");
7245
7246 my $running_jobs = {};
7247 foreach my $stat (@$stats) {
7248 $running_jobs->{$stat->{device}} = $stat;
7249 }
7250
7251 foreach my $job (keys %$jobs) {
7252
7253 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7254 print "$job: Done.\n";
7255 delete $jobs->{$job};
7256 }
7257 }
7258
7259 last if scalar(keys %$jobs) == 0;
7260
7261 sleep 1;
7262 }
7263}
7264
7265sub clone_disk {
7266 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7267 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7268
7269 my $newvolid;
7270
7271 if (!$full) {
7272 print "create linked clone of drive $drivename ($drive->{file})\n";
7273 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
7274 push @$newvollist, $newvolid;
7275 } else {
7276
7277 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7278 $storeid = $storage if $storage;
7279
7280 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7281
7282 print "create full clone of drive $drivename ($drive->{file})\n";
7283 my $name = undef;
7284 my $size = undef;
7285 if (drive_is_cloudinit($drive)) {
7286 $name = "vm-$newvmid-cloudinit";
7287 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7288 if ($scfg->{path}) {
7289 $name .= ".$dst_format";
7290 }
7291 $snapname = undef;
7292 $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7293 } elsif ($drivename eq 'efidisk0') {
7294 $size = get_efivars_size($conf);
7295 } else {
7296 ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7297 }
7298 $newvolid = PVE::Storage::vdisk_alloc(
7299 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7300 );
7301 push @$newvollist, $newvolid;
7302
7303 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7304
7305 if (drive_is_cloudinit($drive)) {
7306 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7307 # if this is the case, we have to complete any block-jobs still there from
7308 # previous drive-mirrors
7309 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7310 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7311 }
7312 goto no_data_clone;
7313 }
7314
7315 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7316 if (!$running || $snapname) {
7317 # TODO: handle bwlimits
7318 if ($drivename eq 'efidisk0') {
7319 # the relevant data on the efidisk may be smaller than the source
7320 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7321 # that is given by the OVMF_VARS.fd
7322 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7323 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7324
7325 # better for Ceph if block size is not too small, see bug #3324
7326 my $bs = 1024*1024;
7327
7328 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7329 "if=$src_path", "of=$dst_path"]);
7330 } else {
7331 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7332 }
7333 } else {
7334
7335 my $kvmver = get_running_qemu_version ($vmid);
7336 if (!min_version($kvmver, 2, 7)) {
7337 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7338 if $drive->{iothread};
7339 }
7340
7341 qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7342 $completion, $qga, $bwlimit);
7343 }
7344 }
7345
7346no_data_clone:
7347 my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7348
7349 my $disk = $drive;
7350 $disk->{format} = undef;
7351 $disk->{file} = $newvolid;
7352 $disk->{size} = $size if defined($size);
7353
7354 return $disk;
7355}
7356
7357sub get_running_qemu_version {
7358 my ($vmid) = @_;
7359 my $res = mon_cmd($vmid, "query-version");
7360 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7361}
7362
7363sub qemu_use_old_bios_files {
7364 my ($machine_type) = @_;
7365
7366 return if !$machine_type;
7367
7368 my $use_old_bios_files = undef;
7369
7370 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7371 $machine_type = $1;
7372 $use_old_bios_files = 1;
7373 } else {
7374 my $version = extract_version($machine_type, kvm_user_version());
7375 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7376 # load new efi bios files on migration. So this hack is required to allow
7377 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7378 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7379 $use_old_bios_files = !min_version($version, 2, 4);
7380 }
7381
7382 return ($use_old_bios_files, $machine_type);
7383}
7384
7385sub get_efivars_size {
7386 my ($conf) = @_;
7387 my $arch = get_vm_arch($conf);
7388 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7389 die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7390 return -s $ovmf_vars;
7391}
7392
7393sub update_efidisk_size {
7394 my ($conf) = @_;
7395
7396 return if !defined($conf->{efidisk0});
7397
7398 my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7399 $disk->{size} = get_efivars_size($conf);
7400 $conf->{efidisk0} = print_drive($disk);
7401
7402 return;
7403}
7404
7405sub create_efidisk($$$$$) {
7406 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7407
7408 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7409 die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7410
7411 my $vars_size_b = -s $ovmf_vars;
7412 my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7413 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7414 PVE::Storage::activate_volumes($storecfg, [$volid]);
7415
7416 qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7417 my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7418
7419 return ($volid, $size/1024);
7420}
7421
7422sub vm_iothreads_list {
7423 my ($vmid) = @_;
7424
7425 my $res = mon_cmd($vmid, 'query-iothreads');
7426
7427 my $iothreads = {};
7428 foreach my $iothread (@$res) {
7429 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7430 }
7431
7432 return $iothreads;
7433}
7434
7435sub scsihw_infos {
7436 my ($conf, $drive) = @_;
7437
7438 my $maxdev = 0;
7439
7440 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7441 $maxdev = 7;
7442 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7443 $maxdev = 1;
7444 } else {
7445 $maxdev = 256;
7446 }
7447
7448 my $controller = int($drive->{index} / $maxdev);
7449 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7450 ? "virtioscsi"
7451 : "scsihw";
7452
7453 return ($maxdev, $controller, $controller_prefix);
7454}
7455
7456sub windows_version {
7457 my ($ostype) = @_;
7458
7459 return 0 if !$ostype;
7460
7461 my $winversion = 0;
7462
7463 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7464 $winversion = 5;
7465 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7466 $winversion = 6;
7467 } elsif ($ostype =~ m/^win(\d+)$/) {
7468 $winversion = $1;
7469 }
7470
7471 return $winversion;
7472}
7473
7474sub resolve_dst_disk_format {
7475 my ($storecfg, $storeid, $src_volname, $format) = @_;
7476 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7477
7478 if (!$format) {
7479 # if no target format is specified, use the source disk format as hint
7480 if ($src_volname) {
7481 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7482 $format = qemu_img_format($scfg, $src_volname);
7483 } else {
7484 return $defFormat;
7485 }
7486 }
7487
7488 # test if requested format is supported - else use default
7489 my $supported = grep { $_ eq $format } @$validFormats;
7490 $format = $defFormat if !$supported;
7491 return $format;
7492}
7493
7494# NOTE: if this logic changes, please update docs & possibly gui logic
7495sub find_vmstate_storage {
7496 my ($conf, $storecfg) = @_;
7497
7498 # first, return storage from conf if set
7499 return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7500
7501 my ($target, $shared, $local);
7502
7503 foreach_storage_used_by_vm($conf, sub {
7504 my ($sid) = @_;
7505 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7506 my $dst = $scfg->{shared} ? \$shared : \$local;
7507 $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7508 });
7509
7510 # second, use shared storage where VM has at least one disk
7511 # third, use local storage where VM has at least one disk
7512 # fall back to local storage
7513 $target = $shared // $local // 'local';
7514
7515 return $target;
7516}
7517
7518sub generate_uuid {
7519 my ($uuid, $uuid_str);
7520 UUID::generate($uuid);
7521 UUID::unparse($uuid, $uuid_str);
7522 return $uuid_str;
7523}
7524
7525sub generate_smbios1_uuid {
7526 return "uuid=".generate_uuid();
7527}
7528
7529sub nbd_stop {
7530 my ($vmid) = @_;
7531
7532 mon_cmd($vmid, 'nbd-server-stop');
7533}
7534
7535sub create_reboot_request {
7536 my ($vmid) = @_;
7537 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7538 or die "failed to create reboot trigger file: $!\n";
7539 close($fh);
7540}
7541
7542sub clear_reboot_request {
7543 my ($vmid) = @_;
7544 my $path = "/run/qemu-server/$vmid.reboot";
7545 my $res = 0;
7546
7547 $res = unlink($path);
7548 die "could not remove reboot request for $vmid: $!"
7549 if !$res && $! != POSIX::ENOENT;
7550
7551 return $res;
7552}
7553
7554sub bootorder_from_legacy {
7555 my ($conf, $bootcfg) = @_;
7556
7557 my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7558 my $bootindex_hash = {};
7559 my $i = 1;
7560 foreach my $o (split(//, $boot)) {
7561 $bootindex_hash->{$o} = $i*100;
7562 $i++;
7563 }
7564
7565 my $bootorder = {};
7566
7567 PVE::QemuConfig->foreach_volume($conf, sub {
7568 my ($ds, $drive) = @_;
7569
7570 if (drive_is_cdrom ($drive, 1)) {
7571 if ($bootindex_hash->{d}) {
7572 $bootorder->{$ds} = $bootindex_hash->{d};
7573 $bootindex_hash->{d} += 1;
7574 }
7575 } elsif ($bootindex_hash->{c}) {
7576 $bootorder->{$ds} = $bootindex_hash->{c}
7577 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7578 $bootindex_hash->{c} += 1;
7579 }
7580 });
7581
7582 if ($bootindex_hash->{n}) {
7583 for (my $i = 0; $i < $MAX_NETS; $i++) {
7584 my $netname = "net$i";
7585 next if !$conf->{$netname};
7586 $bootorder->{$netname} = $bootindex_hash->{n};
7587 $bootindex_hash->{n} += 1;
7588 }
7589 }
7590
7591 return $bootorder;
7592}
7593
7594# Generate default device list for 'boot: order=' property. Matches legacy
7595# default boot order, but with explicit device names. This is important, since
7596# the fallback for when neither 'order' nor the old format is specified relies
7597# on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7598sub get_default_bootdevices {
7599 my ($conf) = @_;
7600
7601 my @ret = ();
7602
7603 # harddisk
7604 my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7605 push @ret, $first if $first;
7606
7607 # cdrom
7608 $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7609 push @ret, $first if $first;
7610
7611 # network
7612 for (my $i = 0; $i < $MAX_NETS; $i++) {
7613 my $netname = "net$i";
7614 next if !$conf->{$netname};
7615 push @ret, $netname;
7616 last;
7617 }
7618
7619 return \@ret;
7620}
7621
7622sub device_bootorder {
7623 my ($conf) = @_;
7624
7625 return bootorder_from_legacy($conf) if !defined($conf->{boot});
7626
7627 my $boot = parse_property_string($boot_fmt, $conf->{boot});
7628
7629 my $bootorder = {};
7630 if (!defined($boot) || $boot->{legacy}) {
7631 $bootorder = bootorder_from_legacy($conf, $boot);
7632 } elsif ($boot->{order}) {
7633 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7634 for my $dev (PVE::Tools::split_list($boot->{order})) {
7635 $bootorder->{$dev} = $i++;
7636 }
7637 }
7638
7639 return $bootorder;
7640}
7641
7642sub register_qmeventd_handle {
7643 my ($vmid) = @_;
7644
7645 my $fh;
7646 my $peer = "/var/run/qmeventd.sock";
7647 my $count = 0;
7648
7649 for (;;) {
7650 $count++;
7651 $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7652 last if $fh;
7653 if ($! != EINTR && $! != EAGAIN) {
7654 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7655 }
7656 if ($count > 4) {
7657 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7658 . "after $count retries\n";
7659 }
7660 usleep(25000);
7661 }
7662
7663 # send handshake to mark VM as backing up
7664 print $fh to_json({vzdump => {vmid => "$vmid"}});
7665
7666 # return handle to be closed later when inhibit is no longer required
7667 return $fh;
7668}
7669
7670# bash completion helper
7671
7672sub complete_backup_archives {
7673 my ($cmdname, $pname, $cvalue) = @_;
7674
7675 my $cfg = PVE::Storage::config();
7676
7677 my $storeid;
7678
7679 if ($cvalue =~ m/^([^:]+):/) {
7680 $storeid = $1;
7681 }
7682
7683 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7684
7685 my $res = [];
7686 foreach my $id (keys %$data) {
7687 foreach my $item (@{$data->{$id}}) {
7688 next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7689 push @$res, $item->{volid} if defined($item->{volid});
7690 }
7691 }
7692
7693 return $res;
7694}
7695
7696my $complete_vmid_full = sub {
7697 my ($running) = @_;
7698
7699 my $idlist = vmstatus();
7700
7701 my $res = [];
7702
7703 foreach my $id (keys %$idlist) {
7704 my $d = $idlist->{$id};
7705 if (defined($running)) {
7706 next if $d->{template};
7707 next if $running && $d->{status} ne 'running';
7708 next if !$running && $d->{status} eq 'running';
7709 }
7710 push @$res, $id;
7711
7712 }
7713 return $res;
7714};
7715
7716sub complete_vmid {
7717 return &$complete_vmid_full();
7718}
7719
7720sub complete_vmid_stopped {
7721 return &$complete_vmid_full(0);
7722}
7723
7724sub complete_vmid_running {
7725 return &$complete_vmid_full(1);
7726}
7727
7728sub complete_storage {
7729
7730 my $cfg = PVE::Storage::config();
7731 my $ids = $cfg->{ids};
7732
7733 my $res = [];
7734 foreach my $sid (keys %$ids) {
7735 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7736 next if !$ids->{$sid}->{content}->{images};
7737 push @$res, $sid;
7738 }
7739
7740 return $res;
7741}
7742
7743sub complete_migration_storage {
7744 my ($cmd, $param, $current_value, $all_args) = @_;
7745
7746 my $targetnode = @$all_args[1];
7747
7748 my $cfg = PVE::Storage::config();
7749 my $ids = $cfg->{ids};
7750
7751 my $res = [];
7752 foreach my $sid (keys %$ids) {
7753 next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7754 next if !$ids->{$sid}->{content}->{images};
7755 push @$res, $sid;
7756 }
7757
7758 return $res;
7759}
7760
7761sub vm_is_paused {
7762 my ($vmid) = @_;
7763 my $qmpstatus = eval {
7764 PVE::QemuConfig::assert_config_exists_on_node($vmid);
7765 mon_cmd($vmid, "query-status");
7766 };
7767 warn "$@\n" if $@;
7768 return $qmpstatus && $qmpstatus->{status} eq "paused";
7769}
7770
7771sub check_volume_storage_type {
7772 my ($storecfg, $vol) = @_;
7773
7774 my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7775 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7776 my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7777
7778 die "storage '$storeid' does not support content-type '$vtype'\n"
7779 if !$scfg->{content}->{$vtype};
7780
7781 return 1;
7782}
7783
77841;
Virtual Machine Manager