]> git.proxmox.com Git - qemu-server.git/blame_incremental - PVE/QemuServer.pm
projects / qemu-server.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
fix #3608: unbreak removal of scsi controller
[qemu-server.git] / PVE / QemuServer.pm
... / ...
CommitLineData
1package PVE::QemuServer;
2
3use strict;
4use warnings;
5
6use Cwd 'abs_path';
7use Digest::SHA;
8use Fcntl ':flock';
9use Fcntl;
10use File::Basename;
11use File::Copy qw(copy);
12use File::Path;
13use File::stat;
14use Getopt::Long;
15use IO::Dir;
16use IO::File;
17use IO::Handle;
18use IO::Select;
19use IO::Socket::UNIX;
20use IPC::Open3;
21use JSON;
22use MIME::Base64;
23use POSIX;
24use Storable qw(dclone);
25use Time::HiRes qw(gettimeofday);
26use URI::Escape;
27use UUID;
28
29use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
30use PVE::CGroup;
31use PVE::DataCenterConfig;
32use PVE::Exception qw(raise raise_param_exc);
33use PVE::Format qw(render_duration render_bytes);
34use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
35use PVE::INotify;
36use PVE::JSONSchema qw(get_standard_option parse_property_string);
37use PVE::ProcFSTools;
38use PVE::PBSClient;
39use PVE::RPCEnvironment;
40use PVE::Storage;
41use PVE::SysFSTools;
42use PVE::Systemd;
43use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44
45use PVE::QMPClient;
46use PVE::QemuConfig;
47use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
48use PVE::QemuServer::Cloudinit;
49use PVE::QemuServer::CGroup;
50use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
51use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
52use PVE::QemuServer::Machine;
53use PVE::QemuServer::Memory;
54use PVE::QemuServer::Monitor qw(mon_cmd);
55use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56use PVE::QemuServer::USB qw(parse_usb_device);
57
58my $have_sdn;
59eval {
60 require PVE::Network::SDN::Zones;
61 $have_sdn = 1;
62};
63
64my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
65my $OVMF = {
66 x86_64 => [
67 "$EDK2_FW_BASE/OVMF_CODE.fd",
68 "$EDK2_FW_BASE/OVMF_VARS.fd"
69 ],
70 aarch64 => [
71 "$EDK2_FW_BASE/AAVMF_CODE.fd",
72 "$EDK2_FW_BASE/AAVMF_VARS.fd"
73 ],
74};
75
76my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
77
78# Note about locking: we use flock on the config file protect
79# against concurent actions.
80# Aditionaly, we have a 'lock' setting in the config file. This
81# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
82# allowed when such lock is set. But you can ignore this kind of
83# lock with the --skiplock flag.
84
85cfs_register_file('/qemu-server/',
86 \&parse_vm_config,
87 \&write_vm_config);
88
89PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
90 description => "Some command save/restore state from this location.",
91 type => 'string',
92 maxLength => 128,
93 optional => 1,
94});
95
96PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
97 description => "Specifies the Qemu machine type.",
98 type => 'string',
99 pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
100 maxLength => 40,
101 optional => 1,
102});
103
104
105sub map_storage {
106 my ($map, $source) = @_;
107
108 return $source if !defined($map);
109
110 return $map->{entries}->{$source}
111 if $map->{entries} && defined($map->{entries}->{$source});
112
113 return $map->{default} if $map->{default};
114
115 # identity (fallback)
116 return $source;
117}
118
119PVE::JSONSchema::register_standard_option('pve-targetstorage', {
120 description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
121 type => 'string',
122 format => 'storagepair-list',
123 optional => 1,
124});
125
126#no warnings 'redefine';
127
128my $nodename_cache;
129sub nodename {
130 $nodename_cache //= PVE::INotify::nodename();
131 return $nodename_cache;
132}
133
134my $watchdog_fmt = {
135 model => {
136 default_key => 1,
137 type => 'string',
138 enum => [qw(i6300esb ib700)],
139 description => "Watchdog type to emulate.",
140 default => 'i6300esb',
141 optional => 1,
142 },
143 action => {
144 type => 'string',
145 enum => [qw(reset shutdown poweroff pause debug none)],
146 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
147 optional => 1,
148 },
149};
150PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
151
152my $agent_fmt = {
153 enabled => {
154 description => "Enable/disable Qemu GuestAgent.",
155 type => 'boolean',
156 default => 0,
157 default_key => 1,
158 },
159 fstrim_cloned_disks => {
160 description => "Run fstrim after moving a disk or migrating the VM.",
161 type => 'boolean',
162 optional => 1,
163 default => 0
164 },
165 type => {
166 description => "Select the agent type",
167 type => 'string',
168 default => 'virtio',
169 optional => 1,
170 enum => [qw(virtio isa)],
171 },
172};
173
174my $vga_fmt = {
175 type => {
176 description => "Select the VGA type.",
177 type => 'string',
178 default => 'std',
179 optional => 1,
180 default_key => 1,
181 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
182 },
183 memory => {
184 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
185 type => 'integer',
186 optional => 1,
187 minimum => 4,
188 maximum => 512,
189 },
190};
191
192my $ivshmem_fmt = {
193 size => {
194 type => 'integer',
195 minimum => 1,
196 description => "The size of the file in MB.",
197 },
198 name => {
199 type => 'string',
200 pattern => '[a-zA-Z0-9\-]+',
201 optional => 1,
202 format_description => 'string',
203 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
204 },
205};
206
207my $audio_fmt = {
208 device => {
209 type => 'string',
210 enum => [qw(ich9-intel-hda intel-hda AC97)],
211 description => "Configure an audio device."
212 },
213 driver => {
214 type => 'string',
215 enum => ['spice', 'none'],
216 default => 'spice',
217 optional => 1,
218 description => "Driver backend for the audio device."
219 },
220};
221
222my $spice_enhancements_fmt = {
223 foldersharing => {
224 type => 'boolean',
225 optional => 1,
226 default => '0',
227 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
228 },
229 videostreaming => {
230 type => 'string',
231 enum => ['off', 'all', 'filter'],
232 default => 'off',
233 optional => 1,
234 description => "Enable video streaming. Uses compression for detected video streams."
235 },
236};
237
238my $rng_fmt = {
239 source => {
240 type => 'string',
241 enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
242 default_key => 1,
243 description => "The file on the host to gather entropy from. In most"
244 . " cases /dev/urandom should be preferred over /dev/random"
245 . " to avoid entropy-starvation issues on the host. Using"
246 . " urandom does *not* decrease security in any meaningful"
247 . " way, as it's still seeded from real entropy, and the"
248 . " bytes provided will most likely be mixed with real"
249 . " entropy on the guest as well. /dev/hwrng can be used"
250 . " to pass through a hardware RNG from the host.",
251 },
252 max_bytes => {
253 type => 'integer',
254 description => "Maximum bytes of entropy injected into the guest every"
255 . " 'period' milliseconds. Prefer a lower value when using"
256 . " /dev/random as source. Use 0 to disable limiting"
257 . " (potentially dangerous!).",
258 optional => 1,
259
260 # default is 1 KiB/s, provides enough entropy to the guest to avoid
261 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
262 # of overwhelming the host, provided we're reading from /dev/urandom
263 default => 1024,
264 },
265 period => {
266 type => 'integer',
267 description => "Every 'period' milliseconds the entropy-injection quota"
268 . " is reset, allowing the guest to retrieve another"
269 . " 'max_bytes' of entropy.",
270 optional => 1,
271 default => 1000,
272 },
273};
274
275my $confdesc = {
276 onboot => {
277 optional => 1,
278 type => 'boolean',
279 description => "Specifies whether a VM will be started during system bootup.",
280 default => 0,
281 },
282 autostart => {
283 optional => 1,
284 type => 'boolean',
285 description => "Automatic restart after crash (currently ignored).",
286 default => 0,
287 },
288 hotplug => {
289 optional => 1,
290 type => 'string', format => 'pve-hotplug-features',
291 description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
292 default => 'network,disk,usb',
293 },
294 reboot => {
295 optional => 1,
296 type => 'boolean',
297 description => "Allow reboot. If set to '0' the VM exit on reboot.",
298 default => 1,
299 },
300 lock => {
301 optional => 1,
302 type => 'string',
303 description => "Lock/unlock the VM.",
304 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
305 },
306 cpulimit => {
307 optional => 1,
308 type => 'number',
309 description => "Limit of CPU usage.",
310 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
311 minimum => 0,
312 maximum => 128,
313 default => 0,
314 },
315 cpuunits => {
316 optional => 1,
317 type => 'integer',
318 description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
319 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
320 ." The larger the number is, the more CPU time this VM gets. Number is relative to"
321 ." weights of all the other running VMs.",
322 minimum => 2,
323 maximum => 262144,
324 default => 'cgroup v1: 1024, cgroup v2: 100',
325 },
326 memory => {
327 optional => 1,
328 type => 'integer',
329 description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
330 minimum => 16,
331 default => 512,
332 },
333 balloon => {
334 optional => 1,
335 type => 'integer',
336 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
337 minimum => 0,
338 },
339 shares => {
340 optional => 1,
341 type => 'integer',
342 description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
343 minimum => 0,
344 maximum => 50000,
345 default => 1000,
346 },
347 keyboard => {
348 optional => 1,
349 type => 'string',
350 description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
351 "It should not be necessary to set it.",
352 enum => PVE::Tools::kvmkeymaplist(),
353 default => undef,
354 },
355 name => {
356 optional => 1,
357 type => 'string', format => 'dns-name',
358 description => "Set a name for the VM. Only used on the configuration web interface.",
359 },
360 scsihw => {
361 optional => 1,
362 type => 'string',
363 description => "SCSI controller model",
364 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
365 default => 'lsi',
366 },
367 description => {
368 optional => 1,
369 type => 'string',
370 description => "Description for the VM. Shown in the web-interface VM's summary."
371 ." This is saved as comment inside the configuration file.",
372 maxLength => 1024 * 8,
373 },
374 ostype => {
375 optional => 1,
376 type => 'string',
377 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
378 description => "Specify guest operating system.",
379 verbose_description => <<EODESC,
380Specify guest operating system. This is used to enable special
381optimization/features for specific operating systems:
382
383[horizontal]
384other;; unspecified OS
385wxp;; Microsoft Windows XP
386w2k;; Microsoft Windows 2000
387w2k3;; Microsoft Windows 2003
388w2k8;; Microsoft Windows 2008
389wvista;; Microsoft Windows Vista
390win7;; Microsoft Windows 7
391win8;; Microsoft Windows 8/2012/2012r2
392win10;; Microsoft Windows 10/2016/2019
393l24;; Linux 2.4 Kernel
394l26;; Linux 2.6 - 5.X Kernel
395solaris;; Solaris/OpenSolaris/OpenIndiania kernel
396EODESC
397 },
398 boot => {
399 optional => 1,
400 type => 'string', format => 'pve-qm-boot',
401 description => "Specify guest boot order. Use with 'order=', usage with"
402 . " no key or 'legacy=' is deprecated.",
403 },
404 bootdisk => {
405 optional => 1,
406 type => 'string', format => 'pve-qm-bootdisk',
407 description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
408 pattern => '(ide|sata|scsi|virtio)\d+',
409 },
410 smp => {
411 optional => 1,
412 type => 'integer',
413 description => "The number of CPUs. Please use option -sockets instead.",
414 minimum => 1,
415 default => 1,
416 },
417 sockets => {
418 optional => 1,
419 type => 'integer',
420 description => "The number of CPU sockets.",
421 minimum => 1,
422 default => 1,
423 },
424 cores => {
425 optional => 1,
426 type => 'integer',
427 description => "The number of cores per socket.",
428 minimum => 1,
429 default => 1,
430 },
431 numa => {
432 optional => 1,
433 type => 'boolean',
434 description => "Enable/disable NUMA.",
435 default => 0,
436 },
437 hugepages => {
438 optional => 1,
439 type => 'string',
440 description => "Enable/disable hugepages memory.",
441 enum => [qw(any 2 1024)],
442 },
443 keephugepages => {
444 optional => 1,
445 type => 'boolean',
446 default => 0,
447 description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
448 ." after VM shutdown and can be used for subsequent starts.",
449 },
450 vcpus => {
451 optional => 1,
452 type => 'integer',
453 description => "Number of hotplugged vcpus.",
454 minimum => 1,
455 default => 0,
456 },
457 acpi => {
458 optional => 1,
459 type => 'boolean',
460 description => "Enable/disable ACPI.",
461 default => 1,
462 },
463 agent => {
464 optional => 1,
465 description => "Enable/disable Qemu GuestAgent and its properties.",
466 type => 'string',
467 format => $agent_fmt,
468 },
469 kvm => {
470 optional => 1,
471 type => 'boolean',
472 description => "Enable/disable KVM hardware virtualization.",
473 default => 1,
474 },
475 tdf => {
476 optional => 1,
477 type => 'boolean',
478 description => "Enable/disable time drift fix.",
479 default => 0,
480 },
481 localtime => {
482 optional => 1,
483 type => 'boolean',
484 description => "Set the real time clock to local time. This is enabled by default if ostype"
485 ." indicates a Microsoft OS.",
486 },
487 freeze => {
488 optional => 1,
489 type => 'boolean',
490 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
491 },
492 vga => {
493 optional => 1,
494 type => 'string', format => $vga_fmt,
495 description => "Configure the VGA hardware.",
496 verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
497 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
498 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
499 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
500 ." display server. For win* OS you can select how many independent displays you want,"
501 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
502 ." using a serial device as terminal.",
503 },
504 watchdog => {
505 optional => 1,
506 type => 'string', format => 'pve-qm-watchdog',
507 description => "Create a virtual hardware watchdog device.",
508 verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
509 ." action), the watchdog must be periodically polled by an agent inside the guest or"
510 ." else the watchdog will reset the guest (or execute the respective action specified)",
511 },
512 startdate => {
513 optional => 1,
514 type => 'string',
515 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
516 description => "Set the initial date of the real time clock. Valid format for date are:"
517 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
518 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
519 default => 'now',
520 },
521 startup => get_standard_option('pve-startup-order'),
522 template => {
523 optional => 1,
524 type => 'boolean',
525 description => "Enable/disable Template.",
526 default => 0,
527 },
528 args => {
529 optional => 1,
530 type => 'string',
531 description => "Arbitrary arguments passed to kvm.",
532 verbose_description => <<EODESCR,
533Arbitrary arguments passed to kvm, for example:
534
535args: -no-reboot -no-hpet
536
537NOTE: this option is for experts only.
538EODESCR
539 },
540 tablet => {
541 optional => 1,
542 type => 'boolean',
543 default => 1,
544 description => "Enable/disable the USB tablet device.",
545 verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
546 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
547 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
548 ." may consider disabling this to save some context switches. This is turned off by"
549 ." default if you use spice (`qm set <vmid> --vga qxl`).",
550 },
551 migrate_speed => {
552 optional => 1,
553 type => 'integer',
554 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
555 minimum => 0,
556 default => 0,
557 },
558 migrate_downtime => {
559 optional => 1,
560 type => 'number',
561 description => "Set maximum tolerated downtime (in seconds) for migrations.",
562 minimum => 0,
563 default => 0.1,
564 },
565 cdrom => {
566 optional => 1,
567 type => 'string', format => 'pve-qm-ide',
568 typetext => '<volume>',
569 description => "This is an alias for option -ide2",
570 },
571 cpu => {
572 optional => 1,
573 description => "Emulated CPU type.",
574 type => 'string',
575 format => 'pve-vm-cpu-conf',
576 },
577 parent => get_standard_option('pve-snapshot-name', {
578 optional => 1,
579 description => "Parent snapshot name. This is used internally, and should not be modified.",
580 }),
581 snaptime => {
582 optional => 1,
583 description => "Timestamp for snapshots.",
584 type => 'integer',
585 minimum => 0,
586 },
587 vmstate => {
588 optional => 1,
589 type => 'string', format => 'pve-volume-id',
590 description => "Reference to a volume which stores the VM state. This is used internally"
591 ." for snapshots.",
592 },
593 vmstatestorage => get_standard_option('pve-storage-id', {
594 description => "Default storage for VM state volumes/files.",
595 optional => 1,
596 }),
597 runningmachine => get_standard_option('pve-qemu-machine', {
598 description => "Specifies the QEMU machine type of the running vm. This is used internally"
599 ." for snapshots.",
600 }),
601 runningcpu => {
602 description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
603 ." internally for snapshots.",
604 optional => 1,
605 type => 'string',
606 pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
607 format_description => 'QEMU -cpu parameter'
608 },
609 machine => get_standard_option('pve-qemu-machine'),
610 arch => {
611 description => "Virtual processor architecture. Defaults to the host.",
612 optional => 1,
613 type => 'string',
614 enum => [qw(x86_64 aarch64)],
615 },
616 smbios1 => {
617 description => "Specify SMBIOS type 1 fields.",
618 type => 'string', format => 'pve-qm-smbios1',
619 maxLength => 512,
620 optional => 1,
621 },
622 protection => {
623 optional => 1,
624 type => 'boolean',
625 description => "Sets the protection flag of the VM. This will disable the remove VM and"
626 ." remove disk operations.",
627 default => 0,
628 },
629 bios => {
630 optional => 1,
631 type => 'string',
632 enum => [ qw(seabios ovmf) ],
633 description => "Select BIOS implementation.",
634 default => 'seabios',
635 },
636 vmgenid => {
637 type => 'string',
638 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
639 format_description => 'UUID',
640 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
641 ." to disable explicitly.",
642 verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
643 ." value identifier to the guest OS. This allows to notify the guest operating system"
644 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
645 ." execution or creation from a template). The guest operating system notices the"
646 ." change, and is then able to react as appropriate by marking its copies of"
647 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
648 ."Note that auto-creation only works when done through API/CLI create or update methods"
649 .", but not when manually editing the config file.",
650 default => "1 (autogenerated)",
651 optional => 1,
652 },
653 hookscript => {
654 type => 'string',
655 format => 'pve-volume-id',
656 optional => 1,
657 description => "Script that will be executed during various steps in the vms lifetime.",
658 },
659 ivshmem => {
660 type => 'string',
661 format => $ivshmem_fmt,
662 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
663 ." the host.",
664 optional => 1,
665 },
666 audio0 => {
667 type => 'string',
668 format => $audio_fmt,
669 description => "Configure a audio device, useful in combination with QXL/Spice.",
670 optional => 1
671 },
672 spice_enhancements => {
673 type => 'string',
674 format => $spice_enhancements_fmt,
675 description => "Configure additional enhancements for SPICE.",
676 optional => 1
677 },
678 tags => {
679 type => 'string', format => 'pve-tag-list',
680 description => 'Tags of the VM. This is only meta information.',
681 optional => 1,
682 },
683 rng0 => {
684 type => 'string',
685 format => $rng_fmt,
686 description => "Configure a VirtIO-based Random Number Generator.",
687 optional => 1,
688 },
689};
690
691my $cicustom_fmt = {
692 meta => {
693 type => 'string',
694 optional => 1,
695 description => 'Specify a custom file containing all meta data passed to the VM via"
696 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
697 format => 'pve-volume-id',
698 format_description => 'volume',
699 },
700 network => {
701 type => 'string',
702 optional => 1,
703 description => 'Specify a custom file containing all network data passed to the VM via'
704 .' cloud-init.',
705 format => 'pve-volume-id',
706 format_description => 'volume',
707 },
708 user => {
709 type => 'string',
710 optional => 1,
711 description => 'Specify a custom file containing all user data passed to the VM via'
712 .' cloud-init.',
713 format => 'pve-volume-id',
714 format_description => 'volume',
715 },
716};
717PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
718
719my $confdesc_cloudinit = {
720 citype => {
721 optional => 1,
722 type => 'string',
723 description => 'Specifies the cloud-init configuration format. The default depends on the'
724 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
725 .' and `configdrive2` for windows.',
726 enum => ['configdrive2', 'nocloud', 'opennebula'],
727 },
728 ciuser => {
729 optional => 1,
730 type => 'string',
731 description => "cloud-init: User name to change ssh keys and password for instead of the"
732 ." image's configured default user.",
733 },
734 cipassword => {
735 optional => 1,
736 type => 'string',
737 description => 'cloud-init: Password to assign the user. Using this is generally not'
738 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
739 .' support hashed passwords.',
740 },
741 cicustom => {
742 optional => 1,
743 type => 'string',
744 description => 'cloud-init: Specify custom files to replace the automatically generated'
745 .' ones at start.',
746 format => 'pve-qm-cicustom',
747 },
748 searchdomain => {
749 optional => 1,
750 type => 'string',
751 description => "cloud-init: Sets DNS search domains for a container. Create will'
752 .' automatically use the setting from the host if neither searchdomain nor nameserver'
753 .' are set.",
754 },
755 nameserver => {
756 optional => 1,
757 type => 'string', format => 'address-list',
758 description => "cloud-init: Sets DNS server IP address for a container. Create will'
759 .' automatically use the setting from the host if neither searchdomain nor nameserver'
760 .' are set.",
761 },
762 sshkeys => {
763 optional => 1,
764 type => 'string',
765 format => 'urlencoded',
766 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
767 },
768};
769
770# what about other qemu settings ?
771#cpu => 'string',
772#machine => 'string',
773#fda => 'file',
774#fdb => 'file',
775#mtdblock => 'file',
776#sd => 'file',
777#pflash => 'file',
778#snapshot => 'bool',
779#bootp => 'file',
780##tftp => 'dir',
781##smb => 'dir',
782#kernel => 'file',
783#append => 'string',
784#initrd => 'file',
785##soundhw => 'string',
786
787while (my ($k, $v) = each %$confdesc) {
788 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
789}
790
791my $MAX_USB_DEVICES = 5;
792my $MAX_NETS = 32;
793my $MAX_SERIAL_PORTS = 4;
794my $MAX_PARALLEL_PORTS = 3;
795my $MAX_NUMA = 8;
796
797my $numa_fmt = {
798 cpus => {
799 type => "string",
800 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
801 description => "CPUs accessing this NUMA node.",
802 format_description => "id[-id];...",
803 },
804 memory => {
805 type => "number",
806 description => "Amount of memory this NUMA node provides.",
807 optional => 1,
808 },
809 hostnodes => {
810 type => "string",
811 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
812 description => "Host NUMA nodes to use.",
813 format_description => "id[-id];...",
814 optional => 1,
815 },
816 policy => {
817 type => 'string',
818 enum => [qw(preferred bind interleave)],
819 description => "NUMA allocation policy.",
820 optional => 1,
821 },
822};
823PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
824my $numadesc = {
825 optional => 1,
826 type => 'string', format => $numa_fmt,
827 description => "NUMA topology.",
828};
829PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
830
831for (my $i = 0; $i < $MAX_NUMA; $i++) {
832 $confdesc->{"numa$i"} = $numadesc;
833}
834
835my $nic_model_list = [
836 'e1000',
837 'e1000-82540em',
838 'e1000-82544gc',
839 'e1000-82545em',
840 'e1000e',
841 'i82551',
842 'i82557b',
843 'i82559er',
844 'ne2k_isa',
845 'ne2k_pci',
846 'pcnet',
847 'rtl8139',
848 'virtio',
849 'vmxnet3',
850];
851my $nic_model_list_txt = join(' ', sort @$nic_model_list);
852
853my $net_fmt_bridge_descr = <<__EOD__;
854Bridge to attach the network device to. The Proxmox VE standard bridge
855is called 'vmbr0'.
856
857If you do not specify a bridge, we create a kvm user (NATed) network
858device, which provides DHCP and DNS services. The following addresses
859are used:
860
861 10.0.2.2 Gateway
862 10.0.2.3 DNS Server
863 10.0.2.4 SMB Server
864
865The DHCP server assign addresses to the guest starting from 10.0.2.15.
866__EOD__
867
868my $net_fmt = {
869 macaddr => get_standard_option('mac-addr', {
870 description => "MAC address. That address must be unique withing your network. This is"
871 ." automatically generated if not specified.",
872 }),
873 model => {
874 type => 'string',
875 description => "Network Card Model. The 'virtio' model provides the best performance with"
876 ." very low CPU overhead. If your guest does not support this driver, it is usually"
877 ." best to use 'e1000'.",
878 enum => $nic_model_list,
879 default_key => 1,
880 },
881 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
882 bridge => {
883 type => 'string',
884 description => $net_fmt_bridge_descr,
885 format_description => 'bridge',
886 pattern => '[-_.\w\d]+',
887 optional => 1,
888 },
889 queues => {
890 type => 'integer',
891 minimum => 0, maximum => 16,
892 description => 'Number of packet queues to be used on the device.',
893 optional => 1,
894 },
895 rate => {
896 type => 'number',
897 minimum => 0,
898 description => "Rate limit in mbps (megabytes per second) as floating point number.",
899 optional => 1,
900 },
901 tag => {
902 type => 'integer',
903 minimum => 1, maximum => 4094,
904 description => 'VLAN tag to apply to packets on this interface.',
905 optional => 1,
906 },
907 trunks => {
908 type => 'string',
909 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
910 description => 'VLAN trunks to pass through this interface.',
911 format_description => 'vlanid[;vlanid...]',
912 optional => 1,
913 },
914 firewall => {
915 type => 'boolean',
916 description => 'Whether this interface should be protected by the firewall.',
917 optional => 1,
918 },
919 link_down => {
920 type => 'boolean',
921 description => 'Whether this interface should be disconnected (like pulling the plug).',
922 optional => 1,
923 },
924 mtu => {
925 type => 'integer',
926 minimum => 1, maximum => 65520,
927 description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
928 optional => 1,
929 },
930};
931
932my $netdesc = {
933 optional => 1,
934 type => 'string', format => $net_fmt,
935 description => "Specify network devices.",
936};
937
938PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
939
940my $ipconfig_fmt = {
941 ip => {
942 type => 'string',
943 format => 'pve-ipv4-config',
944 format_description => 'IPv4Format/CIDR',
945 description => 'IPv4 address in CIDR format.',
946 optional => 1,
947 default => 'dhcp',
948 },
949 gw => {
950 type => 'string',
951 format => 'ipv4',
952 format_description => 'GatewayIPv4',
953 description => 'Default gateway for IPv4 traffic.',
954 optional => 1,
955 requires => 'ip',
956 },
957 ip6 => {
958 type => 'string',
959 format => 'pve-ipv6-config',
960 format_description => 'IPv6Format/CIDR',
961 description => 'IPv6 address in CIDR format.',
962 optional => 1,
963 default => 'dhcp',
964 },
965 gw6 => {
966 type => 'string',
967 format => 'ipv6',
968 format_description => 'GatewayIPv6',
969 description => 'Default gateway for IPv6 traffic.',
970 optional => 1,
971 requires => 'ip6',
972 },
973};
974PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
975my $ipconfigdesc = {
976 optional => 1,
977 type => 'string', format => 'pve-qm-ipconfig',
978 description => <<'EODESCR',
979cloud-init: Specify IP addresses and gateways for the corresponding interface.
980
981IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
982
983The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
984gateway should be provided.
985For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
986cloud-init 19.4 or newer.
987
988If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
989dhcp on IPv4.
990EODESCR
991};
992PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
993
994for (my $i = 0; $i < $MAX_NETS; $i++) {
995 $confdesc->{"net$i"} = $netdesc;
996 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
997}
998
999foreach my $key (keys %$confdesc_cloudinit) {
1000 $confdesc->{$key} = $confdesc_cloudinit->{$key};
1001}
1002
1003PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1004sub verify_volume_id_or_qm_path {
1005 my ($volid, $noerr) = @_;
1006
1007 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1008 return $volid;
1009 }
1010
1011 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1012 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1013 if ($@) {
1014 return if $noerr;
1015 die $@;
1016 }
1017 return $volid;
1018}
1019
1020my $usb_fmt = {
1021 host => {
1022 default_key => 1,
1023 type => 'string', format => 'pve-qm-usb-device',
1024 format_description => 'HOSTUSBDEVICE|spice',
1025 description => <<EODESCR,
1026The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1027
1028 'bus-port(.port)*' (decimal numbers) or
1029 'vendor_id:product_id' (hexadeciaml numbers) or
1030 'spice'
1031
1032You can use the 'lsusb -t' command to list existing usb devices.
1033
1034NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1035machines - use with special care.
1036
1037The value 'spice' can be used to add a usb redirection devices for spice.
1038EODESCR
1039 },
1040 usb3 => {
1041 optional => 1,
1042 type => 'boolean',
1043 description => "Specifies whether if given host option is a USB3 device or port.",
1044 default => 0,
1045 },
1046};
1047
1048my $usbdesc = {
1049 optional => 1,
1050 type => 'string', format => $usb_fmt,
1051 description => "Configure an USB device (n is 0 to 4).",
1052};
1053PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1054
1055my $serialdesc = {
1056 optional => 1,
1057 type => 'string',
1058 pattern => '(/dev/.+|socket)',
1059 description => "Create a serial device inside the VM (n is 0 to 3)",
1060 verbose_description => <<EODESCR,
1061Create a serial device inside the VM (n is 0 to 3), and pass through a
1062host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1063host side (use 'qm terminal' to open a terminal connection).
1064
1065NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1066use with special care.
1067
1068CAUTION: Experimental! User reported problems with this option.
1069EODESCR
1070};
1071
1072my $paralleldesc= {
1073 optional => 1,
1074 type => 'string',
1075 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1076 description => "Map host parallel devices (n is 0 to 2).",
1077 verbose_description => <<EODESCR,
1078Map host parallel devices (n is 0 to 2).
1079
1080NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1081machines - use with special care.
1082
1083CAUTION: Experimental! User reported problems with this option.
1084EODESCR
1085};
1086
1087for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1088 $confdesc->{"parallel$i"} = $paralleldesc;
1089}
1090
1091for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1092 $confdesc->{"serial$i"} = $serialdesc;
1093}
1094
1095for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
1096 $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1097}
1098
1099for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1100 $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1101}
1102
1103for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1104 $confdesc->{"usb$i"} = $usbdesc;
1105}
1106
1107my $boot_fmt = {
1108 legacy => {
1109 optional => 1,
1110 default_key => 1,
1111 type => 'string',
1112 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1113 . " Deprecated, use 'order=' instead.",
1114 pattern => '[acdn]{1,4}',
1115 format_description => "[acdn]{1,4}",
1116
1117 # note: this is also the fallback if boot: is not given at all
1118 default => 'cdn',
1119 },
1120 order => {
1121 optional => 1,
1122 type => 'string',
1123 format => 'pve-qm-bootdev-list',
1124 format_description => "device[;device...]",
1125 description => <<EODESC,
1126The guest will attempt to boot from devices in the order they appear here.
1127
1128Disks, optical drives and passed-through storage USB devices will be directly
1129booted from, NICs will load PXE, and PCIe devices will either behave like disks
1130(e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1131
1132Note that only devices in this list will be marked as bootable and thus loaded
1133by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1134(e.g. software-raid), you need to specify all of them here.
1135
1136Overrides the deprecated 'legacy=[acdn]*' value when given.
1137EODESC
1138 },
1139};
1140PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1141
1142PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1143sub verify_bootdev {
1144 my ($dev, $noerr) = @_;
1145
1146 return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1147
1148 my $check = sub {
1149 my ($base) = @_;
1150 return 0 if $dev !~ m/^$base\d+$/;
1151 return 0 if !$confdesc->{$dev};
1152 return 1;
1153 };
1154
1155 return $dev if $check->("net");
1156 return $dev if $check->("usb");
1157 return $dev if $check->("hostpci");
1158
1159 return if $noerr;
1160 die "invalid boot device '$dev'\n";
1161}
1162
1163sub print_bootorder {
1164 my ($devs) = @_;
1165 return "" if !@$devs;
1166 my $data = { order => join(';', @$devs) };
1167 return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1168}
1169
1170my $kvm_api_version = 0;
1171
1172sub kvm_version {
1173 return $kvm_api_version if $kvm_api_version;
1174
1175 open my $fh, '<', '/dev/kvm' or return;
1176
1177 # 0xae00 => KVM_GET_API_VERSION
1178 $kvm_api_version = ioctl($fh, 0xae00, 0);
1179 close($fh);
1180
1181 return $kvm_api_version;
1182}
1183
1184my $kvm_user_version = {};
1185my $kvm_mtime = {};
1186
1187sub kvm_user_version {
1188 my ($binary) = @_;
1189
1190 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1191 my $st = stat($binary);
1192
1193 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1194 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1195 $cachedmtime == $st->mtime;
1196
1197 $kvm_user_version->{$binary} = 'unknown';
1198 $kvm_mtime->{$binary} = $st->mtime;
1199
1200 my $code = sub {
1201 my $line = shift;
1202 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1203 $kvm_user_version->{$binary} = $2;
1204 }
1205 };
1206
1207 eval { run_command([$binary, '--version'], outfunc => $code); };
1208 warn $@ if $@;
1209
1210 return $kvm_user_version->{$binary};
1211
1212}
1213my sub extract_version {
1214 my ($machine_type, $version) = @_;
1215 $version = kvm_user_version() if !defined($version);
1216 PVE::QemuServer::Machine::extract_version($machine_type, $version)
1217}
1218
1219sub kernel_has_vhost_net {
1220 return -c '/dev/vhost-net';
1221}
1222
1223sub option_exists {
1224 my $key = shift;
1225 return defined($confdesc->{$key});
1226}
1227
1228my $cdrom_path;
1229sub get_cdrom_path {
1230
1231 return $cdrom_path if $cdrom_path;
1232
1233 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1234 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1235 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1236}
1237
1238sub get_iso_path {
1239 my ($storecfg, $vmid, $cdrom) = @_;
1240
1241 if ($cdrom eq 'cdrom') {
1242 return get_cdrom_path();
1243 } elsif ($cdrom eq 'none') {
1244 return '';
1245 } elsif ($cdrom =~ m|^/|) {
1246 return $cdrom;
1247 } else {
1248 return PVE::Storage::path($storecfg, $cdrom);
1249 }
1250}
1251
1252# try to convert old style file names to volume IDs
1253sub filename_to_volume_id {
1254 my ($vmid, $file, $media) = @_;
1255
1256 if (!($file eq 'none' || $file eq 'cdrom' ||
1257 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1258
1259 return if $file =~ m|/|;
1260
1261 if ($media && $media eq 'cdrom') {
1262 $file = "local:iso/$file";
1263 } else {
1264 $file = "local:$vmid/$file";
1265 }
1266 }
1267
1268 return $file;
1269}
1270
1271sub verify_media_type {
1272 my ($opt, $vtype, $media) = @_;
1273
1274 return if !$media;
1275
1276 my $etype;
1277 if ($media eq 'disk') {
1278 $etype = 'images';
1279 } elsif ($media eq 'cdrom') {
1280 $etype = 'iso';
1281 } else {
1282 die "internal error";
1283 }
1284
1285 return if ($vtype eq $etype);
1286
1287 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1288}
1289
1290sub cleanup_drive_path {
1291 my ($opt, $storecfg, $drive) = @_;
1292
1293 # try to convert filesystem paths to volume IDs
1294
1295 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1296 ($drive->{file} !~ m|^/dev/.+|) &&
1297 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1298 ($drive->{file} !~ m/^\d+$/)) {
1299 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1300 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1301 if !$vtype;
1302 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1303 verify_media_type($opt, $vtype, $drive->{media});
1304 $drive->{file} = $volid;
1305 }
1306
1307 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1308}
1309
1310sub parse_hotplug_features {
1311 my ($data) = @_;
1312
1313 my $res = {};
1314
1315 return $res if $data eq '0';
1316
1317 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1318
1319 foreach my $feature (PVE::Tools::split_list($data)) {
1320 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1321 $res->{$1} = 1;
1322 } else {
1323 die "invalid hotplug feature '$feature'\n";
1324 }
1325 }
1326 return $res;
1327}
1328
1329PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1330sub pve_verify_hotplug_features {
1331 my ($value, $noerr) = @_;
1332
1333 return $value if parse_hotplug_features($value);
1334
1335 return if $noerr;
1336
1337 die "unable to parse hotplug option\n";
1338}
1339
1340sub scsi_inquiry {
1341 my($fh, $noerr) = @_;
1342
1343 my $SG_IO = 0x2285;
1344 my $SG_GET_VERSION_NUM = 0x2282;
1345
1346 my $versionbuf = "\x00" x 8;
1347 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1348 if (!$ret) {
1349 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1350 return;
1351 }
1352 my $version = unpack("I", $versionbuf);
1353 if ($version < 30000) {
1354 die "scsi generic interface too old\n" if !$noerr;
1355 return;
1356 }
1357
1358 my $buf = "\x00" x 36;
1359 my $sensebuf = "\x00" x 8;
1360 my $cmd = pack("C x3 C x1", 0x12, 36);
1361
1362 # see /usr/include/scsi/sg.h
1363 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1364
1365 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1366 length($sensebuf), 0, length($buf), $buf,
1367 $cmd, $sensebuf, 6000);
1368
1369 $ret = ioctl($fh, $SG_IO, $packet);
1370 if (!$ret) {
1371 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1372 return;
1373 }
1374
1375 my @res = unpack($sg_io_hdr_t, $packet);
1376 if ($res[17] || $res[18]) {
1377 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1378 return;
1379 }
1380
1381 my $res = {};
1382 (my $byte0, my $byte1, $res->{vendor},
1383 $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1384
1385 $res->{removable} = $byte1 & 128 ? 1 : 0;
1386 $res->{type} = $byte0 & 31;
1387
1388 return $res;
1389}
1390
1391sub path_is_scsi {
1392 my ($path) = @_;
1393
1394 my $fh = IO::File->new("+<$path") || return;
1395 my $res = scsi_inquiry($fh, 1);
1396 close($fh);
1397
1398 return $res;
1399}
1400
1401sub print_tabletdevice_full {
1402 my ($conf, $arch) = @_;
1403
1404 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1405
1406 # we use uhci for old VMs because tablet driver was buggy in older qemu
1407 my $usbbus;
1408 if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1409 $usbbus = 'ehci';
1410 } else {
1411 $usbbus = 'uhci';
1412 }
1413
1414 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1415}
1416
1417sub print_keyboarddevice_full {
1418 my ($conf, $arch, $machine) = @_;
1419
1420 return if $arch ne 'aarch64';
1421
1422 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1423}
1424
1425my sub get_drive_id {
1426 my ($drive) = @_;
1427 return "$drive->{interface}$drive->{index}";
1428}
1429
1430sub print_drivedevice_full {
1431 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1432
1433 my $device = '';
1434 my $maxdev = 0;
1435
1436 my $drive_id = get_drive_id($drive);
1437 if ($drive->{interface} eq 'virtio') {
1438 my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1439 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1440 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1441 } elsif ($drive->{interface} eq 'scsi') {
1442
1443 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1444 my $unit = $drive->{index} % $maxdev;
1445 my $devicetype = 'hd';
1446 my $path = '';
1447 if (drive_is_cdrom($drive)) {
1448 $devicetype = 'cd';
1449 } else {
1450 if ($drive->{file} =~ m|^/|) {
1451 $path = $drive->{file};
1452 if (my $info = path_is_scsi($path)) {
1453 if ($info->{type} == 0 && $drive->{scsiblock}) {
1454 $devicetype = 'block';
1455 } elsif ($info->{type} == 1) { # tape
1456 $devicetype = 'generic';
1457 }
1458 }
1459 } else {
1460 $path = PVE::Storage::path($storecfg, $drive->{file});
1461 }
1462
1463 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1464 my $version = extract_version($machine_type, kvm_user_version());
1465 if ($path =~ m/^iscsi\:\/\// &&
1466 !min_version($version, 4, 1)) {
1467 $devicetype = 'generic';
1468 }
1469 }
1470
1471 if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1472 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1473 } else {
1474 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1475 .",lun=$drive->{index}";
1476 }
1477 $device .= ",drive=drive-$drive_id,id=$drive_id";
1478
1479 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1480 $device .= ",rotation_rate=1";
1481 }
1482 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1483
1484 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1485 my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1486 my $controller = int($drive->{index} / $maxdev);
1487 my $unit = $drive->{index} % $maxdev;
1488 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1489
1490 $device = "ide-$devicetype";
1491 if ($drive->{interface} eq 'ide') {
1492 $device .= ",bus=ide.$controller,unit=$unit";
1493 } else {
1494 $device .= ",bus=ahci$controller.$unit";
1495 }
1496 $device .= ",drive=drive-$drive_id,id=$drive_id";
1497
1498 if ($devicetype eq 'hd') {
1499 if (my $model = $drive->{model}) {
1500 $model = URI::Escape::uri_unescape($model);
1501 $device .= ",model=$model";
1502 }
1503 if ($drive->{ssd}) {
1504 $device .= ",rotation_rate=1";
1505 }
1506 }
1507 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1508 } elsif ($drive->{interface} eq 'usb') {
1509 die "implement me";
1510 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1511 } else {
1512 die "unsupported interface type";
1513 }
1514
1515 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1516
1517 if (my $serial = $drive->{serial}) {
1518 $serial = URI::Escape::uri_unescape($serial);
1519 $device .= ",serial=$serial";
1520 }
1521
1522
1523 return $device;
1524}
1525
1526sub get_initiator_name {
1527 my $initiator;
1528
1529 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1530 while (defined(my $line = <$fh>)) {
1531 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1532 $initiator = $1;
1533 last;
1534 }
1535 $fh->close();
1536
1537 return $initiator;
1538}
1539
1540sub print_drive_commandline_full {
1541 my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1542
1543 my $path;
1544 my $volid = $drive->{file};
1545 my $format = $drive->{format};
1546 my $drive_id = get_drive_id($drive);
1547
1548 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1549 my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1550
1551 if (drive_is_cdrom($drive)) {
1552 $path = get_iso_path($storecfg, $vmid, $volid);
1553 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1554 } else {
1555 if ($storeid) {
1556 $path = PVE::Storage::path($storecfg, $volid);
1557 $format //= qemu_img_format($scfg, $volname);
1558 } else {
1559 $path = $volid;
1560 $format //= "raw";
1561 }
1562 }
1563
1564 my $is_rbd = $path =~ m/^rbd:/;
1565
1566 my $opts = '';
1567 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1568 foreach my $o (@qemu_drive_options) {
1569 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1570 }
1571
1572 # snapshot only accepts on|off
1573 if (defined($drive->{snapshot})) {
1574 my $v = $drive->{snapshot} ? 'on' : 'off';
1575 $opts .= ",snapshot=$v";
1576 }
1577
1578 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1579 my ($dir, $qmpname) = @$type;
1580 if (my $v = $drive->{"mbps$dir"}) {
1581 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1582 }
1583 if (my $v = $drive->{"mbps${dir}_max"}) {
1584 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1585 }
1586 if (my $v = $drive->{"bps${dir}_max_length"}) {
1587 $opts .= ",throttling.bps$qmpname-max-length=$v";
1588 }
1589 if (my $v = $drive->{"iops${dir}"}) {
1590 $opts .= ",throttling.iops$qmpname=$v";
1591 }
1592 if (my $v = $drive->{"iops${dir}_max"}) {
1593 $opts .= ",throttling.iops$qmpname-max=$v";
1594 }
1595 if (my $v = $drive->{"iops${dir}_max_length"}) {
1596 $opts .= ",throttling.iops$qmpname-max-length=$v";
1597 }
1598 }
1599
1600 if ($pbs_name) {
1601 $format = "rbd" if $is_rbd;
1602 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1603 if !$format;
1604 $opts .= ",format=alloc-track,file.driver=$format";
1605 } elsif ($format) {
1606 $opts .= ",format=$format";
1607 }
1608
1609 my $cache_direct = 0;
1610
1611 if (my $cache = $drive->{cache}) {
1612 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1613 } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1614 $opts .= ",cache=none";
1615 $cache_direct = 1;
1616 }
1617
1618 # io_uring with cache mode writeback or writethrough on krbd will hang...
1619 my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1620
1621 # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1622 # sometimes, just plain disable...
1623 my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1624
1625 if (!$drive->{aio}) {
1626 if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1627 # io_uring supports all cache modes
1628 $opts .= ",aio=io_uring";
1629 } else {
1630 # aio native works only with O_DIRECT
1631 if($cache_direct) {
1632 $opts .= ",aio=native";
1633 } else {
1634 $opts .= ",aio=threads";
1635 }
1636 }
1637 }
1638
1639 if (!drive_is_cdrom($drive)) {
1640 my $detectzeroes;
1641 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1642 $detectzeroes = 'off';
1643 } elsif ($drive->{discard}) {
1644 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1645 } else {
1646 # This used to be our default with discard not being specified:
1647 $detectzeroes = 'on';
1648 }
1649
1650 # note: 'detect-zeroes' works per blockdev and we want it to persist
1651 # after the alloc-track is removed, so put it on 'file' directly
1652 my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1653 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1654 }
1655
1656 if ($pbs_name) {
1657 $opts .= ",backing=$pbs_name";
1658 $opts .= ",auto-remove=on";
1659 }
1660
1661 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1662 my $file_param = "file";
1663 if ($pbs_name) {
1664 # non-rbd drivers require the underlying file to be a seperate block
1665 # node, so add a second .file indirection
1666 $file_param .= ".file" if !$is_rbd;
1667 $file_param .= ".filename";
1668 }
1669 my $pathinfo = $path ? "$file_param=$path," : '';
1670
1671 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1672}
1673
1674sub print_pbs_blockdev {
1675 my ($pbs_conf, $pbs_name) = @_;
1676 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1677 $blockdev .= ",repository=$pbs_conf->{repository}";
1678 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1679 $blockdev .= ",archive=$pbs_conf->{archive}";
1680 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1681 return $blockdev;
1682}
1683
1684sub print_netdevice_full {
1685 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1686
1687 my $device = $net->{model};
1688 if ($net->{model} eq 'virtio') {
1689 $device = 'virtio-net-pci';
1690 };
1691
1692 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1693 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1694 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1695 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1696 # and out of each queue plus one config interrupt and control vector queue
1697 my $vectors = $net->{queues} * 2 + 2;
1698 $tmpstr .= ",vectors=$vectors,mq=on";
1699 }
1700 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1701
1702 if (my $mtu = $net->{mtu}) {
1703 if ($net->{model} eq 'virtio' && $net->{bridge}) {
1704 my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1705 if ($mtu == 1) {
1706 $mtu = $bridge_mtu;
1707 } elsif ($mtu < 576) {
1708 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1709 } elsif ($mtu > $bridge_mtu) {
1710 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1711 }
1712 $tmpstr .= ",host_mtu=$mtu";
1713 } else {
1714 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1715 }
1716 }
1717
1718 if ($use_old_bios_files) {
1719 my $romfile;
1720 if ($device eq 'virtio-net-pci') {
1721 $romfile = 'pxe-virtio.rom';
1722 } elsif ($device eq 'e1000') {
1723 $romfile = 'pxe-e1000.rom';
1724 } elsif ($device eq 'e1000e') {
1725 $romfile = 'pxe-e1000e.rom';
1726 } elsif ($device eq 'ne2k') {
1727 $romfile = 'pxe-ne2k_pci.rom';
1728 } elsif ($device eq 'pcnet') {
1729 $romfile = 'pxe-pcnet.rom';
1730 } elsif ($device eq 'rtl8139') {
1731 $romfile = 'pxe-rtl8139.rom';
1732 }
1733 $tmpstr .= ",romfile=$romfile" if $romfile;
1734 }
1735
1736 return $tmpstr;
1737}
1738
1739sub print_netdev_full {
1740 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1741
1742 my $i = '';
1743 if ($netid =~ m/^net(\d+)$/) {
1744 $i = int($1);
1745 }
1746
1747 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1748
1749 my $ifname = "tap${vmid}i$i";
1750
1751 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1752 die "interface name '$ifname' is too long (max 15 character)\n"
1753 if length($ifname) >= 16;
1754
1755 my $vhostparam = '';
1756 if (is_native($arch)) {
1757 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1758 }
1759
1760 my $vmname = $conf->{name} || "vm$vmid";
1761
1762 my $netdev = "";
1763 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1764
1765 if ($net->{bridge}) {
1766 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1767 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1768 } else {
1769 $netdev = "type=user,id=$netid,hostname=$vmname";
1770 }
1771
1772 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1773
1774 return $netdev;
1775}
1776
1777my $vga_map = {
1778 'cirrus' => 'cirrus-vga',
1779 'std' => 'VGA',
1780 'vmware' => 'vmware-svga',
1781 'virtio' => 'virtio-vga',
1782};
1783
1784sub print_vga_device {
1785 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1786
1787 my $type = $vga_map->{$vga->{type}};
1788 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1789 $type = 'virtio-gpu';
1790 }
1791 my $vgamem_mb = $vga->{memory};
1792
1793 my $max_outputs = '';
1794 if ($qxlnum) {
1795 $type = $id ? 'qxl' : 'qxl-vga';
1796
1797 if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1798 # set max outputs so linux can have up to 4 qxl displays with one device
1799 if (min_version($machine_version, 4, 1)) {
1800 $max_outputs = ",max_outputs=4";
1801 }
1802 }
1803 }
1804
1805 die "no devicetype for $vga->{type}\n" if !$type;
1806
1807 my $memory = "";
1808 if ($vgamem_mb) {
1809 if ($vga->{type} eq 'virtio') {
1810 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1811 $memory = ",max_hostmem=$bytes";
1812 } elsif ($qxlnum) {
1813 # from https://www.spice-space.org/multiple-monitors.html
1814 $memory = ",vgamem_mb=$vga->{memory}";
1815 my $ram = $vgamem_mb * 4;
1816 my $vram = $vgamem_mb * 2;
1817 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1818 } else {
1819 $memory = ",vgamem_mb=$vga->{memory}";
1820 }
1821 } elsif ($qxlnum && $id) {
1822 $memory = ",ram_size=67108864,vram_size=33554432";
1823 }
1824
1825 my $edidoff = "";
1826 if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1827 $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1828 }
1829
1830 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1831 my $vgaid = "vga" . ($id // '');
1832 my $pciaddr;
1833
1834 if ($q35 && $vgaid eq 'vga') {
1835 # the first display uses pcie.0 bus on q35 machines
1836 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1837 } else {
1838 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1839 }
1840
1841 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1842}
1843
1844sub parse_number_sets {
1845 my ($set) = @_;
1846 my $res = [];
1847 foreach my $part (split(/;/, $set)) {
1848 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1849 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1850 push @$res, [ $1, $2 ];
1851 } else {
1852 die "invalid range: $part\n";
1853 }
1854 }
1855 return $res;
1856}
1857
1858sub parse_numa {
1859 my ($data) = @_;
1860
1861 my $res = parse_property_string($numa_fmt, $data);
1862 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1863 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1864 return $res;
1865}
1866
1867# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1868sub parse_net {
1869 my ($data) = @_;
1870
1871 my $res = eval { parse_property_string($net_fmt, $data) };
1872 if ($@) {
1873 warn $@;
1874 return;
1875 }
1876 if (!defined($res->{macaddr})) {
1877 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1878 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1879 }
1880 return $res;
1881}
1882
1883# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1884sub parse_ipconfig {
1885 my ($data) = @_;
1886
1887 my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1888 if ($@) {
1889 warn $@;
1890 return;
1891 }
1892
1893 if ($res->{gw} && !$res->{ip}) {
1894 warn 'gateway specified without specifying an IP address';
1895 return;
1896 }
1897 if ($res->{gw6} && !$res->{ip6}) {
1898 warn 'IPv6 gateway specified without specifying an IPv6 address';
1899 return;
1900 }
1901 if ($res->{gw} && $res->{ip} eq 'dhcp') {
1902 warn 'gateway specified together with DHCP';
1903 return;
1904 }
1905 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1906 # gw6 + auto/dhcp
1907 warn "IPv6 gateway specified together with $res->{ip6} address";
1908 return;
1909 }
1910
1911 if (!$res->{ip} && !$res->{ip6}) {
1912 return { ip => 'dhcp', ip6 => 'dhcp' };
1913 }
1914
1915 return $res;
1916}
1917
1918sub print_net {
1919 my $net = shift;
1920
1921 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1922}
1923
1924sub add_random_macs {
1925 my ($settings) = @_;
1926
1927 foreach my $opt (keys %$settings) {
1928 next if $opt !~ m/^net(\d+)$/;
1929 my $net = parse_net($settings->{$opt});
1930 next if !$net;
1931 $settings->{$opt} = print_net($net);
1932 }
1933}
1934
1935sub vm_is_volid_owner {
1936 my ($storecfg, $vmid, $volid) = @_;
1937
1938 if ($volid !~ m|^/|) {
1939 my ($path, $owner);
1940 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1941 if ($owner && ($owner == $vmid)) {
1942 return 1;
1943 }
1944 }
1945
1946 return;
1947}
1948
1949sub vmconfig_register_unused_drive {
1950 my ($storecfg, $vmid, $conf, $drive) = @_;
1951
1952 if (drive_is_cloudinit($drive)) {
1953 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1954 warn $@ if $@;
1955 } elsif (!drive_is_cdrom($drive)) {
1956 my $volid = $drive->{file};
1957 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1958 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1959 }
1960 }
1961}
1962
1963# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1964my $smbios1_fmt = {
1965 uuid => {
1966 type => 'string',
1967 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1968 format_description => 'UUID',
1969 description => "Set SMBIOS1 UUID.",
1970 optional => 1,
1971 },
1972 version => {
1973 type => 'string',
1974 pattern => '[A-Za-z0-9+\/]+={0,2}',
1975 format_description => 'Base64 encoded string',
1976 description => "Set SMBIOS1 version.",
1977 optional => 1,
1978 },
1979 serial => {
1980 type => 'string',
1981 pattern => '[A-Za-z0-9+\/]+={0,2}',
1982 format_description => 'Base64 encoded string',
1983 description => "Set SMBIOS1 serial number.",
1984 optional => 1,
1985 },
1986 manufacturer => {
1987 type => 'string',
1988 pattern => '[A-Za-z0-9+\/]+={0,2}',
1989 format_description => 'Base64 encoded string',
1990 description => "Set SMBIOS1 manufacturer.",
1991 optional => 1,
1992 },
1993 product => {
1994 type => 'string',
1995 pattern => '[A-Za-z0-9+\/]+={0,2}',
1996 format_description => 'Base64 encoded string',
1997 description => "Set SMBIOS1 product ID.",
1998 optional => 1,
1999 },
2000 sku => {
2001 type => 'string',
2002 pattern => '[A-Za-z0-9+\/]+={0,2}',
2003 format_description => 'Base64 encoded string',
2004 description => "Set SMBIOS1 SKU string.",
2005 optional => 1,
2006 },
2007 family => {
2008 type => 'string',
2009 pattern => '[A-Za-z0-9+\/]+={0,2}',
2010 format_description => 'Base64 encoded string',
2011 description => "Set SMBIOS1 family string.",
2012 optional => 1,
2013 },
2014 base64 => {
2015 type => 'boolean',
2016 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2017 optional => 1,
2018 },
2019};
2020
2021sub parse_smbios1 {
2022 my ($data) = @_;
2023
2024 my $res = eval { parse_property_string($smbios1_fmt, $data) };
2025 warn $@ if $@;
2026 return $res;
2027}
2028
2029sub print_smbios1 {
2030 my ($smbios1) = @_;
2031 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2032}
2033
2034PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2035
2036sub parse_watchdog {
2037 my ($value) = @_;
2038
2039 return if !$value;
2040
2041 my $res = eval { parse_property_string($watchdog_fmt, $value) };
2042 warn $@ if $@;
2043 return $res;
2044}
2045
2046sub parse_guest_agent {
2047 my ($conf) = @_;
2048
2049 return {} if !defined($conf->{agent});
2050
2051 my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2052 warn $@ if $@;
2053
2054 # if the agent is disabled ignore the other potentially set properties
2055 return {} if !$res->{enabled};
2056 return $res;
2057}
2058
2059sub get_qga_key {
2060 my ($conf, $key) = @_;
2061 return undef if !defined($conf->{agent});
2062
2063 my $agent = parse_guest_agent($conf);
2064 return $agent->{$key};
2065}
2066
2067sub parse_vga {
2068 my ($value) = @_;
2069
2070 return {} if !$value;
2071 my $res = eval { parse_property_string($vga_fmt, $value) };
2072 warn $@ if $@;
2073 return $res;
2074}
2075
2076sub parse_rng {
2077 my ($value) = @_;
2078
2079 return if !$value;
2080
2081 my $res = eval { parse_property_string($rng_fmt, $value) };
2082 warn $@ if $@;
2083 return $res;
2084}
2085
2086PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2087sub verify_usb_device {
2088 my ($value, $noerr) = @_;
2089
2090 return $value if parse_usb_device($value);
2091
2092 return if $noerr;
2093
2094 die "unable to parse usb device\n";
2095}
2096
2097# add JSON properties for create and set function
2098sub json_config_properties {
2099 my $prop = shift;
2100
2101 foreach my $opt (keys %$confdesc) {
2102 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2103 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2104 $prop->{$opt} = $confdesc->{$opt};
2105 }
2106
2107 return $prop;
2108}
2109
2110# return copy of $confdesc_cloudinit to generate documentation
2111sub cloudinit_config_properties {
2112
2113 return dclone($confdesc_cloudinit);
2114}
2115
2116sub check_type {
2117 my ($key, $value) = @_;
2118
2119 die "unknown setting '$key'\n" if !$confdesc->{$key};
2120
2121 my $type = $confdesc->{$key}->{type};
2122
2123 if (!defined($value)) {
2124 die "got undefined value\n";
2125 }
2126
2127 if ($value =~ m/[\n\r]/) {
2128 die "property contains a line feed\n";
2129 }
2130
2131 if ($type eq 'boolean') {
2132 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2133 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2134 die "type check ('boolean') failed - got '$value'\n";
2135 } elsif ($type eq 'integer') {
2136 return int($1) if $value =~ m/^(\d+)$/;
2137 die "type check ('integer') failed - got '$value'\n";
2138 } elsif ($type eq 'number') {
2139 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2140 die "type check ('number') failed - got '$value'\n";
2141 } elsif ($type eq 'string') {
2142 if (my $fmt = $confdesc->{$key}->{format}) {
2143 PVE::JSONSchema::check_format($fmt, $value);
2144 return $value;
2145 }
2146 $value =~ s/^\"(.*)\"$/$1/;
2147 return $value;
2148 } else {
2149 die "internal error"
2150 }
2151}
2152
2153sub destroy_vm {
2154 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2155
2156 my $conf = PVE::QemuConfig->load_config($vmid);
2157
2158 PVE::QemuConfig->check_lock($conf) if !$skiplock;
2159
2160 if ($conf->{template}) {
2161 # check if any base image is still used by a linked clone
2162 PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2163 my ($ds, $drive) = @_;
2164 return if drive_is_cdrom($drive);
2165
2166 my $volid = $drive->{file};
2167 return if !$volid || $volid =~ m|^/|;
2168
2169 die "base volume '$volid' is still in use by linked cloned\n"
2170 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2171
2172 });
2173 }
2174
2175 my $remove_owned_drive = sub {
2176 my ($ds, $drive) = @_;
2177 return if drive_is_cdrom($drive, 1);
2178
2179 my $volid = $drive->{file};
2180 return if !$volid || $volid =~ m|^/|;
2181
2182 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2183 return if !$path || !$owner || ($owner != $vmid);
2184
2185 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2186 warn "Could not remove disk '$volid', check manually: $@" if $@;
2187 };
2188
2189 # only remove disks owned by this VM (referenced in the config)
2190 my $include_opts = {
2191 include_unused => 1,
2192 extra_keys => ['vmstate'],
2193 };
2194 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2195
2196 for my $snap (values %{$conf->{snapshots}}) {
2197 next if !defined($snap->{vmstate});
2198 my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2199 next if !defined($drive);
2200 $remove_owned_drive->('vmstate', $drive);
2201 }
2202
2203 if ($purge_unreferenced) { # also remove unreferenced disk
2204 my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2205 PVE::Storage::foreach_volid($vmdisks, sub {
2206 my ($volid, $sid, $volname, $d) = @_;
2207 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2208 warn $@ if $@;
2209 });
2210 }
2211
2212 if (defined $replacement_conf) {
2213 PVE::QemuConfig->write_config($vmid, $replacement_conf);
2214 } else {
2215 PVE::QemuConfig->destroy_config($vmid);
2216 }
2217}
2218
2219sub parse_vm_config {
2220 my ($filename, $raw) = @_;
2221
2222 return if !defined($raw);
2223
2224 my $res = {
2225 digest => Digest::SHA::sha1_hex($raw),
2226 snapshots => {},
2227 pending => {},
2228 };
2229
2230 $filename =~ m|/qemu-server/(\d+)\.conf$|
2231 || die "got strange filename '$filename'";
2232
2233 my $vmid = $1;
2234
2235 my $conf = $res;
2236 my $descr;
2237 my $section = '';
2238
2239 my @lines = split(/\n/, $raw);
2240 foreach my $line (@lines) {
2241 next if $line =~ m/^\s*$/;
2242
2243 if ($line =~ m/^\[PENDING\]\s*$/i) {
2244 $section = 'pending';
2245 if (defined($descr)) {
2246 $descr =~ s/\s+$//;
2247 $conf->{description} = $descr;
2248 }
2249 $descr = undef;
2250 $conf = $res->{$section} = {};
2251 next;
2252
2253 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2254 $section = $1;
2255 if (defined($descr)) {
2256 $descr =~ s/\s+$//;
2257 $conf->{description} = $descr;
2258 }
2259 $descr = undef;
2260 $conf = $res->{snapshots}->{$section} = {};
2261 next;
2262 }
2263
2264 if ($line =~ m/^\#(.*)\s*$/) {
2265 $descr = '' if !defined($descr);
2266 $descr .= PVE::Tools::decode_text($1) . "\n";
2267 next;
2268 }
2269
2270 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2271 $descr = '' if !defined($descr);
2272 $descr .= PVE::Tools::decode_text($2);
2273 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2274 $conf->{snapstate} = $1;
2275 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2276 my $key = $1;
2277 my $value = $2;
2278 $conf->{$key} = $value;
2279 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2280 my $value = $1;
2281 if ($section eq 'pending') {
2282 $conf->{delete} = $value; # we parse this later
2283 } else {
2284 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2285 }
2286 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2287 my $key = $1;
2288 my $value = $2;
2289 eval { $value = check_type($key, $value); };
2290 if ($@) {
2291 warn "vm $vmid - unable to parse value of '$key' - $@";
2292 } else {
2293 $key = 'ide2' if $key eq 'cdrom';
2294 my $fmt = $confdesc->{$key}->{format};
2295 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2296 my $v = parse_drive($key, $value);
2297 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2298 $v->{file} = $volid;
2299 $value = print_drive($v);
2300 } else {
2301 warn "vm $vmid - unable to parse value of '$key'\n";
2302 next;
2303 }
2304 }
2305
2306 $conf->{$key} = $value;
2307 }
2308 } else {
2309 warn "vm $vmid - unable to parse config: $line\n";
2310 }
2311 }
2312
2313 if (defined($descr)) {
2314 $descr =~ s/\s+$//;
2315 $conf->{description} = $descr;
2316 }
2317 delete $res->{snapstate}; # just to be sure
2318
2319 return $res;
2320}
2321
2322sub write_vm_config {
2323 my ($filename, $conf) = @_;
2324
2325 delete $conf->{snapstate}; # just to be sure
2326
2327 if ($conf->{cdrom}) {
2328 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2329 $conf->{ide2} = $conf->{cdrom};
2330 delete $conf->{cdrom};
2331 }
2332
2333 # we do not use 'smp' any longer
2334 if ($conf->{sockets}) {
2335 delete $conf->{smp};
2336 } elsif ($conf->{smp}) {
2337 $conf->{sockets} = $conf->{smp};
2338 delete $conf->{cores};
2339 delete $conf->{smp};
2340 }
2341
2342 my $used_volids = {};
2343
2344 my $cleanup_config = sub {
2345 my ($cref, $pending, $snapname) = @_;
2346
2347 foreach my $key (keys %$cref) {
2348 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2349 $key eq 'snapstate' || $key eq 'pending';
2350 my $value = $cref->{$key};
2351 if ($key eq 'delete') {
2352 die "propertry 'delete' is only allowed in [PENDING]\n"
2353 if !$pending;
2354 # fixme: check syntax?
2355 next;
2356 }
2357 eval { $value = check_type($key, $value); };
2358 die "unable to parse value of '$key' - $@" if $@;
2359
2360 $cref->{$key} = $value;
2361
2362 if (!$snapname && is_valid_drivename($key)) {
2363 my $drive = parse_drive($key, $value);
2364 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2365 }
2366 }
2367 };
2368
2369 &$cleanup_config($conf);
2370
2371 &$cleanup_config($conf->{pending}, 1);
2372
2373 foreach my $snapname (keys %{$conf->{snapshots}}) {
2374 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2375 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2376 }
2377
2378 # remove 'unusedX' settings if we re-add a volume
2379 foreach my $key (keys %$conf) {
2380 my $value = $conf->{$key};
2381 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2382 delete $conf->{$key};
2383 }
2384 }
2385
2386 my $generate_raw_config = sub {
2387 my ($conf, $pending) = @_;
2388
2389 my $raw = '';
2390
2391 # add description as comment to top of file
2392 if (defined(my $descr = $conf->{description})) {
2393 if ($descr) {
2394 foreach my $cl (split(/\n/, $descr)) {
2395 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2396 }
2397 } else {
2398 $raw .= "#\n" if $pending;
2399 }
2400 }
2401
2402 foreach my $key (sort keys %$conf) {
2403 next if $key =~ /^(digest|description|pending|snapshots)$/;
2404 $raw .= "$key: $conf->{$key}\n";
2405 }
2406 return $raw;
2407 };
2408
2409 my $raw = &$generate_raw_config($conf);
2410
2411 if (scalar(keys %{$conf->{pending}})){
2412 $raw .= "\n[PENDING]\n";
2413 $raw .= &$generate_raw_config($conf->{pending}, 1);
2414 }
2415
2416 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2417 $raw .= "\n[$snapname]\n";
2418 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2419 }
2420
2421 return $raw;
2422}
2423
2424sub load_defaults {
2425
2426 my $res = {};
2427
2428 # we use static defaults from our JSON schema configuration
2429 foreach my $key (keys %$confdesc) {
2430 if (defined(my $default = $confdesc->{$key}->{default})) {
2431 $res->{$key} = $default;
2432 }
2433 }
2434
2435 return $res;
2436}
2437
2438sub config_list {
2439 my $vmlist = PVE::Cluster::get_vmlist();
2440 my $res = {};
2441 return $res if !$vmlist || !$vmlist->{ids};
2442 my $ids = $vmlist->{ids};
2443 my $nodename = nodename();
2444
2445 foreach my $vmid (keys %$ids) {
2446 my $d = $ids->{$vmid};
2447 next if !$d->{node} || $d->{node} ne $nodename;
2448 next if !$d->{type} || $d->{type} ne 'qemu';
2449 $res->{$vmid}->{exists} = 1;
2450 }
2451 return $res;
2452}
2453
2454# test if VM uses local resources (to prevent migration)
2455sub check_local_resources {
2456 my ($conf, $noerr) = @_;
2457
2458 my @loc_res = ();
2459
2460 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2461 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2462
2463 push @loc_res, "ivshmem" if $conf->{ivshmem};
2464
2465 foreach my $k (keys %$conf) {
2466 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2467 # sockets are safe: they will recreated be on the target side post-migrate
2468 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2469 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2470 }
2471
2472 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2473
2474 return \@loc_res;
2475}
2476
2477# check if used storages are available on all nodes (use by migrate)
2478sub check_storage_availability {
2479 my ($storecfg, $conf, $node) = @_;
2480
2481 PVE::QemuConfig->foreach_volume($conf, sub {
2482 my ($ds, $drive) = @_;
2483
2484 my $volid = $drive->{file};
2485 return if !$volid;
2486
2487 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2488 return if !$sid;
2489
2490 # check if storage is available on both nodes
2491 my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2492 PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2493
2494 my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2495
2496 die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2497 if !$scfg->{content}->{$vtype};
2498 });
2499}
2500
2501# list nodes where all VM images are available (used by has_feature API)
2502sub shared_nodes {
2503 my ($conf, $storecfg) = @_;
2504
2505 my $nodelist = PVE::Cluster::get_nodelist();
2506 my $nodehash = { map { $_ => 1 } @$nodelist };
2507 my $nodename = nodename();
2508
2509 PVE::QemuConfig->foreach_volume($conf, sub {
2510 my ($ds, $drive) = @_;
2511
2512 my $volid = $drive->{file};
2513 return if !$volid;
2514
2515 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2516 if ($storeid) {
2517 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2518 if ($scfg->{disable}) {
2519 $nodehash = {};
2520 } elsif (my $avail = $scfg->{nodes}) {
2521 foreach my $node (keys %$nodehash) {
2522 delete $nodehash->{$node} if !$avail->{$node};
2523 }
2524 } elsif (!$scfg->{shared}) {
2525 foreach my $node (keys %$nodehash) {
2526 delete $nodehash->{$node} if $node ne $nodename
2527 }
2528 }
2529 }
2530 });
2531
2532 return $nodehash
2533}
2534
2535sub check_local_storage_availability {
2536 my ($conf, $storecfg) = @_;
2537
2538 my $nodelist = PVE::Cluster::get_nodelist();
2539 my $nodehash = { map { $_ => {} } @$nodelist };
2540
2541 PVE::QemuConfig->foreach_volume($conf, sub {
2542 my ($ds, $drive) = @_;
2543
2544 my $volid = $drive->{file};
2545 return if !$volid;
2546
2547 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2548 if ($storeid) {
2549 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2550
2551 if ($scfg->{disable}) {
2552 foreach my $node (keys %$nodehash) {
2553 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2554 }
2555 } elsif (my $avail = $scfg->{nodes}) {
2556 foreach my $node (keys %$nodehash) {
2557 if (!$avail->{$node}) {
2558 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2559 }
2560 }
2561 }
2562 }
2563 });
2564
2565 foreach my $node (values %$nodehash) {
2566 if (my $unavail = $node->{unavailable_storages}) {
2567 $node->{unavailable_storages} = [ sort keys %$unavail ];
2568 }
2569 }
2570
2571 return $nodehash
2572}
2573
2574# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2575sub check_running {
2576 my ($vmid, $nocheck, $node) = @_;
2577
2578 PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2579 return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2580}
2581
2582sub vzlist {
2583
2584 my $vzlist = config_list();
2585
2586 my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2587
2588 while (defined(my $de = $fd->read)) {
2589 next if $de !~ m/^(\d+)\.pid$/;
2590 my $vmid = $1;
2591 next if !defined($vzlist->{$vmid});
2592 if (my $pid = check_running($vmid)) {
2593 $vzlist->{$vmid}->{pid} = $pid;
2594 }
2595 }
2596
2597 return $vzlist;
2598}
2599
2600our $vmstatus_return_properties = {
2601 vmid => get_standard_option('pve-vmid'),
2602 status => {
2603 description => "Qemu process status.",
2604 type => 'string',
2605 enum => ['stopped', 'running'],
2606 },
2607 maxmem => {
2608 description => "Maximum memory in bytes.",
2609 type => 'integer',
2610 optional => 1,
2611 renderer => 'bytes',
2612 },
2613 maxdisk => {
2614 description => "Root disk size in bytes.",
2615 type => 'integer',
2616 optional => 1,
2617 renderer => 'bytes',
2618 },
2619 name => {
2620 description => "VM name.",
2621 type => 'string',
2622 optional => 1,
2623 },
2624 qmpstatus => {
2625 description => "Qemu QMP agent status.",
2626 type => 'string',
2627 optional => 1,
2628 },
2629 pid => {
2630 description => "PID of running qemu process.",
2631 type => 'integer',
2632 optional => 1,
2633 },
2634 uptime => {
2635 description => "Uptime.",
2636 type => 'integer',
2637 optional => 1,
2638 renderer => 'duration',
2639 },
2640 cpus => {
2641 description => "Maximum usable CPUs.",
2642 type => 'number',
2643 optional => 1,
2644 },
2645 lock => {
2646 description => "The current config lock, if any.",
2647 type => 'string',
2648 optional => 1,
2649 },
2650 tags => {
2651 description => "The current configured tags, if any",
2652 type => 'string',
2653 optional => 1,
2654 },
2655 'running-machine' => {
2656 description => "The currently running machine type (if running).",
2657 type => 'string',
2658 optional => 1,
2659 },
2660 'running-qemu' => {
2661 description => "The currently running QEMU version (if running).",
2662 type => 'string',
2663 optional => 1,
2664 },
2665};
2666
2667my $last_proc_pid_stat;
2668
2669# get VM status information
2670# This must be fast and should not block ($full == false)
2671# We only query KVM using QMP if $full == true (this can be slow)
2672sub vmstatus {
2673 my ($opt_vmid, $full) = @_;
2674
2675 my $res = {};
2676
2677 my $storecfg = PVE::Storage::config();
2678
2679 my $list = vzlist();
2680 my $defaults = load_defaults();
2681
2682 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2683
2684 my $cpucount = $cpuinfo->{cpus} || 1;
2685
2686 foreach my $vmid (keys %$list) {
2687 next if $opt_vmid && ($vmid ne $opt_vmid);
2688
2689 my $conf = PVE::QemuConfig->load_config($vmid);
2690
2691 my $d = { vmid => int($vmid) };
2692 $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2693
2694 # fixme: better status?
2695 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2696
2697 my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2698 if (defined($size)) {
2699 $d->{disk} = 0; # no info available
2700 $d->{maxdisk} = $size;
2701 } else {
2702 $d->{disk} = 0;
2703 $d->{maxdisk} = 0;
2704 }
2705
2706 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2707 * ($conf->{cores} || $defaults->{cores});
2708 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2709 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2710
2711 $d->{name} = $conf->{name} || "VM $vmid";
2712 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2713 : $defaults->{memory}*(1024*1024);
2714
2715 if ($conf->{balloon}) {
2716 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2717 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2718 : $defaults->{shares};
2719 }
2720
2721 $d->{uptime} = 0;
2722 $d->{cpu} = 0;
2723 $d->{mem} = 0;
2724
2725 $d->{netout} = 0;
2726 $d->{netin} = 0;
2727
2728 $d->{diskread} = 0;
2729 $d->{diskwrite} = 0;
2730
2731 $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2732
2733 $d->{serial} = 1 if conf_has_serial($conf);
2734 $d->{lock} = $conf->{lock} if $conf->{lock};
2735 $d->{tags} = $conf->{tags} if defined($conf->{tags});
2736
2737 $res->{$vmid} = $d;
2738 }
2739
2740 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2741 foreach my $dev (keys %$netdev) {
2742 next if $dev !~ m/^tap([1-9]\d*)i/;
2743 my $vmid = $1;
2744 my $d = $res->{$vmid};
2745 next if !$d;
2746
2747 $d->{netout} += $netdev->{$dev}->{receive};
2748 $d->{netin} += $netdev->{$dev}->{transmit};
2749
2750 if ($full) {
2751 $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2752 $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2753 }
2754
2755 }
2756
2757 my $ctime = gettimeofday;
2758
2759 foreach my $vmid (keys %$list) {
2760
2761 my $d = $res->{$vmid};
2762 my $pid = $d->{pid};
2763 next if !$pid;
2764
2765 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2766 next if !$pstat; # not running
2767
2768 my $used = $pstat->{utime} + $pstat->{stime};
2769
2770 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2771
2772 if ($pstat->{vsize}) {
2773 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2774 }
2775
2776 my $old = $last_proc_pid_stat->{$pid};
2777 if (!$old) {
2778 $last_proc_pid_stat->{$pid} = {
2779 time => $ctime,
2780 used => $used,
2781 cpu => 0,
2782 };
2783 next;
2784 }
2785
2786 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2787
2788 if ($dtime > 1000) {
2789 my $dutime = $used - $old->{used};
2790
2791 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2792 $last_proc_pid_stat->{$pid} = {
2793 time => $ctime,
2794 used => $used,
2795 cpu => $d->{cpu},
2796 };
2797 } else {
2798 $d->{cpu} = $old->{cpu};
2799 }
2800 }
2801
2802 return $res if !$full;
2803
2804 my $qmpclient = PVE::QMPClient->new();
2805
2806 my $ballooncb = sub {
2807 my ($vmid, $resp) = @_;
2808
2809 my $info = $resp->{'return'};
2810 return if !$info->{max_mem};
2811
2812 my $d = $res->{$vmid};
2813
2814 # use memory assigned to VM
2815 $d->{maxmem} = $info->{max_mem};
2816 $d->{balloon} = $info->{actual};
2817
2818 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2819 $d->{mem} = $info->{total_mem} - $info->{free_mem};
2820 $d->{freemem} = $info->{free_mem};
2821 }
2822
2823 $d->{ballooninfo} = $info;
2824 };
2825
2826 my $blockstatscb = sub {
2827 my ($vmid, $resp) = @_;
2828 my $data = $resp->{'return'} || [];
2829 my $totalrdbytes = 0;
2830 my $totalwrbytes = 0;
2831
2832 for my $blockstat (@$data) {
2833 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2834 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2835
2836 $blockstat->{device} =~ s/drive-//;
2837 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2838 }
2839 $res->{$vmid}->{diskread} = $totalrdbytes;
2840 $res->{$vmid}->{diskwrite} = $totalwrbytes;
2841 };
2842
2843 my $machinecb = sub {
2844 my ($vmid, $resp) = @_;
2845 my $data = $resp->{'return'} || [];
2846
2847 $res->{$vmid}->{'running-machine'} =
2848 PVE::QemuServer::Machine::current_from_query_machines($data);
2849 };
2850
2851 my $versioncb = sub {
2852 my ($vmid, $resp) = @_;
2853 my $data = $resp->{'return'} // {};
2854 my $version = 'unknown';
2855
2856 if (my $v = $data->{qemu}) {
2857 $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2858 }
2859
2860 $res->{$vmid}->{'running-qemu'} = $version;
2861 };
2862
2863 my $statuscb = sub {
2864 my ($vmid, $resp) = @_;
2865
2866 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2867 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2868 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2869 # this fails if ballon driver is not loaded, so this must be
2870 # the last commnand (following command are aborted if this fails).
2871 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2872
2873 my $status = 'unknown';
2874 if (!defined($status = $resp->{'return'}->{status})) {
2875 warn "unable to get VM status\n";
2876 return;
2877 }
2878
2879 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2880 };
2881
2882 foreach my $vmid (keys %$list) {
2883 next if $opt_vmid && ($vmid ne $opt_vmid);
2884 next if !$res->{$vmid}->{pid}; # not running
2885 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2886 }
2887
2888 $qmpclient->queue_execute(undef, 2);
2889
2890 foreach my $vmid (keys %$list) {
2891 next if $opt_vmid && ($vmid ne $opt_vmid);
2892 next if !$res->{$vmid}->{pid}; #not running
2893
2894 # we can't use the $qmpclient since it might have already aborted on
2895 # 'query-balloon', but this might also fail for older versions...
2896 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2897 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2898 }
2899
2900 foreach my $vmid (keys %$list) {
2901 next if $opt_vmid && ($vmid ne $opt_vmid);
2902 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2903 }
2904
2905 return $res;
2906}
2907
2908sub conf_has_serial {
2909 my ($conf) = @_;
2910
2911 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2912 if ($conf->{"serial$i"}) {
2913 return 1;
2914 }
2915 }
2916
2917 return 0;
2918}
2919
2920sub conf_has_audio {
2921 my ($conf, $id) = @_;
2922
2923 $id //= 0;
2924 my $audio = $conf->{"audio$id"};
2925 return if !defined($audio);
2926
2927 my $audioproperties = parse_property_string($audio_fmt, $audio);
2928 my $audiodriver = $audioproperties->{driver} // 'spice';
2929
2930 return {
2931 dev => $audioproperties->{device},
2932 dev_id => "audiodev$id",
2933 backend => $audiodriver,
2934 backend_id => "$audiodriver-backend${id}",
2935 };
2936}
2937
2938sub audio_devs {
2939 my ($audio, $audiopciaddr, $machine_version) = @_;
2940
2941 my $devs = [];
2942
2943 my $id = $audio->{dev_id};
2944 my $audiodev = "";
2945 if (min_version($machine_version, 4, 2)) {
2946 $audiodev = ",audiodev=$audio->{backend_id}";
2947 }
2948
2949 if ($audio->{dev} eq 'AC97') {
2950 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2951 } elsif ($audio->{dev} =~ /intel\-hda$/) {
2952 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2953 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2954 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2955 } else {
2956 die "unkown audio device '$audio->{dev}', implement me!";
2957 }
2958
2959 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2960
2961 return $devs;
2962}
2963
2964sub vga_conf_has_spice {
2965 my ($vga) = @_;
2966
2967 my $vgaconf = parse_vga($vga);
2968 my $vgatype = $vgaconf->{type};
2969 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2970
2971 return $1 || 1;
2972}
2973
2974sub is_native($) {
2975 my ($arch) = @_;
2976 return get_host_arch() eq $arch;
2977}
2978
2979sub get_vm_arch {
2980 my ($conf) = @_;
2981 return $conf->{arch} // get_host_arch();
2982}
2983
2984my $default_machines = {
2985 x86_64 => 'pc',
2986 aarch64 => 'virt',
2987};
2988
2989sub get_installed_machine_version {
2990 my ($kvmversion) = @_;
2991 $kvmversion = kvm_user_version() if !defined($kvmversion);
2992 $kvmversion =~ m/^(\d+\.\d+)/;
2993 return $1;
2994}
2995
2996sub windows_get_pinned_machine_version {
2997 my ($machine, $base_version, $kvmversion) = @_;
2998
2999 my $pin_version = $base_version;
3000 if (!defined($base_version) ||
3001 !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3002 ) {
3003 $pin_version = get_installed_machine_version($kvmversion);
3004 }
3005 if (!$machine || $machine eq 'pc') {
3006 $machine = "pc-i440fx-$pin_version";
3007 } elsif ($machine eq 'q35') {
3008 $machine = "pc-q35-$pin_version";
3009 } elsif ($machine eq 'virt') {
3010 $machine = "virt-$pin_version";
3011 } else {
3012 warn "unknown machine type '$machine', not touching that!\n";
3013 }
3014
3015 return $machine;
3016}
3017
3018sub get_vm_machine {
3019 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3020
3021 my $machine = $forcemachine || $conf->{machine};
3022
3023 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3024 $kvmversion //= kvm_user_version();
3025 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3026 # layout which confuses windows quite a bit and may result in various regressions..
3027 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3028 if (windows_version($conf->{ostype})) {
3029 $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3030 }
3031 $arch //= 'x86_64';
3032 $machine ||= $default_machines->{$arch};
3033 if ($add_pve_version) {
3034 my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3035 $machine .= "+pve$pvever";
3036 }
3037 }
3038
3039 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3040 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3041 $machine = $1 if $is_pxe;
3042
3043 # for version-pinned machines that do not include a pve-version (e.g.
3044 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3045 $machine .= '+pve0';
3046
3047 $machine .= '.pxe' if $is_pxe;
3048 }
3049
3050 return $machine;
3051}
3052
3053sub get_ovmf_files($) {
3054 my ($arch) = @_;
3055
3056 my $ovmf = $OVMF->{$arch}
3057 or die "no OVMF images known for architecture '$arch'\n";
3058
3059 return @$ovmf;
3060}
3061
3062my $Arch2Qemu = {
3063 aarch64 => '/usr/bin/qemu-system-aarch64',
3064 x86_64 => '/usr/bin/qemu-system-x86_64',
3065};
3066sub get_command_for_arch($) {
3067 my ($arch) = @_;
3068 return '/usr/bin/kvm' if is_native($arch);
3069
3070 my $cmd = $Arch2Qemu->{$arch}
3071 or die "don't know how to emulate architecture '$arch'\n";
3072 return $cmd;
3073}
3074
3075# To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3076# to use in a QEMU command line (-cpu element), first array_intersect the result
3077# of query_supported_ with query_understood_. This is necessary because:
3078#
3079# a) query_understood_ returns flags the host cannot use and
3080# b) query_supported_ (rather the QMP call) doesn't actually return CPU
3081# flags, but CPU settings - with most of them being flags. Those settings
3082# (and some flags, curiously) cannot be specified as a "-cpu" argument.
3083#
3084# query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3085# expensive. If you need the value returned from this, you can get it much
3086# cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3087# $accel being 'kvm' or 'tcg'.
3088#
3089# pvestatd calls this function on startup and whenever the QEMU/KVM version
3090# changes, automatically populating pmxcfs.
3091#
3092# Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3093# since kvm and tcg machines support different flags
3094#
3095sub query_supported_cpu_flags {
3096 my ($arch) = @_;
3097
3098 $arch //= get_host_arch();
3099 my $default_machine = $default_machines->{$arch};
3100
3101 my $flags = {};
3102
3103 # FIXME: Once this is merged, the code below should work for ARM as well:
3104 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3105 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3106 $arch eq "aarch64";
3107
3108 my $kvm_supported = defined(kvm_version());
3109 my $qemu_cmd = get_command_for_arch($arch);
3110 my $fakevmid = -1;
3111 my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3112
3113 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3114 my $query_supported_run_qemu = sub {
3115 my ($kvm) = @_;
3116
3117 my $flags = {};
3118 my $cmd = [
3119 $qemu_cmd,
3120 '-machine', $default_machine,
3121 '-display', 'none',
3122 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3123 '-mon', 'chardev=qmp,mode=control',
3124 '-pidfile', $pidfile,
3125 '-S', '-daemonize'
3126 ];
3127
3128 if (!$kvm) {
3129 push @$cmd, '-accel', 'tcg';
3130 }
3131
3132 my $rc = run_command($cmd, noerr => 1, quiet => 0);
3133 die "QEMU flag querying VM exited with code " . $rc if $rc;
3134
3135 eval {
3136 my $cmd_result = mon_cmd(
3137 $fakevmid,
3138 'query-cpu-model-expansion',
3139 type => 'full',
3140 model => { name => 'host' }
3141 );
3142
3143 my $props = $cmd_result->{model}->{props};
3144 foreach my $prop (keys %$props) {
3145 next if $props->{$prop} ne '1';
3146 # QEMU returns some flags multiple times, with '_', '.' or '-'
3147 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3148 # We only keep those with underscores, to match /proc/cpuinfo
3149 $prop =~ s/\.|-/_/g;
3150 $flags->{$prop} = 1;
3151 }
3152 };
3153 my $err = $@;
3154
3155 # force stop with 10 sec timeout and 'nocheck'
3156 # always stop, even if QMP failed
3157 vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3158
3159 die $err if $err;
3160
3161 return [ sort keys %$flags ];
3162 };
3163
3164 # We need to query QEMU twice, since KVM and TCG have different supported flags
3165 PVE::QemuConfig->lock_config($fakevmid, sub {
3166 $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3167 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3168
3169 if ($kvm_supported) {
3170 $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3171 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3172 }
3173 });
3174
3175 return $flags;
3176}
3177
3178# Understood CPU flags are written to a file at 'pve-qemu' compile time
3179my $understood_cpu_flag_dir = "/usr/share/kvm";
3180sub query_understood_cpu_flags {
3181 my $arch = get_host_arch();
3182 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3183
3184 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3185 if ! -e $filepath;
3186
3187 my $raw = file_get_contents($filepath);
3188 $raw =~ s/^\s+|\s+$//g;
3189 my @flags = split(/\s+/, $raw);
3190
3191 return \@flags;
3192}
3193
3194my sub get_cpuunits {
3195 my ($conf) = @_;
3196 return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3197}
3198sub config_to_command {
3199 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3200 $pbs_backing) = @_;
3201
3202 my $cmd = [];
3203 my $globalFlags = [];
3204 my $machineFlags = [];
3205 my $rtcFlags = [];
3206 my $devices = [];
3207 my $pciaddr = '';
3208 my $bridges = {};
3209 my $ostype = $conf->{ostype};
3210 my $winversion = windows_version($ostype);
3211 my $kvm = $conf->{kvm};
3212 my $nodename = nodename();
3213
3214 my $arch = get_vm_arch($conf);
3215 my $kvm_binary = get_command_for_arch($arch);
3216 my $kvmver = kvm_user_version($kvm_binary);
3217
3218 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3219 $kvmver //= "undefined";
3220 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3221 }
3222
3223 my $add_pve_version = min_version($kvmver, 4, 1);
3224
3225 my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3226 my $machine_version = extract_version($machine_type, $kvmver);
3227 $kvm //= 1 if is_native($arch);
3228
3229 $machine_version =~ m/(\d+)\.(\d+)/;
3230 my ($machine_major, $machine_minor) = ($1, $2);
3231
3232 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3233 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3234 } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3235 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3236 ." please upgrade node '$nodename'\n"
3237 } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3238 my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3239 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3240 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3241 ." node '$nodename'\n";
3242 }
3243
3244 # if a specific +pve version is required for a feature, use $version_guard
3245 # instead of min_version to allow machines to be run with the minimum
3246 # required version
3247 my $required_pve_version = 0;
3248 my $version_guard = sub {
3249 my ($major, $minor, $pve) = @_;
3250 return 0 if !min_version($machine_version, $major, $minor, $pve);
3251 my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3252 return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3253 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3254 return 1;
3255 };
3256
3257 if ($kvm && !defined kvm_version()) {
3258 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3259 ." or enable in BIOS.\n";
3260 }
3261
3262 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3263 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3264 my $use_old_bios_files = undef;
3265 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3266
3267 my $cpuunits = get_cpuunits($conf);
3268
3269 push @$cmd, $kvm_binary;
3270
3271 push @$cmd, '-id', $vmid;
3272
3273 my $vmname = $conf->{name} || "vm$vmid";
3274
3275 push @$cmd, '-name', $vmname;
3276
3277 push @$cmd, '-no-shutdown';
3278
3279 my $use_virtio = 0;
3280
3281 my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3282 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3283 push @$cmd, '-mon', "chardev=qmp,mode=control";
3284
3285 if (min_version($machine_version, 2, 12)) {
3286 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3287 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3288 }
3289
3290 push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3291
3292 push @$cmd, '-daemonize';
3293
3294 if ($conf->{smbios1}) {
3295 my $smbios_conf = parse_smbios1($conf->{smbios1});
3296 if ($smbios_conf->{base64}) {
3297 # Do not pass base64 flag to qemu
3298 delete $smbios_conf->{base64};
3299 my $smbios_string = "";
3300 foreach my $key (keys %$smbios_conf) {
3301 my $value;
3302 if ($key eq "uuid") {
3303 $value = $smbios_conf->{uuid}
3304 } else {
3305 $value = decode_base64($smbios_conf->{$key});
3306 }
3307 # qemu accepts any binary data, only commas need escaping by double comma
3308 $value =~ s/,/,,/g;
3309 $smbios_string .= "," . $key . "=" . $value if $value;
3310 }
3311 push @$cmd, '-smbios', "type=1" . $smbios_string;
3312 } else {
3313 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3314 }
3315 }
3316
3317 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3318 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3319 die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3320
3321 my ($path, $format);
3322 my $read_only_str = '';
3323 if (my $efidisk = $conf->{efidisk0}) {
3324 my $d = parse_drive('efidisk0', $efidisk);
3325 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3326 $format = $d->{format};
3327 if ($storeid) {
3328 $path = PVE::Storage::path($storecfg, $d->{file});
3329 if (!defined($format)) {
3330 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3331 $format = qemu_img_format($scfg, $volname);
3332 }
3333 } else {
3334 $path = $d->{file};
3335 die "efidisk format must be specified\n"
3336 if !defined($format);
3337 }
3338
3339 $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3340 } else {
3341 warn "no efidisk configured! Using temporary efivars disk.\n";
3342 $path = "/tmp/$vmid-ovmf.fd";
3343 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3344 $format = 'raw';
3345 }
3346
3347 my $size_str = "";
3348
3349 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3350 $size_str = ",size=" . (-s $ovmf_vars);
3351 }
3352
3353 # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3354 my $cache = "";
3355 if ($path =~ m/^rbd:/) {
3356 $cache = ',cache=writeback';
3357 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3358 }
3359
3360 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3361 push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3362 }
3363
3364 # load q35 config
3365 if ($q35) {
3366 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3367 if (min_version($machine_version, 4, 0)) {
3368 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3369 } else {
3370 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3371 }
3372 }
3373
3374 if ($conf->{vmgenid}) {
3375 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3376 }
3377
3378 # add usb controllers
3379 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3380 $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3381 push @$devices, @usbcontrollers if @usbcontrollers;
3382 my $vga = parse_vga($conf->{vga});
3383
3384 my $qxlnum = vga_conf_has_spice($conf->{vga});
3385 $vga->{type} = 'qxl' if $qxlnum;
3386
3387 if (!$vga->{type}) {
3388 if ($arch eq 'aarch64') {
3389 $vga->{type} = 'virtio';
3390 } elsif (min_version($machine_version, 2, 9)) {
3391 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3392 } else {
3393 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3394 }
3395 }
3396
3397 # enable absolute mouse coordinates (needed by vnc)
3398 my $tablet;
3399 if (defined($conf->{tablet})) {
3400 $tablet = $conf->{tablet};
3401 } else {
3402 $tablet = $defaults->{tablet};
3403 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3404 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3405 }
3406
3407 if ($tablet) {
3408 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3409 my $kbd = print_keyboarddevice_full($conf, $arch);
3410 push @$devices, '-device', $kbd if defined($kbd);
3411 }
3412
3413 my $bootorder = device_bootorder($conf);
3414
3415 # host pci device passthrough
3416 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3417 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3418
3419 # usb devices
3420 my $usb_dev_features = {};
3421 $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3422
3423 my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3424 $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3425 push @$devices, @usbdevices if @usbdevices;
3426
3427 # serial devices
3428 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3429 if (my $path = $conf->{"serial$i"}) {
3430 if ($path eq 'socket') {
3431 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3432 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3433 # On aarch64, serial0 is the UART device. Qemu only allows
3434 # connecting UART devices via the '-serial' command line, as
3435 # the device has a fixed slot on the hardware...
3436 if ($arch eq 'aarch64' && $i == 0) {
3437 push @$devices, '-serial', "chardev:serial$i";
3438 } else {
3439 push @$devices, '-device', "isa-serial,chardev=serial$i";
3440 }
3441 } else {
3442 die "no such serial device\n" if ! -c $path;
3443 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3444 push @$devices, '-device', "isa-serial,chardev=serial$i";
3445 }
3446 }
3447 }
3448
3449 # parallel devices
3450 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3451 if (my $path = $conf->{"parallel$i"}) {
3452 die "no such parallel device\n" if ! -c $path;
3453 my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3454 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3455 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3456 }
3457 }
3458
3459 if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3460 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3461 my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3462 push @$devices, @$audio_devs;
3463 }
3464
3465 my $sockets = 1;
3466 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3467 $sockets = $conf->{sockets} if $conf->{sockets};
3468
3469 my $cores = $conf->{cores} || 1;
3470
3471 my $maxcpus = $sockets * $cores;
3472
3473 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3474
3475 my $allowed_vcpus = $cpuinfo->{cpus};
3476
3477 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3478 if ($allowed_vcpus < $maxcpus);
3479
3480 if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3481
3482 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3483 for (my $i = 2; $i <= $vcpus; $i++) {
3484 my $cpustr = print_cpu_device($conf,$i);
3485 push @$cmd, '-device', $cpustr;
3486 }
3487
3488 } else {
3489
3490 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3491 }
3492 push @$cmd, '-nodefaults';
3493
3494 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3495
3496 push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3497
3498 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
3499
3500 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3501 push @$devices, '-device', print_vga_device(
3502 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3503 my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3504 push @$cmd, '-vnc', "unix:$socket,password=on";
3505 } else {
3506 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3507 push @$cmd, '-nographic';
3508 }
3509
3510 # time drift fix
3511 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3512 my $useLocaltime = $conf->{localtime};
3513
3514 if ($winversion >= 5) { # windows
3515 $useLocaltime = 1 if !defined($conf->{localtime});
3516
3517 # use time drift fix when acpi is enabled
3518 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3519 $tdf = 1 if !defined($conf->{tdf});
3520 }
3521 }
3522
3523 if ($winversion >= 6) {
3524 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3525 push @$cmd, '-no-hpet';
3526 }
3527
3528 push @$rtcFlags, 'driftfix=slew' if $tdf;
3529
3530 if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3531 push @$rtcFlags, "base=$conf->{startdate}";
3532 } elsif ($useLocaltime) {
3533 push @$rtcFlags, 'base=localtime';
3534 }
3535
3536 if ($forcecpu) {
3537 push @$cmd, '-cpu', $forcecpu;
3538 } else {
3539 push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3540 }
3541
3542 PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3543
3544 push @$cmd, '-S' if $conf->{freeze};
3545
3546 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3547
3548 my $guest_agent = parse_guest_agent($conf);
3549
3550 if ($guest_agent->{enabled}) {
3551 my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3552 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3553
3554 if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3555 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3556 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3557 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3558 } elsif ($guest_agent->{type} eq 'isa') {
3559 push @$devices, '-device', "isa-serial,chardev=qga0";
3560 }
3561 }
3562
3563 my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3564 if ($rng && $version_guard->(4, 1, 2)) {
3565 check_rng_source($rng->{source});
3566
3567 my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3568 my $period = $rng->{period} // $rng_fmt->{period}->{default};
3569 my $limiter_str = "";
3570 if ($max_bytes) {
3571 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3572 }
3573
3574 my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3575 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3576 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3577 }
3578
3579 my $spice_port;
3580
3581 if ($qxlnum) {
3582 if ($qxlnum > 1) {
3583 if ($winversion){
3584 for (my $i = 1; $i < $qxlnum; $i++){
3585 push @$devices, '-device', print_vga_device(
3586 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3587 }
3588 } else {
3589 # assume other OS works like Linux
3590 my ($ram, $vram) = ("134217728", "67108864");
3591 if ($vga->{memory}) {
3592 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3593 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3594 }
3595 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3596 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3597 }
3598 }
3599
3600 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3601
3602 my $pfamily = PVE::Tools::get_host_address_family($nodename);
3603 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3604 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3605
3606 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3607 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3608 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3609
3610 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3611 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3612
3613 my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3614 my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3615 if ($spice_enhancement->{foldersharing}) {
3616 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3617 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3618 }
3619
3620 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3621 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3622 if $spice_enhancement->{videostreaming};
3623
3624 push @$devices, '-spice', "$spice_opts";
3625 }
3626
3627 # enable balloon by default, unless explicitly disabled
3628 if (!defined($conf->{balloon}) || $conf->{balloon}) {
3629 $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3630 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3631 }
3632
3633 if ($conf->{watchdog}) {
3634 my $wdopts = parse_watchdog($conf->{watchdog});
3635 $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3636 my $watchdog = $wdopts->{model} || 'i6300esb';
3637 push @$devices, '-device', "$watchdog$pciaddr";
3638 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3639 }
3640
3641 my $vollist = [];
3642 my $scsicontroller = {};
3643 my $ahcicontroller = {};
3644 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3645
3646 # Add iscsi initiator name if available
3647 if (my $initiator = get_initiator_name()) {
3648 push @$devices, '-iscsi', "initiator-name=$initiator";
3649 }
3650
3651 PVE::QemuConfig->foreach_volume($conf, sub {
3652 my ($ds, $drive) = @_;
3653
3654 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3655 check_volume_storage_type($storecfg, $drive->{file});
3656 push @$vollist, $drive->{file};
3657 }
3658
3659 # ignore efidisk here, already added in bios/fw handling code above
3660 return if $drive->{interface} eq 'efidisk';
3661
3662 $use_virtio = 1 if $ds =~ m/^virtio/;
3663
3664 $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3665
3666 if ($drive->{interface} eq 'virtio'){
3667 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3668 }
3669
3670 if ($drive->{interface} eq 'scsi') {
3671
3672 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3673
3674 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3675 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3676
3677 $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3678 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3679
3680 my $iothread = '';
3681 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3682 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3683 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3684 } elsif ($drive->{iothread}) {
3685 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3686 }
3687
3688 my $queues = '';
3689 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3690 $queues = ",num_queues=$drive->{queues}";
3691 }
3692
3693 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3694 if !$scsicontroller->{$controller};
3695 $scsicontroller->{$controller}=1;
3696 }
3697
3698 if ($drive->{interface} eq 'sata') {
3699 my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3700 $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3701 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3702 if !$ahcicontroller->{$controller};
3703 $ahcicontroller->{$controller}=1;
3704 }
3705
3706 my $pbs_conf = $pbs_backing->{$ds};
3707 my $pbs_name = undef;
3708 if ($pbs_conf) {
3709 $pbs_name = "drive-$ds-pbs";
3710 push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3711 }
3712
3713 my $drive_cmd = print_drive_commandline_full(
3714 $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3715
3716 # extra protection for templates, but SATA and IDE don't support it..
3717 $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3718
3719 push @$devices, '-drive',$drive_cmd;
3720 push @$devices, '-device', print_drivedevice_full(
3721 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3722 });
3723
3724 for (my $i = 0; $i < $MAX_NETS; $i++) {
3725 my $netname = "net$i";
3726
3727 next if !$conf->{$netname};
3728 my $d = parse_net($conf->{$netname});
3729 next if !$d;
3730
3731 $use_virtio = 1 if $d->{model} eq 'virtio';
3732
3733 $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3734
3735 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3736 push @$devices, '-netdev', $netdevfull;
3737
3738 my $netdevicefull = print_netdevice_full(
3739 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3740
3741 push @$devices, '-device', $netdevicefull;
3742 }
3743
3744 if ($conf->{ivshmem}) {
3745 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3746
3747 my $bus;
3748 if ($q35) {
3749 $bus = print_pcie_addr("ivshmem");
3750 } else {
3751 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3752 }
3753
3754 my $ivshmem_name = $ivshmem->{name} // $vmid;
3755 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3756
3757 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3758 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3759 .",size=$ivshmem->{size}M";
3760 }
3761
3762 # pci.4 is nested in pci.1
3763 $bridges->{1} = 1 if $bridges->{4};
3764
3765 if (!$q35) {
3766 # add pci bridges
3767 if (min_version($machine_version, 2, 3)) {
3768 $bridges->{1} = 1;
3769 $bridges->{2} = 1;
3770 }
3771
3772 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3773
3774 }
3775
3776 for my $k (sort {$b cmp $a} keys %$bridges) {
3777 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3778
3779 my $k_name = $k;
3780 if ($k == 2 && $legacy_igd) {
3781 $k_name = "$k-igd";
3782 }
3783 $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3784
3785 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3786 if ($q35) {
3787 # add after -readconfig pve-q35.cfg
3788 splice @$devices, 2, 0, '-device', $devstr;
3789 } else {
3790 unshift @$devices, '-device', $devstr if $k > 0;
3791 }
3792 }
3793
3794 if (!$kvm) {
3795 push @$machineFlags, 'accel=tcg';
3796 }
3797
3798 my $machine_type_min = $machine_type;
3799 if ($add_pve_version) {
3800 $machine_type_min =~ s/\+pve\d+$//;
3801 $machine_type_min .= "+pve$required_pve_version";
3802 }
3803 push @$machineFlags, "type=${machine_type_min}";
3804
3805 push @$cmd, @$devices;
3806 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3807 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3808 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3809
3810 if (my $vmstate = $conf->{vmstate}) {
3811 my $statepath = PVE::Storage::path($storecfg, $vmstate);
3812 push @$vollist, $vmstate;
3813 push @$cmd, '-loadstate', $statepath;
3814 print "activating and using '$vmstate' as vmstate\n";
3815 }
3816
3817 if (PVE::QemuConfig->is_template($conf)) {
3818 # needed to workaround base volumes being read-only
3819 push @$cmd, '-snapshot';
3820 }
3821
3822 # add custom args
3823 if ($conf->{args}) {
3824 my $aa = PVE::Tools::split_args($conf->{args});
3825 push @$cmd, @$aa;
3826 }
3827
3828 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3829}
3830
3831sub check_rng_source {
3832 my ($source) = @_;
3833
3834 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3835 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3836 if ! -e $source;
3837
3838 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3839 if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3840 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3841 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3842 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3843 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3844 ." to the host.\n";
3845 }
3846}
3847
3848sub spice_port {
3849 my ($vmid) = @_;
3850
3851 my $res = mon_cmd($vmid, 'query-spice');
3852
3853 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3854}
3855
3856sub vm_devices_list {
3857 my ($vmid) = @_;
3858
3859 my $res = mon_cmd($vmid, 'query-pci');
3860 my $devices_to_check = [];
3861 my $devices = {};
3862 foreach my $pcibus (@$res) {
3863 push @$devices_to_check, @{$pcibus->{devices}},
3864 }
3865
3866 while (@$devices_to_check) {
3867 my $to_check = [];
3868 for my $d (@$devices_to_check) {
3869 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3870 next if !$d->{'pci_bridge'};
3871
3872 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3873 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3874 }
3875 $devices_to_check = $to_check;
3876 }
3877
3878 my $resblock = mon_cmd($vmid, 'query-block');
3879 foreach my $block (@$resblock) {
3880 if($block->{device} =~ m/^drive-(\S+)/){
3881 $devices->{$1} = 1;
3882 }
3883 }
3884
3885 my $resmice = mon_cmd($vmid, 'query-mice');
3886 foreach my $mice (@$resmice) {
3887 if ($mice->{name} eq 'QEMU HID Tablet') {
3888 $devices->{tablet} = 1;
3889 last;
3890 }
3891 }
3892
3893 # for usb devices there is no query-usb
3894 # but we can iterate over the entries in
3895 # qom-list path=/machine/peripheral
3896 my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3897 foreach my $per (@$resperipheral) {
3898 if ($per->{name} =~ m/^usb\d+$/) {
3899 $devices->{$per->{name}} = 1;
3900 }
3901 }
3902
3903 return $devices;
3904}
3905
3906sub vm_deviceplug {
3907 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3908
3909 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3910
3911 my $devices_list = vm_devices_list($vmid);
3912 return 1 if defined($devices_list->{$deviceid});
3913
3914 # add PCI bridge if we need it for the device
3915 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3916
3917 if ($deviceid eq 'tablet') {
3918
3919 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3920
3921 } elsif ($deviceid eq 'keyboard') {
3922
3923 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3924
3925 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3926
3927 die "usb hotplug currently not reliable\n";
3928 # since we can't reliably hot unplug all added usb devices and usb
3929 # passthrough breaks live migration we disable usb hotplugging for now
3930 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3931
3932 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3933
3934 qemu_iothread_add($vmid, $deviceid, $device);
3935
3936 qemu_driveadd($storecfg, $vmid, $device);
3937 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3938
3939 qemu_deviceadd($vmid, $devicefull);
3940 eval { qemu_deviceaddverify($vmid, $deviceid); };
3941 if (my $err = $@) {
3942 eval { qemu_drivedel($vmid, $deviceid); };
3943 warn $@ if $@;
3944 die $err;
3945 }
3946
3947 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3948
3949
3950 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3951 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3952 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3953
3954 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3955
3956 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3957 qemu_iothread_add($vmid, $deviceid, $device);
3958 $devicefull .= ",iothread=iothread-$deviceid";
3959 }
3960
3961 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3962 $devicefull .= ",num_queues=$device->{queues}";
3963 }
3964
3965 qemu_deviceadd($vmid, $devicefull);
3966 qemu_deviceaddverify($vmid, $deviceid);
3967
3968 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3969
3970 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3971 qemu_driveadd($storecfg, $vmid, $device);
3972
3973 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3974 eval { qemu_deviceadd($vmid, $devicefull); };
3975 if (my $err = $@) {
3976 eval { qemu_drivedel($vmid, $deviceid); };
3977 warn $@ if $@;
3978 die $err;
3979 }
3980
3981 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3982
3983 return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3984
3985 my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3986 my $use_old_bios_files = undef;
3987 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3988
3989 my $netdevicefull = print_netdevice_full(
3990 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3991 qemu_deviceadd($vmid, $netdevicefull);
3992 eval {
3993 qemu_deviceaddverify($vmid, $deviceid);
3994 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3995 };
3996 if (my $err = $@) {
3997 eval { qemu_netdevdel($vmid, $deviceid); };
3998 warn $@ if $@;
3999 die $err;
4000 }
4001
4002 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4003
4004 my $bridgeid = $2;
4005 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4006 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4007
4008 qemu_deviceadd($vmid, $devicefull);
4009 qemu_deviceaddverify($vmid, $deviceid);
4010
4011 } else {
4012 die "can't hotplug device '$deviceid'\n";
4013 }
4014
4015 return 1;
4016}
4017
4018# fixme: this should raise exceptions on error!
4019sub vm_deviceunplug {
4020 my ($vmid, $conf, $deviceid) = @_;
4021
4022 my $devices_list = vm_devices_list($vmid);
4023 return 1 if !defined($devices_list->{$deviceid});
4024
4025 my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4026 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4027
4028 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4029
4030 qemu_devicedel($vmid, $deviceid);
4031
4032 } elsif ($deviceid =~ m/^usb\d+$/) {
4033
4034 die "usb hotplug currently not reliable\n";
4035 # when unplugging usb devices this way, there may be remaining usb
4036 # controllers/hubs so we disable it for now
4037 #qemu_devicedel($vmid, $deviceid);
4038 #qemu_devicedelverify($vmid, $deviceid);
4039
4040 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4041 my $device = parse_drive($deviceid, $conf->{$deviceid});
4042
4043 qemu_devicedel($vmid, $deviceid);
4044 qemu_devicedelverify($vmid, $deviceid);
4045 qemu_drivedel($vmid, $deviceid);
4046 qemu_iothread_del($vmid, $deviceid, $device);
4047
4048 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4049
4050 qemu_devicedel($vmid, $deviceid);
4051 qemu_devicedelverify($vmid, $deviceid);
4052
4053 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4054 my $device = parse_drive($deviceid, $conf->{$deviceid});
4055
4056 qemu_devicedel($vmid, $deviceid);
4057 qemu_drivedel($vmid, $deviceid);
4058 qemu_deletescsihw($conf, $vmid, $deviceid);
4059
4060 qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4061 if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4062
4063 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4064
4065 qemu_devicedel($vmid, $deviceid);
4066 qemu_devicedelverify($vmid, $deviceid);
4067 qemu_netdevdel($vmid, $deviceid);
4068
4069 } else {
4070 die "can't unplug device '$deviceid'\n";
4071 }
4072
4073 return 1;
4074}
4075
4076sub qemu_deviceadd {
4077 my ($vmid, $devicefull) = @_;
4078
4079 $devicefull = "driver=".$devicefull;
4080 my %options = split(/[=,]/, $devicefull);
4081
4082 mon_cmd($vmid, "device_add" , %options);
4083}
4084
4085sub qemu_devicedel {
4086 my ($vmid, $deviceid) = @_;
4087
4088 my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4089}
4090
4091sub qemu_iothread_add {
4092 my($vmid, $deviceid, $device) = @_;
4093
4094 if ($device->{iothread}) {
4095 my $iothreads = vm_iothreads_list($vmid);
4096 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4097 }
4098}
4099
4100sub qemu_iothread_del {
4101 my($vmid, $deviceid, $device) = @_;
4102
4103 if ($device->{iothread}) {
4104 my $iothreads = vm_iothreads_list($vmid);
4105 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4106 }
4107}
4108
4109sub qemu_objectadd {
4110 my($vmid, $objectid, $qomtype) = @_;
4111
4112 mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4113
4114 return 1;
4115}
4116
4117sub qemu_objectdel {
4118 my($vmid, $objectid) = @_;
4119
4120 mon_cmd($vmid, "object-del", id => $objectid);
4121
4122 return 1;
4123}
4124
4125sub qemu_driveadd {
4126 my ($storecfg, $vmid, $device) = @_;
4127
4128 my $kvmver = get_running_qemu_version($vmid);
4129 my $io_uring = min_version($kvmver, 6, 0);
4130 my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4131 $drive =~ s/\\/\\\\/g;
4132 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4133
4134 # If the command succeeds qemu prints: "OK"
4135 return 1 if $ret =~ m/OK/s;
4136
4137 die "adding drive failed: $ret\n";
4138}
4139
4140sub qemu_drivedel {
4141 my($vmid, $deviceid) = @_;
4142
4143 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4144 $ret =~ s/^\s+//;
4145
4146 return 1 if $ret eq "";
4147
4148 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4149 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4150
4151 die "deleting drive $deviceid failed : $ret\n";
4152}
4153
4154sub qemu_deviceaddverify {
4155 my ($vmid, $deviceid) = @_;
4156
4157 for (my $i = 0; $i <= 5; $i++) {
4158 my $devices_list = vm_devices_list($vmid);
4159 return 1 if defined($devices_list->{$deviceid});
4160 sleep 1;
4161 }
4162
4163 die "error on hotplug device '$deviceid'\n";
4164}
4165
4166
4167sub qemu_devicedelverify {
4168 my ($vmid, $deviceid) = @_;
4169
4170 # need to verify that the device is correctly removed as device_del
4171 # is async and empty return is not reliable
4172
4173 for (my $i = 0; $i <= 5; $i++) {
4174 my $devices_list = vm_devices_list($vmid);
4175 return 1 if !defined($devices_list->{$deviceid});
4176 sleep 1;
4177 }
4178
4179 die "error on hot-unplugging device '$deviceid'\n";
4180}
4181
4182sub qemu_findorcreatescsihw {
4183 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4184
4185 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4186
4187 my $scsihwid="$controller_prefix$controller";
4188 my $devices_list = vm_devices_list($vmid);
4189
4190 if(!defined($devices_list->{$scsihwid})) {
4191 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4192 }
4193
4194 return 1;
4195}
4196
4197sub qemu_deletescsihw {
4198 my ($conf, $vmid, $opt) = @_;
4199
4200 my $device = parse_drive($opt, $conf->{$opt});
4201
4202 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4203 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4204 return 1;
4205 }
4206
4207 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4208
4209 my $devices_list = vm_devices_list($vmid);
4210 foreach my $opt (keys %{$devices_list}) {
4211 if (is_valid_drivename($opt)) {
4212 my $drive = parse_drive($opt, $conf->{$opt});
4213 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4214 return 1;
4215 }
4216 }
4217 }
4218
4219 my $scsihwid="scsihw$controller";
4220
4221 vm_deviceunplug($vmid, $conf, $scsihwid);
4222
4223 return 1;
4224}
4225
4226sub qemu_add_pci_bridge {
4227 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4228
4229 my $bridges = {};
4230
4231 my $bridgeid;
4232
4233 print_pci_addr($device, $bridges, $arch, $machine_type);
4234
4235 while (my ($k, $v) = each %$bridges) {
4236 $bridgeid = $k;
4237 }
4238 return 1 if !defined($bridgeid) || $bridgeid < 1;
4239
4240 my $bridge = "pci.$bridgeid";
4241 my $devices_list = vm_devices_list($vmid);
4242
4243 if (!defined($devices_list->{$bridge})) {
4244 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4245 }
4246
4247 return 1;
4248}
4249
4250sub qemu_set_link_status {
4251 my ($vmid, $device, $up) = @_;
4252
4253 mon_cmd($vmid, "set_link", name => $device,
4254 up => $up ? JSON::true : JSON::false);
4255}
4256
4257sub qemu_netdevadd {
4258 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4259
4260 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4261 my %options = split(/[=,]/, $netdev);
4262
4263 if (defined(my $vhost = $options{vhost})) {
4264 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4265 }
4266
4267 if (defined(my $queues = $options{queues})) {
4268 $options{queues} = $queues + 0;
4269 }
4270
4271 mon_cmd($vmid, "netdev_add", %options);
4272 return 1;
4273}
4274
4275sub qemu_netdevdel {
4276 my ($vmid, $deviceid) = @_;
4277
4278 mon_cmd($vmid, "netdev_del", id => $deviceid);
4279}
4280
4281sub qemu_usb_hotplug {
4282 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4283
4284 return if !$device;
4285
4286 # remove the old one first
4287 vm_deviceunplug($vmid, $conf, $deviceid);
4288
4289 # check if xhci controller is necessary and available
4290 if ($device->{usb3}) {
4291
4292 my $devicelist = vm_devices_list($vmid);
4293
4294 if (!$devicelist->{xhci}) {
4295 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4296 qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4297 }
4298 }
4299 my $d = parse_usb_device($device->{host});
4300 $d->{usb3} = $device->{usb3};
4301
4302 # add the new one
4303 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4304}
4305
4306sub qemu_cpu_hotplug {
4307 my ($vmid, $conf, $vcpus) = @_;
4308
4309 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4310
4311 my $sockets = 1;
4312 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4313 $sockets = $conf->{sockets} if $conf->{sockets};
4314 my $cores = $conf->{cores} || 1;
4315 my $maxcpus = $sockets * $cores;
4316
4317 $vcpus = $maxcpus if !$vcpus;
4318
4319 die "you can't add more vcpus than maxcpus\n"
4320 if $vcpus > $maxcpus;
4321
4322 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4323
4324 if ($vcpus < $currentvcpus) {
4325
4326 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4327
4328 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4329 qemu_devicedel($vmid, "cpu$i");
4330 my $retry = 0;
4331 my $currentrunningvcpus = undef;
4332 while (1) {
4333 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4334 last if scalar(@{$currentrunningvcpus}) == $i-1;
4335 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4336 $retry++;
4337 sleep 1;
4338 }
4339 #update conf after each succesfull cpu unplug
4340 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4341 PVE::QemuConfig->write_config($vmid, $conf);
4342 }
4343 } else {
4344 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4345 }
4346
4347 return;
4348 }
4349
4350 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4351 die "vcpus in running vm does not match its configuration\n"
4352 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4353
4354 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4355
4356 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4357 my $cpustr = print_cpu_device($conf, $i);
4358 qemu_deviceadd($vmid, $cpustr);
4359
4360 my $retry = 0;
4361 my $currentrunningvcpus = undef;
4362 while (1) {
4363 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4364 last if scalar(@{$currentrunningvcpus}) == $i;
4365 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4366 sleep 1;
4367 $retry++;
4368 }
4369 #update conf after each succesfull cpu hotplug
4370 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4371 PVE::QemuConfig->write_config($vmid, $conf);
4372 }
4373 } else {
4374
4375 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4376 mon_cmd($vmid, "cpu-add", id => int($i));
4377 }
4378 }
4379}
4380
4381sub qemu_block_set_io_throttle {
4382 my ($vmid, $deviceid,
4383 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4384 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4385 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4386 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4387
4388 return if !check_running($vmid) ;
4389
4390 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4391 bps => int($bps),
4392 bps_rd => int($bps_rd),
4393 bps_wr => int($bps_wr),
4394 iops => int($iops),
4395 iops_rd => int($iops_rd),
4396 iops_wr => int($iops_wr),
4397 bps_max => int($bps_max),
4398 bps_rd_max => int($bps_rd_max),
4399 bps_wr_max => int($bps_wr_max),
4400 iops_max => int($iops_max),
4401 iops_rd_max => int($iops_rd_max),
4402 iops_wr_max => int($iops_wr_max),
4403 bps_max_length => int($bps_max_length),
4404 bps_rd_max_length => int($bps_rd_max_length),
4405 bps_wr_max_length => int($bps_wr_max_length),
4406 iops_max_length => int($iops_max_length),
4407 iops_rd_max_length => int($iops_rd_max_length),
4408 iops_wr_max_length => int($iops_wr_max_length),
4409 );
4410
4411}
4412
4413sub qemu_block_resize {
4414 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4415
4416 my $running = check_running($vmid);
4417
4418 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4419
4420 return if !$running;
4421
4422 my $padding = (1024 - $size % 1024) % 1024;
4423 $size = $size + $padding;
4424
4425 mon_cmd(
4426 $vmid,
4427 "block_resize",
4428 device => $deviceid,
4429 size => int($size),
4430 timeout => 60,
4431 );
4432}
4433
4434sub qemu_volume_snapshot {
4435 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4436
4437 my $running = check_running($vmid);
4438
4439 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4440 mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4441 } else {
4442 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4443 }
4444}
4445
4446sub qemu_volume_snapshot_delete {
4447 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4448
4449 my $running = check_running($vmid);
4450
4451 if($running) {
4452
4453 $running = undef;
4454 my $conf = PVE::QemuConfig->load_config($vmid);
4455 PVE::QemuConfig->foreach_volume($conf, sub {
4456 my ($ds, $drive) = @_;
4457 $running = 1 if $drive->{file} eq $volid;
4458 });
4459 }
4460
4461 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4462 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4463 } else {
4464 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4465 }
4466}
4467
4468sub set_migration_caps {
4469 my ($vmid, $savevm) = @_;
4470
4471 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4472
4473 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4474 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4475
4476 my $cap_ref = [];
4477
4478 my $enabled_cap = {
4479 "auto-converge" => 1,
4480 "xbzrle" => 1,
4481 "x-rdma-pin-all" => 0,
4482 "zero-blocks" => 0,
4483 "compress" => 0,
4484 "dirty-bitmaps" => $dirty_bitmaps,
4485 };
4486
4487 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4488
4489 for my $supported_capability (@$supported_capabilities) {
4490 push @$cap_ref, {
4491 capability => $supported_capability->{capability},
4492 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4493 };
4494 }
4495
4496 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4497}
4498
4499sub foreach_volid {
4500 my ($conf, $func, @param) = @_;
4501
4502 my $volhash = {};
4503
4504 my $test_volid = sub {
4505 my ($key, $drive, $snapname) = @_;
4506
4507 my $volid = $drive->{file};
4508 return if !$volid;
4509
4510 $volhash->{$volid}->{cdrom} //= 1;
4511 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4512
4513 my $replicate = $drive->{replicate} // 1;
4514 $volhash->{$volid}->{replicate} //= 0;
4515 $volhash->{$volid}->{replicate} = 1 if $replicate;
4516
4517 $volhash->{$volid}->{shared} //= 0;
4518 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4519
4520 $volhash->{$volid}->{referenced_in_config} //= 0;
4521 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4522
4523 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4524 if defined($snapname);
4525
4526 my $size = $drive->{size};
4527 $volhash->{$volid}->{size} //= $size if $size;
4528
4529 $volhash->{$volid}->{is_vmstate} //= 0;
4530 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4531
4532 $volhash->{$volid}->{is_unused} //= 0;
4533 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4534
4535 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4536 };
4537
4538 my $include_opts = {
4539 extra_keys => ['vmstate'],
4540 include_unused => 1,
4541 };
4542
4543 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4544 foreach my $snapname (keys %{$conf->{snapshots}}) {
4545 my $snap = $conf->{snapshots}->{$snapname};
4546 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4547 }
4548
4549 foreach my $volid (keys %$volhash) {
4550 &$func($volid, $volhash->{$volid}, @param);
4551 }
4552}
4553
4554my $fast_plug_option = {
4555 'lock' => 1,
4556 'name' => 1,
4557 'onboot' => 1,
4558 'shares' => 1,
4559 'startup' => 1,
4560 'description' => 1,
4561 'protection' => 1,
4562 'vmstatestorage' => 1,
4563 'hookscript' => 1,
4564 'tags' => 1,
4565};
4566
4567# hotplug changes in [PENDING]
4568# $selection hash can be used to only apply specified options, for
4569# example: { cores => 1 } (only apply changed 'cores')
4570# $errors ref is used to return error messages
4571sub vmconfig_hotplug_pending {
4572 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4573
4574 my $defaults = load_defaults();
4575 my $arch = get_vm_arch($conf);
4576 my $machine_type = get_vm_machine($conf, undef, $arch);
4577
4578 # commit values which do not have any impact on running VM first
4579 # Note: those option cannot raise errors, we we do not care about
4580 # $selection and always apply them.
4581
4582 my $add_error = sub {
4583 my ($opt, $msg) = @_;
4584 $errors->{$opt} = "hotplug problem - $msg";
4585 };
4586
4587 my $changes = 0;
4588 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4589 if ($fast_plug_option->{$opt}) {
4590 $conf->{$opt} = $conf->{pending}->{$opt};
4591 delete $conf->{pending}->{$opt};
4592 $changes = 1;
4593 }
4594 }
4595
4596 if ($changes) {
4597 PVE::QemuConfig->write_config($vmid, $conf);
4598 }
4599
4600 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4601
4602 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4603 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4604 foreach my $opt (sort keys %$pending_delete_hash) {
4605 next if $selection && !$selection->{$opt};
4606 my $force = $pending_delete_hash->{$opt}->{force};
4607 eval {
4608 if ($opt eq 'hotplug') {
4609 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4610 } elsif ($opt eq 'tablet') {
4611 die "skip\n" if !$hotplug_features->{usb};
4612 if ($defaults->{tablet}) {
4613 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4614 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4615 if $arch eq 'aarch64';
4616 } else {
4617 vm_deviceunplug($vmid, $conf, 'tablet');
4618 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4619 }
4620 } elsif ($opt =~ m/^usb\d+/) {
4621 die "skip\n";
4622 # since we cannot reliably hot unplug usb devices we are disabling it
4623 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4624 #vm_deviceunplug($vmid, $conf, $opt);
4625 } elsif ($opt eq 'vcpus') {
4626 die "skip\n" if !$hotplug_features->{cpu};
4627 qemu_cpu_hotplug($vmid, $conf, undef);
4628 } elsif ($opt eq 'balloon') {
4629 # enable balloon device is not hotpluggable
4630 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4631 # here we reset the ballooning value to memory
4632 my $balloon = $conf->{memory} || $defaults->{memory};
4633 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4634 } elsif ($fast_plug_option->{$opt}) {
4635 # do nothing
4636 } elsif ($opt =~ m/^net(\d+)$/) {
4637 die "skip\n" if !$hotplug_features->{network};
4638 vm_deviceunplug($vmid, $conf, $opt);
4639 } elsif (is_valid_drivename($opt)) {
4640 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4641 vm_deviceunplug($vmid, $conf, $opt);
4642 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4643 } elsif ($opt =~ m/^memory$/) {
4644 die "skip\n" if !$hotplug_features->{memory};
4645 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4646 } elsif ($opt eq 'cpuunits') {
4647 $cgroup->change_cpu_shares(undef, 1024);
4648 } elsif ($opt eq 'cpulimit') {
4649 $cgroup->change_cpu_quota(-1, 100000);
4650 } else {
4651 die "skip\n";
4652 }
4653 };
4654 if (my $err = $@) {
4655 &$add_error($opt, $err) if $err ne "skip\n";
4656 } else {
4657 delete $conf->{$opt};
4658 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4659 }
4660 }
4661
4662 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4663 $apply_pending_cloudinit = sub {
4664 return if $apply_pending_cloudinit_done; # once is enough
4665 $apply_pending_cloudinit_done = 1; # once is enough
4666
4667 my ($key, $value) = @_;
4668
4669 my @cloudinit_opts = keys %$confdesc_cloudinit;
4670 foreach my $opt (keys %{$conf->{pending}}) {
4671 next if !grep { $_ eq $opt } @cloudinit_opts;
4672 $conf->{$opt} = delete $conf->{pending}->{$opt};
4673 }
4674
4675 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4676 foreach my $opt (sort keys %$pending_delete_hash) {
4677 next if !grep { $_ eq $opt } @cloudinit_opts;
4678 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4679 delete $conf->{$opt};
4680 }
4681
4682 my $new_conf = { %$conf };
4683 $new_conf->{$key} = $value;
4684 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4685 };
4686
4687 foreach my $opt (keys %{$conf->{pending}}) {
4688 next if $selection && !$selection->{$opt};
4689 my $value = $conf->{pending}->{$opt};
4690 eval {
4691 if ($opt eq 'hotplug') {
4692 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4693 } elsif ($opt eq 'tablet') {
4694 die "skip\n" if !$hotplug_features->{usb};
4695 if ($value == 1) {
4696 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4697 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4698 if $arch eq 'aarch64';
4699 } elsif ($value == 0) {
4700 vm_deviceunplug($vmid, $conf, 'tablet');
4701 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4702 }
4703 } elsif ($opt =~ m/^usb\d+$/) {
4704 die "skip\n";
4705 # since we cannot reliably hot unplug usb devices we disable it for now
4706 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4707 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4708 #die "skip\n" if !$d;
4709 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4710 } elsif ($opt eq 'vcpus') {
4711 die "skip\n" if !$hotplug_features->{cpu};
4712 qemu_cpu_hotplug($vmid, $conf, $value);
4713 } elsif ($opt eq 'balloon') {
4714 # enable/disable balloning device is not hotpluggable
4715 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4716 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4717 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4718
4719 # allow manual ballooning if shares is set to zero
4720 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4721 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4722 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4723 }
4724 } elsif ($opt =~ m/^net(\d+)$/) {
4725 # some changes can be done without hotplug
4726 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4727 $vmid, $opt, $value, $arch, $machine_type);
4728 } elsif (is_valid_drivename($opt)) {
4729 die "skip\n" if $opt eq 'efidisk0';
4730 # some changes can be done without hotplug
4731 my $drive = parse_drive($opt, $value);
4732 if (drive_is_cloudinit($drive)) {
4733 &$apply_pending_cloudinit($opt, $value);
4734 }
4735 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4736 $vmid, $opt, $value, $arch, $machine_type);
4737 } elsif ($opt =~ m/^memory$/) { #dimms
4738 die "skip\n" if !$hotplug_features->{memory};
4739 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4740 } elsif ($opt eq 'cpuunits') {
4741 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4742 } elsif ($opt eq 'cpulimit') {
4743 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4744 $cgroup->change_cpu_quota($cpulimit, 100000);
4745 } else {
4746 die "skip\n"; # skip non-hot-pluggable options
4747 }
4748 };
4749 if (my $err = $@) {
4750 &$add_error($opt, $err) if $err ne "skip\n";
4751 } else {
4752 $conf->{$opt} = $value;
4753 delete $conf->{pending}->{$opt};
4754 }
4755 }
4756
4757 PVE::QemuConfig->write_config($vmid, $conf);
4758}
4759
4760sub try_deallocate_drive {
4761 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4762
4763 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4764 my $volid = $drive->{file};
4765 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4766 my $sid = PVE::Storage::parse_volume_id($volid);
4767 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4768
4769 # check if the disk is really unused
4770 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4771 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4772 PVE::Storage::vdisk_free($storecfg, $volid);
4773 return 1;
4774 } else {
4775 # If vm is not owner of this disk remove from config
4776 return 1;
4777 }
4778 }
4779
4780 return;
4781}
4782
4783sub vmconfig_delete_or_detach_drive {
4784 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4785
4786 my $drive = parse_drive($opt, $conf->{$opt});
4787
4788 my $rpcenv = PVE::RPCEnvironment::get();
4789 my $authuser = $rpcenv->get_user();
4790
4791 if ($force) {
4792 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4793 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4794 } else {
4795 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4796 }
4797}
4798
4799
4800
4801sub vmconfig_apply_pending {
4802 my ($vmid, $conf, $storecfg, $errors) = @_;
4803
4804 my $add_apply_error = sub {
4805 my ($opt, $msg) = @_;
4806 my $err_msg = "unable to apply pending change $opt : $msg";
4807 $errors->{$opt} = $err_msg;
4808 warn $err_msg;
4809 };
4810
4811 # cold plug
4812
4813 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4814 foreach my $opt (sort keys %$pending_delete_hash) {
4815 my $force = $pending_delete_hash->{$opt}->{force};
4816 eval {
4817 if ($opt =~ m/^unused/) {
4818 die "internal error";
4819 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4820 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4821 }
4822 };
4823 if (my $err = $@) {
4824 $add_apply_error->($opt, $err);
4825 } else {
4826 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4827 delete $conf->{$opt};
4828 }
4829 }
4830
4831 PVE::QemuConfig->cleanup_pending($conf);
4832
4833 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4834 next if $opt eq 'delete'; # just to be sure
4835 eval {
4836 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4837 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4838 }
4839 };
4840 if (my $err = $@) {
4841 $add_apply_error->($opt, $err);
4842 } else {
4843 $conf->{$opt} = delete $conf->{pending}->{$opt};
4844 }
4845 }
4846
4847 # write all changes at once to avoid unnecessary i/o
4848 PVE::QemuConfig->write_config($vmid, $conf);
4849}
4850
4851sub vmconfig_update_net {
4852 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4853
4854 my $newnet = parse_net($value);
4855
4856 if ($conf->{$opt}) {
4857 my $oldnet = parse_net($conf->{$opt});
4858
4859 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4860 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4861 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4862 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4863
4864 # for non online change, we try to hot-unplug
4865 die "skip\n" if !$hotplug;
4866 vm_deviceunplug($vmid, $conf, $opt);
4867 } else {
4868
4869 die "internal error" if $opt !~ m/net(\d+)/;
4870 my $iface = "tap${vmid}i$1";
4871
4872 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4873 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4874 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4875 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4876 PVE::Network::tap_unplug($iface);
4877
4878 if ($have_sdn) {
4879 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4880 } else {
4881 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4882 }
4883 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4884 # Rate can be applied on its own but any change above needs to
4885 # include the rate in tap_plug since OVS resets everything.
4886 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4887 }
4888
4889 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4890 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4891 }
4892
4893 return 1;
4894 }
4895 }
4896
4897 if ($hotplug) {
4898 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4899 } else {
4900 die "skip\n";
4901 }
4902}
4903
4904sub vmconfig_update_disk {
4905 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4906
4907 my $drive = parse_drive($opt, $value);
4908
4909 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4910 my $media = $drive->{media} || 'disk';
4911 my $oldmedia = $old_drive->{media} || 'disk';
4912 die "unable to change media type\n" if $media ne $oldmedia;
4913
4914 if (!drive_is_cdrom($old_drive)) {
4915
4916 if ($drive->{file} ne $old_drive->{file}) {
4917
4918 die "skip\n" if !$hotplug;
4919
4920 # unplug and register as unused
4921 vm_deviceunplug($vmid, $conf, $opt);
4922 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4923
4924 } else {
4925 # update existing disk
4926
4927 # skip non hotpluggable value
4928 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4929 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4930 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4931 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4932 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4933 die "skip\n";
4934 }
4935
4936 # apply throttle
4937 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4938 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4939 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4940 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4941 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4942 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4943 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4944 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4945 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4946 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4947 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4948 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4949 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4950 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4951 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4952 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4953 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4954 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4955
4956 qemu_block_set_io_throttle(
4957 $vmid,"drive-$opt",
4958 ($drive->{mbps} || 0)*1024*1024,
4959 ($drive->{mbps_rd} || 0)*1024*1024,
4960 ($drive->{mbps_wr} || 0)*1024*1024,
4961 $drive->{iops} || 0,
4962 $drive->{iops_rd} || 0,
4963 $drive->{iops_wr} || 0,
4964 ($drive->{mbps_max} || 0)*1024*1024,
4965 ($drive->{mbps_rd_max} || 0)*1024*1024,
4966 ($drive->{mbps_wr_max} || 0)*1024*1024,
4967 $drive->{iops_max} || 0,
4968 $drive->{iops_rd_max} || 0,
4969 $drive->{iops_wr_max} || 0,
4970 $drive->{bps_max_length} || 1,
4971 $drive->{bps_rd_max_length} || 1,
4972 $drive->{bps_wr_max_length} || 1,
4973 $drive->{iops_max_length} || 1,
4974 $drive->{iops_rd_max_length} || 1,
4975 $drive->{iops_wr_max_length} || 1,
4976 );
4977
4978 }
4979
4980 return 1;
4981 }
4982
4983 } else { # cdrom
4984
4985 if ($drive->{file} eq 'none') {
4986 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4987 if (drive_is_cloudinit($old_drive)) {
4988 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4989 }
4990 } else {
4991 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4992
4993 # force eject if locked
4994 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4995
4996 if ($path) {
4997 mon_cmd($vmid, "blockdev-change-medium",
4998 id => "$opt", filename => "$path");
4999 }
5000 }
5001
5002 return 1;
5003 }
5004 }
5005
5006 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5007 # hotplug new disks
5008 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5009 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5010}
5011
5012# called in locked context by incoming migration
5013sub vm_migrate_get_nbd_disks {
5014 my ($storecfg, $conf, $replicated_volumes) = @_;
5015
5016 my $local_volumes = {};
5017 PVE::QemuConfig->foreach_volume($conf, sub {
5018 my ($ds, $drive) = @_;
5019
5020 return if drive_is_cdrom($drive);
5021
5022 my $volid = $drive->{file};
5023
5024 return if !$volid;
5025
5026 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5027
5028 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5029 return if $scfg->{shared};
5030
5031 # replicated disks re-use existing state via bitmap
5032 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5033 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5034 });
5035 return $local_volumes;
5036}
5037
5038# called in locked context by incoming migration
5039sub vm_migrate_alloc_nbd_disks {
5040 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5041
5042 my $format = undef;
5043
5044 my $nbd = {};
5045 foreach my $opt (sort keys %$source_volumes) {
5046 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5047
5048 if ($use_existing) {
5049 $nbd->{$opt}->{drivestr} = print_drive($drive);
5050 $nbd->{$opt}->{volid} = $volid;
5051 $nbd->{$opt}->{replicated} = 1;
5052 next;
5053 }
5054
5055 # If a remote storage is specified and the format of the original
5056 # volume is not available there, fall back to the default format.
5057 # Otherwise use the same format as the original.
5058 if (!$storagemap->{identity}) {
5059 $storeid = map_storage($storagemap, $storeid);
5060 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5061 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5062 my $fileFormat = qemu_img_format($scfg, $volname);
5063 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5064 } else {
5065 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5066 $format = qemu_img_format($scfg, $volname);
5067 }
5068
5069 my $size = $drive->{size} / 1024;
5070 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5071 my $newdrive = $drive;
5072 $newdrive->{format} = $format;
5073 $newdrive->{file} = $newvolid;
5074 my $drivestr = print_drive($newdrive);
5075 $nbd->{$opt}->{drivestr} = $drivestr;
5076 $nbd->{$opt}->{volid} = $newvolid;
5077 }
5078
5079 return $nbd;
5080}
5081
5082# see vm_start_nolock for parameters, additionally:
5083# migrate_opts:
5084# storagemap = parsed storage map for allocating NBD disks
5085sub vm_start {
5086 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5087
5088 return PVE::QemuConfig->lock_config($vmid, sub {
5089 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5090
5091 die "you can't start a vm if it's a template\n"
5092 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5093
5094 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5095 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5096
5097 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5098
5099 if ($has_backup_lock && $running) {
5100 # a backup is currently running, attempt to start the guest in the
5101 # existing QEMU instance
5102 return vm_resume($vmid);
5103 }
5104
5105 PVE::QemuConfig->check_lock($conf)
5106 if !($params->{skiplock} || $has_suspended_lock);
5107
5108 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5109
5110 die "VM $vmid already running\n" if $running;
5111
5112 if (my $storagemap = $migrate_opts->{storagemap}) {
5113 my $replicated = $migrate_opts->{replicated_volumes};
5114 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5115 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5116
5117 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5118 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5119 }
5120 }
5121
5122 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5123 });
5124}
5125
5126
5127# params:
5128# statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5129# skiplock => 0/1, skip checking for config lock
5130# skiptemplate => 0/1, skip checking whether VM is template
5131# forcemachine => to force Qemu machine (rollback/migration)
5132# forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5133# timeout => in seconds
5134# paused => start VM in paused state (backup)
5135# resume => resume from hibernation
5136# pbs-backing => {
5137# sata0 => {
5138# repository
5139# snapshot
5140# keyfile
5141# archive
5142# },
5143# virtio2 => ...
5144# }
5145# migrate_opts:
5146# nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5147# migratedfrom => source node
5148# spice_ticket => used for spice migration, passed via tunnel/stdin
5149# network => CIDR of migration network
5150# type => secure/insecure - tunnel over encrypted connection or plain-text
5151# nbd_proto_version => int, 0 for TCP, 1 for UNIX
5152# replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5153sub vm_start_nolock {
5154 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5155
5156 my $statefile = $params->{statefile};
5157 my $resume = $params->{resume};
5158
5159 my $migratedfrom = $migrate_opts->{migratedfrom};
5160 my $migration_type = $migrate_opts->{type};
5161
5162 my $res = {};
5163
5164 # clean up leftover reboot request files
5165 eval { clear_reboot_request($vmid); };
5166 warn $@ if $@;
5167
5168 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5169 vmconfig_apply_pending($vmid, $conf, $storecfg);
5170 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5171 }
5172
5173 # don't regenerate the ISO if the VM is started as part of a live migration
5174 # this way we can reuse the old ISO with the correct config
5175 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5176
5177 my $defaults = load_defaults();
5178
5179 # set environment variable useful inside network script
5180 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5181
5182 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5183
5184 my $forcemachine = $params->{forcemachine};
5185 my $forcecpu = $params->{forcecpu};
5186 if ($resume) {
5187 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5188 $forcemachine = $conf->{runningmachine};
5189 $forcecpu = $conf->{runningcpu};
5190 print "Resuming suspended VM\n";
5191 }
5192
5193 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5194 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5195
5196 my $migration_ip;
5197 my $get_migration_ip = sub {
5198 my ($nodename) = @_;
5199
5200 return $migration_ip if defined($migration_ip);
5201
5202 my $cidr = $migrate_opts->{network};
5203
5204 if (!defined($cidr)) {
5205 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5206 $cidr = $dc_conf->{migration}->{network};
5207 }
5208
5209 if (defined($cidr)) {
5210 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5211
5212 die "could not get IP: no address configured on local " .
5213 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5214
5215 die "could not get IP: multiple addresses configured on local " .
5216 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5217
5218 $migration_ip = @$ips[0];
5219 }
5220
5221 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5222 if !defined($migration_ip);
5223
5224 return $migration_ip;
5225 };
5226
5227 my $migrate_uri;
5228 if ($statefile) {
5229 if ($statefile eq 'tcp') {
5230 my $localip = "localhost";
5231 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5232 my $nodename = nodename();
5233
5234 if (!defined($migration_type)) {
5235 if (defined($datacenterconf->{migration}->{type})) {
5236 $migration_type = $datacenterconf->{migration}->{type};
5237 } else {
5238 $migration_type = 'secure';
5239 }
5240 }
5241
5242 if ($migration_type eq 'insecure') {
5243 $localip = $get_migration_ip->($nodename);
5244 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5245 }
5246
5247 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5248 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5249 $migrate_uri = "tcp:${localip}:${migrate_port}";
5250 push @$cmd, '-incoming', $migrate_uri;
5251 push @$cmd, '-S';
5252
5253 } elsif ($statefile eq 'unix') {
5254 # should be default for secure migrations as a ssh TCP forward
5255 # tunnel is not deterministic reliable ready and fails regurarly
5256 # to set up in time, so use UNIX socket forwards
5257 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5258 unlink $socket_addr;
5259
5260 $migrate_uri = "unix:$socket_addr";
5261
5262 push @$cmd, '-incoming', $migrate_uri;
5263 push @$cmd, '-S';
5264
5265 } elsif (-e $statefile) {
5266 push @$cmd, '-loadstate', $statefile;
5267 } else {
5268 my $statepath = PVE::Storage::path($storecfg, $statefile);
5269 push @$vollist, $statefile;
5270 push @$cmd, '-loadstate', $statepath;
5271 }
5272 } elsif ($params->{paused}) {
5273 push @$cmd, '-S';
5274 }
5275
5276 # host pci devices
5277 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5278 my $d = parse_hostpci($conf->{"hostpci$i"});
5279 next if !$d;
5280 my $pcidevices = $d->{pciid};
5281 foreach my $pcidevice (@$pcidevices) {
5282 my $pciid = $pcidevice->{id};
5283
5284 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5285 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5286 die "no pci device info for device '$pciid'\n" if !$info;
5287
5288 if ($d->{mdev}) {
5289 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5290 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5291 } else {
5292 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5293 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5294 die "can't reset PCI device '$pciid'\n"
5295 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5296 }
5297 }
5298 }
5299
5300 PVE::Storage::activate_volumes($storecfg, $vollist);
5301
5302 eval {
5303 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5304 outfunc => sub {}, errfunc => sub {});
5305 };
5306 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5307 # timeout should be more than enough here...
5308 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5309
5310 my $cpuunits = get_cpuunits($conf);
5311
5312 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5313 my %run_params = (
5314 timeout => $statefile ? undef : $start_timeout,
5315 umask => 0077,
5316 noerr => 1,
5317 );
5318
5319 # when migrating, prefix QEMU output so other side can pick up any
5320 # errors that might occur and show the user
5321 if ($migratedfrom) {
5322 $run_params{quiet} = 1;
5323 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5324 }
5325
5326 my %properties = (
5327 Slice => 'qemu.slice',
5328 KillMode => 'process',
5329 SendSIGKILL => 0,
5330 TimeoutStopUSec => ULONG_MAX, # infinity
5331 );
5332
5333 if (PVE::CGroup::cgroup_mode() == 2) {
5334 $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5335 $properties{CPUWeight} = $cpuunits;
5336 } else {
5337 $properties{CPUShares} = $cpuunits;
5338 }
5339
5340 if (my $cpulimit = $conf->{cpulimit}) {
5341 $properties{CPUQuota} = int($cpulimit * 100);
5342 }
5343 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5344
5345 my $run_qemu = sub {
5346 PVE::Tools::run_fork sub {
5347 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5348
5349 my $exitcode = run_command($cmd, %run_params);
5350 die "QEMU exited with code $exitcode\n" if $exitcode;
5351 };
5352 };
5353
5354 if ($conf->{hugepages}) {
5355
5356 my $code = sub {
5357 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5358 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5359
5360 PVE::QemuServer::Memory::hugepages_mount();
5361 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5362
5363 eval { $run_qemu->() };
5364 if (my $err = $@) {
5365 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5366 if !$conf->{keephugepages};
5367 die $err;
5368 }
5369
5370 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5371 if !$conf->{keephugepages};
5372 };
5373 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5374
5375 } else {
5376 eval { $run_qemu->() };
5377 }
5378
5379 if (my $err = $@) {
5380 # deactivate volumes if start fails
5381 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5382 die "start failed: $err";
5383 }
5384
5385 print "migration listens on $migrate_uri\n" if $migrate_uri;
5386 $res->{migrate_uri} = $migrate_uri;
5387
5388 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5389 eval { mon_cmd($vmid, "cont"); };
5390 warn $@ if $@;
5391 }
5392
5393 #start nbd server for storage migration
5394 if (my $nbd = $migrate_opts->{nbd}) {
5395 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5396
5397 my $migrate_storage_uri;
5398 # nbd_protocol_version > 0 for unix socket support
5399 if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5400 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5401 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5402 $migrate_storage_uri = "nbd:unix:$socket_path";
5403 } else {
5404 my $nodename = nodename();
5405 my $localip = $get_migration_ip->($nodename);
5406 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5407 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5408
5409 mon_cmd($vmid, "nbd-server-start", addr => {
5410 type => 'inet',
5411 data => {
5412 host => "${localip}",
5413 port => "${storage_migrate_port}",
5414 },
5415 });
5416 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5417 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5418 }
5419
5420 $res->{migrate_storage_uri} = $migrate_storage_uri;
5421
5422 foreach my $opt (sort keys %$nbd) {
5423 my $drivestr = $nbd->{$opt}->{drivestr};
5424 my $volid = $nbd->{$opt}->{volid};
5425 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5426 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5427 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5428 print "re-using replicated volume: $opt - $volid\n"
5429 if $nbd->{$opt}->{replicated};
5430
5431 $res->{drives}->{$opt} = $nbd->{$opt};
5432 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5433 }
5434 }
5435
5436 if ($migratedfrom) {
5437 eval {
5438 set_migration_caps($vmid);
5439 };
5440 warn $@ if $@;
5441
5442 if ($spice_port) {
5443 print "spice listens on port $spice_port\n";
5444 $res->{spice_port} = $spice_port;
5445 if ($migrate_opts->{spice_ticket}) {
5446 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5447 $migrate_opts->{spice_ticket});
5448 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5449 }
5450 }
5451
5452 } else {
5453 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5454 if !$statefile && $conf->{balloon};
5455
5456 foreach my $opt (keys %$conf) {
5457 next if $opt !~ m/^net\d+$/;
5458 my $nicconf = parse_net($conf->{$opt});
5459 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5460 }
5461 }
5462
5463 mon_cmd($vmid, 'qom-set',
5464 path => "machine/peripheral/balloon0",
5465 property => "guest-stats-polling-interval",
5466 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5467
5468 if ($resume) {
5469 print "Resumed VM, removing state\n";
5470 if (my $vmstate = $conf->{vmstate}) {
5471 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5472 PVE::Storage::vdisk_free($storecfg, $vmstate);
5473 }
5474 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5475 PVE::QemuConfig->write_config($vmid, $conf);
5476 }
5477
5478 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5479
5480 return $res;
5481}
5482
5483sub vm_commandline {
5484 my ($storecfg, $vmid, $snapname) = @_;
5485
5486 my $conf = PVE::QemuConfig->load_config($vmid);
5487 my $forcemachine;
5488 my $forcecpu;
5489
5490 if ($snapname) {
5491 my $snapshot = $conf->{snapshots}->{$snapname};
5492 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5493
5494 # check for machine or CPU overrides in snapshot
5495 $forcemachine = $snapshot->{runningmachine};
5496 $forcecpu = $snapshot->{runningcpu};
5497
5498 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5499
5500 $conf = $snapshot;
5501 }
5502
5503 my $defaults = load_defaults();
5504
5505 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5506 $forcemachine, $forcecpu);
5507
5508 return PVE::Tools::cmd2string($cmd);
5509}
5510
5511sub vm_reset {
5512 my ($vmid, $skiplock) = @_;
5513
5514 PVE::QemuConfig->lock_config($vmid, sub {
5515
5516 my $conf = PVE::QemuConfig->load_config($vmid);
5517
5518 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5519
5520 mon_cmd($vmid, "system_reset");
5521 });
5522}
5523
5524sub get_vm_volumes {
5525 my ($conf) = @_;
5526
5527 my $vollist = [];
5528 foreach_volid($conf, sub {
5529 my ($volid, $attr) = @_;
5530
5531 return if $volid =~ m|^/|;
5532
5533 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5534 return if !$sid;
5535
5536 push @$vollist, $volid;
5537 });
5538
5539 return $vollist;
5540}
5541
5542sub vm_stop_cleanup {
5543 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5544
5545 eval {
5546
5547 if (!$keepActive) {
5548 my $vollist = get_vm_volumes($conf);
5549 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5550 }
5551
5552 foreach my $ext (qw(mon qmp pid vnc qga)) {
5553 unlink "/var/run/qemu-server/${vmid}.$ext";
5554 }
5555
5556 if ($conf->{ivshmem}) {
5557 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5558 # just delete it for now, VMs which have this already open do not
5559 # are affected, but new VMs will get a separated one. If this
5560 # becomes an issue we either add some sort of ref-counting or just
5561 # add a "don't delete on stop" flag to the ivshmem format.
5562 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5563 }
5564
5565 foreach my $key (keys %$conf) {
5566 next if $key !~ m/^hostpci(\d+)$/;
5567 my $hostpciindex = $1;
5568 my $d = parse_hostpci($conf->{$key});
5569 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5570
5571 foreach my $pci (@{$d->{pciid}}) {
5572 my $pciid = $pci->{id};
5573 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5574 }
5575 }
5576
5577 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5578 };
5579 warn $@ if $@; # avoid errors - just warn
5580}
5581
5582# call only in locked context
5583sub _do_vm_stop {
5584 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5585
5586 my $pid = check_running($vmid, $nocheck);
5587 return if !$pid;
5588
5589 my $conf;
5590 if (!$nocheck) {
5591 $conf = PVE::QemuConfig->load_config($vmid);
5592 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5593 if (!defined($timeout) && $shutdown && $conf->{startup}) {
5594 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5595 $timeout = $opts->{down} if $opts->{down};
5596 }
5597 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5598 }
5599
5600 eval {
5601 if ($shutdown) {
5602 if (defined($conf) && get_qga_key($conf, 'enabled')) {
5603 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5604 } else {
5605 mon_cmd($vmid, "system_powerdown");
5606 }
5607 } else {
5608 mon_cmd($vmid, "quit");
5609 }
5610 };
5611 my $err = $@;
5612
5613 if (!$err) {
5614 $timeout = 60 if !defined($timeout);
5615
5616 my $count = 0;
5617 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5618 $count++;
5619 sleep 1;
5620 }
5621
5622 if ($count >= $timeout) {
5623 if ($force) {
5624 warn "VM still running - terminating now with SIGTERM\n";
5625 kill 15, $pid;
5626 } else {
5627 die "VM quit/powerdown failed - got timeout\n";
5628 }
5629 } else {
5630 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5631 return;
5632 }
5633 } else {
5634 if (!check_running($vmid, $nocheck)) {
5635 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5636 return;
5637 }
5638 if ($force) {
5639 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5640 kill 15, $pid;
5641 } else {
5642 die "VM quit/powerdown failed\n";
5643 }
5644 }
5645
5646 # wait again
5647 $timeout = 10;
5648
5649 my $count = 0;
5650 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5651 $count++;
5652 sleep 1;
5653 }
5654
5655 if ($count >= $timeout) {
5656 warn "VM still running - terminating now with SIGKILL\n";
5657 kill 9, $pid;
5658 sleep 1;
5659 }
5660
5661 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5662}
5663
5664# Note: use $nocheck to skip tests if VM configuration file exists.
5665# We need that when migration VMs to other nodes (files already moved)
5666# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5667sub vm_stop {
5668 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5669
5670 $force = 1 if !defined($force) && !$shutdown;
5671
5672 if ($migratedfrom){
5673 my $pid = check_running($vmid, $nocheck, $migratedfrom);
5674 kill 15, $pid if $pid;
5675 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5676 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5677 return;
5678 }
5679
5680 PVE::QemuConfig->lock_config($vmid, sub {
5681 _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5682 });
5683}
5684
5685sub vm_reboot {
5686 my ($vmid, $timeout) = @_;
5687
5688 PVE::QemuConfig->lock_config($vmid, sub {
5689 eval {
5690
5691 # only reboot if running, as qmeventd starts it again on a stop event
5692 return if !check_running($vmid);
5693
5694 create_reboot_request($vmid);
5695
5696 my $storecfg = PVE::Storage::config();
5697 _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5698
5699 };
5700 if (my $err = $@) {
5701 # avoid that the next normal shutdown will be confused for a reboot
5702 clear_reboot_request($vmid);
5703 die $err;
5704 }
5705 });
5706}
5707
5708# note: if using the statestorage parameter, the caller has to check privileges
5709sub vm_suspend {
5710 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5711
5712 my $conf;
5713 my $path;
5714 my $storecfg;
5715 my $vmstate;
5716
5717 PVE::QemuConfig->lock_config($vmid, sub {
5718
5719 $conf = PVE::QemuConfig->load_config($vmid);
5720
5721 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5722 PVE::QemuConfig->check_lock($conf)
5723 if !($skiplock || $is_backing_up);
5724
5725 die "cannot suspend to disk during backup\n"
5726 if $is_backing_up && $includestate;
5727
5728 if ($includestate) {
5729 $conf->{lock} = 'suspending';
5730 my $date = strftime("%Y-%m-%d", localtime(time()));
5731 $storecfg = PVE::Storage::config();
5732 if (!$statestorage) {
5733 $statestorage = find_vmstate_storage($conf, $storecfg);
5734 # check permissions for the storage
5735 my $rpcenv = PVE::RPCEnvironment::get();
5736 if ($rpcenv->{type} ne 'cli') {
5737 my $authuser = $rpcenv->get_user();
5738 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5739 }
5740 }
5741
5742
5743 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5744 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5745 $path = PVE::Storage::path($storecfg, $vmstate);
5746 PVE::QemuConfig->write_config($vmid, $conf);
5747 } else {
5748 mon_cmd($vmid, "stop");
5749 }
5750 });
5751
5752 if ($includestate) {
5753 # save vm state
5754 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5755
5756 eval {
5757 set_migration_caps($vmid, 1);
5758 mon_cmd($vmid, "savevm-start", statefile => $path);
5759 for(;;) {
5760 my $state = mon_cmd($vmid, "query-savevm");
5761 if (!$state->{status}) {
5762 die "savevm not active\n";
5763 } elsif ($state->{status} eq 'active') {
5764 sleep(1);
5765 next;
5766 } elsif ($state->{status} eq 'completed') {
5767 print "State saved, quitting\n";
5768 last;
5769 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5770 die "query-savevm failed with error '$state->{error}'\n"
5771 } else {
5772 die "query-savevm returned status '$state->{status}'\n";
5773 }
5774 }
5775 };
5776 my $err = $@;
5777
5778 PVE::QemuConfig->lock_config($vmid, sub {
5779 $conf = PVE::QemuConfig->load_config($vmid);
5780 if ($err) {
5781 # cleanup, but leave suspending lock, to indicate something went wrong
5782 eval {
5783 mon_cmd($vmid, "savevm-end");
5784 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5785 PVE::Storage::vdisk_free($storecfg, $vmstate);
5786 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5787 PVE::QemuConfig->write_config($vmid, $conf);
5788 };
5789 warn $@ if $@;
5790 die $err;
5791 }
5792
5793 die "lock changed unexpectedly\n"
5794 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5795
5796 mon_cmd($vmid, "quit");
5797 $conf->{lock} = 'suspended';
5798 PVE::QemuConfig->write_config($vmid, $conf);
5799 });
5800 }
5801}
5802
5803sub vm_resume {
5804 my ($vmid, $skiplock, $nocheck) = @_;
5805
5806 PVE::QemuConfig->lock_config($vmid, sub {
5807 my $res = mon_cmd($vmid, 'query-status');
5808 my $resume_cmd = 'cont';
5809 my $reset = 0;
5810
5811 if ($res->{status}) {
5812 return if $res->{status} eq 'running'; # job done, go home
5813 $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5814 $reset = 1 if $res->{status} eq 'shutdown';
5815 }
5816
5817 if (!$nocheck) {
5818
5819 my $conf = PVE::QemuConfig->load_config($vmid);
5820
5821 PVE::QemuConfig->check_lock($conf)
5822 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5823 }
5824
5825 if ($reset) {
5826 # required if a VM shuts down during a backup and we get a resume
5827 # request before the backup finishes for example
5828 mon_cmd($vmid, "system_reset");
5829 }
5830 mon_cmd($vmid, $resume_cmd);
5831 });
5832}
5833
5834sub vm_sendkey {
5835 my ($vmid, $skiplock, $key) = @_;
5836
5837 PVE::QemuConfig->lock_config($vmid, sub {
5838
5839 my $conf = PVE::QemuConfig->load_config($vmid);
5840
5841 # there is no qmp command, so we use the human monitor command
5842 my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5843 die $res if $res ne '';
5844 });
5845}
5846
5847# vzdump restore implementaion
5848
5849sub tar_archive_read_firstfile {
5850 my $archive = shift;
5851
5852 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5853
5854 # try to detect archive type first
5855 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5856 die "unable to open file '$archive'\n";
5857 my $firstfile = <$fh>;
5858 kill 15, $pid;
5859 close $fh;
5860
5861 die "ERROR: archive contaions no data\n" if !$firstfile;
5862 chomp $firstfile;
5863
5864 return $firstfile;
5865}
5866
5867sub tar_restore_cleanup {
5868 my ($storecfg, $statfile) = @_;
5869
5870 print STDERR "starting cleanup\n";
5871
5872 if (my $fd = IO::File->new($statfile, "r")) {
5873 while (defined(my $line = <$fd>)) {
5874 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5875 my $volid = $2;
5876 eval {
5877 if ($volid =~ m|^/|) {
5878 unlink $volid || die 'unlink failed\n';
5879 } else {
5880 PVE::Storage::vdisk_free($storecfg, $volid);
5881 }
5882 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5883 };
5884 print STDERR "unable to cleanup '$volid' - $@" if $@;
5885 } else {
5886 print STDERR "unable to parse line in statfile - $line";
5887 }
5888 }
5889 $fd->close();
5890 }
5891}
5892
5893sub restore_file_archive {
5894 my ($archive, $vmid, $user, $opts) = @_;
5895
5896 return restore_vma_archive($archive, $vmid, $user, $opts)
5897 if $archive eq '-';
5898
5899 my $info = PVE::Storage::archive_info($archive);
5900 my $format = $opts->{format} // $info->{format};
5901 my $comp = $info->{compression};
5902
5903 # try to detect archive format
5904 if ($format eq 'tar') {
5905 return restore_tar_archive($archive, $vmid, $user, $opts);
5906 } else {
5907 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5908 }
5909}
5910
5911# hepler to remove disks that will not be used after restore
5912my $restore_cleanup_oldconf = sub {
5913 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5914
5915 PVE::QemuConfig->foreach_volume($oldconf, sub {
5916 my ($ds, $drive) = @_;
5917
5918 return if drive_is_cdrom($drive, 1);
5919
5920 my $volid = $drive->{file};
5921 return if !$volid || $volid =~ m|^/|;
5922
5923 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5924 return if !$path || !$owner || ($owner != $vmid);
5925
5926 # Note: only delete disk we want to restore
5927 # other volumes will become unused
5928 if ($virtdev_hash->{$ds}) {
5929 eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5930 if (my $err = $@) {
5931 warn $err;
5932 }
5933 }
5934 });
5935
5936 # delete vmstate files, after the restore we have no snapshots anymore
5937 foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5938 my $snap = $oldconf->{snapshots}->{$snapname};
5939 if ($snap->{vmstate}) {
5940 eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5941 if (my $err = $@) {
5942 warn $err;
5943 }
5944 }
5945 }
5946};
5947
5948# Helper to parse vzdump backup device hints
5949#
5950# $rpcenv: Environment, used to ckeck storage permissions
5951# $user: User ID, to check storage permissions
5952# $storecfg: Storage configuration
5953# $fh: the file handle for reading the configuration
5954# $devinfo: should contain device sizes for all backu-up'ed devices
5955# $options: backup options (pool, default storage)
5956#
5957# Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5958my $parse_backup_hints = sub {
5959 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5960
5961 my $virtdev_hash = {};
5962
5963 while (defined(my $line = <$fh>)) {
5964 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5965 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5966 die "archive does not contain data for drive '$virtdev'\n"
5967 if !$devinfo->{$devname};
5968
5969 if (defined($options->{storage})) {
5970 $storeid = $options->{storage} || 'local';
5971 } elsif (!$storeid) {
5972 $storeid = 'local';
5973 }
5974 $format = 'raw' if !$format;
5975 $devinfo->{$devname}->{devname} = $devname;
5976 $devinfo->{$devname}->{virtdev} = $virtdev;
5977 $devinfo->{$devname}->{format} = $format;
5978 $devinfo->{$devname}->{storeid} = $storeid;
5979
5980 # check permission on storage
5981 my $pool = $options->{pool}; # todo: do we need that?
5982 if ($user ne 'root@pam') {
5983 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5984 }
5985
5986 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5987 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5988 my $virtdev = $1;
5989 my $drive = parse_drive($virtdev, $2);
5990 if (drive_is_cloudinit($drive)) {
5991 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5992 $storeid = $options->{storage} if defined ($options->{storage});
5993 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5994 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5995
5996 $virtdev_hash->{$virtdev} = {
5997 format => $format,
5998 storeid => $storeid,
5999 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6000 is_cloudinit => 1,
6001 };
6002 }
6003 }
6004 }
6005
6006 return $virtdev_hash;
6007};
6008
6009# Helper to allocate and activate all volumes required for a restore
6010#
6011# $storecfg: Storage configuration
6012# $virtdev_hash: as returned by parse_backup_hints()
6013#
6014# Returns: { $virtdev => $volid }
6015my $restore_allocate_devices = sub {
6016 my ($storecfg, $virtdev_hash, $vmid) = @_;
6017
6018 my $map = {};
6019 foreach my $virtdev (sort keys %$virtdev_hash) {
6020 my $d = $virtdev_hash->{$virtdev};
6021 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6022 my $storeid = $d->{storeid};
6023 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6024
6025 # test if requested format is supported
6026 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6027 my $supported = grep { $_ eq $d->{format} } @$validFormats;
6028 $d->{format} = $defFormat if !$supported;
6029
6030 my $name;
6031 if ($d->{is_cloudinit}) {
6032 $name = "vm-$vmid-cloudinit";
6033 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6034 if ($scfg->{path}) {
6035 $name .= ".$d->{format}";
6036 }
6037 }
6038
6039 my $volid = PVE::Storage::vdisk_alloc(
6040 $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6041
6042 print STDERR "new volume ID is '$volid'\n";
6043 $d->{volid} = $volid;
6044
6045 PVE::Storage::activate_volumes($storecfg, [$volid]);
6046
6047 $map->{$virtdev} = $volid;
6048 }
6049
6050 return $map;
6051};
6052
6053sub restore_update_config_line {
6054 my ($cookie, $map, $line, $unique) = @_;
6055
6056 return '' if $line =~ m/^\#qmdump\#/;
6057 return '' if $line =~ m/^\#vzdump\#/;
6058 return '' if $line =~ m/^lock:/;
6059 return '' if $line =~ m/^unused\d+:/;
6060 return '' if $line =~ m/^parent:/;
6061
6062 my $res = '';
6063
6064 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6065 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6066 # try to convert old 1.X settings
6067 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6068 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6069 my ($model, $macaddr) = split(/\=/, $devconfig);
6070 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6071 my $net = {
6072 model => $model,
6073 bridge => "vmbr$ind",
6074 macaddr => $macaddr,
6075 };
6076 my $netstr = print_net($net);
6077
6078 $res .= "net$cookie->{netcount}: $netstr\n";
6079 $cookie->{netcount}++;
6080 }
6081 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6082 my ($id, $netstr) = ($1, $2);
6083 my $net = parse_net($netstr);
6084 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6085 $netstr = print_net($net);
6086 $res .= "$id: $netstr\n";
6087 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6088 my $virtdev = $1;
6089 my $value = $3;
6090 my $di = parse_drive($virtdev, $value);
6091 if (defined($di->{backup}) && !$di->{backup}) {
6092 $res .= "#$line";
6093 } elsif ($map->{$virtdev}) {
6094 delete $di->{format}; # format can change on restore
6095 $di->{file} = $map->{$virtdev};
6096 $value = print_drive($di);
6097 $res .= "$virtdev: $value\n";
6098 } else {
6099 $res .= $line;
6100 }
6101 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6102 my $vmgenid = $1;
6103 if ($vmgenid ne '0') {
6104 # always generate a new vmgenid if there was a valid one setup
6105 $vmgenid = generate_uuid();
6106 }
6107 $res .= "vmgenid: $vmgenid\n";
6108 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6109 my ($uuid, $uuid_str);
6110 UUID::generate($uuid);
6111 UUID::unparse($uuid, $uuid_str);
6112 my $smbios1 = parse_smbios1($2);
6113 $smbios1->{uuid} = $uuid_str;
6114 $res .= $1.print_smbios1($smbios1)."\n";
6115 } else {
6116 $res .= $line;
6117 }
6118
6119 return $res;
6120}
6121
6122my $restore_deactivate_volumes = sub {
6123 my ($storecfg, $devinfo) = @_;
6124
6125 my $vollist = [];
6126 foreach my $devname (keys %$devinfo) {
6127 my $volid = $devinfo->{$devname}->{volid};
6128 push @$vollist, $volid if $volid;
6129 }
6130
6131 PVE::Storage::deactivate_volumes($storecfg, $vollist);
6132};
6133
6134my $restore_destroy_volumes = sub {
6135 my ($storecfg, $devinfo) = @_;
6136
6137 foreach my $devname (keys %$devinfo) {
6138 my $volid = $devinfo->{$devname}->{volid};
6139 next if !$volid;
6140 eval {
6141 if ($volid =~ m|^/|) {
6142 unlink $volid || die 'unlink failed\n';
6143 } else {
6144 PVE::Storage::vdisk_free($storecfg, $volid);
6145 }
6146 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6147 };
6148 print STDERR "unable to cleanup '$volid' - $@" if $@;
6149 }
6150};
6151
6152sub scan_volids {
6153 my ($cfg, $vmid) = @_;
6154
6155 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6156
6157 my $volid_hash = {};
6158 foreach my $storeid (keys %$info) {
6159 foreach my $item (@{$info->{$storeid}}) {
6160 next if !($item->{volid} && $item->{size});
6161 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6162 $volid_hash->{$item->{volid}} = $item;
6163 }
6164 }
6165
6166 return $volid_hash;
6167}
6168
6169sub update_disk_config {
6170 my ($vmid, $conf, $volid_hash) = @_;
6171
6172 my $changes;
6173 my $prefix = "VM $vmid";
6174
6175 # used and unused disks
6176 my $referenced = {};
6177
6178 # Note: it is allowed to define multiple storages with same path (alias), so
6179 # we need to check both 'volid' and real 'path' (two different volid can point
6180 # to the same path).
6181
6182 my $referencedpath = {};
6183
6184 # update size info
6185 PVE::QemuConfig->foreach_volume($conf, sub {
6186 my ($opt, $drive) = @_;
6187
6188 my $volid = $drive->{file};
6189 return if !$volid;
6190 my $volume = $volid_hash->{$volid};
6191
6192 # mark volid as "in-use" for next step
6193 $referenced->{$volid} = 1;
6194 if ($volume && (my $path = $volume->{path})) {
6195 $referencedpath->{$path} = 1;
6196 }
6197
6198 return if drive_is_cdrom($drive);
6199 return if !$volume;
6200
6201 my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6202 if (defined($updated)) {
6203 $changes = 1;
6204 $conf->{$opt} = print_drive($updated);
6205 print "$prefix ($opt): $msg\n";
6206 }
6207 });
6208
6209 # remove 'unusedX' entry if volume is used
6210 PVE::QemuConfig->foreach_unused_volume($conf, sub {
6211 my ($opt, $drive) = @_;
6212
6213 my $volid = $drive->{file};
6214 return if !$volid;
6215
6216 my $path;
6217 $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6218 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6219 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6220 $changes = 1;
6221 delete $conf->{$opt};
6222 }
6223
6224 $referenced->{$volid} = 1;
6225 $referencedpath->{$path} = 1 if $path;
6226 });
6227
6228 foreach my $volid (sort keys %$volid_hash) {
6229 next if $volid =~ m/vm-$vmid-state-/;
6230 next if $referenced->{$volid};
6231 my $path = $volid_hash->{$volid}->{path};
6232 next if !$path; # just to be sure
6233 next if $referencedpath->{$path};
6234 $changes = 1;
6235 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6236 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6237 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6238 }
6239
6240 return $changes;
6241}
6242
6243sub rescan {
6244 my ($vmid, $nolock, $dryrun) = @_;
6245
6246 my $cfg = PVE::Storage::config();
6247
6248 print "rescan volumes...\n";
6249 my $volid_hash = scan_volids($cfg, $vmid);
6250
6251 my $updatefn = sub {
6252 my ($vmid) = @_;
6253
6254 my $conf = PVE::QemuConfig->load_config($vmid);
6255
6256 PVE::QemuConfig->check_lock($conf);
6257
6258 my $vm_volids = {};
6259 foreach my $volid (keys %$volid_hash) {
6260 my $info = $volid_hash->{$volid};
6261 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6262 }
6263
6264 my $changes = update_disk_config($vmid, $conf, $vm_volids);
6265
6266 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6267 };
6268
6269 if (defined($vmid)) {
6270 if ($nolock) {
6271 &$updatefn($vmid);
6272 } else {
6273 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6274 }
6275 } else {
6276 my $vmlist = config_list();
6277 foreach my $vmid (keys %$vmlist) {
6278 if ($nolock) {
6279 &$updatefn($vmid);
6280 } else {
6281 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6282 }
6283 }
6284 }
6285}
6286
6287sub restore_proxmox_backup_archive {
6288 my ($archive, $vmid, $user, $options) = @_;
6289
6290 my $storecfg = PVE::Storage::config();
6291
6292 my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6293 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6294
6295 my $fingerprint = $scfg->{fingerprint};
6296 my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6297
6298 my $repo = PVE::PBSClient::get_repository($scfg);
6299
6300 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6301 my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6302 local $ENV{PBS_PASSWORD} = $password;
6303 local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6304
6305 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6306 PVE::Storage::parse_volname($storecfg, $archive);
6307
6308 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6309
6310 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6311
6312 my $tmpdir = "/var/tmp/vzdumptmp$$";
6313 rmtree $tmpdir;
6314 mkpath $tmpdir;
6315
6316 my $conffile = PVE::QemuConfig->config_file($vmid);
6317 # disable interrupts (always do cleanups)
6318 local $SIG{INT} =
6319 local $SIG{TERM} =
6320 local $SIG{QUIT} =
6321 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6322
6323 # Note: $oldconf is undef if VM does not exists
6324 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6325 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6326 my $new_conf_raw = '';
6327
6328 my $rpcenv = PVE::RPCEnvironment::get();
6329 my $devinfo = {};
6330
6331 eval {
6332 # enable interrupts
6333 local $SIG{INT} =
6334 local $SIG{TERM} =
6335 local $SIG{QUIT} =
6336 local $SIG{HUP} =
6337 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6338
6339 my $cfgfn = "$tmpdir/qemu-server.conf";
6340 my $firewall_config_fn = "$tmpdir/fw.conf";
6341 my $index_fn = "$tmpdir/index.json";
6342
6343 my $cmd = "restore";
6344
6345 my $param = [$pbs_backup_name, "index.json", $index_fn];
6346 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6347 my $index = PVE::Tools::file_get_contents($index_fn);
6348 $index = decode_json($index);
6349
6350 # print Dumper($index);
6351 foreach my $info (@{$index->{files}}) {
6352 if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6353 my $devname = $1;
6354 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6355 $devinfo->{$devname}->{size} = $1;
6356 } else {
6357 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6358 }
6359 }
6360 }
6361
6362 my $is_qemu_server_backup = scalar(
6363 grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6364 );
6365 if (!$is_qemu_server_backup) {
6366 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6367 }
6368 my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6369
6370 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6371 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6372
6373 if ($has_firewall_config) {
6374 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6375 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6376
6377 my $pve_firewall_dir = '/etc/pve/firewall';
6378 mkdir $pve_firewall_dir; # make sure the dir exists
6379 PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6380 }
6381
6382 my $fh = IO::File->new($cfgfn, "r") ||
6383 die "unable to read qemu-server.conf - $!\n";
6384
6385 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6386
6387 # fixme: rate limit?
6388
6389 # create empty/temp config
6390 PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6391
6392 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6393
6394 # allocate volumes
6395 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6396
6397 foreach my $virtdev (sort keys %$virtdev_hash) {
6398 my $d = $virtdev_hash->{$virtdev};
6399 next if $d->{is_cloudinit}; # no need to restore cloudinit
6400
6401 # this fails if storage is unavailable
6402 my $volid = $d->{volid};
6403 my $path = PVE::Storage::path($storecfg, $volid);
6404
6405 # for live-restore we only want to preload the efidisk
6406 next if $options->{live} && $virtdev ne 'efidisk0';
6407
6408 my $pbs_restore_cmd = [
6409 '/usr/bin/pbs-restore',
6410 '--repository', $repo,
6411 $pbs_backup_name,
6412 "$d->{devname}.img.fidx",
6413 $path,
6414 '--verbose',
6415 ];
6416
6417 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6418 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6419
6420 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6421 push @$pbs_restore_cmd, '--skip-zero';
6422 }
6423
6424 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6425 print "restore proxmox backup image: $dbg_cmdstring\n";
6426 run_command($pbs_restore_cmd);
6427 }
6428
6429 $fh->seek(0, 0) || die "seek failed - $!\n";
6430
6431 my $cookie = { netcount => 0 };
6432 while (defined(my $line = <$fh>)) {
6433 $new_conf_raw .= restore_update_config_line(
6434 $cookie,
6435 $map,
6436 $line,
6437 $options->{unique},
6438 );
6439 }
6440
6441 $fh->close();
6442 };
6443 my $err = $@;
6444
6445 if ($err || !$options->{live}) {
6446 $restore_deactivate_volumes->($storecfg, $devinfo);
6447 }
6448
6449 rmtree $tmpdir;
6450
6451 if ($err) {
6452 $restore_destroy_volumes->($storecfg, $devinfo);
6453 die $err;
6454 }
6455
6456 if ($options->{live}) {
6457 # keep lock during live-restore
6458 $new_conf_raw .= "\nlock: create";
6459 }
6460
6461 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6462
6463 PVE::Cluster::cfs_update(); # make sure we read new file
6464
6465 eval { rescan($vmid, 1); };
6466 warn $@ if $@;
6467
6468 PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6469
6470 if ($options->{live}) {
6471 # enable interrupts
6472 local $SIG{INT} =
6473 local $SIG{TERM} =
6474 local $SIG{QUIT} =
6475 local $SIG{HUP} =
6476 local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6477
6478 my $conf = PVE::QemuConfig->load_config($vmid);
6479 die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6480
6481 delete $devinfo->{'drive-efidisk0'}; # this special drive is already restored before start
6482 pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6483
6484 PVE::QemuConfig->remove_lock($vmid, "create");
6485 }
6486}
6487
6488sub pbs_live_restore {
6489 my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6490
6491 print "starting VM for live-restore\n";
6492 print "repository: '$repo', snapshot: '$snap'\n";
6493
6494 my $pbs_backing = {};
6495 for my $ds (keys %$restored_disks) {
6496 $ds =~ m/^drive-(.*)$/;
6497 my $confname = $1;
6498 $pbs_backing->{$confname} = {
6499 repository => $repo,
6500 snapshot => $snap,
6501 archive => "$ds.img.fidx",
6502 };
6503 $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6504
6505 my $drive = parse_drive($confname, $conf->{$confname});
6506 print "restoring '$ds' to '$drive->{file}'\n";
6507 }
6508
6509 my $drives_streamed = 0;
6510 eval {
6511 # make sure HA doesn't interrupt our restore by stopping the VM
6512 if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6513 run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
6514 }
6515
6516 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6517 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6518 vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6519
6520 my $qmeventd_fd = register_qmeventd_handle($vmid);
6521
6522 # begin streaming, i.e. data copy from PBS to target disk for every vol,
6523 # this will effectively collapse the backing image chain consisting of
6524 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6525 # removes itself once all backing images vanish with 'auto-remove=on')
6526 my $jobs = {};
6527 for my $ds (sort keys %$restored_disks) {
6528 my $job_id = "restore-$ds";
6529 mon_cmd($vmid, 'block-stream',
6530 'job-id' => $job_id,
6531 device => "$ds",
6532 );
6533 $jobs->{$job_id} = {};
6534 }
6535
6536 mon_cmd($vmid, 'cont');
6537 qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6538
6539 print "restore-drive jobs finished successfully, removing all tracking block devices"
6540 ." to disconnect from Proxmox Backup Server\n";
6541
6542 for my $ds (sort keys %$restored_disks) {
6543 mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6544 }
6545
6546 close($qmeventd_fd);
6547 };
6548
6549 my $err = $@;
6550
6551 if ($err) {
6552 warn "An error occured during live-restore: $err\n";
6553 _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6554 die "live-restore failed\n";
6555 }
6556}
6557
6558sub restore_vma_archive {
6559 my ($archive, $vmid, $user, $opts, $comp) = @_;
6560
6561 my $readfrom = $archive;
6562
6563 my $cfg = PVE::Storage::config();
6564 my $commands = [];
6565 my $bwlimit = $opts->{bwlimit};
6566
6567 my $dbg_cmdstring = '';
6568 my $add_pipe = sub {
6569 my ($cmd) = @_;
6570 push @$commands, $cmd;
6571 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6572 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6573 $readfrom = '-';
6574 };
6575
6576 my $input = undef;
6577 if ($archive eq '-') {
6578 $input = '<&STDIN';
6579 } else {
6580 # If we use a backup from a PVE defined storage we also consider that
6581 # storage's rate limit:
6582 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6583 if (defined($volid)) {
6584 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6585 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6586 if ($readlimit) {
6587 print STDERR "applying read rate limit: $readlimit\n";
6588 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6589 $add_pipe->($cstream);
6590 }
6591 }
6592 }
6593
6594 if ($comp) {
6595 my $info = PVE::Storage::decompressor_info('vma', $comp);
6596 my $cmd = $info->{decompressor};
6597 push @$cmd, $readfrom;
6598 $add_pipe->($cmd);
6599 }
6600
6601 my $tmpdir = "/var/tmp/vzdumptmp$$";
6602 rmtree $tmpdir;
6603
6604 # disable interrupts (always do cleanups)
6605 local $SIG{INT} =
6606 local $SIG{TERM} =
6607 local $SIG{QUIT} =
6608 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6609
6610 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6611 POSIX::mkfifo($mapfifo, 0600);
6612 my $fifofh;
6613 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6614
6615 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6616
6617 my $oldtimeout;
6618 my $timeout = 5;
6619
6620 my $devinfo = {};
6621
6622 my $rpcenv = PVE::RPCEnvironment::get();
6623
6624 my $conffile = PVE::QemuConfig->config_file($vmid);
6625
6626 # Note: $oldconf is undef if VM does not exist
6627 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6628 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6629 my $new_conf_raw = '';
6630
6631 my %storage_limits;
6632
6633 my $print_devmap = sub {
6634 my $cfgfn = "$tmpdir/qemu-server.conf";
6635
6636 # we can read the config - that is already extracted
6637 my $fh = IO::File->new($cfgfn, "r") ||
6638 die "unable to read qemu-server.conf - $!\n";
6639
6640 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6641 if (-f $fwcfgfn) {
6642 my $pve_firewall_dir = '/etc/pve/firewall';
6643 mkdir $pve_firewall_dir; # make sure the dir exists
6644 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6645 }
6646
6647 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6648
6649 foreach my $info (values %{$virtdev_hash}) {
6650 my $storeid = $info->{storeid};
6651 next if defined($storage_limits{$storeid});
6652
6653 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6654 print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6655 $storage_limits{$storeid} = $limit * 1024;
6656 }
6657
6658 foreach my $devname (keys %$devinfo) {
6659 die "found no device mapping information for device '$devname'\n"
6660 if !$devinfo->{$devname}->{virtdev};
6661 }
6662
6663 # create empty/temp config
6664 if ($oldconf) {
6665 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6666 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6667 }
6668
6669 # allocate volumes
6670 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6671
6672 # print restore information to $fifofh
6673 foreach my $virtdev (sort keys %$virtdev_hash) {
6674 my $d = $virtdev_hash->{$virtdev};
6675 next if $d->{is_cloudinit}; # no need to restore cloudinit
6676
6677 my $storeid = $d->{storeid};
6678 my $volid = $d->{volid};
6679
6680 my $map_opts = '';
6681 if (my $limit = $storage_limits{$storeid}) {
6682 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6683 }
6684
6685 my $write_zeros = 1;
6686 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6687 $write_zeros = 0;
6688 }
6689
6690 my $path = PVE::Storage::path($cfg, $volid);
6691
6692 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6693
6694 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6695 }
6696
6697 $fh->seek(0, 0) || die "seek failed - $!\n";
6698
6699 my $cookie = { netcount => 0 };
6700 while (defined(my $line = <$fh>)) {
6701 $new_conf_raw .= restore_update_config_line(
6702 $cookie,
6703 $map,
6704 $line,
6705 $opts->{unique},
6706 );
6707 }
6708
6709 $fh->close();
6710 };
6711
6712 eval {
6713 # enable interrupts
6714 local $SIG{INT} =
6715 local $SIG{TERM} =
6716 local $SIG{QUIT} =
6717 local $SIG{HUP} =
6718 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6719 local $SIG{ALRM} = sub { die "got timeout\n"; };
6720
6721 $oldtimeout = alarm($timeout);
6722
6723 my $parser = sub {
6724 my $line = shift;
6725
6726 print "$line\n";
6727
6728 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6729 my ($dev_id, $size, $devname) = ($1, $2, $3);
6730 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6731 } elsif ($line =~ m/^CTIME: /) {
6732 # we correctly received the vma config, so we can disable
6733 # the timeout now for disk allocation (set to 10 minutes, so
6734 # that we always timeout if something goes wrong)
6735 alarm(600);
6736 &$print_devmap();
6737 print $fifofh "done\n";
6738 my $tmp = $oldtimeout || 0;
6739 $oldtimeout = undef;
6740 alarm($tmp);
6741 close($fifofh);
6742 $fifofh = undef;
6743 }
6744 };
6745
6746 print "restore vma archive: $dbg_cmdstring\n";
6747 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6748 };
6749 my $err = $@;
6750
6751 alarm($oldtimeout) if $oldtimeout;
6752
6753 $restore_deactivate_volumes->($cfg, $devinfo);
6754
6755 close($fifofh) if $fifofh;
6756 unlink $mapfifo;
6757 rmtree $tmpdir;
6758
6759 if ($err) {
6760 $restore_destroy_volumes->($cfg, $devinfo);
6761 die $err;
6762 }
6763
6764 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6765
6766 PVE::Cluster::cfs_update(); # make sure we read new file
6767
6768 eval { rescan($vmid, 1); };
6769 warn $@ if $@;
6770
6771 PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6772}
6773
6774sub restore_tar_archive {
6775 my ($archive, $vmid, $user, $opts) = @_;
6776
6777 if ($archive ne '-') {
6778 my $firstfile = tar_archive_read_firstfile($archive);
6779 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6780 if $firstfile ne 'qemu-server.conf';
6781 }
6782
6783 my $storecfg = PVE::Storage::config();
6784
6785 # avoid zombie disks when restoring over an existing VM -> cleanup first
6786 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6787 # skiplock=1 because qmrestore has set the 'create' lock itself already
6788 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6789 destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6790
6791 my $tocmd = "/usr/lib/qemu-server/qmextract";
6792
6793 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6794 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6795 $tocmd .= ' --prealloc' if $opts->{prealloc};
6796 $tocmd .= ' --info' if $opts->{info};
6797
6798 # tar option "xf" does not autodetect compression when read from STDIN,
6799 # so we pipe to zcat
6800 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6801 PVE::Tools::shellquote("--to-command=$tocmd");
6802
6803 my $tmpdir = "/var/tmp/vzdumptmp$$";
6804 mkpath $tmpdir;
6805
6806 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6807 local $ENV{VZDUMP_VMID} = $vmid;
6808 local $ENV{VZDUMP_USER} = $user;
6809
6810 my $conffile = PVE::QemuConfig->config_file($vmid);
6811 my $new_conf_raw = '';
6812
6813 # disable interrupts (always do cleanups)
6814 local $SIG{INT} =
6815 local $SIG{TERM} =
6816 local $SIG{QUIT} =
6817 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6818
6819 eval {
6820 # enable interrupts
6821 local $SIG{INT} =
6822 local $SIG{TERM} =
6823 local $SIG{QUIT} =
6824 local $SIG{HUP} =
6825 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6826
6827 if ($archive eq '-') {
6828 print "extracting archive from STDIN\n";
6829 run_command($cmd, input => "<&STDIN");
6830 } else {
6831 print "extracting archive '$archive'\n";
6832 run_command($cmd);
6833 }
6834
6835 return if $opts->{info};
6836
6837 # read new mapping
6838 my $map = {};
6839 my $statfile = "$tmpdir/qmrestore.stat";
6840 if (my $fd = IO::File->new($statfile, "r")) {
6841 while (defined (my $line = <$fd>)) {
6842 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6843 $map->{$1} = $2 if $1;
6844 } else {
6845 print STDERR "unable to parse line in statfile - $line\n";
6846 }
6847 }
6848 $fd->close();
6849 }
6850
6851 my $confsrc = "$tmpdir/qemu-server.conf";
6852
6853 my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6854
6855 my $cookie = { netcount => 0 };
6856 while (defined (my $line = <$srcfd>)) {
6857 $new_conf_raw .= restore_update_config_line(
6858 $cookie,
6859 $map,
6860 $line,
6861 $opts->{unique},
6862 );
6863 }
6864
6865 $srcfd->close();
6866 };
6867 if (my $err = $@) {
6868 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6869 die $err;
6870 }
6871
6872 rmtree $tmpdir;
6873
6874 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6875
6876 PVE::Cluster::cfs_update(); # make sure we read new file
6877
6878 eval { rescan($vmid, 1); };
6879 warn $@ if $@;
6880};
6881
6882sub foreach_storage_used_by_vm {
6883 my ($conf, $func) = @_;
6884
6885 my $sidhash = {};
6886
6887 PVE::QemuConfig->foreach_volume($conf, sub {
6888 my ($ds, $drive) = @_;
6889 return if drive_is_cdrom($drive);
6890
6891 my $volid = $drive->{file};
6892
6893 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6894 $sidhash->{$sid} = $sid if $sid;
6895 });
6896
6897 foreach my $sid (sort keys %$sidhash) {
6898 &$func($sid);
6899 }
6900}
6901
6902my $qemu_snap_storage = {
6903 rbd => 1,
6904};
6905sub do_snapshots_with_qemu {
6906 my ($storecfg, $volid) = @_;
6907
6908 my $storage_name = PVE::Storage::parse_volume_id($volid);
6909 my $scfg = $storecfg->{ids}->{$storage_name};
6910 die "could not find storage '$storage_name'\n" if !defined($scfg);
6911
6912 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6913 return 1;
6914 }
6915
6916 if ($volid =~ m/\.(qcow2|qed)$/){
6917 return 1;
6918 }
6919
6920 return;
6921}
6922
6923sub qga_check_running {
6924 my ($vmid, $nowarn) = @_;
6925
6926 eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6927 if ($@) {
6928 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6929 return 0;
6930 }
6931 return 1;
6932}
6933
6934sub template_create {
6935 my ($vmid, $conf, $disk) = @_;
6936
6937 my $storecfg = PVE::Storage::config();
6938
6939 PVE::QemuConfig->foreach_volume($conf, sub {
6940 my ($ds, $drive) = @_;
6941
6942 return if drive_is_cdrom($drive);
6943 return if $disk && $ds ne $disk;
6944
6945 my $volid = $drive->{file};
6946 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6947
6948 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6949 $drive->{file} = $voliddst;
6950 $conf->{$ds} = print_drive($drive);
6951 PVE::QemuConfig->write_config($vmid, $conf);
6952 });
6953}
6954
6955sub convert_iscsi_path {
6956 my ($path) = @_;
6957
6958 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6959 my $portal = $1;
6960 my $target = $2;
6961 my $lun = $3;
6962
6963 my $initiator_name = get_initiator_name();
6964
6965 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6966 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6967 }
6968
6969 die "cannot convert iscsi path '$path', unkown format\n";
6970}
6971
6972sub qemu_img_convert {
6973 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6974
6975 my $storecfg = PVE::Storage::config();
6976 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6977 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6978
6979 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6980
6981 my $cachemode;
6982 my $src_path;
6983 my $src_is_iscsi = 0;
6984 my $src_format;
6985
6986 if ($src_storeid) {
6987 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6988 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6989 $src_format = qemu_img_format($src_scfg, $src_volname);
6990 $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6991 $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6992 $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6993 } elsif (-f $src_volid) {
6994 $src_path = $src_volid;
6995 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6996 $src_format = $1;
6997 }
6998 }
6999
7000 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7001
7002 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7003 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7004 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7005 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7006
7007 my $cmd = [];
7008 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7009 push @$cmd, '-l', "snapshot.name=$snapname"
7010 if $snapname && $src_format && $src_format eq "qcow2";
7011 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7012 push @$cmd, '-T', $cachemode if defined($cachemode);
7013
7014 if ($src_is_iscsi) {
7015 push @$cmd, '--image-opts';
7016 $src_path = convert_iscsi_path($src_path);
7017 } elsif ($src_format) {
7018 push @$cmd, '-f', $src_format;
7019 }
7020
7021 if ($dst_is_iscsi) {
7022 push @$cmd, '--target-image-opts';
7023 $dst_path = convert_iscsi_path($dst_path);
7024 } else {
7025 push @$cmd, '-O', $dst_format;
7026 }
7027
7028 push @$cmd, $src_path;
7029
7030 if (!$dst_is_iscsi && $is_zero_initialized) {
7031 push @$cmd, "zeroinit:$dst_path";
7032 } else {
7033 push @$cmd, $dst_path;
7034 }
7035
7036 my $parser = sub {
7037 my $line = shift;
7038 if($line =~ m/\((\S+)\/100\%\)/){
7039 my $percent = $1;
7040 my $transferred = int($size * $percent / 100);
7041 my $total_h = render_bytes($size, 1);
7042 my $transferred_h = render_bytes($transferred, 1);
7043
7044 print "transferred $transferred_h of $total_h ($percent%)\n";
7045 }
7046
7047 };
7048
7049 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
7050 my $err = $@;
7051 die "copy failed: $err" if $err;
7052}
7053
7054sub qemu_img_format {
7055 my ($scfg, $volname) = @_;
7056
7057 if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7058 return $1;
7059 } else {
7060 return "raw";
7061 }
7062}
7063
7064sub qemu_drive_mirror {
7065 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7066
7067 $jobs = {} if !$jobs;
7068
7069 my $qemu_target;
7070 my $format;
7071 $jobs->{"drive-$drive"} = {};
7072
7073 if ($dst_volid =~ /^nbd:/) {
7074 $qemu_target = $dst_volid;
7075 $format = "nbd";
7076 } else {
7077 my $storecfg = PVE::Storage::config();
7078 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7079
7080 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7081
7082 $format = qemu_img_format($dst_scfg, $dst_volname);
7083
7084 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7085
7086 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7087 }
7088
7089 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7090 $opts->{format} = $format if $format;
7091
7092 if (defined($src_bitmap)) {
7093 $opts->{sync} = 'incremental';
7094 $opts->{bitmap} = $src_bitmap;
7095 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7096 }
7097
7098 if (defined($bwlimit)) {
7099 $opts->{speed} = $bwlimit * 1024;
7100 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7101 } else {
7102 print "drive mirror is starting for drive-$drive\n";
7103 }
7104
7105 # if a job already runs for this device we get an error, catch it for cleanup
7106 eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7107 if (my $err = $@) {
7108 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7109 warn "$@\n" if $@;
7110 die "mirroring error: $err\n";
7111 }
7112
7113 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7114}
7115
7116# $completion can be either
7117# 'complete': wait until all jobs are ready, block-job-complete them (default)
7118# 'cancel': wait until all jobs are ready, block-job-cancel them
7119# 'skip': wait until all jobs are ready, return with block jobs in ready state
7120# 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7121sub qemu_drive_mirror_monitor {
7122 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7123
7124 $completion //= 'complete';
7125 $op //= "mirror";
7126
7127 eval {
7128 my $err_complete = 0;
7129
7130 my $starttime = time ();
7131 while (1) {
7132 die "block job ('$op') timed out\n" if $err_complete > 300;
7133
7134 my $stats = mon_cmd($vmid, "query-block-jobs");
7135 my $ctime = time();
7136
7137 my $running_jobs = {};
7138 for my $stat (@$stats) {
7139 next if $stat->{type} ne $op;
7140 $running_jobs->{$stat->{device}} = $stat;
7141 }
7142
7143 my $readycounter = 0;
7144
7145 for my $job_id (sort keys %$jobs) {
7146 my $job = $running_jobs->{$job_id};
7147
7148 my $vanished = !defined($job);
7149 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7150 if($complete || ($vanished && $completion eq 'auto')) {
7151 print "$job_id: $op-job finished\n";
7152 delete $jobs->{$job_id};
7153 next;
7154 }
7155
7156 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7157
7158 my $busy = $job->{busy};
7159 my $ready = $job->{ready};
7160 if (my $total = $job->{len}) {
7161 my $transferred = $job->{offset} || 0;
7162 my $remaining = $total - $transferred;
7163 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7164
7165 my $duration = $ctime - $starttime;
7166 my $total_h = render_bytes($total, 1);
7167 my $transferred_h = render_bytes($transferred, 1);
7168
7169 my $status = sprintf(
7170 "transferred $transferred_h of $total_h ($percent%%) in %s",
7171 render_duration($duration),
7172 );
7173
7174 if ($ready) {
7175 if ($busy) {
7176 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7177 } else {
7178 $status .= ", ready";
7179 }
7180 }
7181 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7182 $jobs->{$job_id}->{ready} = $ready;
7183 }
7184
7185 $readycounter++ if $job->{ready};
7186 }
7187
7188 last if scalar(keys %$jobs) == 0;
7189
7190 if ($readycounter == scalar(keys %$jobs)) {
7191 print "all '$op' jobs are ready\n";
7192
7193 # do the complete later (or has already been done)
7194 last if $completion eq 'skip' || $completion eq 'auto';
7195
7196 if ($vmiddst && $vmiddst != $vmid) {
7197 my $agent_running = $qga && qga_check_running($vmid);
7198 if ($agent_running) {
7199 print "freeze filesystem\n";
7200 eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7201 } else {
7202 print "suspend vm\n";
7203 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7204 }
7205
7206 # if we clone a disk for a new target vm, we don't switch the disk
7207 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7208
7209 if ($agent_running) {
7210 print "unfreeze filesystem\n";
7211 eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7212 } else {
7213 print "resume vm\n";
7214 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7215 }
7216
7217 last;
7218 } else {
7219
7220 for my $job_id (sort keys %$jobs) {
7221 # try to switch the disk if source and destination are on the same guest
7222 print "$job_id: Completing block job_id...\n";
7223
7224 my $op;
7225 if ($completion eq 'complete') {
7226 $op = 'block-job-complete';
7227 } elsif ($completion eq 'cancel') {
7228 $op = 'block-job-cancel';
7229 } else {
7230 die "invalid completion value: $completion\n";
7231 }
7232 eval { mon_cmd($vmid, $op, device => $job_id) };
7233 if ($@ =~ m/cannot be completed/) {
7234 print "$job_id: block job cannot be completed, trying again.\n";
7235 $err_complete++;
7236 }else {
7237 print "$job_id: Completed successfully.\n";
7238 $jobs->{$job_id}->{complete} = 1;
7239 }
7240 }
7241 }
7242 }
7243 sleep 1;
7244 }
7245 };
7246 my $err = $@;
7247
7248 if ($err) {
7249 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7250 die "block job ($op) error: $err";
7251 }
7252}
7253
7254sub qemu_blockjobs_cancel {
7255 my ($vmid, $jobs) = @_;
7256
7257 foreach my $job (keys %$jobs) {
7258 print "$job: Cancelling block job\n";
7259 eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7260 $jobs->{$job}->{cancel} = 1;
7261 }
7262
7263 while (1) {
7264 my $stats = mon_cmd($vmid, "query-block-jobs");
7265
7266 my $running_jobs = {};
7267 foreach my $stat (@$stats) {
7268 $running_jobs->{$stat->{device}} = $stat;
7269 }
7270
7271 foreach my $job (keys %$jobs) {
7272
7273 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7274 print "$job: Done.\n";
7275 delete $jobs->{$job};
7276 }
7277 }
7278
7279 last if scalar(keys %$jobs) == 0;
7280
7281 sleep 1;
7282 }
7283}
7284
7285sub clone_disk {
7286 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7287 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7288
7289 my $newvolid;
7290
7291 if (!$full) {
7292 print "create linked clone of drive $drivename ($drive->{file})\n";
7293 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
7294 push @$newvollist, $newvolid;
7295 } else {
7296
7297 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7298 $storeid = $storage if $storage;
7299
7300 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7301
7302 print "create full clone of drive $drivename ($drive->{file})\n";
7303 my $name = undef;
7304 my $size = undef;
7305 if (drive_is_cloudinit($drive)) {
7306 $name = "vm-$newvmid-cloudinit";
7307 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7308 if ($scfg->{path}) {
7309 $name .= ".$dst_format";
7310 }
7311 $snapname = undef;
7312 $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7313 } elsif ($drivename eq 'efidisk0') {
7314 $size = get_efivars_size($conf);
7315 } else {
7316 ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7317 }
7318 $newvolid = PVE::Storage::vdisk_alloc(
7319 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7320 );
7321 push @$newvollist, $newvolid;
7322
7323 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7324
7325 if (drive_is_cloudinit($drive)) {
7326 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7327 # if this is the case, we have to complete any block-jobs still there from
7328 # previous drive-mirrors
7329 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7330 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7331 }
7332 goto no_data_clone;
7333 }
7334
7335 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7336 if (!$running || $snapname) {
7337 # TODO: handle bwlimits
7338 if ($drivename eq 'efidisk0') {
7339 # the relevant data on the efidisk may be smaller than the source
7340 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7341 # that is given by the OVMF_VARS.fd
7342 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7343 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7344
7345 # better for Ceph if block size is not too small, see bug #3324
7346 my $bs = 1024*1024;
7347
7348 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7349 "if=$src_path", "of=$dst_path"]);
7350 } else {
7351 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7352 }
7353 } else {
7354
7355 my $kvmver = get_running_qemu_version ($vmid);
7356 if (!min_version($kvmver, 2, 7)) {
7357 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7358 if $drive->{iothread};
7359 }
7360
7361 qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7362 $completion, $qga, $bwlimit);
7363 }
7364 }
7365
7366no_data_clone:
7367 my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7368
7369 my $disk = $drive;
7370 $disk->{format} = undef;
7371 $disk->{file} = $newvolid;
7372 $disk->{size} = $size if defined($size);
7373
7374 return $disk;
7375}
7376
7377sub get_running_qemu_version {
7378 my ($vmid) = @_;
7379 my $res = mon_cmd($vmid, "query-version");
7380 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7381}
7382
7383sub qemu_use_old_bios_files {
7384 my ($machine_type) = @_;
7385
7386 return if !$machine_type;
7387
7388 my $use_old_bios_files = undef;
7389
7390 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7391 $machine_type = $1;
7392 $use_old_bios_files = 1;
7393 } else {
7394 my $version = extract_version($machine_type, kvm_user_version());
7395 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7396 # load new efi bios files on migration. So this hack is required to allow
7397 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7398 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7399 $use_old_bios_files = !min_version($version, 2, 4);
7400 }
7401
7402 return ($use_old_bios_files, $machine_type);
7403}
7404
7405sub get_efivars_size {
7406 my ($conf) = @_;
7407 my $arch = get_vm_arch($conf);
7408 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7409 die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7410 return -s $ovmf_vars;
7411}
7412
7413sub update_efidisk_size {
7414 my ($conf) = @_;
7415
7416 return if !defined($conf->{efidisk0});
7417
7418 my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7419 $disk->{size} = get_efivars_size($conf);
7420 $conf->{efidisk0} = print_drive($disk);
7421
7422 return;
7423}
7424
7425sub create_efidisk($$$$$) {
7426 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7427
7428 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7429 die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7430
7431 my $vars_size_b = -s $ovmf_vars;
7432 my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7433 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7434 PVE::Storage::activate_volumes($storecfg, [$volid]);
7435
7436 qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7437 my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7438
7439 return ($volid, $size/1024);
7440}
7441
7442sub vm_iothreads_list {
7443 my ($vmid) = @_;
7444
7445 my $res = mon_cmd($vmid, 'query-iothreads');
7446
7447 my $iothreads = {};
7448 foreach my $iothread (@$res) {
7449 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7450 }
7451
7452 return $iothreads;
7453}
7454
7455sub scsihw_infos {
7456 my ($conf, $drive) = @_;
7457
7458 my $maxdev = 0;
7459
7460 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7461 $maxdev = 7;
7462 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7463 $maxdev = 1;
7464 } else {
7465 $maxdev = 256;
7466 }
7467
7468 my $controller = int($drive->{index} / $maxdev);
7469 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7470 ? "virtioscsi"
7471 : "scsihw";
7472
7473 return ($maxdev, $controller, $controller_prefix);
7474}
7475
7476sub windows_version {
7477 my ($ostype) = @_;
7478
7479 return 0 if !$ostype;
7480
7481 my $winversion = 0;
7482
7483 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7484 $winversion = 5;
7485 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7486 $winversion = 6;
7487 } elsif ($ostype =~ m/^win(\d+)$/) {
7488 $winversion = $1;
7489 }
7490
7491 return $winversion;
7492}
7493
7494sub resolve_dst_disk_format {
7495 my ($storecfg, $storeid, $src_volname, $format) = @_;
7496 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7497
7498 if (!$format) {
7499 # if no target format is specified, use the source disk format as hint
7500 if ($src_volname) {
7501 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7502 $format = qemu_img_format($scfg, $src_volname);
7503 } else {
7504 return $defFormat;
7505 }
7506 }
7507
7508 # test if requested format is supported - else use default
7509 my $supported = grep { $_ eq $format } @$validFormats;
7510 $format = $defFormat if !$supported;
7511 return $format;
7512}
7513
7514# NOTE: if this logic changes, please update docs & possibly gui logic
7515sub find_vmstate_storage {
7516 my ($conf, $storecfg) = @_;
7517
7518 # first, return storage from conf if set
7519 return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7520
7521 my ($target, $shared, $local);
7522
7523 foreach_storage_used_by_vm($conf, sub {
7524 my ($sid) = @_;
7525 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7526 my $dst = $scfg->{shared} ? \$shared : \$local;
7527 $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7528 });
7529
7530 # second, use shared storage where VM has at least one disk
7531 # third, use local storage where VM has at least one disk
7532 # fall back to local storage
7533 $target = $shared // $local // 'local';
7534
7535 return $target;
7536}
7537
7538sub generate_uuid {
7539 my ($uuid, $uuid_str);
7540 UUID::generate($uuid);
7541 UUID::unparse($uuid, $uuid_str);
7542 return $uuid_str;
7543}
7544
7545sub generate_smbios1_uuid {
7546 return "uuid=".generate_uuid();
7547}
7548
7549sub nbd_stop {
7550 my ($vmid) = @_;
7551
7552 mon_cmd($vmid, 'nbd-server-stop');
7553}
7554
7555sub create_reboot_request {
7556 my ($vmid) = @_;
7557 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7558 or die "failed to create reboot trigger file: $!\n";
7559 close($fh);
7560}
7561
7562sub clear_reboot_request {
7563 my ($vmid) = @_;
7564 my $path = "/run/qemu-server/$vmid.reboot";
7565 my $res = 0;
7566
7567 $res = unlink($path);
7568 die "could not remove reboot request for $vmid: $!"
7569 if !$res && $! != POSIX::ENOENT;
7570
7571 return $res;
7572}
7573
7574sub bootorder_from_legacy {
7575 my ($conf, $bootcfg) = @_;
7576
7577 my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7578 my $bootindex_hash = {};
7579 my $i = 1;
7580 foreach my $o (split(//, $boot)) {
7581 $bootindex_hash->{$o} = $i*100;
7582 $i++;
7583 }
7584
7585 my $bootorder = {};
7586
7587 PVE::QemuConfig->foreach_volume($conf, sub {
7588 my ($ds, $drive) = @_;
7589
7590 if (drive_is_cdrom ($drive, 1)) {
7591 if ($bootindex_hash->{d}) {
7592 $bootorder->{$ds} = $bootindex_hash->{d};
7593 $bootindex_hash->{d} += 1;
7594 }
7595 } elsif ($bootindex_hash->{c}) {
7596 $bootorder->{$ds} = $bootindex_hash->{c}
7597 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7598 $bootindex_hash->{c} += 1;
7599 }
7600 });
7601
7602 if ($bootindex_hash->{n}) {
7603 for (my $i = 0; $i < $MAX_NETS; $i++) {
7604 my $netname = "net$i";
7605 next if !$conf->{$netname};
7606 $bootorder->{$netname} = $bootindex_hash->{n};
7607 $bootindex_hash->{n} += 1;
7608 }
7609 }
7610
7611 return $bootorder;
7612}
7613
7614# Generate default device list for 'boot: order=' property. Matches legacy
7615# default boot order, but with explicit device names. This is important, since
7616# the fallback for when neither 'order' nor the old format is specified relies
7617# on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7618sub get_default_bootdevices {
7619 my ($conf) = @_;
7620
7621 my @ret = ();
7622
7623 # harddisk
7624 my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7625 push @ret, $first if $first;
7626
7627 # cdrom
7628 $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7629 push @ret, $first if $first;
7630
7631 # network
7632 for (my $i = 0; $i < $MAX_NETS; $i++) {
7633 my $netname = "net$i";
7634 next if !$conf->{$netname};
7635 push @ret, $netname;
7636 last;
7637 }
7638
7639 return \@ret;
7640}
7641
7642sub device_bootorder {
7643 my ($conf) = @_;
7644
7645 return bootorder_from_legacy($conf) if !defined($conf->{boot});
7646
7647 my $boot = parse_property_string($boot_fmt, $conf->{boot});
7648
7649 my $bootorder = {};
7650 if (!defined($boot) || $boot->{legacy}) {
7651 $bootorder = bootorder_from_legacy($conf, $boot);
7652 } elsif ($boot->{order}) {
7653 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7654 for my $dev (PVE::Tools::split_list($boot->{order})) {
7655 $bootorder->{$dev} = $i++;
7656 }
7657 }
7658
7659 return $bootorder;
7660}
7661
7662sub register_qmeventd_handle {
7663 my ($vmid) = @_;
7664
7665 my $fh;
7666 my $peer = "/var/run/qmeventd.sock";
7667 my $count = 0;
7668
7669 for (;;) {
7670 $count++;
7671 $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7672 last if $fh;
7673 if ($! != EINTR && $! != EAGAIN) {
7674 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7675 }
7676 if ($count > 4) {
7677 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7678 . "after $count retries\n";
7679 }
7680 usleep(25000);
7681 }
7682
7683 # send handshake to mark VM as backing up
7684 print $fh to_json({vzdump => {vmid => "$vmid"}});
7685
7686 # return handle to be closed later when inhibit is no longer required
7687 return $fh;
7688}
7689
7690# bash completion helper
7691
7692sub complete_backup_archives {
7693 my ($cmdname, $pname, $cvalue) = @_;
7694
7695 my $cfg = PVE::Storage::config();
7696
7697 my $storeid;
7698
7699 if ($cvalue =~ m/^([^:]+):/) {
7700 $storeid = $1;
7701 }
7702
7703 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7704
7705 my $res = [];
7706 foreach my $id (keys %$data) {
7707 foreach my $item (@{$data->{$id}}) {
7708 next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7709 push @$res, $item->{volid} if defined($item->{volid});
7710 }
7711 }
7712
7713 return $res;
7714}
7715
7716my $complete_vmid_full = sub {
7717 my ($running) = @_;
7718
7719 my $idlist = vmstatus();
7720
7721 my $res = [];
7722
7723 foreach my $id (keys %$idlist) {
7724 my $d = $idlist->{$id};
7725 if (defined($running)) {
7726 next if $d->{template};
7727 next if $running && $d->{status} ne 'running';
7728 next if !$running && $d->{status} eq 'running';
7729 }
7730 push @$res, $id;
7731
7732 }
7733 return $res;
7734};
7735
7736sub complete_vmid {
7737 return &$complete_vmid_full();
7738}
7739
7740sub complete_vmid_stopped {
7741 return &$complete_vmid_full(0);
7742}
7743
7744sub complete_vmid_running {
7745 return &$complete_vmid_full(1);
7746}
7747
7748sub complete_storage {
7749
7750 my $cfg = PVE::Storage::config();
7751 my $ids = $cfg->{ids};
7752
7753 my $res = [];
7754 foreach my $sid (keys %$ids) {
7755 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7756 next if !$ids->{$sid}->{content}->{images};
7757 push @$res, $sid;
7758 }
7759
7760 return $res;
7761}
7762
7763sub complete_migration_storage {
7764 my ($cmd, $param, $current_value, $all_args) = @_;
7765
7766 my $targetnode = @$all_args[1];
7767
7768 my $cfg = PVE::Storage::config();
7769 my $ids = $cfg->{ids};
7770
7771 my $res = [];
7772 foreach my $sid (keys %$ids) {
7773 next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7774 next if !$ids->{$sid}->{content}->{images};
7775 push @$res, $sid;
7776 }
7777
7778 return $res;
7779}
7780
7781sub vm_is_paused {
7782 my ($vmid) = @_;
7783 my $qmpstatus = eval {
7784 PVE::QemuConfig::assert_config_exists_on_node($vmid);
7785 mon_cmd($vmid, "query-status");
7786 };
7787 warn "$@\n" if $@;
7788 return $qmpstatus && $qmpstatus->{status} eq "paused";
7789}
7790
7791sub check_volume_storage_type {
7792 my ($storecfg, $vol) = @_;
7793
7794 my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7795 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7796 my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7797
7798 die "storage '$storeid' does not support content-type '$vtype'\n"
7799 if !$scfg->{content}->{$vtype};
7800
7801 return 1;
7802}
7803
78041;
Virtual Machine Manager