]> git.proxmox.com Git - qemu-server.git/blame_incremental - PVE/QemuServer.pm
projects / qemu-server.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Revert "revert spice_ticket prefix change in 7827de4"
[qemu-server.git] / PVE / QemuServer.pm
... / ...
CommitLineData
1package PVE::QemuServer;
2
3use strict;
4use warnings;
5
6use Cwd 'abs_path';
7use Digest::SHA;
8use Fcntl ':flock';
9use Fcntl;
10use File::Basename;
11use File::Copy qw(copy);
12use File::Path;
13use File::stat;
14use Getopt::Long;
15use IO::Dir;
16use IO::File;
17use IO::Handle;
18use IO::Select;
19use IO::Socket::UNIX;
20use IPC::Open3;
21use JSON;
22use MIME::Base64;
23use POSIX;
24use Storable qw(dclone);
25use Time::HiRes qw(gettimeofday);
26use URI::Escape;
27use UUID;
28
29use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
30use PVE::CGroup;
31use PVE::DataCenterConfig;
32use PVE::Exception qw(raise raise_param_exc);
33use PVE::Format qw(render_duration render_bytes);
34use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
35use PVE::INotify;
36use PVE::JSONSchema qw(get_standard_option parse_property_string);
37use PVE::ProcFSTools;
38use PVE::PBSClient;
39use PVE::RPCEnvironment;
40use PVE::Storage;
41use PVE::SysFSTools;
42use PVE::Systemd;
43use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
44
45use PVE::QMPClient;
46use PVE::QemuConfig;
47use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
48use PVE::QemuServer::Cloudinit;
49use PVE::QemuServer::CGroup;
50use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
51use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
52use PVE::QemuServer::Machine;
53use PVE::QemuServer::Memory;
54use PVE::QemuServer::Monitor qw(mon_cmd);
55use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
56use PVE::QemuServer::USB qw(parse_usb_device);
57
58my $have_sdn;
59eval {
60 require PVE::Network::SDN::Zones;
61 $have_sdn = 1;
62};
63
64my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
65my $OVMF = {
66 x86_64 => [
67 "$EDK2_FW_BASE/OVMF_CODE.fd",
68 "$EDK2_FW_BASE/OVMF_VARS.fd"
69 ],
70 aarch64 => [
71 "$EDK2_FW_BASE/AAVMF_CODE.fd",
72 "$EDK2_FW_BASE/AAVMF_VARS.fd"
73 ],
74};
75
76my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
77
78# Note about locking: we use flock on the config file protect
79# against concurent actions.
80# Aditionaly, we have a 'lock' setting in the config file. This
81# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
82# allowed when such lock is set. But you can ignore this kind of
83# lock with the --skiplock flag.
84
85cfs_register_file('/qemu-server/',
86 \&parse_vm_config,
87 \&write_vm_config);
88
89PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
90 description => "Some command save/restore state from this location.",
91 type => 'string',
92 maxLength => 128,
93 optional => 1,
94});
95
96PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
97 description => "Specifies the Qemu machine type.",
98 type => 'string',
99 pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
100 maxLength => 40,
101 optional => 1,
102});
103
104
105sub map_storage {
106 my ($map, $source) = @_;
107
108 return $source if !defined($map);
109
110 return $map->{entries}->{$source}
111 if $map->{entries} && defined($map->{entries}->{$source});
112
113 return $map->{default} if $map->{default};
114
115 # identity (fallback)
116 return $source;
117}
118
119PVE::JSONSchema::register_standard_option('pve-targetstorage', {
120 description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
121 type => 'string',
122 format => 'storagepair-list',
123 optional => 1,
124});
125
126#no warnings 'redefine';
127
128my $nodename_cache;
129sub nodename {
130 $nodename_cache //= PVE::INotify::nodename();
131 return $nodename_cache;
132}
133
134my $watchdog_fmt = {
135 model => {
136 default_key => 1,
137 type => 'string',
138 enum => [qw(i6300esb ib700)],
139 description => "Watchdog type to emulate.",
140 default => 'i6300esb',
141 optional => 1,
142 },
143 action => {
144 type => 'string',
145 enum => [qw(reset shutdown poweroff pause debug none)],
146 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
147 optional => 1,
148 },
149};
150PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
151
152my $agent_fmt = {
153 enabled => {
154 description => "Enable/disable Qemu GuestAgent.",
155 type => 'boolean',
156 default => 0,
157 default_key => 1,
158 },
159 fstrim_cloned_disks => {
160 description => "Run fstrim after moving a disk or migrating the VM.",
161 type => 'boolean',
162 optional => 1,
163 default => 0
164 },
165 type => {
166 description => "Select the agent type",
167 type => 'string',
168 default => 'virtio',
169 optional => 1,
170 enum => [qw(virtio isa)],
171 },
172};
173
174my $vga_fmt = {
175 type => {
176 description => "Select the VGA type.",
177 type => 'string',
178 default => 'std',
179 optional => 1,
180 default_key => 1,
181 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
182 },
183 memory => {
184 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
185 type => 'integer',
186 optional => 1,
187 minimum => 4,
188 maximum => 512,
189 },
190};
191
192my $ivshmem_fmt = {
193 size => {
194 type => 'integer',
195 minimum => 1,
196 description => "The size of the file in MB.",
197 },
198 name => {
199 type => 'string',
200 pattern => '[a-zA-Z0-9\-]+',
201 optional => 1,
202 format_description => 'string',
203 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
204 },
205};
206
207my $audio_fmt = {
208 device => {
209 type => 'string',
210 enum => [qw(ich9-intel-hda intel-hda AC97)],
211 description => "Configure an audio device."
212 },
213 driver => {
214 type => 'string',
215 enum => ['spice', 'none'],
216 default => 'spice',
217 optional => 1,
218 description => "Driver backend for the audio device."
219 },
220};
221
222my $spice_enhancements_fmt = {
223 foldersharing => {
224 type => 'boolean',
225 optional => 1,
226 default => '0',
227 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
228 },
229 videostreaming => {
230 type => 'string',
231 enum => ['off', 'all', 'filter'],
232 default => 'off',
233 optional => 1,
234 description => "Enable video streaming. Uses compression for detected video streams."
235 },
236};
237
238my $rng_fmt = {
239 source => {
240 type => 'string',
241 enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
242 default_key => 1,
243 description => "The file on the host to gather entropy from. In most"
244 . " cases /dev/urandom should be preferred over /dev/random"
245 . " to avoid entropy-starvation issues on the host. Using"
246 . " urandom does *not* decrease security in any meaningful"
247 . " way, as it's still seeded from real entropy, and the"
248 . " bytes provided will most likely be mixed with real"
249 . " entropy on the guest as well. /dev/hwrng can be used"
250 . " to pass through a hardware RNG from the host.",
251 },
252 max_bytes => {
253 type => 'integer',
254 description => "Maximum bytes of entropy injected into the guest every"
255 . " 'period' milliseconds. Prefer a lower value when using"
256 . " /dev/random as source. Use 0 to disable limiting"
257 . " (potentially dangerous!).",
258 optional => 1,
259
260 # default is 1 KiB/s, provides enough entropy to the guest to avoid
261 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
262 # of overwhelming the host, provided we're reading from /dev/urandom
263 default => 1024,
264 },
265 period => {
266 type => 'integer',
267 description => "Every 'period' milliseconds the entropy-injection quota"
268 . " is reset, allowing the guest to retrieve another"
269 . " 'max_bytes' of entropy.",
270 optional => 1,
271 default => 1000,
272 },
273};
274
275my $confdesc = {
276 onboot => {
277 optional => 1,
278 type => 'boolean',
279 description => "Specifies whether a VM will be started during system bootup.",
280 default => 0,
281 },
282 autostart => {
283 optional => 1,
284 type => 'boolean',
285 description => "Automatic restart after crash (currently ignored).",
286 default => 0,
287 },
288 hotplug => {
289 optional => 1,
290 type => 'string', format => 'pve-hotplug-features',
291 description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
292 default => 'network,disk,usb',
293 },
294 reboot => {
295 optional => 1,
296 type => 'boolean',
297 description => "Allow reboot. If set to '0' the VM exit on reboot.",
298 default => 1,
299 },
300 lock => {
301 optional => 1,
302 type => 'string',
303 description => "Lock/unlock the VM.",
304 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
305 },
306 cpulimit => {
307 optional => 1,
308 type => 'number',
309 description => "Limit of CPU usage.",
310 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
311 minimum => 0,
312 maximum => 128,
313 default => 0,
314 },
315 cpuunits => {
316 optional => 1,
317 type => 'integer',
318 description => "CPU weight for a VM.",
319 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
320 minimum => 2,
321 maximum => 262144,
322 default => 1024,
323 },
324 memory => {
325 optional => 1,
326 type => 'integer',
327 description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
328 minimum => 16,
329 default => 512,
330 },
331 balloon => {
332 optional => 1,
333 type => 'integer',
334 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
335 minimum => 0,
336 },
337 shares => {
338 optional => 1,
339 type => 'integer',
340 description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
341 minimum => 0,
342 maximum => 50000,
343 default => 1000,
344 },
345 keyboard => {
346 optional => 1,
347 type => 'string',
348 description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
349 "It should not be necessary to set it.",
350 enum => PVE::Tools::kvmkeymaplist(),
351 default => undef,
352 },
353 name => {
354 optional => 1,
355 type => 'string', format => 'dns-name',
356 description => "Set a name for the VM. Only used on the configuration web interface.",
357 },
358 scsihw => {
359 optional => 1,
360 type => 'string',
361 description => "SCSI controller model",
362 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
363 default => 'lsi',
364 },
365 description => {
366 optional => 1,
367 type => 'string',
368 description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
369 },
370 ostype => {
371 optional => 1,
372 type => 'string',
373 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
374 description => "Specify guest operating system.",
375 verbose_description => <<EODESC,
376Specify guest operating system. This is used to enable special
377optimization/features for specific operating systems:
378
379[horizontal]
380other;; unspecified OS
381wxp;; Microsoft Windows XP
382w2k;; Microsoft Windows 2000
383w2k3;; Microsoft Windows 2003
384w2k8;; Microsoft Windows 2008
385wvista;; Microsoft Windows Vista
386win7;; Microsoft Windows 7
387win8;; Microsoft Windows 8/2012/2012r2
388win10;; Microsoft Windows 10/2016/2019
389l24;; Linux 2.4 Kernel
390l26;; Linux 2.6 - 5.X Kernel
391solaris;; Solaris/OpenSolaris/OpenIndiania kernel
392EODESC
393 },
394 boot => {
395 optional => 1,
396 type => 'string', format => 'pve-qm-boot',
397 description => "Specify guest boot order. Use with 'order=', usage with"
398 . " no key or 'legacy=' is deprecated.",
399 },
400 bootdisk => {
401 optional => 1,
402 type => 'string', format => 'pve-qm-bootdisk',
403 description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
404 pattern => '(ide|sata|scsi|virtio)\d+',
405 },
406 smp => {
407 optional => 1,
408 type => 'integer',
409 description => "The number of CPUs. Please use option -sockets instead.",
410 minimum => 1,
411 default => 1,
412 },
413 sockets => {
414 optional => 1,
415 type => 'integer',
416 description => "The number of CPU sockets.",
417 minimum => 1,
418 default => 1,
419 },
420 cores => {
421 optional => 1,
422 type => 'integer',
423 description => "The number of cores per socket.",
424 minimum => 1,
425 default => 1,
426 },
427 numa => {
428 optional => 1,
429 type => 'boolean',
430 description => "Enable/disable NUMA.",
431 default => 0,
432 },
433 hugepages => {
434 optional => 1,
435 type => 'string',
436 description => "Enable/disable hugepages memory.",
437 enum => [qw(any 2 1024)],
438 },
439 keephugepages => {
440 optional => 1,
441 type => 'boolean',
442 default => 0,
443 description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
444 ." after VM shutdown and can be used for subsequent starts.",
445 },
446 vcpus => {
447 optional => 1,
448 type => 'integer',
449 description => "Number of hotplugged vcpus.",
450 minimum => 1,
451 default => 0,
452 },
453 acpi => {
454 optional => 1,
455 type => 'boolean',
456 description => "Enable/disable ACPI.",
457 default => 1,
458 },
459 agent => {
460 optional => 1,
461 description => "Enable/disable Qemu GuestAgent and its properties.",
462 type => 'string',
463 format => $agent_fmt,
464 },
465 kvm => {
466 optional => 1,
467 type => 'boolean',
468 description => "Enable/disable KVM hardware virtualization.",
469 default => 1,
470 },
471 tdf => {
472 optional => 1,
473 type => 'boolean',
474 description => "Enable/disable time drift fix.",
475 default => 0,
476 },
477 localtime => {
478 optional => 1,
479 type => 'boolean',
480 description => "Set the real time clock to local time. This is enabled by default if ostype"
481 ." indicates a Microsoft OS.",
482 },
483 freeze => {
484 optional => 1,
485 type => 'boolean',
486 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
487 },
488 vga => {
489 optional => 1,
490 type => 'string', format => $vga_fmt,
491 description => "Configure the VGA hardware.",
492 verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
493 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
494 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
495 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
496 ." display server. For win* OS you can select how many independent displays you want,"
497 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
498 ." using a serial device as terminal.",
499 },
500 watchdog => {
501 optional => 1,
502 type => 'string', format => 'pve-qm-watchdog',
503 description => "Create a virtual hardware watchdog device.",
504 verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
505 ." action), the watchdog must be periodically polled by an agent inside the guest or"
506 ." else the watchdog will reset the guest (or execute the respective action specified)",
507 },
508 startdate => {
509 optional => 1,
510 type => 'string',
511 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
512 description => "Set the initial date of the real time clock. Valid format for date are:"
513 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
514 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
515 default => 'now',
516 },
517 startup => get_standard_option('pve-startup-order'),
518 template => {
519 optional => 1,
520 type => 'boolean',
521 description => "Enable/disable Template.",
522 default => 0,
523 },
524 args => {
525 optional => 1,
526 type => 'string',
527 description => "Arbitrary arguments passed to kvm.",
528 verbose_description => <<EODESCR,
529Arbitrary arguments passed to kvm, for example:
530
531args: -no-reboot -no-hpet
532
533NOTE: this option is for experts only.
534EODESCR
535 },
536 tablet => {
537 optional => 1,
538 type => 'boolean',
539 default => 1,
540 description => "Enable/disable the USB tablet device.",
541 verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
542 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
543 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
544 ." may consider disabling this to save some context switches. This is turned off by"
545 ." default if you use spice (`qm set <vmid> --vga qxl`).",
546 },
547 migrate_speed => {
548 optional => 1,
549 type => 'integer',
550 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
551 minimum => 0,
552 default => 0,
553 },
554 migrate_downtime => {
555 optional => 1,
556 type => 'number',
557 description => "Set maximum tolerated downtime (in seconds) for migrations.",
558 minimum => 0,
559 default => 0.1,
560 },
561 cdrom => {
562 optional => 1,
563 type => 'string', format => 'pve-qm-ide',
564 typetext => '<volume>',
565 description => "This is an alias for option -ide2",
566 },
567 cpu => {
568 optional => 1,
569 description => "Emulated CPU type.",
570 type => 'string',
571 format => 'pve-vm-cpu-conf',
572 },
573 parent => get_standard_option('pve-snapshot-name', {
574 optional => 1,
575 description => "Parent snapshot name. This is used internally, and should not be modified.",
576 }),
577 snaptime => {
578 optional => 1,
579 description => "Timestamp for snapshots.",
580 type => 'integer',
581 minimum => 0,
582 },
583 vmstate => {
584 optional => 1,
585 type => 'string', format => 'pve-volume-id',
586 description => "Reference to a volume which stores the VM state. This is used internally"
587 ." for snapshots.",
588 },
589 vmstatestorage => get_standard_option('pve-storage-id', {
590 description => "Default storage for VM state volumes/files.",
591 optional => 1,
592 }),
593 runningmachine => get_standard_option('pve-qemu-machine', {
594 description => "Specifies the QEMU machine type of the running vm. This is used internally"
595 ." for snapshots.",
596 }),
597 runningcpu => {
598 description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
599 ." internally for snapshots.",
600 optional => 1,
601 type => 'string',
602 pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
603 format_description => 'QEMU -cpu parameter'
604 },
605 machine => get_standard_option('pve-qemu-machine'),
606 arch => {
607 description => "Virtual processor architecture. Defaults to the host.",
608 optional => 1,
609 type => 'string',
610 enum => [qw(x86_64 aarch64)],
611 },
612 smbios1 => {
613 description => "Specify SMBIOS type 1 fields.",
614 type => 'string', format => 'pve-qm-smbios1',
615 maxLength => 512,
616 optional => 1,
617 },
618 protection => {
619 optional => 1,
620 type => 'boolean',
621 description => "Sets the protection flag of the VM. This will disable the remove VM and"
622 ." remove disk operations.",
623 default => 0,
624 },
625 bios => {
626 optional => 1,
627 type => 'string',
628 enum => [ qw(seabios ovmf) ],
629 description => "Select BIOS implementation.",
630 default => 'seabios',
631 },
632 vmgenid => {
633 type => 'string',
634 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
635 format_description => 'UUID',
636 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
637 ." to disable explicitly.",
638 verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
639 ." value identifier to the guest OS. This allows to notify the guest operating system"
640 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
641 ." execution or creation from a template). The guest operating system notices the"
642 ." change, and is then able to react as appropriate by marking its copies of"
643 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
644 ."Note that auto-creation only works when done through API/CLI create or update methods"
645 .", but not when manually editing the config file.",
646 default => "1 (autogenerated)",
647 optional => 1,
648 },
649 hookscript => {
650 type => 'string',
651 format => 'pve-volume-id',
652 optional => 1,
653 description => "Script that will be executed during various steps in the vms lifetime.",
654 },
655 ivshmem => {
656 type => 'string',
657 format => $ivshmem_fmt,
658 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
659 ." the host.",
660 optional => 1,
661 },
662 audio0 => {
663 type => 'string',
664 format => $audio_fmt,
665 description => "Configure a audio device, useful in combination with QXL/Spice.",
666 optional => 1
667 },
668 spice_enhancements => {
669 type => 'string',
670 format => $spice_enhancements_fmt,
671 description => "Configure additional enhancements for SPICE.",
672 optional => 1
673 },
674 tags => {
675 type => 'string', format => 'pve-tag-list',
676 description => 'Tags of the VM. This is only meta information.',
677 optional => 1,
678 },
679 rng0 => {
680 type => 'string',
681 format => $rng_fmt,
682 description => "Configure a VirtIO-based Random Number Generator.",
683 optional => 1,
684 },
685};
686
687my $cicustom_fmt = {
688 meta => {
689 type => 'string',
690 optional => 1,
691 description => 'Specify a custom file containing all meta data passed to the VM via"
692 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
693 format => 'pve-volume-id',
694 format_description => 'volume',
695 },
696 network => {
697 type => 'string',
698 optional => 1,
699 description => 'Specify a custom file containing all network data passed to the VM via'
700 .' cloud-init.',
701 format => 'pve-volume-id',
702 format_description => 'volume',
703 },
704 user => {
705 type => 'string',
706 optional => 1,
707 description => 'Specify a custom file containing all user data passed to the VM via'
708 .' cloud-init.',
709 format => 'pve-volume-id',
710 format_description => 'volume',
711 },
712};
713PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
714
715my $confdesc_cloudinit = {
716 citype => {
717 optional => 1,
718 type => 'string',
719 description => 'Specifies the cloud-init configuration format. The default depends on the'
720 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
721 .' and `configdrive2` for windows.',
722 enum => ['configdrive2', 'nocloud', 'opennebula'],
723 },
724 ciuser => {
725 optional => 1,
726 type => 'string',
727 description => "cloud-init: User name to change ssh keys and password for instead of the"
728 ." image's configured default user.",
729 },
730 cipassword => {
731 optional => 1,
732 type => 'string',
733 description => 'cloud-init: Password to assign the user. Using this is generally not'
734 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
735 .' support hashed passwords.',
736 },
737 cicustom => {
738 optional => 1,
739 type => 'string',
740 description => 'cloud-init: Specify custom files to replace the automatically generated'
741 .' ones at start.',
742 format => 'pve-qm-cicustom',
743 },
744 searchdomain => {
745 optional => 1,
746 type => 'string',
747 description => "cloud-init: Sets DNS search domains for a container. Create will'
748 .' automatically use the setting from the host if neither searchdomain nor nameserver'
749 .' are set.",
750 },
751 nameserver => {
752 optional => 1,
753 type => 'string', format => 'address-list',
754 description => "cloud-init: Sets DNS server IP address for a container. Create will'
755 .' automatically use the setting from the host if neither searchdomain nor nameserver'
756 .' are set.",
757 },
758 sshkeys => {
759 optional => 1,
760 type => 'string',
761 format => 'urlencoded',
762 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
763 },
764};
765
766# what about other qemu settings ?
767#cpu => 'string',
768#machine => 'string',
769#fda => 'file',
770#fdb => 'file',
771#mtdblock => 'file',
772#sd => 'file',
773#pflash => 'file',
774#snapshot => 'bool',
775#bootp => 'file',
776##tftp => 'dir',
777##smb => 'dir',
778#kernel => 'file',
779#append => 'string',
780#initrd => 'file',
781##soundhw => 'string',
782
783while (my ($k, $v) = each %$confdesc) {
784 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
785}
786
787my $MAX_USB_DEVICES = 5;
788my $MAX_NETS = 32;
789my $MAX_SERIAL_PORTS = 4;
790my $MAX_PARALLEL_PORTS = 3;
791my $MAX_NUMA = 8;
792
793my $numa_fmt = {
794 cpus => {
795 type => "string",
796 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
797 description => "CPUs accessing this NUMA node.",
798 format_description => "id[-id];...",
799 },
800 memory => {
801 type => "number",
802 description => "Amount of memory this NUMA node provides.",
803 optional => 1,
804 },
805 hostnodes => {
806 type => "string",
807 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
808 description => "Host NUMA nodes to use.",
809 format_description => "id[-id];...",
810 optional => 1,
811 },
812 policy => {
813 type => 'string',
814 enum => [qw(preferred bind interleave)],
815 description => "NUMA allocation policy.",
816 optional => 1,
817 },
818};
819PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
820my $numadesc = {
821 optional => 1,
822 type => 'string', format => $numa_fmt,
823 description => "NUMA topology.",
824};
825PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
826
827for (my $i = 0; $i < $MAX_NUMA; $i++) {
828 $confdesc->{"numa$i"} = $numadesc;
829}
830
831my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
832 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
833 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
834my $nic_model_list_txt = join(' ', sort @$nic_model_list);
835
836my $net_fmt_bridge_descr = <<__EOD__;
837Bridge to attach the network device to. The Proxmox VE standard bridge
838is called 'vmbr0'.
839
840If you do not specify a bridge, we create a kvm user (NATed) network
841device, which provides DHCP and DNS services. The following addresses
842are used:
843
844 10.0.2.2 Gateway
845 10.0.2.3 DNS Server
846 10.0.2.4 SMB Server
847
848The DHCP server assign addresses to the guest starting from 10.0.2.15.
849__EOD__
850
851my $net_fmt = {
852 macaddr => get_standard_option('mac-addr', {
853 description => "MAC address. That address must be unique withing your network. This is"
854 ." automatically generated if not specified.",
855 }),
856 model => {
857 type => 'string',
858 description => "Network Card Model. The 'virtio' model provides the best performance with"
859 ." very low CPU overhead. If your guest does not support this driver, it is usually"
860 ." best to use 'e1000'.",
861 enum => $nic_model_list,
862 default_key => 1,
863 },
864 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
865 bridge => {
866 type => 'string',
867 description => $net_fmt_bridge_descr,
868 format_description => 'bridge',
869 pattern => '[-_.\w\d]+',
870 optional => 1,
871 },
872 queues => {
873 type => 'integer',
874 minimum => 0, maximum => 16,
875 description => 'Number of packet queues to be used on the device.',
876 optional => 1,
877 },
878 rate => {
879 type => 'number',
880 minimum => 0,
881 description => "Rate limit in mbps (megabytes per second) as floating point number.",
882 optional => 1,
883 },
884 tag => {
885 type => 'integer',
886 minimum => 1, maximum => 4094,
887 description => 'VLAN tag to apply to packets on this interface.',
888 optional => 1,
889 },
890 trunks => {
891 type => 'string',
892 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
893 description => 'VLAN trunks to pass through this interface.',
894 format_description => 'vlanid[;vlanid...]',
895 optional => 1,
896 },
897 firewall => {
898 type => 'boolean',
899 description => 'Whether this interface should be protected by the firewall.',
900 optional => 1,
901 },
902 link_down => {
903 type => 'boolean',
904 description => 'Whether this interface should be disconnected (like pulling the plug).',
905 optional => 1,
906 },
907 mtu => {
908 type => 'integer',
909 minimum => 1, maximum => 65520,
910 description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
911 optional => 1,
912 },
913};
914
915my $netdesc = {
916 optional => 1,
917 type => 'string', format => $net_fmt,
918 description => "Specify network devices.",
919};
920
921PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
922
923my $ipconfig_fmt = {
924 ip => {
925 type => 'string',
926 format => 'pve-ipv4-config',
927 format_description => 'IPv4Format/CIDR',
928 description => 'IPv4 address in CIDR format.',
929 optional => 1,
930 default => 'dhcp',
931 },
932 gw => {
933 type => 'string',
934 format => 'ipv4',
935 format_description => 'GatewayIPv4',
936 description => 'Default gateway for IPv4 traffic.',
937 optional => 1,
938 requires => 'ip',
939 },
940 ip6 => {
941 type => 'string',
942 format => 'pve-ipv6-config',
943 format_description => 'IPv6Format/CIDR',
944 description => 'IPv6 address in CIDR format.',
945 optional => 1,
946 default => 'dhcp',
947 },
948 gw6 => {
949 type => 'string',
950 format => 'ipv6',
951 format_description => 'GatewayIPv6',
952 description => 'Default gateway for IPv6 traffic.',
953 optional => 1,
954 requires => 'ip6',
955 },
956};
957PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
958my $ipconfigdesc = {
959 optional => 1,
960 type => 'string', format => 'pve-qm-ipconfig',
961 description => <<'EODESCR',
962cloud-init: Specify IP addresses and gateways for the corresponding interface.
963
964IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
965
966The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
967gateway should be provided.
968For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
969cloud-init 19.4 or newer.
970
971If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
972dhcp on IPv4.
973EODESCR
974};
975PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
976
977for (my $i = 0; $i < $MAX_NETS; $i++) {
978 $confdesc->{"net$i"} = $netdesc;
979 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
980}
981
982foreach my $key (keys %$confdesc_cloudinit) {
983 $confdesc->{$key} = $confdesc_cloudinit->{$key};
984}
985
986PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
987sub verify_volume_id_or_qm_path {
988 my ($volid, $noerr) = @_;
989
990 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
991 return $volid;
992 }
993
994 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
995 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
996 if ($@) {
997 return if $noerr;
998 die $@;
999 }
1000 return $volid;
1001}
1002
1003my $usb_fmt = {
1004 host => {
1005 default_key => 1,
1006 type => 'string', format => 'pve-qm-usb-device',
1007 format_description => 'HOSTUSBDEVICE|spice',
1008 description => <<EODESCR,
1009The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1010
1011 'bus-port(.port)*' (decimal numbers) or
1012 'vendor_id:product_id' (hexadeciaml numbers) or
1013 'spice'
1014
1015You can use the 'lsusb -t' command to list existing usb devices.
1016
1017NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1018machines - use with special care.
1019
1020The value 'spice' can be used to add a usb redirection devices for spice.
1021EODESCR
1022 },
1023 usb3 => {
1024 optional => 1,
1025 type => 'boolean',
1026 description => "Specifies whether if given host option is a USB3 device or port.",
1027 default => 0,
1028 },
1029};
1030
1031my $usbdesc = {
1032 optional => 1,
1033 type => 'string', format => $usb_fmt,
1034 description => "Configure an USB device (n is 0 to 4).",
1035};
1036PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1037
1038my $serialdesc = {
1039 optional => 1,
1040 type => 'string',
1041 pattern => '(/dev/.+|socket)',
1042 description => "Create a serial device inside the VM (n is 0 to 3)",
1043 verbose_description => <<EODESCR,
1044Create a serial device inside the VM (n is 0 to 3), and pass through a
1045host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1046host side (use 'qm terminal' to open a terminal connection).
1047
1048NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1049use with special care.
1050
1051CAUTION: Experimental! User reported problems with this option.
1052EODESCR
1053};
1054
1055my $paralleldesc= {
1056 optional => 1,
1057 type => 'string',
1058 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1059 description => "Map host parallel devices (n is 0 to 2).",
1060 verbose_description => <<EODESCR,
1061Map host parallel devices (n is 0 to 2).
1062
1063NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1064machines - use with special care.
1065
1066CAUTION: Experimental! User reported problems with this option.
1067EODESCR
1068};
1069
1070for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1071 $confdesc->{"parallel$i"} = $paralleldesc;
1072}
1073
1074for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1075 $confdesc->{"serial$i"} = $serialdesc;
1076}
1077
1078for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
1079 $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1080}
1081
1082for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1083 $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1084}
1085
1086for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1087 $confdesc->{"usb$i"} = $usbdesc;
1088}
1089
1090my $boot_fmt = {
1091 legacy => {
1092 optional => 1,
1093 default_key => 1,
1094 type => 'string',
1095 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1096 . " Deprecated, use 'order=' instead.",
1097 pattern => '[acdn]{1,4}',
1098 format_description => "[acdn]{1,4}",
1099
1100 # note: this is also the fallback if boot: is not given at all
1101 default => 'cdn',
1102 },
1103 order => {
1104 optional => 1,
1105 type => 'string',
1106 format => 'pve-qm-bootdev-list',
1107 format_description => "device[;device...]",
1108 description => <<EODESC,
1109The guest will attempt to boot from devices in the order they appear here.
1110
1111Disks, optical drives and passed-through storage USB devices will be directly
1112booted from, NICs will load PXE, and PCIe devices will either behave like disks
1113(e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1114
1115Note that only devices in this list will be marked as bootable and thus loaded
1116by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1117(e.g. software-raid), you need to specify all of them here.
1118
1119Overrides the deprecated 'legacy=[acdn]*' value when given.
1120EODESC
1121 },
1122};
1123PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1124
1125PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1126sub verify_bootdev {
1127 my ($dev, $noerr) = @_;
1128
1129 return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1130
1131 my $check = sub {
1132 my ($base) = @_;
1133 return 0 if $dev !~ m/^$base\d+$/;
1134 return 0 if !$confdesc->{$dev};
1135 return 1;
1136 };
1137
1138 return $dev if $check->("net");
1139 return $dev if $check->("usb");
1140 return $dev if $check->("hostpci");
1141
1142 return if $noerr;
1143 die "invalid boot device '$dev'\n";
1144}
1145
1146sub print_bootorder {
1147 my ($devs) = @_;
1148 return "" if !@$devs;
1149 my $data = { order => join(';', @$devs) };
1150 return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1151}
1152
1153my $kvm_api_version = 0;
1154
1155sub kvm_version {
1156 return $kvm_api_version if $kvm_api_version;
1157
1158 open my $fh, '<', '/dev/kvm' or return;
1159
1160 # 0xae00 => KVM_GET_API_VERSION
1161 $kvm_api_version = ioctl($fh, 0xae00, 0);
1162 close($fh);
1163
1164 return $kvm_api_version;
1165}
1166
1167my $kvm_user_version = {};
1168my $kvm_mtime = {};
1169
1170sub kvm_user_version {
1171 my ($binary) = @_;
1172
1173 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1174 my $st = stat($binary);
1175
1176 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1177 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1178 $cachedmtime == $st->mtime;
1179
1180 $kvm_user_version->{$binary} = 'unknown';
1181 $kvm_mtime->{$binary} = $st->mtime;
1182
1183 my $code = sub {
1184 my $line = shift;
1185 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186 $kvm_user_version->{$binary} = $2;
1187 }
1188 };
1189
1190 eval { run_command([$binary, '--version'], outfunc => $code); };
1191 warn $@ if $@;
1192
1193 return $kvm_user_version->{$binary};
1194
1195}
1196my sub extract_version {
1197 my ($machine_type, $version) = @_;
1198 $version = kvm_user_version() if !defined($version);
1199 PVE::QemuServer::Machine::extract_version($machine_type, $version)
1200}
1201
1202sub kernel_has_vhost_net {
1203 return -c '/dev/vhost-net';
1204}
1205
1206sub option_exists {
1207 my $key = shift;
1208 return defined($confdesc->{$key});
1209}
1210
1211my $cdrom_path;
1212sub get_cdrom_path {
1213
1214 return $cdrom_path if $cdrom_path;
1215
1216 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1217 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1218 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1219}
1220
1221sub get_iso_path {
1222 my ($storecfg, $vmid, $cdrom) = @_;
1223
1224 if ($cdrom eq 'cdrom') {
1225 return get_cdrom_path();
1226 } elsif ($cdrom eq 'none') {
1227 return '';
1228 } elsif ($cdrom =~ m|^/|) {
1229 return $cdrom;
1230 } else {
1231 return PVE::Storage::path($storecfg, $cdrom);
1232 }
1233}
1234
1235# try to convert old style file names to volume IDs
1236sub filename_to_volume_id {
1237 my ($vmid, $file, $media) = @_;
1238
1239 if (!($file eq 'none' || $file eq 'cdrom' ||
1240 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1241
1242 return if $file =~ m|/|;
1243
1244 if ($media && $media eq 'cdrom') {
1245 $file = "local:iso/$file";
1246 } else {
1247 $file = "local:$vmid/$file";
1248 }
1249 }
1250
1251 return $file;
1252}
1253
1254sub verify_media_type {
1255 my ($opt, $vtype, $media) = @_;
1256
1257 return if !$media;
1258
1259 my $etype;
1260 if ($media eq 'disk') {
1261 $etype = 'images';
1262 } elsif ($media eq 'cdrom') {
1263 $etype = 'iso';
1264 } else {
1265 die "internal error";
1266 }
1267
1268 return if ($vtype eq $etype);
1269
1270 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1271}
1272
1273sub cleanup_drive_path {
1274 my ($opt, $storecfg, $drive) = @_;
1275
1276 # try to convert filesystem paths to volume IDs
1277
1278 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1279 ($drive->{file} !~ m|^/dev/.+|) &&
1280 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1281 ($drive->{file} !~ m/^\d+$/)) {
1282 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1283 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1284 if !$vtype;
1285 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1286 verify_media_type($opt, $vtype, $drive->{media});
1287 $drive->{file} = $volid;
1288 }
1289
1290 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1291}
1292
1293sub parse_hotplug_features {
1294 my ($data) = @_;
1295
1296 my $res = {};
1297
1298 return $res if $data eq '0';
1299
1300 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1301
1302 foreach my $feature (PVE::Tools::split_list($data)) {
1303 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1304 $res->{$1} = 1;
1305 } else {
1306 die "invalid hotplug feature '$feature'\n";
1307 }
1308 }
1309 return $res;
1310}
1311
1312PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1313sub pve_verify_hotplug_features {
1314 my ($value, $noerr) = @_;
1315
1316 return $value if parse_hotplug_features($value);
1317
1318 return if $noerr;
1319
1320 die "unable to parse hotplug option\n";
1321}
1322
1323sub scsi_inquiry {
1324 my($fh, $noerr) = @_;
1325
1326 my $SG_IO = 0x2285;
1327 my $SG_GET_VERSION_NUM = 0x2282;
1328
1329 my $versionbuf = "\x00" x 8;
1330 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1331 if (!$ret) {
1332 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1333 return;
1334 }
1335 my $version = unpack("I", $versionbuf);
1336 if ($version < 30000) {
1337 die "scsi generic interface too old\n" if !$noerr;
1338 return;
1339 }
1340
1341 my $buf = "\x00" x 36;
1342 my $sensebuf = "\x00" x 8;
1343 my $cmd = pack("C x3 C x1", 0x12, 36);
1344
1345 # see /usr/include/scsi/sg.h
1346 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1347
1348 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1349 length($sensebuf), 0, length($buf), $buf,
1350 $cmd, $sensebuf, 6000);
1351
1352 $ret = ioctl($fh, $SG_IO, $packet);
1353 if (!$ret) {
1354 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1355 return;
1356 }
1357
1358 my @res = unpack($sg_io_hdr_t, $packet);
1359 if ($res[17] || $res[18]) {
1360 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1361 return;
1362 }
1363
1364 my $res = {};
1365 (my $byte0, my $byte1, $res->{vendor},
1366 $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1367
1368 $res->{removable} = $byte1 & 128 ? 1 : 0;
1369 $res->{type} = $byte0 & 31;
1370
1371 return $res;
1372}
1373
1374sub path_is_scsi {
1375 my ($path) = @_;
1376
1377 my $fh = IO::File->new("+<$path") || return;
1378 my $res = scsi_inquiry($fh, 1);
1379 close($fh);
1380
1381 return $res;
1382}
1383
1384sub print_tabletdevice_full {
1385 my ($conf, $arch) = @_;
1386
1387 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1388
1389 # we use uhci for old VMs because tablet driver was buggy in older qemu
1390 my $usbbus;
1391 if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1392 $usbbus = 'ehci';
1393 } else {
1394 $usbbus = 'uhci';
1395 }
1396
1397 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1398}
1399
1400sub print_keyboarddevice_full {
1401 my ($conf, $arch, $machine) = @_;
1402
1403 return if $arch ne 'aarch64';
1404
1405 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1406}
1407
1408my sub get_drive_id {
1409 my ($drive) = @_;
1410 return "$drive->{interface}$drive->{index}";
1411}
1412
1413sub print_drivedevice_full {
1414 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1415
1416 my $device = '';
1417 my $maxdev = 0;
1418
1419 my $drive_id = get_drive_id($drive);
1420 if ($drive->{interface} eq 'virtio') {
1421 my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1422 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1423 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1424 } elsif ($drive->{interface} eq 'scsi') {
1425
1426 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1427 my $unit = $drive->{index} % $maxdev;
1428 my $devicetype = 'hd';
1429 my $path = '';
1430 if (drive_is_cdrom($drive)) {
1431 $devicetype = 'cd';
1432 } else {
1433 if ($drive->{file} =~ m|^/|) {
1434 $path = $drive->{file};
1435 if (my $info = path_is_scsi($path)) {
1436 if ($info->{type} == 0 && $drive->{scsiblock}) {
1437 $devicetype = 'block';
1438 } elsif ($info->{type} == 1) { # tape
1439 $devicetype = 'generic';
1440 }
1441 }
1442 } else {
1443 $path = PVE::Storage::path($storecfg, $drive->{file});
1444 }
1445
1446 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1447 my $version = extract_version($machine_type, kvm_user_version());
1448 if ($path =~ m/^iscsi\:\/\// &&
1449 !min_version($version, 4, 1)) {
1450 $devicetype = 'generic';
1451 }
1452 }
1453
1454 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1455 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1456 } else {
1457 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1458 .",lun=$drive->{index}";
1459 }
1460 $device .= ",drive=drive-$drive_id,id=$drive_id";
1461
1462 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1463 $device .= ",rotation_rate=1";
1464 }
1465 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1466
1467 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1468 my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1469 my $controller = int($drive->{index} / $maxdev);
1470 my $unit = $drive->{index} % $maxdev;
1471 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1472
1473 $device = "ide-$devicetype";
1474 if ($drive->{interface} eq 'ide') {
1475 $device .= ",bus=ide.$controller,unit=$unit";
1476 } else {
1477 $device .= ",bus=ahci$controller.$unit";
1478 }
1479 $device .= ",drive=drive-$drive_id,id=$drive_id";
1480
1481 if ($devicetype eq 'hd') {
1482 if (my $model = $drive->{model}) {
1483 $model = URI::Escape::uri_unescape($model);
1484 $device .= ",model=$model";
1485 }
1486 if ($drive->{ssd}) {
1487 $device .= ",rotation_rate=1";
1488 }
1489 }
1490 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1491 } elsif ($drive->{interface} eq 'usb') {
1492 die "implement me";
1493 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1494 } else {
1495 die "unsupported interface type";
1496 }
1497
1498 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1499
1500 if (my $serial = $drive->{serial}) {
1501 $serial = URI::Escape::uri_unescape($serial);
1502 $device .= ",serial=$serial";
1503 }
1504
1505
1506 return $device;
1507}
1508
1509sub get_initiator_name {
1510 my $initiator;
1511
1512 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1513 while (defined(my $line = <$fh>)) {
1514 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1515 $initiator = $1;
1516 last;
1517 }
1518 $fh->close();
1519
1520 return $initiator;
1521}
1522
1523sub print_drive_commandline_full {
1524 my ($storecfg, $vmid, $drive, $pbs_name) = @_;
1525
1526 my $path;
1527 my $volid = $drive->{file};
1528 my $format = $drive->{format};
1529 my $drive_id = get_drive_id($drive);
1530
1531 if (drive_is_cdrom($drive)) {
1532 $path = get_iso_path($storecfg, $vmid, $volid);
1533 die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1534 } else {
1535 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1536 if ($storeid) {
1537 $path = PVE::Storage::path($storecfg, $volid);
1538 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1539 $format //= qemu_img_format($scfg, $volname);
1540 } else {
1541 $path = $volid;
1542 $format //= "raw";
1543 }
1544 }
1545
1546 my $is_rbd = $path =~ m/^rbd:/;
1547
1548 my $opts = '';
1549 my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1550 foreach my $o (@qemu_drive_options) {
1551 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1552 }
1553
1554 # snapshot only accepts on|off
1555 if (defined($drive->{snapshot})) {
1556 my $v = $drive->{snapshot} ? 'on' : 'off';
1557 $opts .= ",snapshot=$v";
1558 }
1559
1560 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1561 my ($dir, $qmpname) = @$type;
1562 if (my $v = $drive->{"mbps$dir"}) {
1563 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1564 }
1565 if (my $v = $drive->{"mbps${dir}_max"}) {
1566 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1567 }
1568 if (my $v = $drive->{"bps${dir}_max_length"}) {
1569 $opts .= ",throttling.bps$qmpname-max-length=$v";
1570 }
1571 if (my $v = $drive->{"iops${dir}"}) {
1572 $opts .= ",throttling.iops$qmpname=$v";
1573 }
1574 if (my $v = $drive->{"iops${dir}_max"}) {
1575 $opts .= ",throttling.iops$qmpname-max=$v";
1576 }
1577 if (my $v = $drive->{"iops${dir}_max_length"}) {
1578 $opts .= ",throttling.iops$qmpname-max-length=$v";
1579 }
1580 }
1581
1582 if ($pbs_name) {
1583 $format = "rbd" if $is_rbd;
1584 die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1585 if !$format;
1586 $opts .= ",format=alloc-track,file.driver=$format";
1587 } elsif ($format) {
1588 $opts .= ",format=$format";
1589 }
1590
1591 my $cache_direct = 0;
1592
1593 if (my $cache = $drive->{cache}) {
1594 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1595 } elsif (!drive_is_cdrom($drive)) {
1596 $opts .= ",cache=none";
1597 $cache_direct = 1;
1598 }
1599
1600 # aio native works only with O_DIRECT
1601 if (!$drive->{aio}) {
1602 if($cache_direct) {
1603 $opts .= ",aio=native";
1604 } else {
1605 $opts .= ",aio=threads";
1606 }
1607 }
1608
1609 if (!drive_is_cdrom($drive)) {
1610 my $detectzeroes;
1611 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1612 $detectzeroes = 'off';
1613 } elsif ($drive->{discard}) {
1614 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1615 } else {
1616 # This used to be our default with discard not being specified:
1617 $detectzeroes = 'on';
1618 }
1619
1620 # note: 'detect-zeroes' works per blockdev and we want it to persist
1621 # after the alloc-track is removed, so put it on 'file' directly
1622 my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1623 $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1624 }
1625
1626 if ($pbs_name) {
1627 $opts .= ",backing=$pbs_name";
1628 $opts .= ",auto-remove=on";
1629 }
1630
1631 # my $file_param = $pbs_name ? "file.file.filename" : "file";
1632 my $file_param = "file";
1633 if ($pbs_name) {
1634 # non-rbd drivers require the underlying file to be a seperate block
1635 # node, so add a second .file indirection
1636 $file_param .= ".file" if !$is_rbd;
1637 $file_param .= ".filename";
1638 }
1639 my $pathinfo = $path ? "$file_param=$path," : '';
1640
1641 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1642}
1643
1644sub print_pbs_blockdev {
1645 my ($pbs_conf, $pbs_name) = @_;
1646 my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1647 $blockdev .= ",repository=$pbs_conf->{repository}";
1648 $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1649 $blockdev .= ",archive=$pbs_conf->{archive}";
1650 $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1651 return $blockdev;
1652}
1653
1654sub print_netdevice_full {
1655 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1656
1657 my $device = $net->{model};
1658 if ($net->{model} eq 'virtio') {
1659 $device = 'virtio-net-pci';
1660 };
1661
1662 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1663 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1664 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1665 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1666 # and out of each queue plus one config interrupt and control vector queue
1667 my $vectors = $net->{queues} * 2 + 2;
1668 $tmpstr .= ",vectors=$vectors,mq=on";
1669 }
1670 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1671
1672 if (my $mtu = $net->{mtu}) {
1673 if ($net->{model} eq 'virtio' && $net->{bridge}) {
1674 my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1675 if ($mtu == 1) {
1676 $mtu = $bridge_mtu;
1677 } elsif ($mtu < 576) {
1678 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1679 } elsif ($mtu > $bridge_mtu) {
1680 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1681 }
1682 $tmpstr .= ",host_mtu=$mtu";
1683 } else {
1684 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1685 }
1686 }
1687
1688 if ($use_old_bios_files) {
1689 my $romfile;
1690 if ($device eq 'virtio-net-pci') {
1691 $romfile = 'pxe-virtio.rom';
1692 } elsif ($device eq 'e1000') {
1693 $romfile = 'pxe-e1000.rom';
1694 } elsif ($device eq 'ne2k') {
1695 $romfile = 'pxe-ne2k_pci.rom';
1696 } elsif ($device eq 'pcnet') {
1697 $romfile = 'pxe-pcnet.rom';
1698 } elsif ($device eq 'rtl8139') {
1699 $romfile = 'pxe-rtl8139.rom';
1700 }
1701 $tmpstr .= ",romfile=$romfile" if $romfile;
1702 }
1703
1704 return $tmpstr;
1705}
1706
1707sub print_netdev_full {
1708 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1709
1710 my $i = '';
1711 if ($netid =~ m/^net(\d+)$/) {
1712 $i = int($1);
1713 }
1714
1715 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717 my $ifname = "tap${vmid}i$i";
1718
1719 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720 die "interface name '$ifname' is too long (max 15 character)\n"
1721 if length($ifname) >= 16;
1722
1723 my $vhostparam = '';
1724 if (is_native($arch)) {
1725 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1726 }
1727
1728 my $vmname = $conf->{name} || "vm$vmid";
1729
1730 my $netdev = "";
1731 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1732
1733 if ($net->{bridge}) {
1734 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1735 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1736 } else {
1737 $netdev = "type=user,id=$netid,hostname=$vmname";
1738 }
1739
1740 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1741
1742 return $netdev;
1743}
1744
1745my $vga_map = {
1746 'cirrus' => 'cirrus-vga',
1747 'std' => 'VGA',
1748 'vmware' => 'vmware-svga',
1749 'virtio' => 'virtio-vga',
1750};
1751
1752sub print_vga_device {
1753 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1754
1755 my $type = $vga_map->{$vga->{type}};
1756 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1757 $type = 'virtio-gpu';
1758 }
1759 my $vgamem_mb = $vga->{memory};
1760
1761 my $max_outputs = '';
1762 if ($qxlnum) {
1763 $type = $id ? 'qxl' : 'qxl-vga';
1764
1765 if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1766 # set max outputs so linux can have up to 4 qxl displays with one device
1767 if (min_version($machine_version, 4, 1)) {
1768 $max_outputs = ",max_outputs=4";
1769 }
1770 }
1771 }
1772
1773 die "no devicetype for $vga->{type}\n" if !$type;
1774
1775 my $memory = "";
1776 if ($vgamem_mb) {
1777 if ($vga->{type} eq 'virtio') {
1778 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1779 $memory = ",max_hostmem=$bytes";
1780 } elsif ($qxlnum) {
1781 # from https://www.spice-space.org/multiple-monitors.html
1782 $memory = ",vgamem_mb=$vga->{memory}";
1783 my $ram = $vgamem_mb * 4;
1784 my $vram = $vgamem_mb * 2;
1785 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1786 } else {
1787 $memory = ",vgamem_mb=$vga->{memory}";
1788 }
1789 } elsif ($qxlnum && $id) {
1790 $memory = ",ram_size=67108864,vram_size=33554432";
1791 }
1792
1793 my $edidoff = "";
1794 if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1795 $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1796 }
1797
1798 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1799 my $vgaid = "vga" . ($id // '');
1800 my $pciaddr;
1801
1802 if ($q35 && $vgaid eq 'vga') {
1803 # the first display uses pcie.0 bus on q35 machines
1804 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1805 } else {
1806 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1807 }
1808
1809 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1810}
1811
1812sub parse_number_sets {
1813 my ($set) = @_;
1814 my $res = [];
1815 foreach my $part (split(/;/, $set)) {
1816 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1817 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1818 push @$res, [ $1, $2 ];
1819 } else {
1820 die "invalid range: $part\n";
1821 }
1822 }
1823 return $res;
1824}
1825
1826sub parse_numa {
1827 my ($data) = @_;
1828
1829 my $res = parse_property_string($numa_fmt, $data);
1830 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1831 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1832 return $res;
1833}
1834
1835# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1836sub parse_net {
1837 my ($data) = @_;
1838
1839 my $res = eval { parse_property_string($net_fmt, $data) };
1840 if ($@) {
1841 warn $@;
1842 return;
1843 }
1844 if (!defined($res->{macaddr})) {
1845 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1846 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1847 }
1848 return $res;
1849}
1850
1851# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1852sub parse_ipconfig {
1853 my ($data) = @_;
1854
1855 my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1856 if ($@) {
1857 warn $@;
1858 return;
1859 }
1860
1861 if ($res->{gw} && !$res->{ip}) {
1862 warn 'gateway specified without specifying an IP address';
1863 return;
1864 }
1865 if ($res->{gw6} && !$res->{ip6}) {
1866 warn 'IPv6 gateway specified without specifying an IPv6 address';
1867 return;
1868 }
1869 if ($res->{gw} && $res->{ip} eq 'dhcp') {
1870 warn 'gateway specified together with DHCP';
1871 return;
1872 }
1873 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1874 # gw6 + auto/dhcp
1875 warn "IPv6 gateway specified together with $res->{ip6} address";
1876 return;
1877 }
1878
1879 if (!$res->{ip} && !$res->{ip6}) {
1880 return { ip => 'dhcp', ip6 => 'dhcp' };
1881 }
1882
1883 return $res;
1884}
1885
1886sub print_net {
1887 my $net = shift;
1888
1889 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1890}
1891
1892sub add_random_macs {
1893 my ($settings) = @_;
1894
1895 foreach my $opt (keys %$settings) {
1896 next if $opt !~ m/^net(\d+)$/;
1897 my $net = parse_net($settings->{$opt});
1898 next if !$net;
1899 $settings->{$opt} = print_net($net);
1900 }
1901}
1902
1903sub vm_is_volid_owner {
1904 my ($storecfg, $vmid, $volid) = @_;
1905
1906 if ($volid !~ m|^/|) {
1907 my ($path, $owner);
1908 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1909 if ($owner && ($owner == $vmid)) {
1910 return 1;
1911 }
1912 }
1913
1914 return;
1915}
1916
1917sub vmconfig_register_unused_drive {
1918 my ($storecfg, $vmid, $conf, $drive) = @_;
1919
1920 if (drive_is_cloudinit($drive)) {
1921 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1922 warn $@ if $@;
1923 } elsif (!drive_is_cdrom($drive)) {
1924 my $volid = $drive->{file};
1925 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1926 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1927 }
1928 }
1929}
1930
1931# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1932my $smbios1_fmt = {
1933 uuid => {
1934 type => 'string',
1935 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1936 format_description => 'UUID',
1937 description => "Set SMBIOS1 UUID.",
1938 optional => 1,
1939 },
1940 version => {
1941 type => 'string',
1942 pattern => '[A-Za-z0-9+\/]+={0,2}',
1943 format_description => 'Base64 encoded string',
1944 description => "Set SMBIOS1 version.",
1945 optional => 1,
1946 },
1947 serial => {
1948 type => 'string',
1949 pattern => '[A-Za-z0-9+\/]+={0,2}',
1950 format_description => 'Base64 encoded string',
1951 description => "Set SMBIOS1 serial number.",
1952 optional => 1,
1953 },
1954 manufacturer => {
1955 type => 'string',
1956 pattern => '[A-Za-z0-9+\/]+={0,2}',
1957 format_description => 'Base64 encoded string',
1958 description => "Set SMBIOS1 manufacturer.",
1959 optional => 1,
1960 },
1961 product => {
1962 type => 'string',
1963 pattern => '[A-Za-z0-9+\/]+={0,2}',
1964 format_description => 'Base64 encoded string',
1965 description => "Set SMBIOS1 product ID.",
1966 optional => 1,
1967 },
1968 sku => {
1969 type => 'string',
1970 pattern => '[A-Za-z0-9+\/]+={0,2}',
1971 format_description => 'Base64 encoded string',
1972 description => "Set SMBIOS1 SKU string.",
1973 optional => 1,
1974 },
1975 family => {
1976 type => 'string',
1977 pattern => '[A-Za-z0-9+\/]+={0,2}',
1978 format_description => 'Base64 encoded string',
1979 description => "Set SMBIOS1 family string.",
1980 optional => 1,
1981 },
1982 base64 => {
1983 type => 'boolean',
1984 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1985 optional => 1,
1986 },
1987};
1988
1989sub parse_smbios1 {
1990 my ($data) = @_;
1991
1992 my $res = eval { parse_property_string($smbios1_fmt, $data) };
1993 warn $@ if $@;
1994 return $res;
1995}
1996
1997sub print_smbios1 {
1998 my ($smbios1) = @_;
1999 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2000}
2001
2002PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2003
2004sub parse_watchdog {
2005 my ($value) = @_;
2006
2007 return if !$value;
2008
2009 my $res = eval { parse_property_string($watchdog_fmt, $value) };
2010 warn $@ if $@;
2011 return $res;
2012}
2013
2014sub parse_guest_agent {
2015 my ($conf) = @_;
2016
2017 return {} if !defined($conf->{agent});
2018
2019 my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2020 warn $@ if $@;
2021
2022 # if the agent is disabled ignore the other potentially set properties
2023 return {} if !$res->{enabled};
2024 return $res;
2025}
2026
2027sub get_qga_key {
2028 my ($conf, $key) = @_;
2029 return undef if !defined($conf->{agent});
2030
2031 my $agent = parse_guest_agent($conf);
2032 return $agent->{$key};
2033}
2034
2035sub parse_vga {
2036 my ($value) = @_;
2037
2038 return {} if !$value;
2039 my $res = eval { parse_property_string($vga_fmt, $value) };
2040 warn $@ if $@;
2041 return $res;
2042}
2043
2044sub parse_rng {
2045 my ($value) = @_;
2046
2047 return if !$value;
2048
2049 my $res = eval { parse_property_string($rng_fmt, $value) };
2050 warn $@ if $@;
2051 return $res;
2052}
2053
2054PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2055sub verify_usb_device {
2056 my ($value, $noerr) = @_;
2057
2058 return $value if parse_usb_device($value);
2059
2060 return if $noerr;
2061
2062 die "unable to parse usb device\n";
2063}
2064
2065# add JSON properties for create and set function
2066sub json_config_properties {
2067 my $prop = shift;
2068
2069 foreach my $opt (keys %$confdesc) {
2070 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2071 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2072 $prop->{$opt} = $confdesc->{$opt};
2073 }
2074
2075 return $prop;
2076}
2077
2078# return copy of $confdesc_cloudinit to generate documentation
2079sub cloudinit_config_properties {
2080
2081 return dclone($confdesc_cloudinit);
2082}
2083
2084sub check_type {
2085 my ($key, $value) = @_;
2086
2087 die "unknown setting '$key'\n" if !$confdesc->{$key};
2088
2089 my $type = $confdesc->{$key}->{type};
2090
2091 if (!defined($value)) {
2092 die "got undefined value\n";
2093 }
2094
2095 if ($value =~ m/[\n\r]/) {
2096 die "property contains a line feed\n";
2097 }
2098
2099 if ($type eq 'boolean') {
2100 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2101 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2102 die "type check ('boolean') failed - got '$value'\n";
2103 } elsif ($type eq 'integer') {
2104 return int($1) if $value =~ m/^(\d+)$/;
2105 die "type check ('integer') failed - got '$value'\n";
2106 } elsif ($type eq 'number') {
2107 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2108 die "type check ('number') failed - got '$value'\n";
2109 } elsif ($type eq 'string') {
2110 if (my $fmt = $confdesc->{$key}->{format}) {
2111 PVE::JSONSchema::check_format($fmt, $value);
2112 return $value;
2113 }
2114 $value =~ s/^\"(.*)\"$/$1/;
2115 return $value;
2116 } else {
2117 die "internal error"
2118 }
2119}
2120
2121sub destroy_vm {
2122 my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2123
2124 my $conf = PVE::QemuConfig->load_config($vmid);
2125
2126 PVE::QemuConfig->check_lock($conf) if !$skiplock;
2127
2128 if ($conf->{template}) {
2129 # check if any base image is still used by a linked clone
2130 PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2131 my ($ds, $drive) = @_;
2132 return if drive_is_cdrom($drive);
2133
2134 my $volid = $drive->{file};
2135 return if !$volid || $volid =~ m|^/|;
2136
2137 die "base volume '$volid' is still in use by linked cloned\n"
2138 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2139
2140 });
2141 }
2142
2143 my $remove_owned_drive = sub {
2144 my ($ds, $drive) = @_;
2145 return if drive_is_cdrom($drive, 1);
2146
2147 my $volid = $drive->{file};
2148 return if !$volid || $volid =~ m|^/|;
2149
2150 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2151 return if !$path || !$owner || ($owner != $vmid);
2152
2153 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2154 warn "Could not remove disk '$volid', check manually: $@" if $@;
2155 };
2156
2157 # only remove disks owned by this VM (referenced in the config)
2158 my $include_opts = {
2159 include_unused => 1,
2160 extra_keys => ['vmstate'],
2161 };
2162 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2163
2164 for my $snap (values %{$conf->{snapshots}}) {
2165 next if !defined($snap->{vmstate});
2166 my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2167 next if !defined($drive);
2168 $remove_owned_drive->('vmstate', $drive);
2169 }
2170
2171 if ($purge_unreferenced) { # also remove unreferenced disk
2172 my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2173 PVE::Storage::foreach_volid($vmdisks, sub {
2174 my ($volid, $sid, $volname, $d) = @_;
2175 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2176 warn $@ if $@;
2177 });
2178 }
2179
2180 if (defined $replacement_conf) {
2181 PVE::QemuConfig->write_config($vmid, $replacement_conf);
2182 } else {
2183 PVE::QemuConfig->destroy_config($vmid);
2184 }
2185}
2186
2187sub parse_vm_config {
2188 my ($filename, $raw) = @_;
2189
2190 return if !defined($raw);
2191
2192 my $res = {
2193 digest => Digest::SHA::sha1_hex($raw),
2194 snapshots => {},
2195 pending => {},
2196 };
2197
2198 $filename =~ m|/qemu-server/(\d+)\.conf$|
2199 || die "got strange filename '$filename'";
2200
2201 my $vmid = $1;
2202
2203 my $conf = $res;
2204 my $descr;
2205 my $section = '';
2206
2207 my @lines = split(/\n/, $raw);
2208 foreach my $line (@lines) {
2209 next if $line =~ m/^\s*$/;
2210
2211 if ($line =~ m/^\[PENDING\]\s*$/i) {
2212 $section = 'pending';
2213 if (defined($descr)) {
2214 $descr =~ s/\s+$//;
2215 $conf->{description} = $descr;
2216 }
2217 $descr = undef;
2218 $conf = $res->{$section} = {};
2219 next;
2220
2221 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2222 $section = $1;
2223 if (defined($descr)) {
2224 $descr =~ s/\s+$//;
2225 $conf->{description} = $descr;
2226 }
2227 $descr = undef;
2228 $conf = $res->{snapshots}->{$section} = {};
2229 next;
2230 }
2231
2232 if ($line =~ m/^\#(.*)\s*$/) {
2233 $descr = '' if !defined($descr);
2234 $descr .= PVE::Tools::decode_text($1) . "\n";
2235 next;
2236 }
2237
2238 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2239 $descr = '' if !defined($descr);
2240 $descr .= PVE::Tools::decode_text($2);
2241 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2242 $conf->{snapstate} = $1;
2243 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2244 my $key = $1;
2245 my $value = $2;
2246 $conf->{$key} = $value;
2247 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2248 my $value = $1;
2249 if ($section eq 'pending') {
2250 $conf->{delete} = $value; # we parse this later
2251 } else {
2252 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2253 }
2254 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2255 my $key = $1;
2256 my $value = $2;
2257 eval { $value = check_type($key, $value); };
2258 if ($@) {
2259 warn "vm $vmid - unable to parse value of '$key' - $@";
2260 } else {
2261 $key = 'ide2' if $key eq 'cdrom';
2262 my $fmt = $confdesc->{$key}->{format};
2263 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2264 my $v = parse_drive($key, $value);
2265 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2266 $v->{file} = $volid;
2267 $value = print_drive($v);
2268 } else {
2269 warn "vm $vmid - unable to parse value of '$key'\n";
2270 next;
2271 }
2272 }
2273
2274 $conf->{$key} = $value;
2275 }
2276 } else {
2277 warn "vm $vmid - unable to parse config: $line\n";
2278 }
2279 }
2280
2281 if (defined($descr)) {
2282 $descr =~ s/\s+$//;
2283 $conf->{description} = $descr;
2284 }
2285 delete $res->{snapstate}; # just to be sure
2286
2287 return $res;
2288}
2289
2290sub write_vm_config {
2291 my ($filename, $conf) = @_;
2292
2293 delete $conf->{snapstate}; # just to be sure
2294
2295 if ($conf->{cdrom}) {
2296 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2297 $conf->{ide2} = $conf->{cdrom};
2298 delete $conf->{cdrom};
2299 }
2300
2301 # we do not use 'smp' any longer
2302 if ($conf->{sockets}) {
2303 delete $conf->{smp};
2304 } elsif ($conf->{smp}) {
2305 $conf->{sockets} = $conf->{smp};
2306 delete $conf->{cores};
2307 delete $conf->{smp};
2308 }
2309
2310 my $used_volids = {};
2311
2312 my $cleanup_config = sub {
2313 my ($cref, $pending, $snapname) = @_;
2314
2315 foreach my $key (keys %$cref) {
2316 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2317 $key eq 'snapstate' || $key eq 'pending';
2318 my $value = $cref->{$key};
2319 if ($key eq 'delete') {
2320 die "propertry 'delete' is only allowed in [PENDING]\n"
2321 if !$pending;
2322 # fixme: check syntax?
2323 next;
2324 }
2325 eval { $value = check_type($key, $value); };
2326 die "unable to parse value of '$key' - $@" if $@;
2327
2328 $cref->{$key} = $value;
2329
2330 if (!$snapname && is_valid_drivename($key)) {
2331 my $drive = parse_drive($key, $value);
2332 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2333 }
2334 }
2335 };
2336
2337 &$cleanup_config($conf);
2338
2339 &$cleanup_config($conf->{pending}, 1);
2340
2341 foreach my $snapname (keys %{$conf->{snapshots}}) {
2342 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2343 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2344 }
2345
2346 # remove 'unusedX' settings if we re-add a volume
2347 foreach my $key (keys %$conf) {
2348 my $value = $conf->{$key};
2349 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2350 delete $conf->{$key};
2351 }
2352 }
2353
2354 my $generate_raw_config = sub {
2355 my ($conf, $pending) = @_;
2356
2357 my $raw = '';
2358
2359 # add description as comment to top of file
2360 if (defined(my $descr = $conf->{description})) {
2361 if ($descr) {
2362 foreach my $cl (split(/\n/, $descr)) {
2363 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2364 }
2365 } else {
2366 $raw .= "#\n" if $pending;
2367 }
2368 }
2369
2370 foreach my $key (sort keys %$conf) {
2371 next if $key =~ /^(digest|description|pending|snapshots)$/;
2372 $raw .= "$key: $conf->{$key}\n";
2373 }
2374 return $raw;
2375 };
2376
2377 my $raw = &$generate_raw_config($conf);
2378
2379 if (scalar(keys %{$conf->{pending}})){
2380 $raw .= "\n[PENDING]\n";
2381 $raw .= &$generate_raw_config($conf->{pending}, 1);
2382 }
2383
2384 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2385 $raw .= "\n[$snapname]\n";
2386 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2387 }
2388
2389 return $raw;
2390}
2391
2392sub load_defaults {
2393
2394 my $res = {};
2395
2396 # we use static defaults from our JSON schema configuration
2397 foreach my $key (keys %$confdesc) {
2398 if (defined(my $default = $confdesc->{$key}->{default})) {
2399 $res->{$key} = $default;
2400 }
2401 }
2402
2403 return $res;
2404}
2405
2406sub config_list {
2407 my $vmlist = PVE::Cluster::get_vmlist();
2408 my $res = {};
2409 return $res if !$vmlist || !$vmlist->{ids};
2410 my $ids = $vmlist->{ids};
2411 my $nodename = nodename();
2412
2413 foreach my $vmid (keys %$ids) {
2414 my $d = $ids->{$vmid};
2415 next if !$d->{node} || $d->{node} ne $nodename;
2416 next if !$d->{type} || $d->{type} ne 'qemu';
2417 $res->{$vmid}->{exists} = 1;
2418 }
2419 return $res;
2420}
2421
2422# test if VM uses local resources (to prevent migration)
2423sub check_local_resources {
2424 my ($conf, $noerr) = @_;
2425
2426 my @loc_res = ();
2427
2428 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2429 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2430
2431 push @loc_res, "ivshmem" if $conf->{ivshmem};
2432
2433 foreach my $k (keys %$conf) {
2434 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2435 # sockets are safe: they will recreated be on the target side post-migrate
2436 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2437 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2438 }
2439
2440 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2441
2442 return \@loc_res;
2443}
2444
2445# check if used storages are available on all nodes (use by migrate)
2446sub check_storage_availability {
2447 my ($storecfg, $conf, $node) = @_;
2448
2449 PVE::QemuConfig->foreach_volume($conf, sub {
2450 my ($ds, $drive) = @_;
2451
2452 my $volid = $drive->{file};
2453 return if !$volid;
2454
2455 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2456 return if !$sid;
2457
2458 # check if storage is available on both nodes
2459 my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2460 PVE::Storage::storage_check_node($storecfg, $sid, $node);
2461 });
2462}
2463
2464# list nodes where all VM images are available (used by has_feature API)
2465sub shared_nodes {
2466 my ($conf, $storecfg) = @_;
2467
2468 my $nodelist = PVE::Cluster::get_nodelist();
2469 my $nodehash = { map { $_ => 1 } @$nodelist };
2470 my $nodename = nodename();
2471
2472 PVE::QemuConfig->foreach_volume($conf, sub {
2473 my ($ds, $drive) = @_;
2474
2475 my $volid = $drive->{file};
2476 return if !$volid;
2477
2478 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2479 if ($storeid) {
2480 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2481 if ($scfg->{disable}) {
2482 $nodehash = {};
2483 } elsif (my $avail = $scfg->{nodes}) {
2484 foreach my $node (keys %$nodehash) {
2485 delete $nodehash->{$node} if !$avail->{$node};
2486 }
2487 } elsif (!$scfg->{shared}) {
2488 foreach my $node (keys %$nodehash) {
2489 delete $nodehash->{$node} if $node ne $nodename
2490 }
2491 }
2492 }
2493 });
2494
2495 return $nodehash
2496}
2497
2498sub check_local_storage_availability {
2499 my ($conf, $storecfg) = @_;
2500
2501 my $nodelist = PVE::Cluster::get_nodelist();
2502 my $nodehash = { map { $_ => {} } @$nodelist };
2503
2504 PVE::QemuConfig->foreach_volume($conf, sub {
2505 my ($ds, $drive) = @_;
2506
2507 my $volid = $drive->{file};
2508 return if !$volid;
2509
2510 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2511 if ($storeid) {
2512 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2513
2514 if ($scfg->{disable}) {
2515 foreach my $node (keys %$nodehash) {
2516 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2517 }
2518 } elsif (my $avail = $scfg->{nodes}) {
2519 foreach my $node (keys %$nodehash) {
2520 if (!$avail->{$node}) {
2521 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2522 }
2523 }
2524 }
2525 }
2526 });
2527
2528 foreach my $node (values %$nodehash) {
2529 if (my $unavail = $node->{unavailable_storages}) {
2530 $node->{unavailable_storages} = [ sort keys %$unavail ];
2531 }
2532 }
2533
2534 return $nodehash
2535}
2536
2537# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2538sub check_running {
2539 my ($vmid, $nocheck, $node) = @_;
2540
2541 PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2542 return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2543}
2544
2545sub vzlist {
2546
2547 my $vzlist = config_list();
2548
2549 my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2550
2551 while (defined(my $de = $fd->read)) {
2552 next if $de !~ m/^(\d+)\.pid$/;
2553 my $vmid = $1;
2554 next if !defined($vzlist->{$vmid});
2555 if (my $pid = check_running($vmid)) {
2556 $vzlist->{$vmid}->{pid} = $pid;
2557 }
2558 }
2559
2560 return $vzlist;
2561}
2562
2563our $vmstatus_return_properties = {
2564 vmid => get_standard_option('pve-vmid'),
2565 status => {
2566 description => "Qemu process status.",
2567 type => 'string',
2568 enum => ['stopped', 'running'],
2569 },
2570 maxmem => {
2571 description => "Maximum memory in bytes.",
2572 type => 'integer',
2573 optional => 1,
2574 renderer => 'bytes',
2575 },
2576 maxdisk => {
2577 description => "Root disk size in bytes.",
2578 type => 'integer',
2579 optional => 1,
2580 renderer => 'bytes',
2581 },
2582 name => {
2583 description => "VM name.",
2584 type => 'string',
2585 optional => 1,
2586 },
2587 qmpstatus => {
2588 description => "Qemu QMP agent status.",
2589 type => 'string',
2590 optional => 1,
2591 },
2592 pid => {
2593 description => "PID of running qemu process.",
2594 type => 'integer',
2595 optional => 1,
2596 },
2597 uptime => {
2598 description => "Uptime.",
2599 type => 'integer',
2600 optional => 1,
2601 renderer => 'duration',
2602 },
2603 cpus => {
2604 description => "Maximum usable CPUs.",
2605 type => 'number',
2606 optional => 1,
2607 },
2608 lock => {
2609 description => "The current config lock, if any.",
2610 type => 'string',
2611 optional => 1,
2612 },
2613 tags => {
2614 description => "The current configured tags, if any",
2615 type => 'string',
2616 optional => 1,
2617 },
2618 'running-machine' => {
2619 description => "The currently running machine type (if running).",
2620 type => 'string',
2621 optional => 1,
2622 },
2623 'running-qemu' => {
2624 description => "The currently running QEMU version (if running).",
2625 type => 'string',
2626 optional => 1,
2627 },
2628};
2629
2630my $last_proc_pid_stat;
2631
2632# get VM status information
2633# This must be fast and should not block ($full == false)
2634# We only query KVM using QMP if $full == true (this can be slow)
2635sub vmstatus {
2636 my ($opt_vmid, $full) = @_;
2637
2638 my $res = {};
2639
2640 my $storecfg = PVE::Storage::config();
2641
2642 my $list = vzlist();
2643 my $defaults = load_defaults();
2644
2645 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2646
2647 my $cpucount = $cpuinfo->{cpus} || 1;
2648
2649 foreach my $vmid (keys %$list) {
2650 next if $opt_vmid && ($vmid ne $opt_vmid);
2651
2652 my $conf = PVE::QemuConfig->load_config($vmid);
2653
2654 my $d = { vmid => $vmid };
2655 $d->{pid} = $list->{$vmid}->{pid};
2656
2657 # fixme: better status?
2658 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2659
2660 my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2661 if (defined($size)) {
2662 $d->{disk} = 0; # no info available
2663 $d->{maxdisk} = $size;
2664 } else {
2665 $d->{disk} = 0;
2666 $d->{maxdisk} = 0;
2667 }
2668
2669 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2670 * ($conf->{cores} || $defaults->{cores});
2671 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2672 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2673
2674 $d->{name} = $conf->{name} || "VM $vmid";
2675 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2676 : $defaults->{memory}*(1024*1024);
2677
2678 if ($conf->{balloon}) {
2679 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2680 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2681 : $defaults->{shares};
2682 }
2683
2684 $d->{uptime} = 0;
2685 $d->{cpu} = 0;
2686 $d->{mem} = 0;
2687
2688 $d->{netout} = 0;
2689 $d->{netin} = 0;
2690
2691 $d->{diskread} = 0;
2692 $d->{diskwrite} = 0;
2693
2694 $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2695
2696 $d->{serial} = 1 if conf_has_serial($conf);
2697 $d->{lock} = $conf->{lock} if $conf->{lock};
2698 $d->{tags} = $conf->{tags} if defined($conf->{tags});
2699
2700 $res->{$vmid} = $d;
2701 }
2702
2703 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2704 foreach my $dev (keys %$netdev) {
2705 next if $dev !~ m/^tap([1-9]\d*)i/;
2706 my $vmid = $1;
2707 my $d = $res->{$vmid};
2708 next if !$d;
2709
2710 $d->{netout} += $netdev->{$dev}->{receive};
2711 $d->{netin} += $netdev->{$dev}->{transmit};
2712
2713 if ($full) {
2714 $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2715 $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2716 }
2717
2718 }
2719
2720 my $ctime = gettimeofday;
2721
2722 foreach my $vmid (keys %$list) {
2723
2724 my $d = $res->{$vmid};
2725 my $pid = $d->{pid};
2726 next if !$pid;
2727
2728 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2729 next if !$pstat; # not running
2730
2731 my $used = $pstat->{utime} + $pstat->{stime};
2732
2733 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2734
2735 if ($pstat->{vsize}) {
2736 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2737 }
2738
2739 my $old = $last_proc_pid_stat->{$pid};
2740 if (!$old) {
2741 $last_proc_pid_stat->{$pid} = {
2742 time => $ctime,
2743 used => $used,
2744 cpu => 0,
2745 };
2746 next;
2747 }
2748
2749 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2750
2751 if ($dtime > 1000) {
2752 my $dutime = $used - $old->{used};
2753
2754 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2755 $last_proc_pid_stat->{$pid} = {
2756 time => $ctime,
2757 used => $used,
2758 cpu => $d->{cpu},
2759 };
2760 } else {
2761 $d->{cpu} = $old->{cpu};
2762 }
2763 }
2764
2765 return $res if !$full;
2766
2767 my $qmpclient = PVE::QMPClient->new();
2768
2769 my $ballooncb = sub {
2770 my ($vmid, $resp) = @_;
2771
2772 my $info = $resp->{'return'};
2773 return if !$info->{max_mem};
2774
2775 my $d = $res->{$vmid};
2776
2777 # use memory assigned to VM
2778 $d->{maxmem} = $info->{max_mem};
2779 $d->{balloon} = $info->{actual};
2780
2781 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2782 $d->{mem} = $info->{total_mem} - $info->{free_mem};
2783 $d->{freemem} = $info->{free_mem};
2784 }
2785
2786 $d->{ballooninfo} = $info;
2787 };
2788
2789 my $blockstatscb = sub {
2790 my ($vmid, $resp) = @_;
2791 my $data = $resp->{'return'} || [];
2792 my $totalrdbytes = 0;
2793 my $totalwrbytes = 0;
2794
2795 for my $blockstat (@$data) {
2796 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2797 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2798
2799 $blockstat->{device} =~ s/drive-//;
2800 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2801 }
2802 $res->{$vmid}->{diskread} = $totalrdbytes;
2803 $res->{$vmid}->{diskwrite} = $totalwrbytes;
2804 };
2805
2806 my $machinecb = sub {
2807 my ($vmid, $resp) = @_;
2808 my $data = $resp->{'return'} || [];
2809
2810 $res->{$vmid}->{'running-machine'} =
2811 PVE::QemuServer::Machine::current_from_query_machines($data);
2812 };
2813
2814 my $versioncb = sub {
2815 my ($vmid, $resp) = @_;
2816 my $data = $resp->{'return'} // {};
2817 my $version = 'unknown';
2818
2819 if (my $v = $data->{qemu}) {
2820 $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2821 }
2822
2823 $res->{$vmid}->{'running-qemu'} = $version;
2824 };
2825
2826 my $statuscb = sub {
2827 my ($vmid, $resp) = @_;
2828
2829 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2830 $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2831 $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2832 # this fails if ballon driver is not loaded, so this must be
2833 # the last commnand (following command are aborted if this fails).
2834 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2835
2836 my $status = 'unknown';
2837 if (!defined($status = $resp->{'return'}->{status})) {
2838 warn "unable to get VM status\n";
2839 return;
2840 }
2841
2842 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2843 };
2844
2845 foreach my $vmid (keys %$list) {
2846 next if $opt_vmid && ($vmid ne $opt_vmid);
2847 next if !$res->{$vmid}->{pid}; # not running
2848 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2849 }
2850
2851 $qmpclient->queue_execute(undef, 2);
2852
2853 foreach my $vmid (keys %$list) {
2854 next if $opt_vmid && ($vmid ne $opt_vmid);
2855 next if !$res->{$vmid}->{pid}; #not running
2856
2857 # we can't use the $qmpclient since it might have already aborted on
2858 # 'query-balloon', but this might also fail for older versions...
2859 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2860 $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2861 }
2862
2863 foreach my $vmid (keys %$list) {
2864 next if $opt_vmid && ($vmid ne $opt_vmid);
2865 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2866 }
2867
2868 return $res;
2869}
2870
2871sub conf_has_serial {
2872 my ($conf) = @_;
2873
2874 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2875 if ($conf->{"serial$i"}) {
2876 return 1;
2877 }
2878 }
2879
2880 return 0;
2881}
2882
2883sub conf_has_audio {
2884 my ($conf, $id) = @_;
2885
2886 $id //= 0;
2887 my $audio = $conf->{"audio$id"};
2888 return if !defined($audio);
2889
2890 my $audioproperties = parse_property_string($audio_fmt, $audio);
2891 my $audiodriver = $audioproperties->{driver} // 'spice';
2892
2893 return {
2894 dev => $audioproperties->{device},
2895 dev_id => "audiodev$id",
2896 backend => $audiodriver,
2897 backend_id => "$audiodriver-backend${id}",
2898 };
2899}
2900
2901sub audio_devs {
2902 my ($audio, $audiopciaddr, $machine_version) = @_;
2903
2904 my $devs = [];
2905
2906 my $id = $audio->{dev_id};
2907 my $audiodev = "";
2908 if (min_version($machine_version, 4, 2)) {
2909 $audiodev = ",audiodev=$audio->{backend_id}";
2910 }
2911
2912 if ($audio->{dev} eq 'AC97') {
2913 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2914 } elsif ($audio->{dev} =~ /intel\-hda$/) {
2915 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2916 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2917 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2918 } else {
2919 die "unkown audio device '$audio->{dev}', implement me!";
2920 }
2921
2922 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2923
2924 return $devs;
2925}
2926
2927sub vga_conf_has_spice {
2928 my ($vga) = @_;
2929
2930 my $vgaconf = parse_vga($vga);
2931 my $vgatype = $vgaconf->{type};
2932 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2933
2934 return $1 || 1;
2935}
2936
2937sub is_native($) {
2938 my ($arch) = @_;
2939 return get_host_arch() eq $arch;
2940}
2941
2942sub get_vm_arch {
2943 my ($conf) = @_;
2944 return $conf->{arch} // get_host_arch();
2945}
2946
2947my $default_machines = {
2948 x86_64 => 'pc',
2949 aarch64 => 'virt',
2950};
2951
2952sub get_installed_machine_version {
2953 my ($kvmversion) = @_;
2954 $kvmversion = kvm_user_version() if !defined($kvmversion);
2955 $kvmversion =~ m/^(\d+\.\d+)/;
2956 return $1;
2957}
2958
2959sub windows_get_pinned_machine_version {
2960 my ($machine, $base_version, $kvmversion) = @_;
2961
2962 my $pin_version = $base_version;
2963 if (!defined($base_version) ||
2964 !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2965 ) {
2966 $pin_version = get_installed_machine_version($kvmversion);
2967 }
2968 if (!$machine || $machine eq 'pc') {
2969 $machine = "pc-i440fx-$pin_version";
2970 } elsif ($machine eq 'q35') {
2971 $machine = "pc-q35-$pin_version";
2972 } elsif ($machine eq 'virt') {
2973 $machine = "virt-$pin_version";
2974 } else {
2975 warn "unknown machine type '$machine', not touching that!\n";
2976 }
2977
2978 return $machine;
2979}
2980
2981sub get_vm_machine {
2982 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2983
2984 my $machine = $forcemachine || $conf->{machine};
2985
2986 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2987 $kvmversion //= kvm_user_version();
2988 # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2989 # layout which confuses windows quite a bit and may result in various regressions..
2990 # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2991 if (windows_version($conf->{ostype})) {
2992 $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2993 }
2994 $arch //= 'x86_64';
2995 $machine ||= $default_machines->{$arch};
2996 if ($add_pve_version) {
2997 my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2998 $machine .= "+pve$pvever";
2999 }
3000 }
3001
3002 if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3003 my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3004 $machine = $1 if $is_pxe;
3005
3006 # for version-pinned machines that do not include a pve-version (e.g.
3007 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3008 $machine .= '+pve0';
3009
3010 $machine .= '.pxe' if $is_pxe;
3011 }
3012
3013 return $machine;
3014}
3015
3016sub get_ovmf_files($) {
3017 my ($arch) = @_;
3018
3019 my $ovmf = $OVMF->{$arch}
3020 or die "no OVMF images known for architecture '$arch'\n";
3021
3022 return @$ovmf;
3023}
3024
3025my $Arch2Qemu = {
3026 aarch64 => '/usr/bin/qemu-system-aarch64',
3027 x86_64 => '/usr/bin/qemu-system-x86_64',
3028};
3029sub get_command_for_arch($) {
3030 my ($arch) = @_;
3031 return '/usr/bin/kvm' if is_native($arch);
3032
3033 my $cmd = $Arch2Qemu->{$arch}
3034 or die "don't know how to emulate architecture '$arch'\n";
3035 return $cmd;
3036}
3037
3038# To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3039# to use in a QEMU command line (-cpu element), first array_intersect the result
3040# of query_supported_ with query_understood_. This is necessary because:
3041#
3042# a) query_understood_ returns flags the host cannot use and
3043# b) query_supported_ (rather the QMP call) doesn't actually return CPU
3044# flags, but CPU settings - with most of them being flags. Those settings
3045# (and some flags, curiously) cannot be specified as a "-cpu" argument.
3046#
3047# query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3048# expensive. If you need the value returned from this, you can get it much
3049# cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3050# $accel being 'kvm' or 'tcg'.
3051#
3052# pvestatd calls this function on startup and whenever the QEMU/KVM version
3053# changes, automatically populating pmxcfs.
3054#
3055# Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3056# since kvm and tcg machines support different flags
3057#
3058sub query_supported_cpu_flags {
3059 my ($arch) = @_;
3060
3061 $arch //= get_host_arch();
3062 my $default_machine = $default_machines->{$arch};
3063
3064 my $flags = {};
3065
3066 # FIXME: Once this is merged, the code below should work for ARM as well:
3067 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3068 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3069 $arch eq "aarch64";
3070
3071 my $kvm_supported = defined(kvm_version());
3072 my $qemu_cmd = get_command_for_arch($arch);
3073 my $fakevmid = -1;
3074 my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3075
3076 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3077 my $query_supported_run_qemu = sub {
3078 my ($kvm) = @_;
3079
3080 my $flags = {};
3081 my $cmd = [
3082 $qemu_cmd,
3083 '-machine', $default_machine,
3084 '-display', 'none',
3085 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3086 '-mon', 'chardev=qmp,mode=control',
3087 '-pidfile', $pidfile,
3088 '-S', '-daemonize'
3089 ];
3090
3091 if (!$kvm) {
3092 push @$cmd, '-accel', 'tcg';
3093 }
3094
3095 my $rc = run_command($cmd, noerr => 1, quiet => 0);
3096 die "QEMU flag querying VM exited with code " . $rc if $rc;
3097
3098 eval {
3099 my $cmd_result = mon_cmd(
3100 $fakevmid,
3101 'query-cpu-model-expansion',
3102 type => 'full',
3103 model => { name => 'host' }
3104 );
3105
3106 my $props = $cmd_result->{model}->{props};
3107 foreach my $prop (keys %$props) {
3108 next if $props->{$prop} ne '1';
3109 # QEMU returns some flags multiple times, with '_', '.' or '-'
3110 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3111 # We only keep those with underscores, to match /proc/cpuinfo
3112 $prop =~ s/\.|-/_/g;
3113 $flags->{$prop} = 1;
3114 }
3115 };
3116 my $err = $@;
3117
3118 # force stop with 10 sec timeout and 'nocheck'
3119 # always stop, even if QMP failed
3120 vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3121
3122 die $err if $err;
3123
3124 return [ sort keys %$flags ];
3125 };
3126
3127 # We need to query QEMU twice, since KVM and TCG have different supported flags
3128 PVE::QemuConfig->lock_config($fakevmid, sub {
3129 $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3130 warn "warning: failed querying supported tcg flags: $@\n" if $@;
3131
3132 if ($kvm_supported) {
3133 $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3134 warn "warning: failed querying supported kvm flags: $@\n" if $@;
3135 }
3136 });
3137
3138 return $flags;
3139}
3140
3141# Understood CPU flags are written to a file at 'pve-qemu' compile time
3142my $understood_cpu_flag_dir = "/usr/share/kvm";
3143sub query_understood_cpu_flags {
3144 my $arch = get_host_arch();
3145 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3146
3147 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3148 if ! -e $filepath;
3149
3150 my $raw = file_get_contents($filepath);
3151 $raw =~ s/^\s+|\s+$//g;
3152 my @flags = split(/\s+/, $raw);
3153
3154 return \@flags;
3155}
3156
3157sub config_to_command {
3158 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3159 $pbs_backing) = @_;
3160
3161 my $cmd = [];
3162 my $globalFlags = [];
3163 my $machineFlags = [];
3164 my $rtcFlags = [];
3165 my $devices = [];
3166 my $pciaddr = '';
3167 my $bridges = {};
3168 my $ostype = $conf->{ostype};
3169 my $winversion = windows_version($ostype);
3170 my $kvm = $conf->{kvm};
3171 my $nodename = nodename();
3172
3173 my $arch = get_vm_arch($conf);
3174 my $kvm_binary = get_command_for_arch($arch);
3175 my $kvmver = kvm_user_version($kvm_binary);
3176
3177 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3178 $kvmver //= "undefined";
3179 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3180 }
3181
3182 my $add_pve_version = min_version($kvmver, 4, 1);
3183
3184 my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3185 my $machine_version = extract_version($machine_type, $kvmver);
3186 $kvm //= 1 if is_native($arch);
3187
3188 $machine_version =~ m/(\d+)\.(\d+)/;
3189 my ($machine_major, $machine_minor) = ($1, $2);
3190
3191 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3192 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3193 } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3194 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3195 ." please upgrade node '$nodename'\n"
3196 } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3197 my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3198 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3199 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3200 ." node '$nodename'\n";
3201 }
3202
3203 # if a specific +pve version is required for a feature, use $version_guard
3204 # instead of min_version to allow machines to be run with the minimum
3205 # required version
3206 my $required_pve_version = 0;
3207 my $version_guard = sub {
3208 my ($major, $minor, $pve) = @_;
3209 return 0 if !min_version($machine_version, $major, $minor, $pve);
3210 my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3211 return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3212 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3213 return 1;
3214 };
3215
3216 if ($kvm && !defined kvm_version()) {
3217 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3218 ." or enable in BIOS.\n";
3219 }
3220
3221 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3222 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3223 my $use_old_bios_files = undef;
3224 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3225
3226 my $cpuunits = defined($conf->{cpuunits}) ?
3227 $conf->{cpuunits} : $defaults->{cpuunits};
3228
3229 push @$cmd, $kvm_binary;
3230
3231 push @$cmd, '-id', $vmid;
3232
3233 my $vmname = $conf->{name} || "vm$vmid";
3234
3235 push @$cmd, '-name', $vmname;
3236
3237 push @$cmd, '-no-shutdown';
3238
3239 my $use_virtio = 0;
3240
3241 my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3242 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3243 push @$cmd, '-mon', "chardev=qmp,mode=control";
3244
3245 if (min_version($machine_version, 2, 12)) {
3246 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3247 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3248 }
3249
3250 push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3251
3252 push @$cmd, '-daemonize';
3253
3254 if ($conf->{smbios1}) {
3255 my $smbios_conf = parse_smbios1($conf->{smbios1});
3256 if ($smbios_conf->{base64}) {
3257 # Do not pass base64 flag to qemu
3258 delete $smbios_conf->{base64};
3259 my $smbios_string = "";
3260 foreach my $key (keys %$smbios_conf) {
3261 my $value;
3262 if ($key eq "uuid") {
3263 $value = $smbios_conf->{uuid}
3264 } else {
3265 $value = decode_base64($smbios_conf->{$key});
3266 }
3267 # qemu accepts any binary data, only commas need escaping by double comma
3268 $value =~ s/,/,,/g;
3269 $smbios_string .= "," . $key . "=" . $value if $value;
3270 }
3271 push @$cmd, '-smbios', "type=1" . $smbios_string;
3272 } else {
3273 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3274 }
3275 }
3276
3277 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3278 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3279 die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3280
3281 my ($path, $format);
3282 if (my $efidisk = $conf->{efidisk0}) {
3283 my $d = parse_drive('efidisk0', $efidisk);
3284 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3285 $format = $d->{format};
3286 if ($storeid) {
3287 $path = PVE::Storage::path($storecfg, $d->{file});
3288 if (!defined($format)) {
3289 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3290 $format = qemu_img_format($scfg, $volname);
3291 }
3292 } else {
3293 $path = $d->{file};
3294 die "efidisk format must be specified\n"
3295 if !defined($format);
3296 }
3297 } else {
3298 warn "no efidisk configured! Using temporary efivars disk.\n";
3299 $path = "/tmp/$vmid-ovmf.fd";
3300 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3301 $format = 'raw';
3302 }
3303
3304 my $size_str = "";
3305
3306 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3307 $size_str = ",size=" . (-s $ovmf_vars);
3308 }
3309
3310 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3311 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3312 }
3313
3314 # load q35 config
3315 if ($q35) {
3316 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3317 if (min_version($machine_version, 4, 0)) {
3318 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3319 } else {
3320 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3321 }
3322 }
3323
3324 if ($conf->{vmgenid}) {
3325 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3326 }
3327
3328 # add usb controllers
3329 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3330 $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3331 push @$devices, @usbcontrollers if @usbcontrollers;
3332 my $vga = parse_vga($conf->{vga});
3333
3334 my $qxlnum = vga_conf_has_spice($conf->{vga});
3335 $vga->{type} = 'qxl' if $qxlnum;
3336
3337 if (!$vga->{type}) {
3338 if ($arch eq 'aarch64') {
3339 $vga->{type} = 'virtio';
3340 } elsif (min_version($machine_version, 2, 9)) {
3341 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3342 } else {
3343 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3344 }
3345 }
3346
3347 # enable absolute mouse coordinates (needed by vnc)
3348 my $tablet;
3349 if (defined($conf->{tablet})) {
3350 $tablet = $conf->{tablet};
3351 } else {
3352 $tablet = $defaults->{tablet};
3353 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3354 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3355 }
3356
3357 if ($tablet) {
3358 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3359 my $kbd = print_keyboarddevice_full($conf, $arch);
3360 push @$devices, '-device', $kbd if defined($kbd);
3361 }
3362
3363 my $bootorder = device_bootorder($conf);
3364
3365 # host pci device passthrough
3366 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3367 $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3368
3369 # usb devices
3370 my $usb_dev_features = {};
3371 $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3372
3373 my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3374 $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3375 push @$devices, @usbdevices if @usbdevices;
3376
3377 # serial devices
3378 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3379 if (my $path = $conf->{"serial$i"}) {
3380 if ($path eq 'socket') {
3381 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3382 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3383 # On aarch64, serial0 is the UART device. Qemu only allows
3384 # connecting UART devices via the '-serial' command line, as
3385 # the device has a fixed slot on the hardware...
3386 if ($arch eq 'aarch64' && $i == 0) {
3387 push @$devices, '-serial', "chardev:serial$i";
3388 } else {
3389 push @$devices, '-device', "isa-serial,chardev=serial$i";
3390 }
3391 } else {
3392 die "no such serial device\n" if ! -c $path;
3393 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3394 push @$devices, '-device', "isa-serial,chardev=serial$i";
3395 }
3396 }
3397 }
3398
3399 # parallel devices
3400 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3401 if (my $path = $conf->{"parallel$i"}) {
3402 die "no such parallel device\n" if ! -c $path;
3403 my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3404 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3405 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3406 }
3407 }
3408
3409 if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3410 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3411 my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3412 push @$devices, @$audio_devs;
3413 }
3414
3415 my $sockets = 1;
3416 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3417 $sockets = $conf->{sockets} if $conf->{sockets};
3418
3419 my $cores = $conf->{cores} || 1;
3420
3421 my $maxcpus = $sockets * $cores;
3422
3423 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3424
3425 my $allowed_vcpus = $cpuinfo->{cpus};
3426
3427 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3428 if ($allowed_vcpus < $maxcpus);
3429
3430 if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3431
3432 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3433 for (my $i = 2; $i <= $vcpus; $i++) {
3434 my $cpustr = print_cpu_device($conf,$i);
3435 push @$cmd, '-device', $cpustr;
3436 }
3437
3438 } else {
3439
3440 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3441 }
3442 push @$cmd, '-nodefaults';
3443
3444 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3445
3446 push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3447
3448 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
3449
3450 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3451 push @$devices, '-device', print_vga_device(
3452 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3453 my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3454 push @$cmd, '-vnc', "unix:$socket,password=on";
3455 } else {
3456 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3457 push @$cmd, '-nographic';
3458 }
3459
3460 # time drift fix
3461 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3462 my $useLocaltime = $conf->{localtime};
3463
3464 if ($winversion >= 5) { # windows
3465 $useLocaltime = 1 if !defined($conf->{localtime});
3466
3467 # use time drift fix when acpi is enabled
3468 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3469 $tdf = 1 if !defined($conf->{tdf});
3470 }
3471 }
3472
3473 if ($winversion >= 6) {
3474 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3475 push @$cmd, '-no-hpet';
3476 }
3477
3478 push @$rtcFlags, 'driftfix=slew' if $tdf;
3479
3480 if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3481 push @$rtcFlags, "base=$conf->{startdate}";
3482 } elsif ($useLocaltime) {
3483 push @$rtcFlags, 'base=localtime';
3484 }
3485
3486 if ($forcecpu) {
3487 push @$cmd, '-cpu', $forcecpu;
3488 } else {
3489 push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3490 }
3491
3492 PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3493
3494 push @$cmd, '-S' if $conf->{freeze};
3495
3496 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3497
3498 my $guest_agent = parse_guest_agent($conf);
3499
3500 if ($guest_agent->{enabled}) {
3501 my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3502 push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3503
3504 if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3505 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3506 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3507 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3508 } elsif ($guest_agent->{type} eq 'isa') {
3509 push @$devices, '-device', "isa-serial,chardev=qga0";
3510 }
3511 }
3512
3513 my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3514 if ($rng && $version_guard->(4, 1, 2)) {
3515 check_rng_source($rng->{source});
3516
3517 my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3518 my $period = $rng->{period} // $rng_fmt->{period}->{default};
3519 my $limiter_str = "";
3520 if ($max_bytes) {
3521 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3522 }
3523
3524 my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3525 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3526 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3527 }
3528
3529 my $spice_port;
3530
3531 if ($qxlnum) {
3532 if ($qxlnum > 1) {
3533 if ($winversion){
3534 for (my $i = 1; $i < $qxlnum; $i++){
3535 push @$devices, '-device', print_vga_device(
3536 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3537 }
3538 } else {
3539 # assume other OS works like Linux
3540 my ($ram, $vram) = ("134217728", "67108864");
3541 if ($vga->{memory}) {
3542 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3543 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3544 }
3545 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3546 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3547 }
3548 }
3549
3550 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3551
3552 my $pfamily = PVE::Tools::get_host_address_family($nodename);
3553 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3554 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3555
3556 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3557 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3558 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3559
3560 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3561 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3562
3563 my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3564 my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3565 if ($spice_enhancement->{foldersharing}) {
3566 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3567 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3568 }
3569
3570 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3571 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3572 if $spice_enhancement->{videostreaming};
3573
3574 push @$devices, '-spice', "$spice_opts";
3575 }
3576
3577 # enable balloon by default, unless explicitly disabled
3578 if (!defined($conf->{balloon}) || $conf->{balloon}) {
3579 $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3580 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3581 }
3582
3583 if ($conf->{watchdog}) {
3584 my $wdopts = parse_watchdog($conf->{watchdog});
3585 $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3586 my $watchdog = $wdopts->{model} || 'i6300esb';
3587 push @$devices, '-device', "$watchdog$pciaddr";
3588 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3589 }
3590
3591 my $vollist = [];
3592 my $scsicontroller = {};
3593 my $ahcicontroller = {};
3594 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3595
3596 # Add iscsi initiator name if available
3597 if (my $initiator = get_initiator_name()) {
3598 push @$devices, '-iscsi', "initiator-name=$initiator";
3599 }
3600
3601 PVE::QemuConfig->foreach_volume($conf, sub {
3602 my ($ds, $drive) = @_;
3603
3604 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3605 push @$vollist, $drive->{file};
3606 }
3607
3608 # ignore efidisk here, already added in bios/fw handling code above
3609 return if $drive->{interface} eq 'efidisk';
3610
3611 $use_virtio = 1 if $ds =~ m/^virtio/;
3612
3613 $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3614
3615 if ($drive->{interface} eq 'virtio'){
3616 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3617 }
3618
3619 if ($drive->{interface} eq 'scsi') {
3620
3621 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3622
3623 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3624 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3625
3626 $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3627 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3628
3629 my $iothread = '';
3630 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3631 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3632 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3633 } elsif ($drive->{iothread}) {
3634 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3635 }
3636
3637 my $queues = '';
3638 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3639 $queues = ",num_queues=$drive->{queues}";
3640 }
3641
3642 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3643 if !$scsicontroller->{$controller};
3644 $scsicontroller->{$controller}=1;
3645 }
3646
3647 if ($drive->{interface} eq 'sata') {
3648 my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3649 $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3650 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3651 if !$ahcicontroller->{$controller};
3652 $ahcicontroller->{$controller}=1;
3653 }
3654
3655 my $pbs_conf = $pbs_backing->{$ds};
3656 my $pbs_name = undef;
3657 if ($pbs_conf) {
3658 $pbs_name = "drive-$ds-pbs";
3659 push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3660 }
3661
3662 my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive, $pbs_name);
3663
3664 # extra protection for templates, but SATA and IDE don't support it..
3665 my $read_only = PVE::QemuConfig->is_template($conf)
3666 && $drive->{interface} ne 'sata'
3667 && $drive->{interface} ne 'ide';
3668
3669 $drive_cmd .= ',readonly=on' if $read_only;
3670
3671 push @$devices, '-drive',$drive_cmd;
3672 push @$devices, '-device', print_drivedevice_full(
3673 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3674 });
3675
3676 for (my $i = 0; $i < $MAX_NETS; $i++) {
3677 my $netname = "net$i";
3678
3679 next if !$conf->{$netname};
3680 my $d = parse_net($conf->{$netname});
3681 next if !$d;
3682
3683 $use_virtio = 1 if $d->{model} eq 'virtio';
3684
3685 $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3686
3687 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3688 push @$devices, '-netdev', $netdevfull;
3689
3690 my $netdevicefull = print_netdevice_full(
3691 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3692
3693 push @$devices, '-device', $netdevicefull;
3694 }
3695
3696 if ($conf->{ivshmem}) {
3697 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3698
3699 my $bus;
3700 if ($q35) {
3701 $bus = print_pcie_addr("ivshmem");
3702 } else {
3703 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3704 }
3705
3706 my $ivshmem_name = $ivshmem->{name} // $vmid;
3707 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3708
3709 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3710 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3711 .",size=$ivshmem->{size}M";
3712 }
3713
3714 # pci.4 is nested in pci.1
3715 $bridges->{1} = 1 if $bridges->{4};
3716
3717 if (!$q35) {
3718 # add pci bridges
3719 if (min_version($machine_version, 2, 3)) {
3720 $bridges->{1} = 1;
3721 $bridges->{2} = 1;
3722 }
3723
3724 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3725
3726 }
3727
3728 for my $k (sort {$b cmp $a} keys %$bridges) {
3729 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3730
3731 my $k_name = $k;
3732 if ($k == 2 && $legacy_igd) {
3733 $k_name = "$k-igd";
3734 }
3735 $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3736
3737 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3738 if ($q35) {
3739 # add after -readconfig pve-q35.cfg
3740 splice @$devices, 2, 0, '-device', $devstr;
3741 } else {
3742 unshift @$devices, '-device', $devstr if $k > 0;
3743 }
3744 }
3745
3746 if (!$kvm) {
3747 push @$machineFlags, 'accel=tcg';
3748 }
3749
3750 my $machine_type_min = $machine_type;
3751 if ($add_pve_version) {
3752 $machine_type_min =~ s/\+pve\d+$//;
3753 $machine_type_min .= "+pve$required_pve_version";
3754 }
3755 push @$machineFlags, "type=${machine_type_min}";
3756
3757 push @$cmd, @$devices;
3758 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3759 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3760 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3761
3762 if (my $vmstate = $conf->{vmstate}) {
3763 my $statepath = PVE::Storage::path($storecfg, $vmstate);
3764 push @$vollist, $vmstate;
3765 push @$cmd, '-loadstate', $statepath;
3766 print "activating and using '$vmstate' as vmstate\n";
3767 }
3768
3769 # add custom args
3770 if ($conf->{args}) {
3771 my $aa = PVE::Tools::split_args($conf->{args});
3772 push @$cmd, @$aa;
3773 }
3774
3775 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3776}
3777
3778sub check_rng_source {
3779 my ($source) = @_;
3780
3781 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3782 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3783 if ! -e $source;
3784
3785 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3786 if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3787 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3788 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3789 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3790 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3791 ." to the host.\n";
3792 }
3793}
3794
3795sub spice_port {
3796 my ($vmid) = @_;
3797
3798 my $res = mon_cmd($vmid, 'query-spice');
3799
3800 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3801}
3802
3803sub vm_devices_list {
3804 my ($vmid) = @_;
3805
3806 my $res = mon_cmd($vmid, 'query-pci');
3807 my $devices_to_check = [];
3808 my $devices = {};
3809 foreach my $pcibus (@$res) {
3810 push @$devices_to_check, @{$pcibus->{devices}},
3811 }
3812
3813 while (@$devices_to_check) {
3814 my $to_check = [];
3815 for my $d (@$devices_to_check) {
3816 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3817 next if !$d->{'pci_bridge'};
3818
3819 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3820 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3821 }
3822 $devices_to_check = $to_check;
3823 }
3824
3825 my $resblock = mon_cmd($vmid, 'query-block');
3826 foreach my $block (@$resblock) {
3827 if($block->{device} =~ m/^drive-(\S+)/){
3828 $devices->{$1} = 1;
3829 }
3830 }
3831
3832 my $resmice = mon_cmd($vmid, 'query-mice');
3833 foreach my $mice (@$resmice) {
3834 if ($mice->{name} eq 'QEMU HID Tablet') {
3835 $devices->{tablet} = 1;
3836 last;
3837 }
3838 }
3839
3840 # for usb devices there is no query-usb
3841 # but we can iterate over the entries in
3842 # qom-list path=/machine/peripheral
3843 my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3844 foreach my $per (@$resperipheral) {
3845 if ($per->{name} =~ m/^usb\d+$/) {
3846 $devices->{$per->{name}} = 1;
3847 }
3848 }
3849
3850 return $devices;
3851}
3852
3853sub vm_deviceplug {
3854 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3855
3856 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3857
3858 my $devices_list = vm_devices_list($vmid);
3859 return 1 if defined($devices_list->{$deviceid});
3860
3861 # add PCI bridge if we need it for the device
3862 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3863
3864 if ($deviceid eq 'tablet') {
3865
3866 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3867
3868 } elsif ($deviceid eq 'keyboard') {
3869
3870 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3871
3872 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3873
3874 die "usb hotplug currently not reliable\n";
3875 # since we can't reliably hot unplug all added usb devices and usb
3876 # passthrough breaks live migration we disable usb hotplugging for now
3877 #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3878
3879 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3880
3881 qemu_iothread_add($vmid, $deviceid, $device);
3882
3883 qemu_driveadd($storecfg, $vmid, $device);
3884 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3885
3886 qemu_deviceadd($vmid, $devicefull);
3887 eval { qemu_deviceaddverify($vmid, $deviceid); };
3888 if (my $err = $@) {
3889 eval { qemu_drivedel($vmid, $deviceid); };
3890 warn $@ if $@;
3891 die $err;
3892 }
3893
3894 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3895
3896
3897 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3898 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3899 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3900
3901 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3902
3903 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3904 qemu_iothread_add($vmid, $deviceid, $device);
3905 $devicefull .= ",iothread=iothread-$deviceid";
3906 }
3907
3908 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3909 $devicefull .= ",num_queues=$device->{queues}";
3910 }
3911
3912 qemu_deviceadd($vmid, $devicefull);
3913 qemu_deviceaddverify($vmid, $deviceid);
3914
3915 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3916
3917 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3918 qemu_driveadd($storecfg, $vmid, $device);
3919
3920 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3921 eval { qemu_deviceadd($vmid, $devicefull); };
3922 if (my $err = $@) {
3923 eval { qemu_drivedel($vmid, $deviceid); };
3924 warn $@ if $@;
3925 die $err;
3926 }
3927
3928 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3929
3930 return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3931
3932 my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3933 my $use_old_bios_files = undef;
3934 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3935
3936 my $netdevicefull = print_netdevice_full(
3937 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3938 qemu_deviceadd($vmid, $netdevicefull);
3939 eval {
3940 qemu_deviceaddverify($vmid, $deviceid);
3941 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3942 };
3943 if (my $err = $@) {
3944 eval { qemu_netdevdel($vmid, $deviceid); };
3945 warn $@ if $@;
3946 die $err;
3947 }
3948
3949 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3950
3951 my $bridgeid = $2;
3952 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3953 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3954
3955 qemu_deviceadd($vmid, $devicefull);
3956 qemu_deviceaddverify($vmid, $deviceid);
3957
3958 } else {
3959 die "can't hotplug device '$deviceid'\n";
3960 }
3961
3962 return 1;
3963}
3964
3965# fixme: this should raise exceptions on error!
3966sub vm_deviceunplug {
3967 my ($vmid, $conf, $deviceid) = @_;
3968
3969 my $devices_list = vm_devices_list($vmid);
3970 return 1 if !defined($devices_list->{$deviceid});
3971
3972 my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3973 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3974
3975 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3976
3977 qemu_devicedel($vmid, $deviceid);
3978
3979 } elsif ($deviceid =~ m/^usb\d+$/) {
3980
3981 die "usb hotplug currently not reliable\n";
3982 # when unplugging usb devices this way, there may be remaining usb
3983 # controllers/hubs so we disable it for now
3984 #qemu_devicedel($vmid, $deviceid);
3985 #qemu_devicedelverify($vmid, $deviceid);
3986
3987 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3988
3989 qemu_devicedel($vmid, $deviceid);
3990 qemu_devicedelverify($vmid, $deviceid);
3991 qemu_drivedel($vmid, $deviceid);
3992 qemu_iothread_del($conf, $vmid, $deviceid);
3993
3994 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3995
3996 qemu_devicedel($vmid, $deviceid);
3997 qemu_devicedelverify($vmid, $deviceid);
3998 qemu_iothread_del($conf, $vmid, $deviceid);
3999
4000 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4001
4002 qemu_devicedel($vmid, $deviceid);
4003 qemu_drivedel($vmid, $deviceid);
4004 qemu_deletescsihw($conf, $vmid, $deviceid);
4005
4006 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4007
4008 qemu_devicedel($vmid, $deviceid);
4009 qemu_devicedelverify($vmid, $deviceid);
4010 qemu_netdevdel($vmid, $deviceid);
4011
4012 } else {
4013 die "can't unplug device '$deviceid'\n";
4014 }
4015
4016 return 1;
4017}
4018
4019sub qemu_deviceadd {
4020 my ($vmid, $devicefull) = @_;
4021
4022 $devicefull = "driver=".$devicefull;
4023 my %options = split(/[=,]/, $devicefull);
4024
4025 mon_cmd($vmid, "device_add" , %options);
4026}
4027
4028sub qemu_devicedel {
4029 my ($vmid, $deviceid) = @_;
4030
4031 my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4032}
4033
4034sub qemu_iothread_add {
4035 my($vmid, $deviceid, $device) = @_;
4036
4037 if ($device->{iothread}) {
4038 my $iothreads = vm_iothreads_list($vmid);
4039 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4040 }
4041}
4042
4043sub qemu_iothread_del {
4044 my($conf, $vmid, $deviceid) = @_;
4045
4046 my $confid = $deviceid;
4047 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4048 $confid = 'scsi' . $1;
4049 }
4050 my $device = parse_drive($confid, $conf->{$confid});
4051 if ($device->{iothread}) {
4052 my $iothreads = vm_iothreads_list($vmid);
4053 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4054 }
4055}
4056
4057sub qemu_objectadd {
4058 my($vmid, $objectid, $qomtype) = @_;
4059
4060 mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4061
4062 return 1;
4063}
4064
4065sub qemu_objectdel {
4066 my($vmid, $objectid) = @_;
4067
4068 mon_cmd($vmid, "object-del", id => $objectid);
4069
4070 return 1;
4071}
4072
4073sub qemu_driveadd {
4074 my ($storecfg, $vmid, $device) = @_;
4075
4076 my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4077 $drive =~ s/\\/\\\\/g;
4078 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4079
4080 # If the command succeeds qemu prints: "OK"
4081 return 1 if $ret =~ m/OK/s;
4082
4083 die "adding drive failed: $ret\n";
4084}
4085
4086sub qemu_drivedel {
4087 my($vmid, $deviceid) = @_;
4088
4089 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4090 $ret =~ s/^\s+//;
4091
4092 return 1 if $ret eq "";
4093
4094 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4095 return 1 if $ret =~ m/Device \'.*?\' not found/s;
4096
4097 die "deleting drive $deviceid failed : $ret\n";
4098}
4099
4100sub qemu_deviceaddverify {
4101 my ($vmid, $deviceid) = @_;
4102
4103 for (my $i = 0; $i <= 5; $i++) {
4104 my $devices_list = vm_devices_list($vmid);
4105 return 1 if defined($devices_list->{$deviceid});
4106 sleep 1;
4107 }
4108
4109 die "error on hotplug device '$deviceid'\n";
4110}
4111
4112
4113sub qemu_devicedelverify {
4114 my ($vmid, $deviceid) = @_;
4115
4116 # need to verify that the device is correctly removed as device_del
4117 # is async and empty return is not reliable
4118
4119 for (my $i = 0; $i <= 5; $i++) {
4120 my $devices_list = vm_devices_list($vmid);
4121 return 1 if !defined($devices_list->{$deviceid});
4122 sleep 1;
4123 }
4124
4125 die "error on hot-unplugging device '$deviceid'\n";
4126}
4127
4128sub qemu_findorcreatescsihw {
4129 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4130
4131 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4132
4133 my $scsihwid="$controller_prefix$controller";
4134 my $devices_list = vm_devices_list($vmid);
4135
4136 if(!defined($devices_list->{$scsihwid})) {
4137 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4138 }
4139
4140 return 1;
4141}
4142
4143sub qemu_deletescsihw {
4144 my ($conf, $vmid, $opt) = @_;
4145
4146 my $device = parse_drive($opt, $conf->{$opt});
4147
4148 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4149 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4150 return 1;
4151 }
4152
4153 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4154
4155 my $devices_list = vm_devices_list($vmid);
4156 foreach my $opt (keys %{$devices_list}) {
4157 if (is_valid_drivename($opt)) {
4158 my $drive = parse_drive($opt, $conf->{$opt});
4159 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4160 return 1;
4161 }
4162 }
4163 }
4164
4165 my $scsihwid="scsihw$controller";
4166
4167 vm_deviceunplug($vmid, $conf, $scsihwid);
4168
4169 return 1;
4170}
4171
4172sub qemu_add_pci_bridge {
4173 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4174
4175 my $bridges = {};
4176
4177 my $bridgeid;
4178
4179 print_pci_addr($device, $bridges, $arch, $machine_type);
4180
4181 while (my ($k, $v) = each %$bridges) {
4182 $bridgeid = $k;
4183 }
4184 return 1 if !defined($bridgeid) || $bridgeid < 1;
4185
4186 my $bridge = "pci.$bridgeid";
4187 my $devices_list = vm_devices_list($vmid);
4188
4189 if (!defined($devices_list->{$bridge})) {
4190 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4191 }
4192
4193 return 1;
4194}
4195
4196sub qemu_set_link_status {
4197 my ($vmid, $device, $up) = @_;
4198
4199 mon_cmd($vmid, "set_link", name => $device,
4200 up => $up ? JSON::true : JSON::false);
4201}
4202
4203sub qemu_netdevadd {
4204 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4205
4206 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4207 my %options = split(/[=,]/, $netdev);
4208
4209 if (defined(my $vhost = $options{vhost})) {
4210 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4211 }
4212
4213 if (defined(my $queues = $options{queues})) {
4214 $options{queues} = $queues + 0;
4215 }
4216
4217 mon_cmd($vmid, "netdev_add", %options);
4218 return 1;
4219}
4220
4221sub qemu_netdevdel {
4222 my ($vmid, $deviceid) = @_;
4223
4224 mon_cmd($vmid, "netdev_del", id => $deviceid);
4225}
4226
4227sub qemu_usb_hotplug {
4228 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4229
4230 return if !$device;
4231
4232 # remove the old one first
4233 vm_deviceunplug($vmid, $conf, $deviceid);
4234
4235 # check if xhci controller is necessary and available
4236 if ($device->{usb3}) {
4237
4238 my $devicelist = vm_devices_list($vmid);
4239
4240 if (!$devicelist->{xhci}) {
4241 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4242 qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4243 }
4244 }
4245 my $d = parse_usb_device($device->{host});
4246 $d->{usb3} = $device->{usb3};
4247
4248 # add the new one
4249 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4250}
4251
4252sub qemu_cpu_hotplug {
4253 my ($vmid, $conf, $vcpus) = @_;
4254
4255 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4256
4257 my $sockets = 1;
4258 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4259 $sockets = $conf->{sockets} if $conf->{sockets};
4260 my $cores = $conf->{cores} || 1;
4261 my $maxcpus = $sockets * $cores;
4262
4263 $vcpus = $maxcpus if !$vcpus;
4264
4265 die "you can't add more vcpus than maxcpus\n"
4266 if $vcpus > $maxcpus;
4267
4268 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4269
4270 if ($vcpus < $currentvcpus) {
4271
4272 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4273
4274 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4275 qemu_devicedel($vmid, "cpu$i");
4276 my $retry = 0;
4277 my $currentrunningvcpus = undef;
4278 while (1) {
4279 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4280 last if scalar(@{$currentrunningvcpus}) == $i-1;
4281 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4282 $retry++;
4283 sleep 1;
4284 }
4285 #update conf after each succesfull cpu unplug
4286 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4287 PVE::QemuConfig->write_config($vmid, $conf);
4288 }
4289 } else {
4290 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4291 }
4292
4293 return;
4294 }
4295
4296 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4297 die "vcpus in running vm does not match its configuration\n"
4298 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4299
4300 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4301
4302 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4303 my $cpustr = print_cpu_device($conf, $i);
4304 qemu_deviceadd($vmid, $cpustr);
4305
4306 my $retry = 0;
4307 my $currentrunningvcpus = undef;
4308 while (1) {
4309 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4310 last if scalar(@{$currentrunningvcpus}) == $i;
4311 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4312 sleep 1;
4313 $retry++;
4314 }
4315 #update conf after each succesfull cpu hotplug
4316 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4317 PVE::QemuConfig->write_config($vmid, $conf);
4318 }
4319 } else {
4320
4321 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4322 mon_cmd($vmid, "cpu-add", id => int($i));
4323 }
4324 }
4325}
4326
4327sub qemu_block_set_io_throttle {
4328 my ($vmid, $deviceid,
4329 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4330 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4331 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4332 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4333
4334 return if !check_running($vmid) ;
4335
4336 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4337 bps => int($bps),
4338 bps_rd => int($bps_rd),
4339 bps_wr => int($bps_wr),
4340 iops => int($iops),
4341 iops_rd => int($iops_rd),
4342 iops_wr => int($iops_wr),
4343 bps_max => int($bps_max),
4344 bps_rd_max => int($bps_rd_max),
4345 bps_wr_max => int($bps_wr_max),
4346 iops_max => int($iops_max),
4347 iops_rd_max => int($iops_rd_max),
4348 iops_wr_max => int($iops_wr_max),
4349 bps_max_length => int($bps_max_length),
4350 bps_rd_max_length => int($bps_rd_max_length),
4351 bps_wr_max_length => int($bps_wr_max_length),
4352 iops_max_length => int($iops_max_length),
4353 iops_rd_max_length => int($iops_rd_max_length),
4354 iops_wr_max_length => int($iops_wr_max_length),
4355 );
4356
4357}
4358
4359sub qemu_block_resize {
4360 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4361
4362 my $running = check_running($vmid);
4363
4364 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4365
4366 return if !$running;
4367
4368 my $padding = (1024 - $size % 1024) % 1024;
4369 $size = $size + $padding;
4370
4371 mon_cmd(
4372 $vmid,
4373 "block_resize",
4374 device => $deviceid,
4375 size => int($size),
4376 timeout => 60,
4377 );
4378}
4379
4380sub qemu_volume_snapshot {
4381 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4382
4383 my $running = check_running($vmid);
4384
4385 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4386 mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4387 } else {
4388 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4389 }
4390}
4391
4392sub qemu_volume_snapshot_delete {
4393 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4394
4395 my $running = check_running($vmid);
4396
4397 if($running) {
4398
4399 $running = undef;
4400 my $conf = PVE::QemuConfig->load_config($vmid);
4401 PVE::QemuConfig->foreach_volume($conf, sub {
4402 my ($ds, $drive) = @_;
4403 $running = 1 if $drive->{file} eq $volid;
4404 });
4405 }
4406
4407 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4408 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4409 } else {
4410 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4411 }
4412}
4413
4414sub set_migration_caps {
4415 my ($vmid, $savevm) = @_;
4416
4417 my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4418
4419 my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4420 my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4421
4422 my $cap_ref = [];
4423
4424 my $enabled_cap = {
4425 "auto-converge" => 1,
4426 "xbzrle" => 1,
4427 "x-rdma-pin-all" => 0,
4428 "zero-blocks" => 0,
4429 "compress" => 0,
4430 "dirty-bitmaps" => $dirty_bitmaps,
4431 };
4432
4433 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4434
4435 for my $supported_capability (@$supported_capabilities) {
4436 push @$cap_ref, {
4437 capability => $supported_capability->{capability},
4438 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4439 };
4440 }
4441
4442 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4443}
4444
4445sub foreach_volid {
4446 my ($conf, $func, @param) = @_;
4447
4448 my $volhash = {};
4449
4450 my $test_volid = sub {
4451 my ($key, $drive, $snapname) = @_;
4452
4453 my $volid = $drive->{file};
4454 return if !$volid;
4455
4456 $volhash->{$volid}->{cdrom} //= 1;
4457 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4458
4459 my $replicate = $drive->{replicate} // 1;
4460 $volhash->{$volid}->{replicate} //= 0;
4461 $volhash->{$volid}->{replicate} = 1 if $replicate;
4462
4463 $volhash->{$volid}->{shared} //= 0;
4464 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4465
4466 $volhash->{$volid}->{referenced_in_config} //= 0;
4467 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4468
4469 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4470 if defined($snapname);
4471
4472 my $size = $drive->{size};
4473 $volhash->{$volid}->{size} //= $size if $size;
4474
4475 $volhash->{$volid}->{is_vmstate} //= 0;
4476 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4477
4478 $volhash->{$volid}->{is_unused} //= 0;
4479 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4480
4481 $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4482 };
4483
4484 my $include_opts = {
4485 extra_keys => ['vmstate'],
4486 include_unused => 1,
4487 };
4488
4489 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4490 foreach my $snapname (keys %{$conf->{snapshots}}) {
4491 my $snap = $conf->{snapshots}->{$snapname};
4492 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4493 }
4494
4495 foreach my $volid (keys %$volhash) {
4496 &$func($volid, $volhash->{$volid}, @param);
4497 }
4498}
4499
4500my $fast_plug_option = {
4501 'lock' => 1,
4502 'name' => 1,
4503 'onboot' => 1,
4504 'shares' => 1,
4505 'startup' => 1,
4506 'description' => 1,
4507 'protection' => 1,
4508 'vmstatestorage' => 1,
4509 'hookscript' => 1,
4510 'tags' => 1,
4511};
4512
4513# hotplug changes in [PENDING]
4514# $selection hash can be used to only apply specified options, for
4515# example: { cores => 1 } (only apply changed 'cores')
4516# $errors ref is used to return error messages
4517sub vmconfig_hotplug_pending {
4518 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4519
4520 my $defaults = load_defaults();
4521 my $arch = get_vm_arch($conf);
4522 my $machine_type = get_vm_machine($conf, undef, $arch);
4523
4524 # commit values which do not have any impact on running VM first
4525 # Note: those option cannot raise errors, we we do not care about
4526 # $selection and always apply them.
4527
4528 my $add_error = sub {
4529 my ($opt, $msg) = @_;
4530 $errors->{$opt} = "hotplug problem - $msg";
4531 };
4532
4533 my $changes = 0;
4534 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4535 if ($fast_plug_option->{$opt}) {
4536 $conf->{$opt} = $conf->{pending}->{$opt};
4537 delete $conf->{pending}->{$opt};
4538 $changes = 1;
4539 }
4540 }
4541
4542 if ($changes) {
4543 PVE::QemuConfig->write_config($vmid, $conf);
4544 }
4545
4546 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4547
4548 my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4549 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4550 foreach my $opt (sort keys %$pending_delete_hash) {
4551 next if $selection && !$selection->{$opt};
4552 my $force = $pending_delete_hash->{$opt}->{force};
4553 eval {
4554 if ($opt eq 'hotplug') {
4555 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4556 } elsif ($opt eq 'tablet') {
4557 die "skip\n" if !$hotplug_features->{usb};
4558 if ($defaults->{tablet}) {
4559 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4560 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4561 if $arch eq 'aarch64';
4562 } else {
4563 vm_deviceunplug($vmid, $conf, 'tablet');
4564 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4565 }
4566 } elsif ($opt =~ m/^usb\d+/) {
4567 die "skip\n";
4568 # since we cannot reliably hot unplug usb devices we are disabling it
4569 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4570 #vm_deviceunplug($vmid, $conf, $opt);
4571 } elsif ($opt eq 'vcpus') {
4572 die "skip\n" if !$hotplug_features->{cpu};
4573 qemu_cpu_hotplug($vmid, $conf, undef);
4574 } elsif ($opt eq 'balloon') {
4575 # enable balloon device is not hotpluggable
4576 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4577 # here we reset the ballooning value to memory
4578 my $balloon = $conf->{memory} || $defaults->{memory};
4579 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4580 } elsif ($fast_plug_option->{$opt}) {
4581 # do nothing
4582 } elsif ($opt =~ m/^net(\d+)$/) {
4583 die "skip\n" if !$hotplug_features->{network};
4584 vm_deviceunplug($vmid, $conf, $opt);
4585 } elsif (is_valid_drivename($opt)) {
4586 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4587 vm_deviceunplug($vmid, $conf, $opt);
4588 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4589 } elsif ($opt =~ m/^memory$/) {
4590 die "skip\n" if !$hotplug_features->{memory};
4591 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4592 } elsif ($opt eq 'cpuunits') {
4593 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4594 } elsif ($opt eq 'cpulimit') {
4595 $cgroup->change_cpu_quota(-1, 100000);
4596 } else {
4597 die "skip\n";
4598 }
4599 };
4600 if (my $err = $@) {
4601 &$add_error($opt, $err) if $err ne "skip\n";
4602 } else {
4603 delete $conf->{$opt};
4604 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4605 }
4606 }
4607
4608 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4609 $apply_pending_cloudinit = sub {
4610 return if $apply_pending_cloudinit_done; # once is enough
4611 $apply_pending_cloudinit_done = 1; # once is enough
4612
4613 my ($key, $value) = @_;
4614
4615 my @cloudinit_opts = keys %$confdesc_cloudinit;
4616 foreach my $opt (keys %{$conf->{pending}}) {
4617 next if !grep { $_ eq $opt } @cloudinit_opts;
4618 $conf->{$opt} = delete $conf->{pending}->{$opt};
4619 }
4620
4621 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4622 foreach my $opt (sort keys %$pending_delete_hash) {
4623 next if !grep { $_ eq $opt } @cloudinit_opts;
4624 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4625 delete $conf->{$opt};
4626 }
4627
4628 my $new_conf = { %$conf };
4629 $new_conf->{$key} = $value;
4630 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4631 };
4632
4633 foreach my $opt (keys %{$conf->{pending}}) {
4634 next if $selection && !$selection->{$opt};
4635 my $value = $conf->{pending}->{$opt};
4636 eval {
4637 if ($opt eq 'hotplug') {
4638 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4639 } elsif ($opt eq 'tablet') {
4640 die "skip\n" if !$hotplug_features->{usb};
4641 if ($value == 1) {
4642 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4643 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4644 if $arch eq 'aarch64';
4645 } elsif ($value == 0) {
4646 vm_deviceunplug($vmid, $conf, 'tablet');
4647 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4648 }
4649 } elsif ($opt =~ m/^usb\d+$/) {
4650 die "skip\n";
4651 # since we cannot reliably hot unplug usb devices we disable it for now
4652 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4653 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4654 #die "skip\n" if !$d;
4655 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4656 } elsif ($opt eq 'vcpus') {
4657 die "skip\n" if !$hotplug_features->{cpu};
4658 qemu_cpu_hotplug($vmid, $conf, $value);
4659 } elsif ($opt eq 'balloon') {
4660 # enable/disable balloning device is not hotpluggable
4661 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4662 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4663 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4664
4665 # allow manual ballooning if shares is set to zero
4666 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4667 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4668 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4669 }
4670 } elsif ($opt =~ m/^net(\d+)$/) {
4671 # some changes can be done without hotplug
4672 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4673 $vmid, $opt, $value, $arch, $machine_type);
4674 } elsif (is_valid_drivename($opt)) {
4675 die "skip\n" if $opt eq 'efidisk0';
4676 # some changes can be done without hotplug
4677 my $drive = parse_drive($opt, $value);
4678 if (drive_is_cloudinit($drive)) {
4679 &$apply_pending_cloudinit($opt, $value);
4680 }
4681 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4682 $vmid, $opt, $value, $arch, $machine_type);
4683 } elsif ($opt =~ m/^memory$/) { #dimms
4684 die "skip\n" if !$hotplug_features->{memory};
4685 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4686 } elsif ($opt eq 'cpuunits') {
4687 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4688 } elsif ($opt eq 'cpulimit') {
4689 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4690 $cgroup->change_cpu_quota($cpulimit, 100000);
4691 } else {
4692 die "skip\n"; # skip non-hot-pluggable options
4693 }
4694 };
4695 if (my $err = $@) {
4696 &$add_error($opt, $err) if $err ne "skip\n";
4697 } else {
4698 $conf->{$opt} = $value;
4699 delete $conf->{pending}->{$opt};
4700 }
4701 }
4702
4703 PVE::QemuConfig->write_config($vmid, $conf);
4704}
4705
4706sub try_deallocate_drive {
4707 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4708
4709 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4710 my $volid = $drive->{file};
4711 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4712 my $sid = PVE::Storage::parse_volume_id($volid);
4713 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4714
4715 # check if the disk is really unused
4716 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4717 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4718 PVE::Storage::vdisk_free($storecfg, $volid);
4719 return 1;
4720 } else {
4721 # If vm is not owner of this disk remove from config
4722 return 1;
4723 }
4724 }
4725
4726 return;
4727}
4728
4729sub vmconfig_delete_or_detach_drive {
4730 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4731
4732 my $drive = parse_drive($opt, $conf->{$opt});
4733
4734 my $rpcenv = PVE::RPCEnvironment::get();
4735 my $authuser = $rpcenv->get_user();
4736
4737 if ($force) {
4738 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4739 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4740 } else {
4741 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4742 }
4743}
4744
4745
4746
4747sub vmconfig_apply_pending {
4748 my ($vmid, $conf, $storecfg, $errors) = @_;
4749
4750 my $add_apply_error = sub {
4751 my ($opt, $msg) = @_;
4752 my $err_msg = "unable to apply pending change $opt : $msg";
4753 $errors->{$opt} = $err_msg;
4754 warn $err_msg;
4755 };
4756
4757 # cold plug
4758
4759 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4760 foreach my $opt (sort keys %$pending_delete_hash) {
4761 my $force = $pending_delete_hash->{$opt}->{force};
4762 eval {
4763 if ($opt =~ m/^unused/) {
4764 die "internal error";
4765 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4766 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4767 }
4768 };
4769 if (my $err = $@) {
4770 $add_apply_error->($opt, $err);
4771 } else {
4772 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4773 delete $conf->{$opt};
4774 }
4775 }
4776
4777 PVE::QemuConfig->cleanup_pending($conf);
4778
4779 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4780 next if $opt eq 'delete'; # just to be sure
4781 eval {
4782 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4783 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4784 }
4785 };
4786 if (my $err = $@) {
4787 $add_apply_error->($opt, $err);
4788 } else {
4789 $conf->{$opt} = delete $conf->{pending}->{$opt};
4790 }
4791 }
4792
4793 # write all changes at once to avoid unnecessary i/o
4794 PVE::QemuConfig->write_config($vmid, $conf);
4795}
4796
4797sub vmconfig_update_net {
4798 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4799
4800 my $newnet = parse_net($value);
4801
4802 if ($conf->{$opt}) {
4803 my $oldnet = parse_net($conf->{$opt});
4804
4805 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4806 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4807 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4808 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4809
4810 # for non online change, we try to hot-unplug
4811 die "skip\n" if !$hotplug;
4812 vm_deviceunplug($vmid, $conf, $opt);
4813 } else {
4814
4815 die "internal error" if $opt !~ m/net(\d+)/;
4816 my $iface = "tap${vmid}i$1";
4817
4818 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4819 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4820 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4821 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4822 PVE::Network::tap_unplug($iface);
4823
4824 if ($have_sdn) {
4825 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4826 } else {
4827 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4828 }
4829 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4830 # Rate can be applied on its own but any change above needs to
4831 # include the rate in tap_plug since OVS resets everything.
4832 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4833 }
4834
4835 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4836 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4837 }
4838
4839 return 1;
4840 }
4841 }
4842
4843 if ($hotplug) {
4844 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4845 } else {
4846 die "skip\n";
4847 }
4848}
4849
4850sub vmconfig_update_disk {
4851 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4852
4853 my $drive = parse_drive($opt, $value);
4854
4855 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4856 my $media = $drive->{media} || 'disk';
4857 my $oldmedia = $old_drive->{media} || 'disk';
4858 die "unable to change media type\n" if $media ne $oldmedia;
4859
4860 if (!drive_is_cdrom($old_drive)) {
4861
4862 if ($drive->{file} ne $old_drive->{file}) {
4863
4864 die "skip\n" if !$hotplug;
4865
4866 # unplug and register as unused
4867 vm_deviceunplug($vmid, $conf, $opt);
4868 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4869
4870 } else {
4871 # update existing disk
4872
4873 # skip non hotpluggable value
4874 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4875 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4876 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4877 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4878 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4879 die "skip\n";
4880 }
4881
4882 # apply throttle
4883 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4884 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4885 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4886 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4887 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4888 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4889 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4890 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4891 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4892 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4893 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4894 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4895 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4896 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4897 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4898 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4899 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4900 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4901
4902 qemu_block_set_io_throttle(
4903 $vmid,"drive-$opt",
4904 ($drive->{mbps} || 0)*1024*1024,
4905 ($drive->{mbps_rd} || 0)*1024*1024,
4906 ($drive->{mbps_wr} || 0)*1024*1024,
4907 $drive->{iops} || 0,
4908 $drive->{iops_rd} || 0,
4909 $drive->{iops_wr} || 0,
4910 ($drive->{mbps_max} || 0)*1024*1024,
4911 ($drive->{mbps_rd_max} || 0)*1024*1024,
4912 ($drive->{mbps_wr_max} || 0)*1024*1024,
4913 $drive->{iops_max} || 0,
4914 $drive->{iops_rd_max} || 0,
4915 $drive->{iops_wr_max} || 0,
4916 $drive->{bps_max_length} || 1,
4917 $drive->{bps_rd_max_length} || 1,
4918 $drive->{bps_wr_max_length} || 1,
4919 $drive->{iops_max_length} || 1,
4920 $drive->{iops_rd_max_length} || 1,
4921 $drive->{iops_wr_max_length} || 1,
4922 );
4923
4924 }
4925
4926 return 1;
4927 }
4928
4929 } else { # cdrom
4930
4931 if ($drive->{file} eq 'none') {
4932 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4933 if (drive_is_cloudinit($old_drive)) {
4934 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4935 }
4936 } else {
4937 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4938
4939 # force eject if locked
4940 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4941
4942 if ($path) {
4943 mon_cmd($vmid, "blockdev-change-medium",
4944 id => "$opt", filename => "$path");
4945 }
4946 }
4947
4948 return 1;
4949 }
4950 }
4951
4952 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4953 # hotplug new disks
4954 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4955 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4956}
4957
4958# called in locked context by incoming migration
4959sub vm_migrate_get_nbd_disks {
4960 my ($storecfg, $conf, $replicated_volumes) = @_;
4961
4962 my $local_volumes = {};
4963 PVE::QemuConfig->foreach_volume($conf, sub {
4964 my ($ds, $drive) = @_;
4965
4966 return if drive_is_cdrom($drive);
4967
4968 my $volid = $drive->{file};
4969
4970 return if !$volid;
4971
4972 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4973
4974 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4975 return if $scfg->{shared};
4976
4977 # replicated disks re-use existing state via bitmap
4978 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4979 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4980 });
4981 return $local_volumes;
4982}
4983
4984# called in locked context by incoming migration
4985sub vm_migrate_alloc_nbd_disks {
4986 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4987
4988 my $format = undef;
4989
4990 my $nbd = {};
4991 foreach my $opt (sort keys %$source_volumes) {
4992 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4993
4994 if ($use_existing) {
4995 $nbd->{$opt}->{drivestr} = print_drive($drive);
4996 $nbd->{$opt}->{volid} = $volid;
4997 $nbd->{$opt}->{replicated} = 1;
4998 next;
4999 }
5000
5001 # If a remote storage is specified and the format of the original
5002 # volume is not available there, fall back to the default format.
5003 # Otherwise use the same format as the original.
5004 if (!$storagemap->{identity}) {
5005 $storeid = map_storage($storagemap, $storeid);
5006 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5007 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5008 my $fileFormat = qemu_img_format($scfg, $volname);
5009 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5010 } else {
5011 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5012 $format = qemu_img_format($scfg, $volname);
5013 }
5014
5015 my $size = $drive->{size} / 1024;
5016 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5017 my $newdrive = $drive;
5018 $newdrive->{format} = $format;
5019 $newdrive->{file} = $newvolid;
5020 my $drivestr = print_drive($newdrive);
5021 $nbd->{$opt}->{drivestr} = $drivestr;
5022 $nbd->{$opt}->{volid} = $newvolid;
5023 }
5024
5025 return $nbd;
5026}
5027
5028# see vm_start_nolock for parameters, additionally:
5029# migrate_opts:
5030# storagemap = parsed storage map for allocating NBD disks
5031sub vm_start {
5032 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5033
5034 return PVE::QemuConfig->lock_config($vmid, sub {
5035 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5036
5037 die "you can't start a vm if it's a template\n"
5038 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5039
5040 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5041 my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5042
5043 my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5044
5045 if ($has_backup_lock && $running) {
5046 # a backup is currently running, attempt to start the guest in the
5047 # existing QEMU instance
5048 return vm_resume($vmid);
5049 }
5050
5051 PVE::QemuConfig->check_lock($conf)
5052 if !($params->{skiplock} || $has_suspended_lock);
5053
5054 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5055
5056 die "VM $vmid already running\n" if $running;
5057
5058 if (my $storagemap = $migrate_opts->{storagemap}) {
5059 my $replicated = $migrate_opts->{replicated_volumes};
5060 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5061 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5062
5063 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5064 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5065 }
5066 }
5067
5068 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5069 });
5070}
5071
5072
5073# params:
5074# statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5075# skiplock => 0/1, skip checking for config lock
5076# skiptemplate => 0/1, skip checking whether VM is template
5077# forcemachine => to force Qemu machine (rollback/migration)
5078# forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5079# timeout => in seconds
5080# paused => start VM in paused state (backup)
5081# resume => resume from hibernation
5082# pbs-backing => {
5083# sata0 => {
5084# repository
5085# snapshot
5086# keyfile
5087# archive
5088# },
5089# virtio2 => ...
5090# }
5091# migrate_opts:
5092# nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5093# migratedfrom => source node
5094# spice_ticket => used for spice migration, passed via tunnel/stdin
5095# network => CIDR of migration network
5096# type => secure/insecure - tunnel over encrypted connection or plain-text
5097# nbd_proto_version => int, 0 for TCP, 1 for UNIX
5098# replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5099sub vm_start_nolock {
5100 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5101
5102 my $statefile = $params->{statefile};
5103 my $resume = $params->{resume};
5104
5105 my $migratedfrom = $migrate_opts->{migratedfrom};
5106 my $migration_type = $migrate_opts->{type};
5107
5108 my $res = {};
5109
5110 # clean up leftover reboot request files
5111 eval { clear_reboot_request($vmid); };
5112 warn $@ if $@;
5113
5114 if (!$statefile && scalar(keys %{$conf->{pending}})) {
5115 vmconfig_apply_pending($vmid, $conf, $storecfg);
5116 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5117 }
5118
5119 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5120
5121 my $defaults = load_defaults();
5122
5123 # set environment variable useful inside network script
5124 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5125
5126 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5127
5128 my $forcemachine = $params->{forcemachine};
5129 my $forcecpu = $params->{forcecpu};
5130 if ($resume) {
5131 # enforce machine and CPU type on suspended vm to ensure HW compatibility
5132 $forcemachine = $conf->{runningmachine};
5133 $forcecpu = $conf->{runningcpu};
5134 print "Resuming suspended VM\n";
5135 }
5136
5137 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5138 $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5139
5140 my $migration_ip;
5141 my $get_migration_ip = sub {
5142 my ($nodename) = @_;
5143
5144 return $migration_ip if defined($migration_ip);
5145
5146 my $cidr = $migrate_opts->{network};
5147
5148 if (!defined($cidr)) {
5149 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5150 $cidr = $dc_conf->{migration}->{network};
5151 }
5152
5153 if (defined($cidr)) {
5154 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5155
5156 die "could not get IP: no address configured on local " .
5157 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5158
5159 die "could not get IP: multiple addresses configured on local " .
5160 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5161
5162 $migration_ip = @$ips[0];
5163 }
5164
5165 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5166 if !defined($migration_ip);
5167
5168 return $migration_ip;
5169 };
5170
5171 my $migrate_uri;
5172 if ($statefile) {
5173 if ($statefile eq 'tcp') {
5174 my $localip = "localhost";
5175 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5176 my $nodename = nodename();
5177
5178 if (!defined($migration_type)) {
5179 if (defined($datacenterconf->{migration}->{type})) {
5180 $migration_type = $datacenterconf->{migration}->{type};
5181 } else {
5182 $migration_type = 'secure';
5183 }
5184 }
5185
5186 if ($migration_type eq 'insecure') {
5187 $localip = $get_migration_ip->($nodename);
5188 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5189 }
5190
5191 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5192 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5193 $migrate_uri = "tcp:${localip}:${migrate_port}";
5194 push @$cmd, '-incoming', $migrate_uri;
5195 push @$cmd, '-S';
5196
5197 } elsif ($statefile eq 'unix') {
5198 # should be default for secure migrations as a ssh TCP forward
5199 # tunnel is not deterministic reliable ready and fails regurarly
5200 # to set up in time, so use UNIX socket forwards
5201 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5202 unlink $socket_addr;
5203
5204 $migrate_uri = "unix:$socket_addr";
5205
5206 push @$cmd, '-incoming', $migrate_uri;
5207 push @$cmd, '-S';
5208
5209 } elsif (-e $statefile) {
5210 push @$cmd, '-loadstate', $statefile;
5211 } else {
5212 my $statepath = PVE::Storage::path($storecfg, $statefile);
5213 push @$vollist, $statefile;
5214 push @$cmd, '-loadstate', $statepath;
5215 }
5216 } elsif ($params->{paused}) {
5217 push @$cmd, '-S';
5218 }
5219
5220 # host pci devices
5221 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5222 my $d = parse_hostpci($conf->{"hostpci$i"});
5223 next if !$d;
5224 my $pcidevices = $d->{pciid};
5225 foreach my $pcidevice (@$pcidevices) {
5226 my $pciid = $pcidevice->{id};
5227
5228 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5229 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5230 die "no pci device info for device '$pciid'\n" if !$info;
5231
5232 if ($d->{mdev}) {
5233 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5234 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5235 } else {
5236 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5237 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5238 die "can't reset PCI device '$pciid'\n"
5239 if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5240 }
5241 }
5242 }
5243
5244 PVE::Storage::activate_volumes($storecfg, $vollist);
5245
5246 eval {
5247 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5248 outfunc => sub {}, errfunc => sub {});
5249 };
5250 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5251 # timeout should be more than enough here...
5252 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5253
5254 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5255 : $defaults->{cpuunits};
5256
5257 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5258 my %run_params = (
5259 timeout => $statefile ? undef : $start_timeout,
5260 umask => 0077,
5261 noerr => 1,
5262 );
5263
5264 # when migrating, prefix QEMU output so other side can pick up any
5265 # errors that might occur and show the user
5266 if ($migratedfrom) {
5267 $run_params{quiet} = 1;
5268 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5269 }
5270
5271 my %properties = (
5272 Slice => 'qemu.slice',
5273 KillMode => 'none'
5274 );
5275
5276 if (PVE::CGroup::cgroup_mode() == 2) {
5277 $properties{CPUWeight} = $cpuunits;
5278 } else {
5279 $properties{CPUShares} = $cpuunits;
5280 }
5281
5282 if (my $cpulimit = $conf->{cpulimit}) {
5283 $properties{CPUQuota} = int($cpulimit * 100);
5284 }
5285 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5286
5287 my $run_qemu = sub {
5288 PVE::Tools::run_fork sub {
5289 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5290
5291 my $exitcode = run_command($cmd, %run_params);
5292 die "QEMU exited with code $exitcode\n" if $exitcode;
5293 };
5294 };
5295
5296 if ($conf->{hugepages}) {
5297
5298 my $code = sub {
5299 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5300 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5301
5302 PVE::QemuServer::Memory::hugepages_mount();
5303 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5304
5305 eval { $run_qemu->() };
5306 if (my $err = $@) {
5307 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5308 if !$conf->{keephugepages};
5309 die $err;
5310 }
5311
5312 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5313 if !$conf->{keephugepages};
5314 };
5315 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5316
5317 } else {
5318 eval { $run_qemu->() };
5319 }
5320
5321 if (my $err = $@) {
5322 # deactivate volumes if start fails
5323 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5324 die "start failed: $err";
5325 }
5326
5327 print "migration listens on $migrate_uri\n" if $migrate_uri;
5328 $res->{migrate_uri} = $migrate_uri;
5329
5330 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5331 eval { mon_cmd($vmid, "cont"); };
5332 warn $@ if $@;
5333 }
5334
5335 #start nbd server for storage migration
5336 if (my $nbd = $migrate_opts->{nbd}) {
5337 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5338
5339 my $migrate_storage_uri;
5340 # nbd_protocol_version > 0 for unix socket support
5341 if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5342 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5343 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5344 $migrate_storage_uri = "nbd:unix:$socket_path";
5345 } else {
5346 my $nodename = nodename();
5347 my $localip = $get_migration_ip->($nodename);
5348 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5349 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5350
5351 mon_cmd($vmid, "nbd-server-start", addr => {
5352 type => 'inet',
5353 data => {
5354 host => "${localip}",
5355 port => "${storage_migrate_port}",
5356 },
5357 });
5358 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5359 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5360 }
5361
5362 $res->{migrate_storage_uri} = $migrate_storage_uri;
5363
5364 foreach my $opt (sort keys %$nbd) {
5365 my $drivestr = $nbd->{$opt}->{drivestr};
5366 my $volid = $nbd->{$opt}->{volid};
5367 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5368 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5369 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5370 print "re-using replicated volume: $opt - $volid\n"
5371 if $nbd->{$opt}->{replicated};
5372
5373 $res->{drives}->{$opt} = $nbd->{$opt};
5374 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5375 }
5376 }
5377
5378 if ($migratedfrom) {
5379 eval {
5380 set_migration_caps($vmid);
5381 };
5382 warn $@ if $@;
5383
5384 if ($spice_port) {
5385 print "spice listens on port $spice_port\n";
5386 $res->{spice_port} = $spice_port;
5387 if ($migrate_opts->{spice_ticket}) {
5388 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5389 $migrate_opts->{spice_ticket});
5390 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5391 }
5392 }
5393
5394 } else {
5395 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5396 if !$statefile && $conf->{balloon};
5397
5398 foreach my $opt (keys %$conf) {
5399 next if $opt !~ m/^net\d+$/;
5400 my $nicconf = parse_net($conf->{$opt});
5401 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5402 }
5403 }
5404
5405 mon_cmd($vmid, 'qom-set',
5406 path => "machine/peripheral/balloon0",
5407 property => "guest-stats-polling-interval",
5408 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5409
5410 if ($resume) {
5411 print "Resumed VM, removing state\n";
5412 if (my $vmstate = $conf->{vmstate}) {
5413 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5414 PVE::Storage::vdisk_free($storecfg, $vmstate);
5415 }
5416 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5417 PVE::QemuConfig->write_config($vmid, $conf);
5418 }
5419
5420 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5421
5422 return $res;
5423}
5424
5425sub vm_commandline {
5426 my ($storecfg, $vmid, $snapname) = @_;
5427
5428 my $conf = PVE::QemuConfig->load_config($vmid);
5429 my $forcemachine;
5430 my $forcecpu;
5431
5432 if ($snapname) {
5433 my $snapshot = $conf->{snapshots}->{$snapname};
5434 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5435
5436 # check for machine or CPU overrides in snapshot
5437 $forcemachine = $snapshot->{runningmachine};
5438 $forcecpu = $snapshot->{runningcpu};
5439
5440 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5441
5442 $conf = $snapshot;
5443 }
5444
5445 my $defaults = load_defaults();
5446
5447 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5448 $forcemachine, $forcecpu);
5449
5450 return PVE::Tools::cmd2string($cmd);
5451}
5452
5453sub vm_reset {
5454 my ($vmid, $skiplock) = @_;
5455
5456 PVE::QemuConfig->lock_config($vmid, sub {
5457
5458 my $conf = PVE::QemuConfig->load_config($vmid);
5459
5460 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5461
5462 mon_cmd($vmid, "system_reset");
5463 });
5464}
5465
5466sub get_vm_volumes {
5467 my ($conf) = @_;
5468
5469 my $vollist = [];
5470 foreach_volid($conf, sub {
5471 my ($volid, $attr) = @_;
5472
5473 return if $volid =~ m|^/|;
5474
5475 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5476 return if !$sid;
5477
5478 push @$vollist, $volid;
5479 });
5480
5481 return $vollist;
5482}
5483
5484sub vm_stop_cleanup {
5485 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5486
5487 eval {
5488
5489 if (!$keepActive) {
5490 my $vollist = get_vm_volumes($conf);
5491 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5492 }
5493
5494 foreach my $ext (qw(mon qmp pid vnc qga)) {
5495 unlink "/var/run/qemu-server/${vmid}.$ext";
5496 }
5497
5498 if ($conf->{ivshmem}) {
5499 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5500 # just delete it for now, VMs which have this already open do not
5501 # are affected, but new VMs will get a separated one. If this
5502 # becomes an issue we either add some sort of ref-counting or just
5503 # add a "don't delete on stop" flag to the ivshmem format.
5504 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5505 }
5506
5507 foreach my $key (keys %$conf) {
5508 next if $key !~ m/^hostpci(\d+)$/;
5509 my $hostpciindex = $1;
5510 my $d = parse_hostpci($conf->{$key});
5511 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5512
5513 foreach my $pci (@{$d->{pciid}}) {
5514 my $pciid = $pci->{id};
5515 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5516 }
5517 }
5518
5519 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5520 };
5521 warn $@ if $@; # avoid errors - just warn
5522}
5523
5524# call only in locked context
5525sub _do_vm_stop {
5526 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5527
5528 my $pid = check_running($vmid, $nocheck);
5529 return if !$pid;
5530
5531 my $conf;
5532 if (!$nocheck) {
5533 $conf = PVE::QemuConfig->load_config($vmid);
5534 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5535 if (!defined($timeout) && $shutdown && $conf->{startup}) {
5536 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5537 $timeout = $opts->{down} if $opts->{down};
5538 }
5539 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5540 }
5541
5542 eval {
5543 if ($shutdown) {
5544 if (defined($conf) && get_qga_key($conf, 'enabled')) {
5545 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5546 } else {
5547 mon_cmd($vmid, "system_powerdown");
5548 }
5549 } else {
5550 mon_cmd($vmid, "quit");
5551 }
5552 };
5553 my $err = $@;
5554
5555 if (!$err) {
5556 $timeout = 60 if !defined($timeout);
5557
5558 my $count = 0;
5559 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5560 $count++;
5561 sleep 1;
5562 }
5563
5564 if ($count >= $timeout) {
5565 if ($force) {
5566 warn "VM still running - terminating now with SIGTERM\n";
5567 kill 15, $pid;
5568 } else {
5569 die "VM quit/powerdown failed - got timeout\n";
5570 }
5571 } else {
5572 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5573 return;
5574 }
5575 } else {
5576 if (!check_running($vmid, $nocheck)) {
5577 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5578 return;
5579 }
5580 if ($force) {
5581 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5582 kill 15, $pid;
5583 } else {
5584 die "VM quit/powerdown failed\n";
5585 }
5586 }
5587
5588 # wait again
5589 $timeout = 10;
5590
5591 my $count = 0;
5592 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5593 $count++;
5594 sleep 1;
5595 }
5596
5597 if ($count >= $timeout) {
5598 warn "VM still running - terminating now with SIGKILL\n";
5599 kill 9, $pid;
5600 sleep 1;
5601 }
5602
5603 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5604}
5605
5606# Note: use $nocheck to skip tests if VM configuration file exists.
5607# We need that when migration VMs to other nodes (files already moved)
5608# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5609sub vm_stop {
5610 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5611
5612 $force = 1 if !defined($force) && !$shutdown;
5613
5614 if ($migratedfrom){
5615 my $pid = check_running($vmid, $nocheck, $migratedfrom);
5616 kill 15, $pid if $pid;
5617 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5618 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5619 return;
5620 }
5621
5622 PVE::QemuConfig->lock_config($vmid, sub {
5623 _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5624 });
5625}
5626
5627sub vm_reboot {
5628 my ($vmid, $timeout) = @_;
5629
5630 PVE::QemuConfig->lock_config($vmid, sub {
5631 eval {
5632
5633 # only reboot if running, as qmeventd starts it again on a stop event
5634 return if !check_running($vmid);
5635
5636 create_reboot_request($vmid);
5637
5638 my $storecfg = PVE::Storage::config();
5639 _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5640
5641 };
5642 if (my $err = $@) {
5643 # avoid that the next normal shutdown will be confused for a reboot
5644 clear_reboot_request($vmid);
5645 die $err;
5646 }
5647 });
5648}
5649
5650# note: if using the statestorage parameter, the caller has to check privileges
5651sub vm_suspend {
5652 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5653
5654 my $conf;
5655 my $path;
5656 my $storecfg;
5657 my $vmstate;
5658
5659 PVE::QemuConfig->lock_config($vmid, sub {
5660
5661 $conf = PVE::QemuConfig->load_config($vmid);
5662
5663 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5664 PVE::QemuConfig->check_lock($conf)
5665 if !($skiplock || $is_backing_up);
5666
5667 die "cannot suspend to disk during backup\n"
5668 if $is_backing_up && $includestate;
5669
5670 if ($includestate) {
5671 $conf->{lock} = 'suspending';
5672 my $date = strftime("%Y-%m-%d", localtime(time()));
5673 $storecfg = PVE::Storage::config();
5674 if (!$statestorage) {
5675 $statestorage = find_vmstate_storage($conf, $storecfg);
5676 # check permissions for the storage
5677 my $rpcenv = PVE::RPCEnvironment::get();
5678 if ($rpcenv->{type} ne 'cli') {
5679 my $authuser = $rpcenv->get_user();
5680 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5681 }
5682 }
5683
5684
5685 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5686 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5687 $path = PVE::Storage::path($storecfg, $vmstate);
5688 PVE::QemuConfig->write_config($vmid, $conf);
5689 } else {
5690 mon_cmd($vmid, "stop");
5691 }
5692 });
5693
5694 if ($includestate) {
5695 # save vm state
5696 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5697
5698 eval {
5699 set_migration_caps($vmid, 1);
5700 mon_cmd($vmid, "savevm-start", statefile => $path);
5701 for(;;) {
5702 my $state = mon_cmd($vmid, "query-savevm");
5703 if (!$state->{status}) {
5704 die "savevm not active\n";
5705 } elsif ($state->{status} eq 'active') {
5706 sleep(1);
5707 next;
5708 } elsif ($state->{status} eq 'completed') {
5709 print "State saved, quitting\n";
5710 last;
5711 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5712 die "query-savevm failed with error '$state->{error}'\n"
5713 } else {
5714 die "query-savevm returned status '$state->{status}'\n";
5715 }
5716 }
5717 };
5718 my $err = $@;
5719
5720 PVE::QemuConfig->lock_config($vmid, sub {
5721 $conf = PVE::QemuConfig->load_config($vmid);
5722 if ($err) {
5723 # cleanup, but leave suspending lock, to indicate something went wrong
5724 eval {
5725 mon_cmd($vmid, "savevm-end");
5726 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5727 PVE::Storage::vdisk_free($storecfg, $vmstate);
5728 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5729 PVE::QemuConfig->write_config($vmid, $conf);
5730 };
5731 warn $@ if $@;
5732 die $err;
5733 }
5734
5735 die "lock changed unexpectedly\n"
5736 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5737
5738 mon_cmd($vmid, "quit");
5739 $conf->{lock} = 'suspended';
5740 PVE::QemuConfig->write_config($vmid, $conf);
5741 });
5742 }
5743}
5744
5745sub vm_resume {
5746 my ($vmid, $skiplock, $nocheck) = @_;
5747
5748 PVE::QemuConfig->lock_config($vmid, sub {
5749 my $res = mon_cmd($vmid, 'query-status');
5750 my $resume_cmd = 'cont';
5751 my $reset = 0;
5752
5753 if ($res->{status}) {
5754 return if $res->{status} eq 'running'; # job done, go home
5755 $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5756 $reset = 1 if $res->{status} eq 'shutdown';
5757 }
5758
5759 if (!$nocheck) {
5760
5761 my $conf = PVE::QemuConfig->load_config($vmid);
5762
5763 PVE::QemuConfig->check_lock($conf)
5764 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5765 }
5766
5767 if ($reset) {
5768 # required if a VM shuts down during a backup and we get a resume
5769 # request before the backup finishes for example
5770 mon_cmd($vmid, "system_reset");
5771 }
5772 mon_cmd($vmid, $resume_cmd);
5773 });
5774}
5775
5776sub vm_sendkey {
5777 my ($vmid, $skiplock, $key) = @_;
5778
5779 PVE::QemuConfig->lock_config($vmid, sub {
5780
5781 my $conf = PVE::QemuConfig->load_config($vmid);
5782
5783 # there is no qmp command, so we use the human monitor command
5784 my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5785 die $res if $res ne '';
5786 });
5787}
5788
5789# vzdump restore implementaion
5790
5791sub tar_archive_read_firstfile {
5792 my $archive = shift;
5793
5794 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5795
5796 # try to detect archive type first
5797 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5798 die "unable to open file '$archive'\n";
5799 my $firstfile = <$fh>;
5800 kill 15, $pid;
5801 close $fh;
5802
5803 die "ERROR: archive contaions no data\n" if !$firstfile;
5804 chomp $firstfile;
5805
5806 return $firstfile;
5807}
5808
5809sub tar_restore_cleanup {
5810 my ($storecfg, $statfile) = @_;
5811
5812 print STDERR "starting cleanup\n";
5813
5814 if (my $fd = IO::File->new($statfile, "r")) {
5815 while (defined(my $line = <$fd>)) {
5816 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5817 my $volid = $2;
5818 eval {
5819 if ($volid =~ m|^/|) {
5820 unlink $volid || die 'unlink failed\n';
5821 } else {
5822 PVE::Storage::vdisk_free($storecfg, $volid);
5823 }
5824 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5825 };
5826 print STDERR "unable to cleanup '$volid' - $@" if $@;
5827 } else {
5828 print STDERR "unable to parse line in statfile - $line";
5829 }
5830 }
5831 $fd->close();
5832 }
5833}
5834
5835sub restore_file_archive {
5836 my ($archive, $vmid, $user, $opts) = @_;
5837
5838 return restore_vma_archive($archive, $vmid, $user, $opts)
5839 if $archive eq '-';
5840
5841 my $info = PVE::Storage::archive_info($archive);
5842 my $format = $opts->{format} // $info->{format};
5843 my $comp = $info->{compression};
5844
5845 # try to detect archive format
5846 if ($format eq 'tar') {
5847 return restore_tar_archive($archive, $vmid, $user, $opts);
5848 } else {
5849 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5850 }
5851}
5852
5853# hepler to remove disks that will not be used after restore
5854my $restore_cleanup_oldconf = sub {
5855 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5856
5857 PVE::QemuConfig->foreach_volume($oldconf, sub {
5858 my ($ds, $drive) = @_;
5859
5860 return if drive_is_cdrom($drive, 1);
5861
5862 my $volid = $drive->{file};
5863 return if !$volid || $volid =~ m|^/|;
5864
5865 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5866 return if !$path || !$owner || ($owner != $vmid);
5867
5868 # Note: only delete disk we want to restore
5869 # other volumes will become unused
5870 if ($virtdev_hash->{$ds}) {
5871 eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5872 if (my $err = $@) {
5873 warn $err;
5874 }
5875 }
5876 });
5877
5878 # delete vmstate files, after the restore we have no snapshots anymore
5879 foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5880 my $snap = $oldconf->{snapshots}->{$snapname};
5881 if ($snap->{vmstate}) {
5882 eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5883 if (my $err = $@) {
5884 warn $err;
5885 }
5886 }
5887 }
5888};
5889
5890# Helper to parse vzdump backup device hints
5891#
5892# $rpcenv: Environment, used to ckeck storage permissions
5893# $user: User ID, to check storage permissions
5894# $storecfg: Storage configuration
5895# $fh: the file handle for reading the configuration
5896# $devinfo: should contain device sizes for all backu-up'ed devices
5897# $options: backup options (pool, default storage)
5898#
5899# Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5900my $parse_backup_hints = sub {
5901 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5902
5903 my $virtdev_hash = {};
5904
5905 while (defined(my $line = <$fh>)) {
5906 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5907 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5908 die "archive does not contain data for drive '$virtdev'\n"
5909 if !$devinfo->{$devname};
5910
5911 if (defined($options->{storage})) {
5912 $storeid = $options->{storage} || 'local';
5913 } elsif (!$storeid) {
5914 $storeid = 'local';
5915 }
5916 $format = 'raw' if !$format;
5917 $devinfo->{$devname}->{devname} = $devname;
5918 $devinfo->{$devname}->{virtdev} = $virtdev;
5919 $devinfo->{$devname}->{format} = $format;
5920 $devinfo->{$devname}->{storeid} = $storeid;
5921
5922 # check permission on storage
5923 my $pool = $options->{pool}; # todo: do we need that?
5924 if ($user ne 'root@pam') {
5925 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5926 }
5927
5928 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5929 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5930 my $virtdev = $1;
5931 my $drive = parse_drive($virtdev, $2);
5932 if (drive_is_cloudinit($drive)) {
5933 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5934 $storeid = $options->{storage} if defined ($options->{storage});
5935 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5936 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5937
5938 $virtdev_hash->{$virtdev} = {
5939 format => $format,
5940 storeid => $storeid,
5941 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5942 is_cloudinit => 1,
5943 };
5944 }
5945 }
5946 }
5947
5948 return $virtdev_hash;
5949};
5950
5951# Helper to allocate and activate all volumes required for a restore
5952#
5953# $storecfg: Storage configuration
5954# $virtdev_hash: as returned by parse_backup_hints()
5955#
5956# Returns: { $virtdev => $volid }
5957my $restore_allocate_devices = sub {
5958 my ($storecfg, $virtdev_hash, $vmid) = @_;
5959
5960 my $map = {};
5961 foreach my $virtdev (sort keys %$virtdev_hash) {
5962 my $d = $virtdev_hash->{$virtdev};
5963 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5964 my $storeid = $d->{storeid};
5965 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5966
5967 # test if requested format is supported
5968 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5969 my $supported = grep { $_ eq $d->{format} } @$validFormats;
5970 $d->{format} = $defFormat if !$supported;
5971
5972 my $name;
5973 if ($d->{is_cloudinit}) {
5974 $name = "vm-$vmid-cloudinit";
5975 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5976 if ($scfg->{path}) {
5977 $name .= ".$d->{format}";
5978 }
5979 }
5980
5981 my $volid = PVE::Storage::vdisk_alloc(
5982 $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5983
5984 print STDERR "new volume ID is '$volid'\n";
5985 $d->{volid} = $volid;
5986
5987 PVE::Storage::activate_volumes($storecfg, [$volid]);
5988
5989 $map->{$virtdev} = $volid;
5990 }
5991
5992 return $map;
5993};
5994
5995sub restore_update_config_line {
5996 my ($cookie, $map, $line, $unique) = @_;
5997
5998 return '' if $line =~ m/^\#qmdump\#/;
5999 return '' if $line =~ m/^\#vzdump\#/;
6000 return '' if $line =~ m/^lock:/;
6001 return '' if $line =~ m/^unused\d+:/;
6002 return '' if $line =~ m/^parent:/;
6003
6004 my $res = '';
6005
6006 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6007 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6008 # try to convert old 1.X settings
6009 my ($id, $ind, $ethcfg) = ($1, $2, $3);
6010 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6011 my ($model, $macaddr) = split(/\=/, $devconfig);
6012 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6013 my $net = {
6014 model => $model,
6015 bridge => "vmbr$ind",
6016 macaddr => $macaddr,
6017 };
6018 my $netstr = print_net($net);
6019
6020 $res .= "net$cookie->{netcount}: $netstr\n";
6021 $cookie->{netcount}++;
6022 }
6023 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6024 my ($id, $netstr) = ($1, $2);
6025 my $net = parse_net($netstr);
6026 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6027 $netstr = print_net($net);
6028 $res .= "$id: $netstr\n";
6029 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6030 my $virtdev = $1;
6031 my $value = $3;
6032 my $di = parse_drive($virtdev, $value);
6033 if (defined($di->{backup}) && !$di->{backup}) {
6034 $res .= "#$line";
6035 } elsif ($map->{$virtdev}) {
6036 delete $di->{format}; # format can change on restore
6037 $di->{file} = $map->{$virtdev};
6038 $value = print_drive($di);
6039 $res .= "$virtdev: $value\n";
6040 } else {
6041 $res .= $line;
6042 }
6043 } elsif (($line =~ m/^vmgenid: (.*)/)) {
6044 my $vmgenid = $1;
6045 if ($vmgenid ne '0') {
6046 # always generate a new vmgenid if there was a valid one setup
6047 $vmgenid = generate_uuid();
6048 }
6049 $res .= "vmgenid: $vmgenid\n";
6050 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6051 my ($uuid, $uuid_str);
6052 UUID::generate($uuid);
6053 UUID::unparse($uuid, $uuid_str);
6054 my $smbios1 = parse_smbios1($2);
6055 $smbios1->{uuid} = $uuid_str;
6056 $res .= $1.print_smbios1($smbios1)."\n";
6057 } else {
6058 $res .= $line;
6059 }
6060
6061 return $res;
6062}
6063
6064my $restore_deactivate_volumes = sub {
6065 my ($storecfg, $devinfo) = @_;
6066
6067 my $vollist = [];
6068 foreach my $devname (keys %$devinfo) {
6069 my $volid = $devinfo->{$devname}->{volid};
6070 push @$vollist, $volid if $volid;
6071 }
6072
6073 PVE::Storage::deactivate_volumes($storecfg, $vollist);
6074};
6075
6076my $restore_destroy_volumes = sub {
6077 my ($storecfg, $devinfo) = @_;
6078
6079 foreach my $devname (keys %$devinfo) {
6080 my $volid = $devinfo->{$devname}->{volid};
6081 next if !$volid;
6082 eval {
6083 if ($volid =~ m|^/|) {
6084 unlink $volid || die 'unlink failed\n';
6085 } else {
6086 PVE::Storage::vdisk_free($storecfg, $volid);
6087 }
6088 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6089 };
6090 print STDERR "unable to cleanup '$volid' - $@" if $@;
6091 }
6092};
6093
6094# FIXME For PVE 7.0, remove $content_type and always use 'images'
6095sub scan_volids {
6096 my ($cfg, $vmid, $content_type) = @_;
6097
6098 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, $content_type);
6099
6100 my $volid_hash = {};
6101 foreach my $storeid (keys %$info) {
6102 foreach my $item (@{$info->{$storeid}}) {
6103 next if !($item->{volid} && $item->{size});
6104 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6105 $volid_hash->{$item->{volid}} = $item;
6106 }
6107 }
6108
6109 return $volid_hash;
6110}
6111
6112sub update_disk_config {
6113 my ($vmid, $conf, $volid_hash) = @_;
6114
6115 my $changes;
6116 my $prefix = "VM $vmid";
6117
6118 # used and unused disks
6119 my $referenced = {};
6120
6121 # Note: it is allowed to define multiple storages with same path (alias), so
6122 # we need to check both 'volid' and real 'path' (two different volid can point
6123 # to the same path).
6124
6125 my $referencedpath = {};
6126
6127 # update size info
6128 PVE::QemuConfig->foreach_volume($conf, sub {
6129 my ($opt, $drive) = @_;
6130
6131 my $volid = $drive->{file};
6132 return if !$volid;
6133 my $volume = $volid_hash->{$volid};
6134
6135 # mark volid as "in-use" for next step
6136 $referenced->{$volid} = 1;
6137 if ($volume && (my $path = $volume->{path})) {
6138 $referencedpath->{$path} = 1;
6139 }
6140
6141 return if drive_is_cdrom($drive);
6142 return if !$volume;
6143
6144 my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6145 if (defined($updated)) {
6146 $changes = 1;
6147 $conf->{$opt} = print_drive($updated);
6148 print "$prefix ($opt): $msg\n";
6149 }
6150 });
6151
6152 # remove 'unusedX' entry if volume is used
6153 PVE::QemuConfig->foreach_unused_volume($conf, sub {
6154 my ($opt, $drive) = @_;
6155
6156 my $volid = $drive->{file};
6157 return if !$volid;
6158
6159 my $path;
6160 $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6161 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6162 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6163 $changes = 1;
6164 delete $conf->{$opt};
6165 }
6166
6167 $referenced->{$volid} = 1;
6168 $referencedpath->{$path} = 1 if $path;
6169 });
6170
6171 foreach my $volid (sort keys %$volid_hash) {
6172 next if $volid =~ m/vm-$vmid-state-/;
6173 next if $referenced->{$volid};
6174 my $path = $volid_hash->{$volid}->{path};
6175 next if !$path; # just to be sure
6176 next if $referencedpath->{$path};
6177 $changes = 1;
6178 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6179 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6180 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6181 }
6182
6183 return $changes;
6184}
6185
6186sub rescan {
6187 my ($vmid, $nolock, $dryrun) = @_;
6188
6189 my $cfg = PVE::Storage::config();
6190
6191 print "rescan volumes...\n";
6192 my $volid_hash = scan_volids($cfg, $vmid, 'images');
6193
6194 my $updatefn = sub {
6195 my ($vmid) = @_;
6196
6197 my $conf = PVE::QemuConfig->load_config($vmid);
6198
6199 PVE::QemuConfig->check_lock($conf);
6200
6201 my $vm_volids = {};
6202 foreach my $volid (keys %$volid_hash) {
6203 my $info = $volid_hash->{$volid};
6204 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6205 }
6206
6207 my $changes = update_disk_config($vmid, $conf, $vm_volids);
6208
6209 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6210 };
6211
6212 if (defined($vmid)) {
6213 if ($nolock) {
6214 &$updatefn($vmid);
6215 } else {
6216 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6217 }
6218 } else {
6219 my $vmlist = config_list();
6220 foreach my $vmid (keys %$vmlist) {
6221 if ($nolock) {
6222 &$updatefn($vmid);
6223 } else {
6224 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6225 }
6226 }
6227 }
6228}
6229
6230sub restore_proxmox_backup_archive {
6231 my ($archive, $vmid, $user, $options) = @_;
6232
6233 my $storecfg = PVE::Storage::config();
6234
6235 my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6236 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6237
6238 my $fingerprint = $scfg->{fingerprint};
6239 my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6240
6241 my $repo = PVE::PBSClient::get_repository($scfg);
6242
6243 # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6244 my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6245 local $ENV{PBS_PASSWORD} = $password;
6246 local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6247
6248 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6249 PVE::Storage::parse_volname($storecfg, $archive);
6250
6251 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6252
6253 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6254
6255 my $tmpdir = "/var/tmp/vzdumptmp$$";
6256 rmtree $tmpdir;
6257 mkpath $tmpdir;
6258
6259 my $conffile = PVE::QemuConfig->config_file($vmid);
6260 # disable interrupts (always do cleanups)
6261 local $SIG{INT} =
6262 local $SIG{TERM} =
6263 local $SIG{QUIT} =
6264 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6265
6266 # Note: $oldconf is undef if VM does not exists
6267 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6268 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6269 my $new_conf_raw = '';
6270
6271 my $rpcenv = PVE::RPCEnvironment::get();
6272 my $devinfo = {};
6273
6274 eval {
6275 # enable interrupts
6276 local $SIG{INT} =
6277 local $SIG{TERM} =
6278 local $SIG{QUIT} =
6279 local $SIG{HUP} =
6280 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6281
6282 my $cfgfn = "$tmpdir/qemu-server.conf";
6283 my $firewall_config_fn = "$tmpdir/fw.conf";
6284 my $index_fn = "$tmpdir/index.json";
6285
6286 my $cmd = "restore";
6287
6288 my $param = [$pbs_backup_name, "index.json", $index_fn];
6289 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6290 my $index = PVE::Tools::file_get_contents($index_fn);
6291 $index = decode_json($index);
6292
6293 # print Dumper($index);
6294 foreach my $info (@{$index->{files}}) {
6295 if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6296 my $devname = $1;
6297 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6298 $devinfo->{$devname}->{size} = $1;
6299 } else {
6300 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6301 }
6302 }
6303 }
6304
6305 my $is_qemu_server_backup = scalar(
6306 grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6307 );
6308 if (!$is_qemu_server_backup) {
6309 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6310 }
6311 my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6312
6313 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6314 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6315
6316 if ($has_firewall_config) {
6317 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6318 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6319
6320 my $pve_firewall_dir = '/etc/pve/firewall';
6321 mkdir $pve_firewall_dir; # make sure the dir exists
6322 PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6323 }
6324
6325 my $fh = IO::File->new($cfgfn, "r") ||
6326 die "unable to read qemu-server.conf - $!\n";
6327
6328 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6329
6330 # fixme: rate limit?
6331
6332 # create empty/temp config
6333 PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6334
6335 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6336
6337 # allocate volumes
6338 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6339
6340 if (!$options->{live}) {
6341 foreach my $virtdev (sort keys %$virtdev_hash) {
6342 my $d = $virtdev_hash->{$virtdev};
6343 next if $d->{is_cloudinit}; # no need to restore cloudinit
6344
6345 my $volid = $d->{volid};
6346
6347 my $path = PVE::Storage::path($storecfg, $volid);
6348
6349 my $pbs_restore_cmd = [
6350 '/usr/bin/pbs-restore',
6351 '--repository', $repo,
6352 $pbs_backup_name,
6353 "$d->{devname}.img.fidx",
6354 $path,
6355 '--verbose',
6356 ];
6357
6358 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6359 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6360
6361 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6362 push @$pbs_restore_cmd, '--skip-zero';
6363 }
6364
6365 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6366 print "restore proxmox backup image: $dbg_cmdstring\n";
6367 run_command($pbs_restore_cmd);
6368 }
6369 }
6370
6371 $fh->seek(0, 0) || die "seek failed - $!\n";
6372
6373 my $cookie = { netcount => 0 };
6374 while (defined(my $line = <$fh>)) {
6375 $new_conf_raw .= restore_update_config_line(
6376 $cookie,
6377 $map,
6378 $line,
6379 $options->{unique},
6380 );
6381 }
6382
6383 $fh->close();
6384 };
6385 my $err = $@;
6386
6387 if ($err || !$options->{live}) {
6388 $restore_deactivate_volumes->($storecfg, $devinfo);
6389 }
6390
6391 rmtree $tmpdir;
6392
6393 if ($err) {
6394 $restore_destroy_volumes->($storecfg, $devinfo);
6395 die $err;
6396 }
6397
6398 if ($options->{live}) {
6399 # keep lock during live-restore
6400 $new_conf_raw .= "\nlock: create";
6401 }
6402
6403 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6404
6405 PVE::Cluster::cfs_update(); # make sure we read new file
6406
6407 eval { rescan($vmid, 1); };
6408 warn $@ if $@;
6409
6410 PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6411
6412 if ($options->{live}) {
6413 # enable interrupts
6414 local $SIG{INT} =
6415 local $SIG{TERM} =
6416 local $SIG{QUIT} =
6417 local $SIG{HUP} =
6418 local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6419
6420 my $conf = PVE::QemuConfig->load_config($vmid);
6421 die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6422
6423 pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6424
6425 PVE::QemuConfig->remove_lock($vmid, "create");
6426 }
6427}
6428
6429sub pbs_live_restore {
6430 my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6431
6432 print "starting VM for live-restore\n";
6433 print "repository: '$repo', snapshot: '$snap'\n";
6434
6435 my $pbs_backing = {};
6436 for my $ds (keys %$restored_disks) {
6437 $ds =~ m/^drive-(.*)$/;
6438 my $confname = $1;
6439 $pbs_backing->{$confname} = {
6440 repository => $repo,
6441 snapshot => $snap,
6442 archive => "$ds.img.fidx",
6443 };
6444 $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6445
6446 my $drive = parse_drive($confname, $conf->{$confname});
6447 print "restoring '$ds' to '$drive->{file}'\n";
6448 }
6449
6450 my $drives_streamed = 0;
6451 eval {
6452 # make sure HA doesn't interrupt our restore by stopping the VM
6453 if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6454 run_command(['ha-manager', 'set', "vm:$vmid", '--state', 'started']);
6455 }
6456
6457 # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6458 # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6459 vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6460
6461 my $qmeventd_fd = register_qmeventd_handle($vmid);
6462
6463 # begin streaming, i.e. data copy from PBS to target disk for every vol,
6464 # this will effectively collapse the backing image chain consisting of
6465 # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6466 # removes itself once all backing images vanish with 'auto-remove=on')
6467 my $jobs = {};
6468 for my $ds (sort keys %$restored_disks) {
6469 my $job_id = "restore-$ds";
6470 mon_cmd($vmid, 'block-stream',
6471 'job-id' => $job_id,
6472 device => "$ds",
6473 );
6474 $jobs->{$job_id} = {};
6475 }
6476
6477 mon_cmd($vmid, 'cont');
6478 qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6479
6480 print "restore-drive jobs finished successfully, removing all tracking block devices"
6481 ." to disconnect from Proxmox Backup Server\n";
6482
6483 for my $ds (sort keys %$restored_disks) {
6484 mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6485 }
6486
6487 close($qmeventd_fd);
6488 };
6489
6490 my $err = $@;
6491
6492 if ($err) {
6493 warn "An error occured during live-restore: $err\n";
6494 _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6495 die "live-restore failed\n";
6496 }
6497}
6498
6499sub restore_vma_archive {
6500 my ($archive, $vmid, $user, $opts, $comp) = @_;
6501
6502 my $readfrom = $archive;
6503
6504 my $cfg = PVE::Storage::config();
6505 my $commands = [];
6506 my $bwlimit = $opts->{bwlimit};
6507
6508 my $dbg_cmdstring = '';
6509 my $add_pipe = sub {
6510 my ($cmd) = @_;
6511 push @$commands, $cmd;
6512 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6513 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6514 $readfrom = '-';
6515 };
6516
6517 my $input = undef;
6518 if ($archive eq '-') {
6519 $input = '<&STDIN';
6520 } else {
6521 # If we use a backup from a PVE defined storage we also consider that
6522 # storage's rate limit:
6523 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6524 if (defined($volid)) {
6525 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6526 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6527 if ($readlimit) {
6528 print STDERR "applying read rate limit: $readlimit\n";
6529 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6530 $add_pipe->($cstream);
6531 }
6532 }
6533 }
6534
6535 if ($comp) {
6536 my $info = PVE::Storage::decompressor_info('vma', $comp);
6537 my $cmd = $info->{decompressor};
6538 push @$cmd, $readfrom;
6539 $add_pipe->($cmd);
6540 }
6541
6542 my $tmpdir = "/var/tmp/vzdumptmp$$";
6543 rmtree $tmpdir;
6544
6545 # disable interrupts (always do cleanups)
6546 local $SIG{INT} =
6547 local $SIG{TERM} =
6548 local $SIG{QUIT} =
6549 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6550
6551 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6552 POSIX::mkfifo($mapfifo, 0600);
6553 my $fifofh;
6554 my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6555
6556 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6557
6558 my $oldtimeout;
6559 my $timeout = 5;
6560
6561 my $devinfo = {};
6562
6563 my $rpcenv = PVE::RPCEnvironment::get();
6564
6565 my $conffile = PVE::QemuConfig->config_file($vmid);
6566
6567 # Note: $oldconf is undef if VM does not exist
6568 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6569 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6570 my $new_conf_raw = '';
6571
6572 my %storage_limits;
6573
6574 my $print_devmap = sub {
6575 my $cfgfn = "$tmpdir/qemu-server.conf";
6576
6577 # we can read the config - that is already extracted
6578 my $fh = IO::File->new($cfgfn, "r") ||
6579 die "unable to read qemu-server.conf - $!\n";
6580
6581 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6582 if (-f $fwcfgfn) {
6583 my $pve_firewall_dir = '/etc/pve/firewall';
6584 mkdir $pve_firewall_dir; # make sure the dir exists
6585 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6586 }
6587
6588 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6589
6590 foreach my $info (values %{$virtdev_hash}) {
6591 my $storeid = $info->{storeid};
6592 next if defined($storage_limits{$storeid});
6593
6594 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6595 print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6596 $storage_limits{$storeid} = $limit * 1024;
6597 }
6598
6599 foreach my $devname (keys %$devinfo) {
6600 die "found no device mapping information for device '$devname'\n"
6601 if !$devinfo->{$devname}->{virtdev};
6602 }
6603
6604 # create empty/temp config
6605 if ($oldconf) {
6606 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6607 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6608 }
6609
6610 # allocate volumes
6611 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6612
6613 # print restore information to $fifofh
6614 foreach my $virtdev (sort keys %$virtdev_hash) {
6615 my $d = $virtdev_hash->{$virtdev};
6616 next if $d->{is_cloudinit}; # no need to restore cloudinit
6617
6618 my $storeid = $d->{storeid};
6619 my $volid = $d->{volid};
6620
6621 my $map_opts = '';
6622 if (my $limit = $storage_limits{$storeid}) {
6623 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6624 }
6625
6626 my $write_zeros = 1;
6627 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6628 $write_zeros = 0;
6629 }
6630
6631 my $path = PVE::Storage::path($cfg, $volid);
6632
6633 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6634
6635 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6636 }
6637
6638 $fh->seek(0, 0) || die "seek failed - $!\n";
6639
6640 my $cookie = { netcount => 0 };
6641 while (defined(my $line = <$fh>)) {
6642 $new_conf_raw .= restore_update_config_line(
6643 $cookie,
6644 $map,
6645 $line,
6646 $opts->{unique},
6647 );
6648 }
6649
6650 $fh->close();
6651 };
6652
6653 eval {
6654 # enable interrupts
6655 local $SIG{INT} =
6656 local $SIG{TERM} =
6657 local $SIG{QUIT} =
6658 local $SIG{HUP} =
6659 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6660 local $SIG{ALRM} = sub { die "got timeout\n"; };
6661
6662 $oldtimeout = alarm($timeout);
6663
6664 my $parser = sub {
6665 my $line = shift;
6666
6667 print "$line\n";
6668
6669 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6670 my ($dev_id, $size, $devname) = ($1, $2, $3);
6671 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6672 } elsif ($line =~ m/^CTIME: /) {
6673 # we correctly received the vma config, so we can disable
6674 # the timeout now for disk allocation (set to 10 minutes, so
6675 # that we always timeout if something goes wrong)
6676 alarm(600);
6677 &$print_devmap();
6678 print $fifofh "done\n";
6679 my $tmp = $oldtimeout || 0;
6680 $oldtimeout = undef;
6681 alarm($tmp);
6682 close($fifofh);
6683 $fifofh = undef;
6684 }
6685 };
6686
6687 print "restore vma archive: $dbg_cmdstring\n";
6688 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6689 };
6690 my $err = $@;
6691
6692 alarm($oldtimeout) if $oldtimeout;
6693
6694 $restore_deactivate_volumes->($cfg, $devinfo);
6695
6696 close($fifofh) if $fifofh;
6697 unlink $mapfifo;
6698 rmtree $tmpdir;
6699
6700 if ($err) {
6701 $restore_destroy_volumes->($cfg, $devinfo);
6702 die $err;
6703 }
6704
6705 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6706
6707 PVE::Cluster::cfs_update(); # make sure we read new file
6708
6709 eval { rescan($vmid, 1); };
6710 warn $@ if $@;
6711
6712 PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6713}
6714
6715sub restore_tar_archive {
6716 my ($archive, $vmid, $user, $opts) = @_;
6717
6718 if ($archive ne '-') {
6719 my $firstfile = tar_archive_read_firstfile($archive);
6720 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6721 if $firstfile ne 'qemu-server.conf';
6722 }
6723
6724 my $storecfg = PVE::Storage::config();
6725
6726 # avoid zombie disks when restoring over an existing VM -> cleanup first
6727 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6728 # skiplock=1 because qmrestore has set the 'create' lock itself already
6729 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6730 destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6731
6732 my $tocmd = "/usr/lib/qemu-server/qmextract";
6733
6734 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6735 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6736 $tocmd .= ' --prealloc' if $opts->{prealloc};
6737 $tocmd .= ' --info' if $opts->{info};
6738
6739 # tar option "xf" does not autodetect compression when read from STDIN,
6740 # so we pipe to zcat
6741 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6742 PVE::Tools::shellquote("--to-command=$tocmd");
6743
6744 my $tmpdir = "/var/tmp/vzdumptmp$$";
6745 mkpath $tmpdir;
6746
6747 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6748 local $ENV{VZDUMP_VMID} = $vmid;
6749 local $ENV{VZDUMP_USER} = $user;
6750
6751 my $conffile = PVE::QemuConfig->config_file($vmid);
6752 my $new_conf_raw = '';
6753
6754 # disable interrupts (always do cleanups)
6755 local $SIG{INT} =
6756 local $SIG{TERM} =
6757 local $SIG{QUIT} =
6758 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6759
6760 eval {
6761 # enable interrupts
6762 local $SIG{INT} =
6763 local $SIG{TERM} =
6764 local $SIG{QUIT} =
6765 local $SIG{HUP} =
6766 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6767
6768 if ($archive eq '-') {
6769 print "extracting archive from STDIN\n";
6770 run_command($cmd, input => "<&STDIN");
6771 } else {
6772 print "extracting archive '$archive'\n";
6773 run_command($cmd);
6774 }
6775
6776 return if $opts->{info};
6777
6778 # read new mapping
6779 my $map = {};
6780 my $statfile = "$tmpdir/qmrestore.stat";
6781 if (my $fd = IO::File->new($statfile, "r")) {
6782 while (defined (my $line = <$fd>)) {
6783 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6784 $map->{$1} = $2 if $1;
6785 } else {
6786 print STDERR "unable to parse line in statfile - $line\n";
6787 }
6788 }
6789 $fd->close();
6790 }
6791
6792 my $confsrc = "$tmpdir/qemu-server.conf";
6793
6794 my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6795
6796 my $cookie = { netcount => 0 };
6797 while (defined (my $line = <$srcfd>)) {
6798 $new_conf_raw .= restore_update_config_line(
6799 $cookie,
6800 $map,
6801 $line,
6802 $opts->{unique},
6803 );
6804 }
6805
6806 $srcfd->close();
6807 };
6808 if (my $err = $@) {
6809 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6810 die $err;
6811 }
6812
6813 rmtree $tmpdir;
6814
6815 PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6816
6817 PVE::Cluster::cfs_update(); # make sure we read new file
6818
6819 eval { rescan($vmid, 1); };
6820 warn $@ if $@;
6821};
6822
6823sub foreach_storage_used_by_vm {
6824 my ($conf, $func) = @_;
6825
6826 my $sidhash = {};
6827
6828 PVE::QemuConfig->foreach_volume($conf, sub {
6829 my ($ds, $drive) = @_;
6830 return if drive_is_cdrom($drive);
6831
6832 my $volid = $drive->{file};
6833
6834 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6835 $sidhash->{$sid} = $sid if $sid;
6836 });
6837
6838 foreach my $sid (sort keys %$sidhash) {
6839 &$func($sid);
6840 }
6841}
6842
6843my $qemu_snap_storage = {
6844 rbd => 1,
6845};
6846sub do_snapshots_with_qemu {
6847 my ($storecfg, $volid) = @_;
6848
6849 my $storage_name = PVE::Storage::parse_volume_id($volid);
6850 my $scfg = $storecfg->{ids}->{$storage_name};
6851 die "could not find storage '$storage_name'\n" if !defined($scfg);
6852
6853 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6854 return 1;
6855 }
6856
6857 if ($volid =~ m/\.(qcow2|qed)$/){
6858 return 1;
6859 }
6860
6861 return;
6862}
6863
6864sub qga_check_running {
6865 my ($vmid, $nowarn) = @_;
6866
6867 eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6868 if ($@) {
6869 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6870 return 0;
6871 }
6872 return 1;
6873}
6874
6875sub template_create {
6876 my ($vmid, $conf, $disk) = @_;
6877
6878 my $storecfg = PVE::Storage::config();
6879
6880 PVE::QemuConfig->foreach_volume($conf, sub {
6881 my ($ds, $drive) = @_;
6882
6883 return if drive_is_cdrom($drive);
6884 return if $disk && $ds ne $disk;
6885
6886 my $volid = $drive->{file};
6887 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6888
6889 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6890 $drive->{file} = $voliddst;
6891 $conf->{$ds} = print_drive($drive);
6892 PVE::QemuConfig->write_config($vmid, $conf);
6893 });
6894}
6895
6896sub convert_iscsi_path {
6897 my ($path) = @_;
6898
6899 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6900 my $portal = $1;
6901 my $target = $2;
6902 my $lun = $3;
6903
6904 my $initiator_name = get_initiator_name();
6905
6906 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6907 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6908 }
6909
6910 die "cannot convert iscsi path '$path', unkown format\n";
6911}
6912
6913sub qemu_img_convert {
6914 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6915
6916 my $storecfg = PVE::Storage::config();
6917 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6918 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6919
6920 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6921
6922 my $cachemode;
6923 my $src_path;
6924 my $src_is_iscsi = 0;
6925 my $src_format;
6926
6927 if ($src_storeid) {
6928 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6929 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6930 $src_format = qemu_img_format($src_scfg, $src_volname);
6931 $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6932 $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6933 $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6934 } elsif (-f $src_volid) {
6935 $src_path = $src_volid;
6936 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6937 $src_format = $1;
6938 }
6939 }
6940
6941 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6942
6943 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6944 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6945 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6946 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6947
6948 my $cmd = [];
6949 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6950 push @$cmd, '-l', "snapshot.name=$snapname"
6951 if $snapname && $src_format && $src_format eq "qcow2";
6952 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6953 push @$cmd, '-T', $cachemode if defined($cachemode);
6954
6955 if ($src_is_iscsi) {
6956 push @$cmd, '--image-opts';
6957 $src_path = convert_iscsi_path($src_path);
6958 } elsif ($src_format) {
6959 push @$cmd, '-f', $src_format;
6960 }
6961
6962 if ($dst_is_iscsi) {
6963 push @$cmd, '--target-image-opts';
6964 $dst_path = convert_iscsi_path($dst_path);
6965 } else {
6966 push @$cmd, '-O', $dst_format;
6967 }
6968
6969 push @$cmd, $src_path;
6970
6971 if (!$dst_is_iscsi && $is_zero_initialized) {
6972 push @$cmd, "zeroinit:$dst_path";
6973 } else {
6974 push @$cmd, $dst_path;
6975 }
6976
6977 my $parser = sub {
6978 my $line = shift;
6979 if($line =~ m/\((\S+)\/100\%\)/){
6980 my $percent = $1;
6981 my $transferred = int($size * $percent / 100);
6982 my $total_h = render_bytes($size, 1);
6983 my $transferred_h = render_bytes($transferred, 1);
6984
6985 print "transferred $transferred_h of $total_h ($percent%)\n";
6986 }
6987
6988 };
6989
6990 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
6991 my $err = $@;
6992 die "copy failed: $err" if $err;
6993}
6994
6995sub qemu_img_format {
6996 my ($scfg, $volname) = @_;
6997
6998 if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6999 return $1;
7000 } else {
7001 return "raw";
7002 }
7003}
7004
7005sub qemu_drive_mirror {
7006 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7007
7008 $jobs = {} if !$jobs;
7009
7010 my $qemu_target;
7011 my $format;
7012 $jobs->{"drive-$drive"} = {};
7013
7014 if ($dst_volid =~ /^nbd:/) {
7015 $qemu_target = $dst_volid;
7016 $format = "nbd";
7017 } else {
7018 my $storecfg = PVE::Storage::config();
7019 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7020
7021 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7022
7023 $format = qemu_img_format($dst_scfg, $dst_volname);
7024
7025 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7026
7027 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7028 }
7029
7030 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7031 $opts->{format} = $format if $format;
7032
7033 if (defined($src_bitmap)) {
7034 $opts->{sync} = 'incremental';
7035 $opts->{bitmap} = $src_bitmap;
7036 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7037 }
7038
7039 if (defined($bwlimit)) {
7040 $opts->{speed} = $bwlimit * 1024;
7041 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7042 } else {
7043 print "drive mirror is starting for drive-$drive\n";
7044 }
7045
7046 # if a job already runs for this device we get an error, catch it for cleanup
7047 eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7048 if (my $err = $@) {
7049 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7050 warn "$@\n" if $@;
7051 die "mirroring error: $err\n";
7052 }
7053
7054 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7055}
7056
7057# $completion can be either
7058# 'complete': wait until all jobs are ready, block-job-complete them (default)
7059# 'cancel': wait until all jobs are ready, block-job-cancel them
7060# 'skip': wait until all jobs are ready, return with block jobs in ready state
7061# 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7062sub qemu_drive_mirror_monitor {
7063 my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7064
7065 $completion //= 'complete';
7066 $op //= "mirror";
7067
7068 eval {
7069 my $err_complete = 0;
7070
7071 my $starttime = time ();
7072 while (1) {
7073 die "block job ('$op') timed out\n" if $err_complete > 300;
7074
7075 my $stats = mon_cmd($vmid, "query-block-jobs");
7076 my $ctime = time();
7077
7078 my $running_jobs = {};
7079 for my $stat (@$stats) {
7080 next if $stat->{type} ne $op;
7081 $running_jobs->{$stat->{device}} = $stat;
7082 }
7083
7084 my $readycounter = 0;
7085
7086 for my $job_id (sort keys %$jobs) {
7087 my $job = $running_jobs->{$job_id};
7088
7089 my $vanished = !defined($job);
7090 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7091 if($complete || ($vanished && $completion eq 'auto')) {
7092 print "$job_id: $op-job finished\n";
7093 delete $jobs->{$job_id};
7094 next;
7095 }
7096
7097 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7098
7099 my $busy = $job->{busy};
7100 my $ready = $job->{ready};
7101 if (my $total = $job->{len}) {
7102 my $transferred = $job->{offset} || 0;
7103 my $remaining = $total - $transferred;
7104 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7105
7106 my $duration = $ctime - $starttime;
7107 my $total_h = render_bytes($total, 1);
7108 my $transferred_h = render_bytes($transferred, 1);
7109
7110 my $status = sprintf(
7111 "transferred $transferred_h of $total_h ($percent%%) in %s",
7112 render_duration($duration),
7113 );
7114
7115 if ($ready) {
7116 if ($busy) {
7117 $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7118 } else {
7119 $status .= ", ready";
7120 }
7121 }
7122 print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7123 $jobs->{$job_id}->{ready} = $ready;
7124 }
7125
7126 $readycounter++ if $job->{ready};
7127 }
7128
7129 last if scalar(keys %$jobs) == 0;
7130
7131 if ($readycounter == scalar(keys %$jobs)) {
7132 print "all '$op' jobs are ready\n";
7133
7134 # do the complete later (or has already been done)
7135 last if $completion eq 'skip' || $completion eq 'auto';
7136
7137 if ($vmiddst && $vmiddst != $vmid) {
7138 my $agent_running = $qga && qga_check_running($vmid);
7139 if ($agent_running) {
7140 print "freeze filesystem\n";
7141 eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7142 } else {
7143 print "suspend vm\n";
7144 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7145 }
7146
7147 # if we clone a disk for a new target vm, we don't switch the disk
7148 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7149
7150 if ($agent_running) {
7151 print "unfreeze filesystem\n";
7152 eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7153 } else {
7154 print "resume vm\n";
7155 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7156 }
7157
7158 last;
7159 } else {
7160
7161 for my $job_id (sort keys %$jobs) {
7162 # try to switch the disk if source and destination are on the same guest
7163 print "$job_id: Completing block job_id...\n";
7164
7165 my $op;
7166 if ($completion eq 'complete') {
7167 $op = 'block-job-complete';
7168 } elsif ($completion eq 'cancel') {
7169 $op = 'block-job-cancel';
7170 } else {
7171 die "invalid completion value: $completion\n";
7172 }
7173 eval { mon_cmd($vmid, $op, device => $job_id) };
7174 if ($@ =~ m/cannot be completed/) {
7175 print "$job_id: block job cannot be completed, trying again.\n";
7176 $err_complete++;
7177 }else {
7178 print "$job_id: Completed successfully.\n";
7179 $jobs->{$job_id}->{complete} = 1;
7180 }
7181 }
7182 }
7183 }
7184 sleep 1;
7185 }
7186 };
7187 my $err = $@;
7188
7189 if ($err) {
7190 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7191 die "block job ($op) error: $err";
7192 }
7193}
7194
7195sub qemu_blockjobs_cancel {
7196 my ($vmid, $jobs) = @_;
7197
7198 foreach my $job (keys %$jobs) {
7199 print "$job: Cancelling block job\n";
7200 eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7201 $jobs->{$job}->{cancel} = 1;
7202 }
7203
7204 while (1) {
7205 my $stats = mon_cmd($vmid, "query-block-jobs");
7206
7207 my $running_jobs = {};
7208 foreach my $stat (@$stats) {
7209 $running_jobs->{$stat->{device}} = $stat;
7210 }
7211
7212 foreach my $job (keys %$jobs) {
7213
7214 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7215 print "$job: Done.\n";
7216 delete $jobs->{$job};
7217 }
7218 }
7219
7220 last if scalar(keys %$jobs) == 0;
7221
7222 sleep 1;
7223 }
7224}
7225
7226sub clone_disk {
7227 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7228 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7229
7230 my $newvolid;
7231
7232 if (!$full) {
7233 print "create linked clone of drive $drivename ($drive->{file})\n";
7234 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
7235 push @$newvollist, $newvolid;
7236 } else {
7237
7238 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7239 $storeid = $storage if $storage;
7240
7241 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7242
7243 print "create full clone of drive $drivename ($drive->{file})\n";
7244 my $name = undef;
7245 my $size = undef;
7246 if (drive_is_cloudinit($drive)) {
7247 $name = "vm-$newvmid-cloudinit";
7248 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7249 if ($scfg->{path}) {
7250 $name .= ".$dst_format";
7251 }
7252 $snapname = undef;
7253 $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7254 } elsif ($drivename eq 'efidisk0') {
7255 $size = get_efivars_size($conf);
7256 } else {
7257 ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7258 }
7259 $newvolid = PVE::Storage::vdisk_alloc(
7260 $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7261 );
7262 push @$newvollist, $newvolid;
7263
7264 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7265
7266 if (drive_is_cloudinit($drive)) {
7267 # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7268 # if this is the case, we have to complete any block-jobs still there from
7269 # previous drive-mirrors
7270 if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7271 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7272 }
7273 goto no_data_clone;
7274 }
7275
7276 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7277 if (!$running || $snapname) {
7278 # TODO: handle bwlimits
7279 if ($drivename eq 'efidisk0') {
7280 # the relevant data on the efidisk may be smaller than the source
7281 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7282 # that is given by the OVMF_VARS.fd
7283 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7284 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7285
7286 # better for Ceph if block size is not too small, see bug #3324
7287 my $bs = 1024*1024;
7288
7289 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7290 "if=$src_path", "of=$dst_path"]);
7291 } else {
7292 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7293 }
7294 } else {
7295
7296 my $kvmver = get_running_qemu_version ($vmid);
7297 if (!min_version($kvmver, 2, 7)) {
7298 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7299 if $drive->{iothread};
7300 }
7301
7302 qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7303 $completion, $qga, $bwlimit);
7304 }
7305 }
7306
7307no_data_clone:
7308 my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7309
7310 my $disk = $drive;
7311 $disk->{format} = undef;
7312 $disk->{file} = $newvolid;
7313 $disk->{size} = $size if defined($size);
7314
7315 return $disk;
7316}
7317
7318sub get_running_qemu_version {
7319 my ($vmid) = @_;
7320 my $res = mon_cmd($vmid, "query-version");
7321 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7322}
7323
7324sub qemu_use_old_bios_files {
7325 my ($machine_type) = @_;
7326
7327 return if !$machine_type;
7328
7329 my $use_old_bios_files = undef;
7330
7331 if ($machine_type =~ m/^(\S+)\.pxe$/) {
7332 $machine_type = $1;
7333 $use_old_bios_files = 1;
7334 } else {
7335 my $version = extract_version($machine_type, kvm_user_version());
7336 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7337 # load new efi bios files on migration. So this hack is required to allow
7338 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7339 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7340 $use_old_bios_files = !min_version($version, 2, 4);
7341 }
7342
7343 return ($use_old_bios_files, $machine_type);
7344}
7345
7346sub get_efivars_size {
7347 my ($conf) = @_;
7348 my $arch = get_vm_arch($conf);
7349 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7350 die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7351 return -s $ovmf_vars;
7352}
7353
7354sub update_efidisk_size {
7355 my ($conf) = @_;
7356
7357 return if !defined($conf->{efidisk0});
7358
7359 my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7360 $disk->{size} = get_efivars_size($conf);
7361 $conf->{efidisk0} = print_drive($disk);
7362
7363 return;
7364}
7365
7366sub create_efidisk($$$$$) {
7367 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7368
7369 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7370 die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7371
7372 my $vars_size_b = -s $ovmf_vars;
7373 my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7374 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7375 PVE::Storage::activate_volumes($storecfg, [$volid]);
7376
7377 qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7378 my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7379
7380 return ($volid, $size/1024);
7381}
7382
7383sub vm_iothreads_list {
7384 my ($vmid) = @_;
7385
7386 my $res = mon_cmd($vmid, 'query-iothreads');
7387
7388 my $iothreads = {};
7389 foreach my $iothread (@$res) {
7390 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7391 }
7392
7393 return $iothreads;
7394}
7395
7396sub scsihw_infos {
7397 my ($conf, $drive) = @_;
7398
7399 my $maxdev = 0;
7400
7401 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7402 $maxdev = 7;
7403 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7404 $maxdev = 1;
7405 } else {
7406 $maxdev = 256;
7407 }
7408
7409 my $controller = int($drive->{index} / $maxdev);
7410 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7411 ? "virtioscsi"
7412 : "scsihw";
7413
7414 return ($maxdev, $controller, $controller_prefix);
7415}
7416
7417sub windows_version {
7418 my ($ostype) = @_;
7419
7420 return 0 if !$ostype;
7421
7422 my $winversion = 0;
7423
7424 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7425 $winversion = 5;
7426 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7427 $winversion = 6;
7428 } elsif ($ostype =~ m/^win(\d+)$/) {
7429 $winversion = $1;
7430 }
7431
7432 return $winversion;
7433}
7434
7435sub resolve_dst_disk_format {
7436 my ($storecfg, $storeid, $src_volname, $format) = @_;
7437 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7438
7439 if (!$format) {
7440 # if no target format is specified, use the source disk format as hint
7441 if ($src_volname) {
7442 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7443 $format = qemu_img_format($scfg, $src_volname);
7444 } else {
7445 return $defFormat;
7446 }
7447 }
7448
7449 # test if requested format is supported - else use default
7450 my $supported = grep { $_ eq $format } @$validFormats;
7451 $format = $defFormat if !$supported;
7452 return $format;
7453}
7454
7455# NOTE: if this logic changes, please update docs & possibly gui logic
7456sub find_vmstate_storage {
7457 my ($conf, $storecfg) = @_;
7458
7459 # first, return storage from conf if set
7460 return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7461
7462 my ($target, $shared, $local);
7463
7464 foreach_storage_used_by_vm($conf, sub {
7465 my ($sid) = @_;
7466 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7467 my $dst = $scfg->{shared} ? \$shared : \$local;
7468 $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7469 });
7470
7471 # second, use shared storage where VM has at least one disk
7472 # third, use local storage where VM has at least one disk
7473 # fall back to local storage
7474 $target = $shared // $local // 'local';
7475
7476 return $target;
7477}
7478
7479sub generate_uuid {
7480 my ($uuid, $uuid_str);
7481 UUID::generate($uuid);
7482 UUID::unparse($uuid, $uuid_str);
7483 return $uuid_str;
7484}
7485
7486sub generate_smbios1_uuid {
7487 return "uuid=".generate_uuid();
7488}
7489
7490sub nbd_stop {
7491 my ($vmid) = @_;
7492
7493 mon_cmd($vmid, 'nbd-server-stop');
7494}
7495
7496sub create_reboot_request {
7497 my ($vmid) = @_;
7498 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7499 or die "failed to create reboot trigger file: $!\n";
7500 close($fh);
7501}
7502
7503sub clear_reboot_request {
7504 my ($vmid) = @_;
7505 my $path = "/run/qemu-server/$vmid.reboot";
7506 my $res = 0;
7507
7508 $res = unlink($path);
7509 die "could not remove reboot request for $vmid: $!"
7510 if !$res && $! != POSIX::ENOENT;
7511
7512 return $res;
7513}
7514
7515sub bootorder_from_legacy {
7516 my ($conf, $bootcfg) = @_;
7517
7518 my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7519 my $bootindex_hash = {};
7520 my $i = 1;
7521 foreach my $o (split(//, $boot)) {
7522 $bootindex_hash->{$o} = $i*100;
7523 $i++;
7524 }
7525
7526 my $bootorder = {};
7527
7528 PVE::QemuConfig->foreach_volume($conf, sub {
7529 my ($ds, $drive) = @_;
7530
7531 if (drive_is_cdrom ($drive, 1)) {
7532 if ($bootindex_hash->{d}) {
7533 $bootorder->{$ds} = $bootindex_hash->{d};
7534 $bootindex_hash->{d} += 1;
7535 }
7536 } elsif ($bootindex_hash->{c}) {
7537 $bootorder->{$ds} = $bootindex_hash->{c}
7538 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7539 $bootindex_hash->{c} += 1;
7540 }
7541 });
7542
7543 if ($bootindex_hash->{n}) {
7544 for (my $i = 0; $i < $MAX_NETS; $i++) {
7545 my $netname = "net$i";
7546 next if !$conf->{$netname};
7547 $bootorder->{$netname} = $bootindex_hash->{n};
7548 $bootindex_hash->{n} += 1;
7549 }
7550 }
7551
7552 return $bootorder;
7553}
7554
7555# Generate default device list for 'boot: order=' property. Matches legacy
7556# default boot order, but with explicit device names. This is important, since
7557# the fallback for when neither 'order' nor the old format is specified relies
7558# on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7559sub get_default_bootdevices {
7560 my ($conf) = @_;
7561
7562 my @ret = ();
7563
7564 # harddisk
7565 my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7566 push @ret, $first if $first;
7567
7568 # cdrom
7569 $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7570 push @ret, $first if $first;
7571
7572 # network
7573 for (my $i = 0; $i < $MAX_NETS; $i++) {
7574 my $netname = "net$i";
7575 next if !$conf->{$netname};
7576 push @ret, $netname;
7577 last;
7578 }
7579
7580 return \@ret;
7581}
7582
7583sub device_bootorder {
7584 my ($conf) = @_;
7585
7586 return bootorder_from_legacy($conf) if !defined($conf->{boot});
7587
7588 my $boot = parse_property_string($boot_fmt, $conf->{boot});
7589
7590 my $bootorder = {};
7591 if (!defined($boot) || $boot->{legacy}) {
7592 $bootorder = bootorder_from_legacy($conf, $boot);
7593 } elsif ($boot->{order}) {
7594 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7595 for my $dev (PVE::Tools::split_list($boot->{order})) {
7596 $bootorder->{$dev} = $i++;
7597 }
7598 }
7599
7600 return $bootorder;
7601}
7602
7603sub register_qmeventd_handle {
7604 my ($vmid) = @_;
7605
7606 my $fh;
7607 my $peer = "/var/run/qmeventd.sock";
7608 my $count = 0;
7609
7610 for (;;) {
7611 $count++;
7612 $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7613 last if $fh;
7614 if ($! != EINTR && $! != EAGAIN) {
7615 die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7616 }
7617 if ($count > 4) {
7618 die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7619 . "after $count retries\n";
7620 }
7621 usleep(25000);
7622 }
7623
7624 # send handshake to mark VM as backing up
7625 print $fh to_json({vzdump => {vmid => "$vmid"}});
7626
7627 # return handle to be closed later when inhibit is no longer required
7628 return $fh;
7629}
7630
7631# bash completion helper
7632
7633sub complete_backup_archives {
7634 my ($cmdname, $pname, $cvalue) = @_;
7635
7636 my $cfg = PVE::Storage::config();
7637
7638 my $storeid;
7639
7640 if ($cvalue =~ m/^([^:]+):/) {
7641 $storeid = $1;
7642 }
7643
7644 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7645
7646 my $res = [];
7647 foreach my $id (keys %$data) {
7648 foreach my $item (@{$data->{$id}}) {
7649 next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7650 push @$res, $item->{volid} if defined($item->{volid});
7651 }
7652 }
7653
7654 return $res;
7655}
7656
7657my $complete_vmid_full = sub {
7658 my ($running) = @_;
7659
7660 my $idlist = vmstatus();
7661
7662 my $res = [];
7663
7664 foreach my $id (keys %$idlist) {
7665 my $d = $idlist->{$id};
7666 if (defined($running)) {
7667 next if $d->{template};
7668 next if $running && $d->{status} ne 'running';
7669 next if !$running && $d->{status} eq 'running';
7670 }
7671 push @$res, $id;
7672
7673 }
7674 return $res;
7675};
7676
7677sub complete_vmid {
7678 return &$complete_vmid_full();
7679}
7680
7681sub complete_vmid_stopped {
7682 return &$complete_vmid_full(0);
7683}
7684
7685sub complete_vmid_running {
7686 return &$complete_vmid_full(1);
7687}
7688
7689sub complete_storage {
7690
7691 my $cfg = PVE::Storage::config();
7692 my $ids = $cfg->{ids};
7693
7694 my $res = [];
7695 foreach my $sid (keys %$ids) {
7696 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7697 next if !$ids->{$sid}->{content}->{images};
7698 push @$res, $sid;
7699 }
7700
7701 return $res;
7702}
7703
7704sub complete_migration_storage {
7705 my ($cmd, $param, $current_value, $all_args) = @_;
7706
7707 my $targetnode = @$all_args[1];
7708
7709 my $cfg = PVE::Storage::config();
7710 my $ids = $cfg->{ids};
7711
7712 my $res = [];
7713 foreach my $sid (keys %$ids) {
7714 next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7715 next if !$ids->{$sid}->{content}->{images};
7716 push @$res, $sid;
7717 }
7718
7719 return $res;
7720}
7721
7722sub vm_is_paused {
7723 my ($vmid) = @_;
7724 my $qmpstatus = eval {
7725 PVE::QemuConfig::assert_config_exists_on_node($vmid);
7726 mon_cmd($vmid, "query-status");
7727 };
7728 warn "$@\n" if $@;
7729 return $qmpstatus && $qmpstatus->{status} eq "paused";
7730}
7731
77321;
Virtual Machine Manager