]> git.proxmox.com Git - qemu-server.git/blame_incremental - PVE/QemuServer.pm
projects / qemu-server.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
drop legacy __read_avail code
[qemu-server.git] / PVE / QemuServer.pm
... / ...
CommitLineData
1package PVE::QemuServer;
2
3use strict;
4use warnings;
5
6use Cwd 'abs_path';
7use Digest::SHA;
8use Fcntl ':flock';
9use Fcntl;
10use File::Basename;
11use File::Copy qw(copy);
12use File::Path;
13use File::stat;
14use Getopt::Long;
15use IO::Dir;
16use IO::File;
17use IO::Handle;
18use IO::Select;
19use IO::Socket::UNIX;
20use IPC::Open3;
21use JSON;
22use MIME::Base64;
23use POSIX;
24use Storable qw(dclone);
25use Time::HiRes qw(gettimeofday);
26use URI::Escape;
27use UUID;
28
29use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
30use PVE::DataCenterConfig;
31use PVE::Exception qw(raise raise_param_exc);
32use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
33use PVE::INotify;
34use PVE::JSONSchema qw(get_standard_option parse_property_string);
35use PVE::ProcFSTools;
36use PVE::RPCEnvironment;
37use PVE::Storage;
38use PVE::SysFSTools;
39use PVE::Systemd;
40use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
41
42use PVE::QMPClient;
43use PVE::QemuConfig;
44use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
45use PVE::QemuServer::Cloudinit;
46use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
47use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
48use PVE::QemuServer::Machine;
49use PVE::QemuServer::Memory;
50use PVE::QemuServer::Monitor qw(mon_cmd);
51use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
52use PVE::QemuServer::USB qw(parse_usb_device);
53
54my $have_sdn;
55eval {
56 require PVE::Network::SDN::Zones;
57 $have_sdn = 1;
58};
59
60my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
61my $OVMF = {
62 x86_64 => [
63 "$EDK2_FW_BASE/OVMF_CODE.fd",
64 "$EDK2_FW_BASE/OVMF_VARS.fd"
65 ],
66 aarch64 => [
67 "$EDK2_FW_BASE/AAVMF_CODE.fd",
68 "$EDK2_FW_BASE/AAVMF_VARS.fd"
69 ],
70};
71
72my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
73
74# Note about locking: we use flock on the config file protect
75# against concurent actions.
76# Aditionaly, we have a 'lock' setting in the config file. This
77# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
78# allowed when such lock is set. But you can ignore this kind of
79# lock with the --skiplock flag.
80
81cfs_register_file('/qemu-server/',
82 \&parse_vm_config,
83 \&write_vm_config);
84
85PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
86 description => "Some command save/restore state from this location.",
87 type => 'string',
88 maxLength => 128,
89 optional => 1,
90});
91
92PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
93 description => "Specifies the Qemu machine type.",
94 type => 'string',
95 pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
96 maxLength => 40,
97 optional => 1,
98});
99
100
101sub map_storage {
102 my ($map, $source) = @_;
103
104 return $source if !defined($map);
105
106 return $map->{entries}->{$source}
107 if $map->{entries} && defined($map->{entries}->{$source});
108
109 return $map->{default} if $map->{default};
110
111 # identity (fallback)
112 return $source;
113}
114
115PVE::JSONSchema::register_standard_option('pve-targetstorage', {
116 description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
117 type => 'string',
118 format => 'storagepair-list',
119 optional => 1,
120});
121
122#no warnings 'redefine';
123
124sub cgroups_write {
125 my ($controller, $vmid, $option, $value) = @_;
126
127 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
128 PVE::ProcFSTools::write_proc_entry($path, $value);
129
130}
131
132my $nodename_cache;
133sub nodename {
134 $nodename_cache //= PVE::INotify::nodename();
135 return $nodename_cache;
136}
137
138my $watchdog_fmt = {
139 model => {
140 default_key => 1,
141 type => 'string',
142 enum => [qw(i6300esb ib700)],
143 description => "Watchdog type to emulate.",
144 default => 'i6300esb',
145 optional => 1,
146 },
147 action => {
148 type => 'string',
149 enum => [qw(reset shutdown poweroff pause debug none)],
150 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
151 optional => 1,
152 },
153};
154PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
155
156my $agent_fmt = {
157 enabled => {
158 description => "Enable/disable Qemu GuestAgent.",
159 type => 'boolean',
160 default => 0,
161 default_key => 1,
162 },
163 fstrim_cloned_disks => {
164 description => "Run fstrim after cloning/moving a disk.",
165 type => 'boolean',
166 optional => 1,
167 default => 0
168 },
169 type => {
170 description => "Select the agent type",
171 type => 'string',
172 default => 'virtio',
173 optional => 1,
174 enum => [qw(virtio isa)],
175 },
176};
177
178my $vga_fmt = {
179 type => {
180 description => "Select the VGA type.",
181 type => 'string',
182 default => 'std',
183 optional => 1,
184 default_key => 1,
185 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
186 },
187 memory => {
188 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
189 type => 'integer',
190 optional => 1,
191 minimum => 4,
192 maximum => 512,
193 },
194};
195
196my $ivshmem_fmt = {
197 size => {
198 type => 'integer',
199 minimum => 1,
200 description => "The size of the file in MB.",
201 },
202 name => {
203 type => 'string',
204 pattern => '[a-zA-Z0-9\-]+',
205 optional => 1,
206 format_description => 'string',
207 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
208 },
209};
210
211my $audio_fmt = {
212 device => {
213 type => 'string',
214 enum => [qw(ich9-intel-hda intel-hda AC97)],
215 description => "Configure an audio device."
216 },
217 driver => {
218 type => 'string',
219 enum => ['spice'],
220 default => 'spice',
221 optional => 1,
222 description => "Driver backend for the audio device."
223 },
224};
225
226my $spice_enhancements_fmt = {
227 foldersharing => {
228 type => 'boolean',
229 optional => 1,
230 default => '0',
231 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
232 },
233 videostreaming => {
234 type => 'string',
235 enum => ['off', 'all', 'filter'],
236 default => 'off',
237 optional => 1,
238 description => "Enable video streaming. Uses compression for detected video streams."
239 },
240};
241
242my $rng_fmt = {
243 source => {
244 type => 'string',
245 enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
246 default_key => 1,
247 description => "The file on the host to gather entropy from. In most"
248 . " cases /dev/urandom should be preferred over /dev/random"
249 . " to avoid entropy-starvation issues on the host. Using"
250 . " urandom does *not* decrease security in any meaningful"
251 . " way, as it's still seeded from real entropy, and the"
252 . " bytes provided will most likely be mixed with real"
253 . " entropy on the guest as well. /dev/hwrng can be used"
254 . " to pass through a hardware RNG from the host.",
255 },
256 max_bytes => {
257 type => 'integer',
258 description => "Maximum bytes of entropy injected into the guest every"
259 . " 'period' milliseconds. Prefer a lower value when using"
260 . " /dev/random as source. Use 0 to disable limiting"
261 . " (potentially dangerous!).",
262 optional => 1,
263
264 # default is 1 KiB/s, provides enough entropy to the guest to avoid
265 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
266 # of overwhelming the host, provided we're reading from /dev/urandom
267 default => 1024,
268 },
269 period => {
270 type => 'integer',
271 description => "Every 'period' milliseconds the entropy-injection quota"
272 . " is reset, allowing the guest to retrieve another"
273 . " 'max_bytes' of entropy.",
274 optional => 1,
275 default => 1000,
276 },
277};
278
279my $confdesc = {
280 onboot => {
281 optional => 1,
282 type => 'boolean',
283 description => "Specifies whether a VM will be started during system bootup.",
284 default => 0,
285 },
286 autostart => {
287 optional => 1,
288 type => 'boolean',
289 description => "Automatic restart after crash (currently ignored).",
290 default => 0,
291 },
292 hotplug => {
293 optional => 1,
294 type => 'string', format => 'pve-hotplug-features',
295 description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
296 default => 'network,disk,usb',
297 },
298 reboot => {
299 optional => 1,
300 type => 'boolean',
301 description => "Allow reboot. If set to '0' the VM exit on reboot.",
302 default => 1,
303 },
304 lock => {
305 optional => 1,
306 type => 'string',
307 description => "Lock/unlock the VM.",
308 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
309 },
310 cpulimit => {
311 optional => 1,
312 type => 'number',
313 description => "Limit of CPU usage.",
314 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
315 minimum => 0,
316 maximum => 128,
317 default => 0,
318 },
319 cpuunits => {
320 optional => 1,
321 type => 'integer',
322 description => "CPU weight for a VM.",
323 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
324 minimum => 2,
325 maximum => 262144,
326 default => 1024,
327 },
328 memory => {
329 optional => 1,
330 type => 'integer',
331 description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
332 minimum => 16,
333 default => 512,
334 },
335 balloon => {
336 optional => 1,
337 type => 'integer',
338 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
339 minimum => 0,
340 },
341 shares => {
342 optional => 1,
343 type => 'integer',
344 description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
345 minimum => 0,
346 maximum => 50000,
347 default => 1000,
348 },
349 keyboard => {
350 optional => 1,
351 type => 'string',
352 description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
353 "It should not be necessary to set it.",
354 enum => PVE::Tools::kvmkeymaplist(),
355 default => undef,
356 },
357 name => {
358 optional => 1,
359 type => 'string', format => 'dns-name',
360 description => "Set a name for the VM. Only used on the configuration web interface.",
361 },
362 scsihw => {
363 optional => 1,
364 type => 'string',
365 description => "SCSI controller model",
366 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
367 default => 'lsi',
368 },
369 description => {
370 optional => 1,
371 type => 'string',
372 description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
373 },
374 ostype => {
375 optional => 1,
376 type => 'string',
377 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
378 description => "Specify guest operating system.",
379 verbose_description => <<EODESC,
380Specify guest operating system. This is used to enable special
381optimization/features for specific operating systems:
382
383[horizontal]
384other;; unspecified OS
385wxp;; Microsoft Windows XP
386w2k;; Microsoft Windows 2000
387w2k3;; Microsoft Windows 2003
388w2k8;; Microsoft Windows 2008
389wvista;; Microsoft Windows Vista
390win7;; Microsoft Windows 7
391win8;; Microsoft Windows 8/2012/2012r2
392win10;; Microsoft Windows 10/2016
393l24;; Linux 2.4 Kernel
394l26;; Linux 2.6 - 5.X Kernel
395solaris;; Solaris/OpenSolaris/OpenIndiania kernel
396EODESC
397 },
398 boot => {
399 optional => 1,
400 type => 'string', format => 'pve-qm-boot',
401 description => "Specify guest boot order. Use with 'order=', usage with"
402 . " no key or 'legacy=' is deprecated.",
403 },
404 bootdisk => {
405 optional => 1,
406 type => 'string', format => 'pve-qm-bootdisk',
407 description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
408 pattern => '(ide|sata|scsi|virtio)\d+',
409 },
410 smp => {
411 optional => 1,
412 type => 'integer',
413 description => "The number of CPUs. Please use option -sockets instead.",
414 minimum => 1,
415 default => 1,
416 },
417 sockets => {
418 optional => 1,
419 type => 'integer',
420 description => "The number of CPU sockets.",
421 minimum => 1,
422 default => 1,
423 },
424 cores => {
425 optional => 1,
426 type => 'integer',
427 description => "The number of cores per socket.",
428 minimum => 1,
429 default => 1,
430 },
431 numa => {
432 optional => 1,
433 type => 'boolean',
434 description => "Enable/disable NUMA.",
435 default => 0,
436 },
437 hugepages => {
438 optional => 1,
439 type => 'string',
440 description => "Enable/disable hugepages memory.",
441 enum => [qw(any 2 1024)],
442 },
443 keephugepages => {
444 optional => 1,
445 type => 'boolean',
446 default => 0,
447 description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
448 ." after VM shutdown and can be used for subsequent starts.",
449 },
450 vcpus => {
451 optional => 1,
452 type => 'integer',
453 description => "Number of hotplugged vcpus.",
454 minimum => 1,
455 default => 0,
456 },
457 acpi => {
458 optional => 1,
459 type => 'boolean',
460 description => "Enable/disable ACPI.",
461 default => 1,
462 },
463 agent => {
464 optional => 1,
465 description => "Enable/disable Qemu GuestAgent and its properties.",
466 type => 'string',
467 format => $agent_fmt,
468 },
469 kvm => {
470 optional => 1,
471 type => 'boolean',
472 description => "Enable/disable KVM hardware virtualization.",
473 default => 1,
474 },
475 tdf => {
476 optional => 1,
477 type => 'boolean',
478 description => "Enable/disable time drift fix.",
479 default => 0,
480 },
481 localtime => {
482 optional => 1,
483 type => 'boolean',
484 description => "Set the real time clock to local time. This is enabled by default if ostype"
485 ." indicates a Microsoft OS.",
486 },
487 freeze => {
488 optional => 1,
489 type => 'boolean',
490 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
491 },
492 vga => {
493 optional => 1,
494 type => 'string', format => $vga_fmt,
495 description => "Configure the VGA hardware.",
496 verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
497 ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
498 ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
499 ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
500 ." display server. For win* OS you can select how many independent displays you want,"
501 ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
502 ." using a serial device as terminal.",
503 },
504 watchdog => {
505 optional => 1,
506 type => 'string', format => 'pve-qm-watchdog',
507 description => "Create a virtual hardware watchdog device.",
508 verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
509 ." action), the watchdog must be periodically polled by an agent inside the guest or"
510 ." else the watchdog will reset the guest (or execute the respective action specified)",
511 },
512 startdate => {
513 optional => 1,
514 type => 'string',
515 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
516 description => "Set the initial date of the real time clock. Valid format for date are:"
517 ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
518 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
519 default => 'now',
520 },
521 startup => get_standard_option('pve-startup-order'),
522 template => {
523 optional => 1,
524 type => 'boolean',
525 description => "Enable/disable Template.",
526 default => 0,
527 },
528 args => {
529 optional => 1,
530 type => 'string',
531 description => "Arbitrary arguments passed to kvm.",
532 verbose_description => <<EODESCR,
533Arbitrary arguments passed to kvm, for example:
534
535args: -no-reboot -no-hpet
536
537NOTE: this option is for experts only.
538EODESCR
539 },
540 tablet => {
541 optional => 1,
542 type => 'boolean',
543 default => 1,
544 description => "Enable/disable the USB tablet device.",
545 verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
546 ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
547 ." normal VNC clients. If you're running lots of console-only guests on one host, you"
548 ." may consider disabling this to save some context switches. This is turned off by"
549 ." default if you use spice (`qm set <vmid> --vga qxl`).",
550 },
551 migrate_speed => {
552 optional => 1,
553 type => 'integer',
554 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
555 minimum => 0,
556 default => 0,
557 },
558 migrate_downtime => {
559 optional => 1,
560 type => 'number',
561 description => "Set maximum tolerated downtime (in seconds) for migrations.",
562 minimum => 0,
563 default => 0.1,
564 },
565 cdrom => {
566 optional => 1,
567 type => 'string', format => 'pve-qm-ide',
568 typetext => '<volume>',
569 description => "This is an alias for option -ide2",
570 },
571 cpu => {
572 optional => 1,
573 description => "Emulated CPU type.",
574 type => 'string',
575 format => 'pve-vm-cpu-conf',
576 },
577 parent => get_standard_option('pve-snapshot-name', {
578 optional => 1,
579 description => "Parent snapshot name. This is used internally, and should not be modified.",
580 }),
581 snaptime => {
582 optional => 1,
583 description => "Timestamp for snapshots.",
584 type => 'integer',
585 minimum => 0,
586 },
587 vmstate => {
588 optional => 1,
589 type => 'string', format => 'pve-volume-id',
590 description => "Reference to a volume which stores the VM state. This is used internally"
591 ." for snapshots.",
592 },
593 vmstatestorage => get_standard_option('pve-storage-id', {
594 description => "Default storage for VM state volumes/files.",
595 optional => 1,
596 }),
597 runningmachine => get_standard_option('pve-qemu-machine', {
598 description => "Specifies the QEMU machine type of the running vm. This is used internally"
599 ." for snapshots.",
600 }),
601 runningcpu => {
602 description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
603 ." internally for snapshots.",
604 optional => 1,
605 type => 'string',
606 pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
607 format_description => 'QEMU -cpu parameter'
608 },
609 machine => get_standard_option('pve-qemu-machine'),
610 arch => {
611 description => "Virtual processor architecture. Defaults to the host.",
612 optional => 1,
613 type => 'string',
614 enum => [qw(x86_64 aarch64)],
615 },
616 smbios1 => {
617 description => "Specify SMBIOS type 1 fields.",
618 type => 'string', format => 'pve-qm-smbios1',
619 maxLength => 512,
620 optional => 1,
621 },
622 protection => {
623 optional => 1,
624 type => 'boolean',
625 description => "Sets the protection flag of the VM. This will disable the remove VM and"
626 ." remove disk operations.",
627 default => 0,
628 },
629 bios => {
630 optional => 1,
631 type => 'string',
632 enum => [ qw(seabios ovmf) ],
633 description => "Select BIOS implementation.",
634 default => 'seabios',
635 },
636 vmgenid => {
637 type => 'string',
638 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
639 format_description => 'UUID',
640 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
641 ." to disable explicitly.",
642 verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
643 ." value identifier to the guest OS. This allows to notify the guest operating system"
644 ." when the virtual machine is executed with a different configuration (e.g. snapshot"
645 ." execution or creation from a template). The guest operating system notices the"
646 ." change, and is then able to react as appropriate by marking its copies of"
647 ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
648 ."Note that auto-creation only works when done through API/CLI create or update methods"
649 .", but not when manually editing the config file.",
650 default => "1 (autogenerated)",
651 optional => 1,
652 },
653 hookscript => {
654 type => 'string',
655 format => 'pve-volume-id',
656 optional => 1,
657 description => "Script that will be executed during various steps in the vms lifetime.",
658 },
659 ivshmem => {
660 type => 'string',
661 format => $ivshmem_fmt,
662 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
663 ." the host.",
664 optional => 1,
665 },
666 audio0 => {
667 type => 'string',
668 format => $audio_fmt,
669 description => "Configure a audio device, useful in combination with QXL/Spice.",
670 optional => 1
671 },
672 spice_enhancements => {
673 type => 'string',
674 format => $spice_enhancements_fmt,
675 description => "Configure additional enhancements for SPICE.",
676 optional => 1
677 },
678 tags => {
679 type => 'string', format => 'pve-tag-list',
680 description => 'Tags of the VM. This is only meta information.',
681 optional => 1,
682 },
683 rng0 => {
684 type => 'string',
685 format => $rng_fmt,
686 description => "Configure a VirtIO-based Random Number Generator.",
687 optional => 1,
688 },
689};
690
691my $cicustom_fmt = {
692 meta => {
693 type => 'string',
694 optional => 1,
695 description => 'Specify a custom file containing all meta data passed to the VM via"
696 ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
697 format => 'pve-volume-id',
698 format_description => 'volume',
699 },
700 network => {
701 type => 'string',
702 optional => 1,
703 description => 'Specify a custom file containing all network data passed to the VM via'
704 .' cloud-init.',
705 format => 'pve-volume-id',
706 format_description => 'volume',
707 },
708 user => {
709 type => 'string',
710 optional => 1,
711 description => 'Specify a custom file containing all user data passed to the VM via'
712 .' cloud-init.',
713 format => 'pve-volume-id',
714 format_description => 'volume',
715 },
716};
717PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
718
719my $confdesc_cloudinit = {
720 citype => {
721 optional => 1,
722 type => 'string',
723 description => 'Specifies the cloud-init configuration format. The default depends on the'
724 .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
725 .' and `configdrive2` for windows.',
726 enum => ['configdrive2', 'nocloud'],
727 },
728 ciuser => {
729 optional => 1,
730 type => 'string',
731 description => "cloud-init: User name to change ssh keys and password for instead of the"
732 ." image's configured default user.",
733 },
734 cipassword => {
735 optional => 1,
736 type => 'string',
737 description => 'cloud-init: Password to assign the user. Using this is generally not'
738 .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
739 .' support hashed passwords.',
740 },
741 cicustom => {
742 optional => 1,
743 type => 'string',
744 description => 'cloud-init: Specify custom files to replace the automatically generated'
745 .' ones at start.',
746 format => 'pve-qm-cicustom',
747 },
748 searchdomain => {
749 optional => 1,
750 type => 'string',
751 description => "cloud-init: Sets DNS search domains for a container. Create will'
752 .' automatically use the setting from the host if neither searchdomain nor nameserver'
753 .' are set.",
754 },
755 nameserver => {
756 optional => 1,
757 type => 'string', format => 'address-list',
758 description => "cloud-init: Sets DNS server IP address for a container. Create will'
759 .' automatically use the setting from the host if neither searchdomain nor nameserver'
760 .' are set.",
761 },
762 sshkeys => {
763 optional => 1,
764 type => 'string',
765 format => 'urlencoded',
766 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
767 },
768};
769
770# what about other qemu settings ?
771#cpu => 'string',
772#machine => 'string',
773#fda => 'file',
774#fdb => 'file',
775#mtdblock => 'file',
776#sd => 'file',
777#pflash => 'file',
778#snapshot => 'bool',
779#bootp => 'file',
780##tftp => 'dir',
781##smb => 'dir',
782#kernel => 'file',
783#append => 'string',
784#initrd => 'file',
785##soundhw => 'string',
786
787while (my ($k, $v) = each %$confdesc) {
788 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
789}
790
791my $MAX_USB_DEVICES = 5;
792my $MAX_NETS = 32;
793my $MAX_SERIAL_PORTS = 4;
794my $MAX_PARALLEL_PORTS = 3;
795my $MAX_NUMA = 8;
796
797my $numa_fmt = {
798 cpus => {
799 type => "string",
800 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
801 description => "CPUs accessing this NUMA node.",
802 format_description => "id[-id];...",
803 },
804 memory => {
805 type => "number",
806 description => "Amount of memory this NUMA node provides.",
807 optional => 1,
808 },
809 hostnodes => {
810 type => "string",
811 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
812 description => "Host NUMA nodes to use.",
813 format_description => "id[-id];...",
814 optional => 1,
815 },
816 policy => {
817 type => 'string',
818 enum => [qw(preferred bind interleave)],
819 description => "NUMA allocation policy.",
820 optional => 1,
821 },
822};
823PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
824my $numadesc = {
825 optional => 1,
826 type => 'string', format => $numa_fmt,
827 description => "NUMA topology.",
828};
829PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
830
831for (my $i = 0; $i < $MAX_NUMA; $i++) {
832 $confdesc->{"numa$i"} = $numadesc;
833}
834
835my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
836 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
837 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
838my $nic_model_list_txt = join(' ', sort @$nic_model_list);
839
840my $net_fmt_bridge_descr = <<__EOD__;
841Bridge to attach the network device to. The Proxmox VE standard bridge
842is called 'vmbr0'.
843
844If you do not specify a bridge, we create a kvm user (NATed) network
845device, which provides DHCP and DNS services. The following addresses
846are used:
847
848 10.0.2.2 Gateway
849 10.0.2.3 DNS Server
850 10.0.2.4 SMB Server
851
852The DHCP server assign addresses to the guest starting from 10.0.2.15.
853__EOD__
854
855my $net_fmt = {
856 macaddr => get_standard_option('mac-addr', {
857 description => "MAC address. That address must be unique withing your network. This is"
858 ." automatically generated if not specified.",
859 }),
860 model => {
861 type => 'string',
862 description => "Network Card Model. The 'virtio' model provides the best performance with"
863 ." very low CPU overhead. If your guest does not support this driver, it is usually"
864 ." best to use 'e1000'.",
865 enum => $nic_model_list,
866 default_key => 1,
867 },
868 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
869 bridge => {
870 type => 'string',
871 description => $net_fmt_bridge_descr,
872 format_description => 'bridge',
873 pattern => '[-_.\w\d]+',
874 optional => 1,
875 },
876 queues => {
877 type => 'integer',
878 minimum => 0, maximum => 16,
879 description => 'Number of packet queues to be used on the device.',
880 optional => 1,
881 },
882 rate => {
883 type => 'number',
884 minimum => 0,
885 description => "Rate limit in mbps (megabytes per second) as floating point number.",
886 optional => 1,
887 },
888 tag => {
889 type => 'integer',
890 minimum => 1, maximum => 4094,
891 description => 'VLAN tag to apply to packets on this interface.',
892 optional => 1,
893 },
894 trunks => {
895 type => 'string',
896 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
897 description => 'VLAN trunks to pass through this interface.',
898 format_description => 'vlanid[;vlanid...]',
899 optional => 1,
900 },
901 firewall => {
902 type => 'boolean',
903 description => 'Whether this interface should be protected by the firewall.',
904 optional => 1,
905 },
906 link_down => {
907 type => 'boolean',
908 description => 'Whether this interface should be disconnected (like pulling the plug).',
909 optional => 1,
910 },
911 mtu => {
912 type => 'integer',
913 minimum => 1, maximum => 65520,
914 description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
915 optional => 1,
916 },
917};
918
919my $netdesc = {
920 optional => 1,
921 type => 'string', format => $net_fmt,
922 description => "Specify network devices.",
923};
924
925PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
926
927my $ipconfig_fmt = {
928 ip => {
929 type => 'string',
930 format => 'pve-ipv4-config',
931 format_description => 'IPv4Format/CIDR',
932 description => 'IPv4 address in CIDR format.',
933 optional => 1,
934 default => 'dhcp',
935 },
936 gw => {
937 type => 'string',
938 format => 'ipv4',
939 format_description => 'GatewayIPv4',
940 description => 'Default gateway for IPv4 traffic.',
941 optional => 1,
942 requires => 'ip',
943 },
944 ip6 => {
945 type => 'string',
946 format => 'pve-ipv6-config',
947 format_description => 'IPv6Format/CIDR',
948 description => 'IPv6 address in CIDR format.',
949 optional => 1,
950 default => 'dhcp',
951 },
952 gw6 => {
953 type => 'string',
954 format => 'ipv6',
955 format_description => 'GatewayIPv6',
956 description => 'Default gateway for IPv6 traffic.',
957 optional => 1,
958 requires => 'ip6',
959 },
960};
961PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
962my $ipconfigdesc = {
963 optional => 1,
964 type => 'string', format => 'pve-qm-ipconfig',
965 description => <<'EODESCR',
966cloud-init: Specify IP addresses and gateways for the corresponding interface.
967
968IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
969
970The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
971gateway should be provided.
972For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
973
974If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
975dhcp on IPv4.
976EODESCR
977};
978PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
979
980for (my $i = 0; $i < $MAX_NETS; $i++) {
981 $confdesc->{"net$i"} = $netdesc;
982 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
983}
984
985foreach my $key (keys %$confdesc_cloudinit) {
986 $confdesc->{$key} = $confdesc_cloudinit->{$key};
987}
988
989PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
990sub verify_volume_id_or_qm_path {
991 my ($volid, $noerr) = @_;
992
993 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
994 return $volid;
995 }
996
997 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
998 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
999 if ($@) {
1000 return if $noerr;
1001 die $@;
1002 }
1003 return $volid;
1004}
1005
1006my $usb_fmt = {
1007 host => {
1008 default_key => 1,
1009 type => 'string', format => 'pve-qm-usb-device',
1010 format_description => 'HOSTUSBDEVICE|spice',
1011 description => <<EODESCR,
1012The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1013
1014 'bus-port(.port)*' (decimal numbers) or
1015 'vendor_id:product_id' (hexadeciaml numbers) or
1016 'spice'
1017
1018You can use the 'lsusb -t' command to list existing usb devices.
1019
1020NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1021machines - use with special care.
1022
1023The value 'spice' can be used to add a usb redirection devices for spice.
1024EODESCR
1025 },
1026 usb3 => {
1027 optional => 1,
1028 type => 'boolean',
1029 description => "Specifies whether if given host option is a USB3 device or port.",
1030 default => 0,
1031 },
1032};
1033
1034my $usbdesc = {
1035 optional => 1,
1036 type => 'string', format => $usb_fmt,
1037 description => "Configure an USB device (n is 0 to 4).",
1038};
1039PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1040
1041my $serialdesc = {
1042 optional => 1,
1043 type => 'string',
1044 pattern => '(/dev/.+|socket)',
1045 description => "Create a serial device inside the VM (n is 0 to 3)",
1046 verbose_description => <<EODESCR,
1047Create a serial device inside the VM (n is 0 to 3), and pass through a
1048host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1049host side (use 'qm terminal' to open a terminal connection).
1050
1051NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1052use with special care.
1053
1054CAUTION: Experimental! User reported problems with this option.
1055EODESCR
1056};
1057
1058my $paralleldesc= {
1059 optional => 1,
1060 type => 'string',
1061 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1062 description => "Map host parallel devices (n is 0 to 2).",
1063 verbose_description => <<EODESCR,
1064Map host parallel devices (n is 0 to 2).
1065
1066NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067machines - use with special care.
1068
1069CAUTION: Experimental! User reported problems with this option.
1070EODESCR
1071};
1072
1073for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1074 $confdesc->{"parallel$i"} = $paralleldesc;
1075}
1076
1077for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1078 $confdesc->{"serial$i"} = $serialdesc;
1079}
1080
1081for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
1082 $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1083}
1084
1085for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1086 $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1087}
1088
1089for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1090 $confdesc->{"usb$i"} = $usbdesc;
1091}
1092
1093my $boot_fmt = {
1094 legacy => {
1095 optional => 1,
1096 default_key => 1,
1097 type => 'string',
1098 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1099 . " Deprecated, use 'order=' instead.",
1100 pattern => '[acdn]{1,4}',
1101 format_description => "[acdn]{1,4}",
1102
1103 # note: this is also the fallback if boot: is not given at all
1104 default => 'cdn',
1105 },
1106 order => {
1107 optional => 1,
1108 type => 'string',
1109 format => 'pve-qm-bootdev-list',
1110 format_description => "device[;device...]",
1111 description => <<EODESC,
1112The guest will attempt to boot from devices in the order they appear here.
1113
1114Disks, optical drives and passed-through storage USB devices will be directly
1115booted from, NICs will load PXE, and PCIe devices will either behave like disks
1116(e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1117
1118Note that only devices in this list will be marked as bootable and thus loaded
1119by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1120(e.g. software-raid), you need to specify all of them here.
1121
1122Overrides the deprecated 'legacy=[acdn]*' value when given.
1123EODESC
1124 },
1125};
1126PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1127
1128PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1129sub verify_bootdev {
1130 my ($dev, $noerr) = @_;
1131
1132 return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1133
1134 my $check = sub {
1135 my ($base) = @_;
1136 return 0 if $dev !~ m/^$base\d+$/;
1137 return 0 if !$confdesc->{$dev};
1138 return 1;
1139 };
1140
1141 return $dev if $check->("net");
1142 return $dev if $check->("usb");
1143 return $dev if $check->("hostpci");
1144
1145 return if $noerr;
1146 die "invalid boot device '$dev'\n";
1147}
1148
1149sub print_bootorder {
1150 my ($devs) = @_;
1151 my $data = { order => join(';', @$devs) };
1152 return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1153}
1154
1155my $kvm_api_version = 0;
1156
1157sub kvm_version {
1158 return $kvm_api_version if $kvm_api_version;
1159
1160 open my $fh, '<', '/dev/kvm'
1161 or return;
1162
1163 # 0xae00 => KVM_GET_API_VERSION
1164 $kvm_api_version = ioctl($fh, 0xae00, 0);
1165
1166 return $kvm_api_version;
1167}
1168
1169my $kvm_user_version = {};
1170my $kvm_mtime = {};
1171
1172sub kvm_user_version {
1173 my ($binary) = @_;
1174
1175 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1176 my $st = stat($binary);
1177
1178 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1179 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1180 $cachedmtime == $st->mtime;
1181
1182 $kvm_user_version->{$binary} = 'unknown';
1183 $kvm_mtime->{$binary} = $st->mtime;
1184
1185 my $code = sub {
1186 my $line = shift;
1187 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188 $kvm_user_version->{$binary} = $2;
1189 }
1190 };
1191
1192 eval { run_command([$binary, '--version'], outfunc => $code); };
1193 warn $@ if $@;
1194
1195 return $kvm_user_version->{$binary};
1196
1197}
1198my sub extract_version {
1199 my ($machine_type, $version) = @_;
1200 $version = kvm_user_version() if !defined($version);
1201 PVE::QemuServer::Machine::extract_version($machine_type, $version)
1202}
1203
1204sub kernel_has_vhost_net {
1205 return -c '/dev/vhost-net';
1206}
1207
1208sub option_exists {
1209 my $key = shift;
1210 return defined($confdesc->{$key});
1211}
1212
1213my $cdrom_path;
1214sub get_cdrom_path {
1215
1216 return $cdrom_path if $cdrom_path;
1217
1218 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221}
1222
1223sub get_iso_path {
1224 my ($storecfg, $vmid, $cdrom) = @_;
1225
1226 if ($cdrom eq 'cdrom') {
1227 return get_cdrom_path();
1228 } elsif ($cdrom eq 'none') {
1229 return '';
1230 } elsif ($cdrom =~ m|^/|) {
1231 return $cdrom;
1232 } else {
1233 return PVE::Storage::path($storecfg, $cdrom);
1234 }
1235}
1236
1237# try to convert old style file names to volume IDs
1238sub filename_to_volume_id {
1239 my ($vmid, $file, $media) = @_;
1240
1241 if (!($file eq 'none' || $file eq 'cdrom' ||
1242 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244 return if $file =~ m|/|;
1245
1246 if ($media && $media eq 'cdrom') {
1247 $file = "local:iso/$file";
1248 } else {
1249 $file = "local:$vmid/$file";
1250 }
1251 }
1252
1253 return $file;
1254}
1255
1256sub verify_media_type {
1257 my ($opt, $vtype, $media) = @_;
1258
1259 return if !$media;
1260
1261 my $etype;
1262 if ($media eq 'disk') {
1263 $etype = 'images';
1264 } elsif ($media eq 'cdrom') {
1265 $etype = 'iso';
1266 } else {
1267 die "internal error";
1268 }
1269
1270 return if ($vtype eq $etype);
1271
1272 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273}
1274
1275sub cleanup_drive_path {
1276 my ($opt, $storecfg, $drive) = @_;
1277
1278 # try to convert filesystem paths to volume IDs
1279
1280 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281 ($drive->{file} !~ m|^/dev/.+|) &&
1282 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283 ($drive->{file} !~ m/^\d+$/)) {
1284 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1286 if !$vtype;
1287 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1288 verify_media_type($opt, $vtype, $drive->{media});
1289 $drive->{file} = $volid;
1290 }
1291
1292 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1293}
1294
1295sub parse_hotplug_features {
1296 my ($data) = @_;
1297
1298 my $res = {};
1299
1300 return $res if $data eq '0';
1301
1302 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1303
1304 foreach my $feature (PVE::Tools::split_list($data)) {
1305 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1306 $res->{$1} = 1;
1307 } else {
1308 die "invalid hotplug feature '$feature'\n";
1309 }
1310 }
1311 return $res;
1312}
1313
1314PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1315sub pve_verify_hotplug_features {
1316 my ($value, $noerr) = @_;
1317
1318 return $value if parse_hotplug_features($value);
1319
1320 return if $noerr;
1321
1322 die "unable to parse hotplug option\n";
1323}
1324
1325sub scsi_inquiry {
1326 my($fh, $noerr) = @_;
1327
1328 my $SG_IO = 0x2285;
1329 my $SG_GET_VERSION_NUM = 0x2282;
1330
1331 my $versionbuf = "\x00" x 8;
1332 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1333 if (!$ret) {
1334 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1335 return;
1336 }
1337 my $version = unpack("I", $versionbuf);
1338 if ($version < 30000) {
1339 die "scsi generic interface too old\n" if !$noerr;
1340 return;
1341 }
1342
1343 my $buf = "\x00" x 36;
1344 my $sensebuf = "\x00" x 8;
1345 my $cmd = pack("C x3 C x1", 0x12, 36);
1346
1347 # see /usr/include/scsi/sg.h
1348 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1349
1350 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1351 length($sensebuf), 0, length($buf), $buf,
1352 $cmd, $sensebuf, 6000);
1353
1354 $ret = ioctl($fh, $SG_IO, $packet);
1355 if (!$ret) {
1356 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1357 return;
1358 }
1359
1360 my @res = unpack($sg_io_hdr_t, $packet);
1361 if ($res[17] || $res[18]) {
1362 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1363 return;
1364 }
1365
1366 my $res = {};
1367 (my $byte0, my $byte1, $res->{vendor},
1368 $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1369
1370 $res->{removable} = $byte1 & 128 ? 1 : 0;
1371 $res->{type} = $byte0 & 31;
1372
1373 return $res;
1374}
1375
1376sub path_is_scsi {
1377 my ($path) = @_;
1378
1379 my $fh = IO::File->new("+<$path") || return;
1380 my $res = scsi_inquiry($fh, 1);
1381 close($fh);
1382
1383 return $res;
1384}
1385
1386sub print_tabletdevice_full {
1387 my ($conf, $arch) = @_;
1388
1389 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1390
1391 # we use uhci for old VMs because tablet driver was buggy in older qemu
1392 my $usbbus;
1393 if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1394 $usbbus = 'ehci';
1395 } else {
1396 $usbbus = 'uhci';
1397 }
1398
1399 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1400}
1401
1402sub print_keyboarddevice_full {
1403 my ($conf, $arch, $machine) = @_;
1404
1405 return if $arch ne 'aarch64';
1406
1407 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1408}
1409
1410sub print_drivedevice_full {
1411 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1412
1413 my $device = '';
1414 my $maxdev = 0;
1415
1416 my $drive_id = "$drive->{interface}$drive->{index}";
1417 if ($drive->{interface} eq 'virtio') {
1418 my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1419 $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1420 $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1421 } elsif ($drive->{interface} eq 'scsi') {
1422
1423 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1424 my $unit = $drive->{index} % $maxdev;
1425 my $devicetype = 'hd';
1426 my $path = '';
1427 if (drive_is_cdrom($drive)) {
1428 $devicetype = 'cd';
1429 } else {
1430 if ($drive->{file} =~ m|^/|) {
1431 $path = $drive->{file};
1432 if (my $info = path_is_scsi($path)) {
1433 if ($info->{type} == 0 && $drive->{scsiblock}) {
1434 $devicetype = 'block';
1435 } elsif ($info->{type} == 1) { # tape
1436 $devicetype = 'generic';
1437 }
1438 }
1439 } else {
1440 $path = PVE::Storage::path($storecfg, $drive->{file});
1441 }
1442
1443 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1444 my $version = extract_version($machine_type, kvm_user_version());
1445 if ($path =~ m/^iscsi\:\/\// &&
1446 !min_version($version, 4, 1)) {
1447 $devicetype = 'generic';
1448 }
1449 }
1450
1451 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1452 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1453 } else {
1454 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1455 .",lun=$drive->{index}";
1456 }
1457 $device .= ",drive=drive-$drive_id,id=$drive_id";
1458
1459 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1460 $device .= ",rotation_rate=1";
1461 }
1462 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1463
1464 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1465 my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1466 my $controller = int($drive->{index} / $maxdev);
1467 my $unit = $drive->{index} % $maxdev;
1468 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1469
1470 $device = "ide-$devicetype";
1471 if ($drive->{interface} eq 'ide') {
1472 $device .= ",bus=ide.$controller,unit=$unit";
1473 } else {
1474 $device .= ",bus=ahci$controller.$unit";
1475 }
1476 $device .= ",drive=drive-$drive_id,id=$drive_id";
1477
1478 if ($devicetype eq 'hd') {
1479 if (my $model = $drive->{model}) {
1480 $model = URI::Escape::uri_unescape($model);
1481 $device .= ",model=$model";
1482 }
1483 if ($drive->{ssd}) {
1484 $device .= ",rotation_rate=1";
1485 }
1486 }
1487 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1488 } elsif ($drive->{interface} eq 'usb') {
1489 die "implement me";
1490 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1491 } else {
1492 die "unsupported interface type";
1493 }
1494
1495 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1496
1497 if (my $serial = $drive->{serial}) {
1498 $serial = URI::Escape::uri_unescape($serial);
1499 $device .= ",serial=$serial";
1500 }
1501
1502
1503 return $device;
1504}
1505
1506sub get_initiator_name {
1507 my $initiator;
1508
1509 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1510 while (defined(my $line = <$fh>)) {
1511 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1512 $initiator = $1;
1513 last;
1514 }
1515 $fh->close();
1516
1517 return $initiator;
1518}
1519
1520sub print_drive_commandline_full {
1521 my ($storecfg, $vmid, $drive) = @_;
1522
1523 my $path;
1524 my $volid = $drive->{file};
1525 my $format;
1526
1527 if (drive_is_cdrom($drive)) {
1528 $path = get_iso_path($storecfg, $vmid, $volid);
1529 } else {
1530 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1531 if ($storeid) {
1532 $path = PVE::Storage::path($storecfg, $volid);
1533 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1534 $format = qemu_img_format($scfg, $volname);
1535 } else {
1536 $path = $volid;
1537 $format = "raw";
1538 }
1539 }
1540
1541 my $opts = '';
1542 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1543 foreach my $o (@qemu_drive_options) {
1544 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1545 }
1546
1547 # snapshot only accepts on|off
1548 if (defined($drive->{snapshot})) {
1549 my $v = $drive->{snapshot} ? 'on' : 'off';
1550 $opts .= ",snapshot=$v";
1551 }
1552
1553 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1554 my ($dir, $qmpname) = @$type;
1555 if (my $v = $drive->{"mbps$dir"}) {
1556 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1557 }
1558 if (my $v = $drive->{"mbps${dir}_max"}) {
1559 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1560 }
1561 if (my $v = $drive->{"bps${dir}_max_length"}) {
1562 $opts .= ",throttling.bps$qmpname-max-length=$v";
1563 }
1564 if (my $v = $drive->{"iops${dir}"}) {
1565 $opts .= ",throttling.iops$qmpname=$v";
1566 }
1567 if (my $v = $drive->{"iops${dir}_max"}) {
1568 $opts .= ",throttling.iops$qmpname-max=$v";
1569 }
1570 if (my $v = $drive->{"iops${dir}_max_length"}) {
1571 $opts .= ",throttling.iops$qmpname-max-length=$v";
1572 }
1573 }
1574
1575 $opts .= ",format=$format" if $format && !$drive->{format};
1576
1577 my $cache_direct = 0;
1578
1579 if (my $cache = $drive->{cache}) {
1580 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1581 } elsif (!drive_is_cdrom($drive)) {
1582 $opts .= ",cache=none";
1583 $cache_direct = 1;
1584 }
1585
1586 # aio native works only with O_DIRECT
1587 if (!$drive->{aio}) {
1588 if($cache_direct) {
1589 $opts .= ",aio=native";
1590 } else {
1591 $opts .= ",aio=threads";
1592 }
1593 }
1594
1595 if (!drive_is_cdrom($drive)) {
1596 my $detectzeroes;
1597 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1598 $detectzeroes = 'off';
1599 } elsif ($drive->{discard}) {
1600 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1601 } else {
1602 # This used to be our default with discard not being specified:
1603 $detectzeroes = 'on';
1604 }
1605 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1606 }
1607
1608 my $pathinfo = $path ? "file=$path," : '';
1609
1610 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1611}
1612
1613sub print_netdevice_full {
1614 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1615
1616 my $device = $net->{model};
1617 if ($net->{model} eq 'virtio') {
1618 $device = 'virtio-net-pci';
1619 };
1620
1621 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1622 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1623 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1624 # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1625 # and out of each queue plus one config interrupt and control vector queue
1626 my $vectors = $net->{queues} * 2 + 2;
1627 $tmpstr .= ",vectors=$vectors,mq=on";
1628 }
1629 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1630
1631 if (my $mtu = $net->{mtu}) {
1632 if ($net->{model} eq 'virtio' && $net->{bridge}) {
1633 my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1634 if ($mtu == 1) {
1635 $mtu = $bridge_mtu;
1636 } elsif ($mtu < 576) {
1637 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1638 } elsif ($mtu > $bridge_mtu) {
1639 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1640 }
1641 $tmpstr .= ",host_mtu=$mtu";
1642 } else {
1643 warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1644 }
1645 }
1646
1647 if ($use_old_bios_files) {
1648 my $romfile;
1649 if ($device eq 'virtio-net-pci') {
1650 $romfile = 'pxe-virtio.rom';
1651 } elsif ($device eq 'e1000') {
1652 $romfile = 'pxe-e1000.rom';
1653 } elsif ($device eq 'ne2k') {
1654 $romfile = 'pxe-ne2k_pci.rom';
1655 } elsif ($device eq 'pcnet') {
1656 $romfile = 'pxe-pcnet.rom';
1657 } elsif ($device eq 'rtl8139') {
1658 $romfile = 'pxe-rtl8139.rom';
1659 }
1660 $tmpstr .= ",romfile=$romfile" if $romfile;
1661 }
1662
1663 return $tmpstr;
1664}
1665
1666sub print_netdev_full {
1667 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1668
1669 my $i = '';
1670 if ($netid =~ m/^net(\d+)$/) {
1671 $i = int($1);
1672 }
1673
1674 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1675
1676 my $ifname = "tap${vmid}i$i";
1677
1678 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1679 die "interface name '$ifname' is too long (max 15 character)\n"
1680 if length($ifname) >= 16;
1681
1682 my $vhostparam = '';
1683 if (is_native($arch)) {
1684 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1685 }
1686
1687 my $vmname = $conf->{name} || "vm$vmid";
1688
1689 my $netdev = "";
1690 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1691
1692 if ($net->{bridge}) {
1693 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1694 .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1695 } else {
1696 $netdev = "type=user,id=$netid,hostname=$vmname";
1697 }
1698
1699 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1700
1701 return $netdev;
1702}
1703
1704my $vga_map = {
1705 'cirrus' => 'cirrus-vga',
1706 'std' => 'VGA',
1707 'vmware' => 'vmware-svga',
1708 'virtio' => 'virtio-vga',
1709};
1710
1711sub print_vga_device {
1712 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1713
1714 my $type = $vga_map->{$vga->{type}};
1715 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1716 $type = 'virtio-gpu';
1717 }
1718 my $vgamem_mb = $vga->{memory};
1719
1720 my $max_outputs = '';
1721 if ($qxlnum) {
1722 $type = $id ? 'qxl' : 'qxl-vga';
1723
1724 if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1725 # set max outputs so linux can have up to 4 qxl displays with one device
1726 if (min_version($machine_version, 4, 1)) {
1727 $max_outputs = ",max_outputs=4";
1728 }
1729 }
1730 }
1731
1732 die "no devicetype for $vga->{type}\n" if !$type;
1733
1734 my $memory = "";
1735 if ($vgamem_mb) {
1736 if ($vga->{type} eq 'virtio') {
1737 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1738 $memory = ",max_hostmem=$bytes";
1739 } elsif ($qxlnum) {
1740 # from https://www.spice-space.org/multiple-monitors.html
1741 $memory = ",vgamem_mb=$vga->{memory}";
1742 my $ram = $vgamem_mb * 4;
1743 my $vram = $vgamem_mb * 2;
1744 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1745 } else {
1746 $memory = ",vgamem_mb=$vga->{memory}";
1747 }
1748 } elsif ($qxlnum && $id) {
1749 $memory = ",ram_size=67108864,vram_size=33554432";
1750 }
1751
1752 my $edidoff = "";
1753 if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1754 $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1755 }
1756
1757 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1758 my $vgaid = "vga" . ($id // '');
1759 my $pciaddr;
1760
1761 if ($q35 && $vgaid eq 'vga') {
1762 # the first display uses pcie.0 bus on q35 machines
1763 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1764 } else {
1765 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1766 }
1767
1768 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1769}
1770
1771sub parse_number_sets {
1772 my ($set) = @_;
1773 my $res = [];
1774 foreach my $part (split(/;/, $set)) {
1775 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1776 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1777 push @$res, [ $1, $2 ];
1778 } else {
1779 die "invalid range: $part\n";
1780 }
1781 }
1782 return $res;
1783}
1784
1785sub parse_numa {
1786 my ($data) = @_;
1787
1788 my $res = parse_property_string($numa_fmt, $data);
1789 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1790 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1791 return $res;
1792}
1793
1794# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1795sub parse_net {
1796 my ($data) = @_;
1797
1798 my $res = eval { parse_property_string($net_fmt, $data) };
1799 if ($@) {
1800 warn $@;
1801 return;
1802 }
1803 if (!defined($res->{macaddr})) {
1804 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1805 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1806 }
1807 return $res;
1808}
1809
1810# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1811sub parse_ipconfig {
1812 my ($data) = @_;
1813
1814 my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1815 if ($@) {
1816 warn $@;
1817 return;
1818 }
1819
1820 if ($res->{gw} && !$res->{ip}) {
1821 warn 'gateway specified without specifying an IP address';
1822 return;
1823 }
1824 if ($res->{gw6} && !$res->{ip6}) {
1825 warn 'IPv6 gateway specified without specifying an IPv6 address';
1826 return;
1827 }
1828 if ($res->{gw} && $res->{ip} eq 'dhcp') {
1829 warn 'gateway specified together with DHCP';
1830 return;
1831 }
1832 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1833 # gw6 + auto/dhcp
1834 warn "IPv6 gateway specified together with $res->{ip6} address";
1835 return;
1836 }
1837
1838 if (!$res->{ip} && !$res->{ip6}) {
1839 return { ip => 'dhcp', ip6 => 'dhcp' };
1840 }
1841
1842 return $res;
1843}
1844
1845sub print_net {
1846 my $net = shift;
1847
1848 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1849}
1850
1851sub add_random_macs {
1852 my ($settings) = @_;
1853
1854 foreach my $opt (keys %$settings) {
1855 next if $opt !~ m/^net(\d+)$/;
1856 my $net = parse_net($settings->{$opt});
1857 next if !$net;
1858 $settings->{$opt} = print_net($net);
1859 }
1860}
1861
1862sub vm_is_volid_owner {
1863 my ($storecfg, $vmid, $volid) = @_;
1864
1865 if ($volid !~ m|^/|) {
1866 my ($path, $owner);
1867 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1868 if ($owner && ($owner == $vmid)) {
1869 return 1;
1870 }
1871 }
1872
1873 return;
1874}
1875
1876sub vmconfig_register_unused_drive {
1877 my ($storecfg, $vmid, $conf, $drive) = @_;
1878
1879 if (drive_is_cloudinit($drive)) {
1880 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1881 warn $@ if $@;
1882 } elsif (!drive_is_cdrom($drive)) {
1883 my $volid = $drive->{file};
1884 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1885 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1886 }
1887 }
1888}
1889
1890# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1891my $smbios1_fmt = {
1892 uuid => {
1893 type => 'string',
1894 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1895 format_description => 'UUID',
1896 description => "Set SMBIOS1 UUID.",
1897 optional => 1,
1898 },
1899 version => {
1900 type => 'string',
1901 pattern => '[A-Za-z0-9+\/]+={0,2}',
1902 format_description => 'Base64 encoded string',
1903 description => "Set SMBIOS1 version.",
1904 optional => 1,
1905 },
1906 serial => {
1907 type => 'string',
1908 pattern => '[A-Za-z0-9+\/]+={0,2}',
1909 format_description => 'Base64 encoded string',
1910 description => "Set SMBIOS1 serial number.",
1911 optional => 1,
1912 },
1913 manufacturer => {
1914 type => 'string',
1915 pattern => '[A-Za-z0-9+\/]+={0,2}',
1916 format_description => 'Base64 encoded string',
1917 description => "Set SMBIOS1 manufacturer.",
1918 optional => 1,
1919 },
1920 product => {
1921 type => 'string',
1922 pattern => '[A-Za-z0-9+\/]+={0,2}',
1923 format_description => 'Base64 encoded string',
1924 description => "Set SMBIOS1 product ID.",
1925 optional => 1,
1926 },
1927 sku => {
1928 type => 'string',
1929 pattern => '[A-Za-z0-9+\/]+={0,2}',
1930 format_description => 'Base64 encoded string',
1931 description => "Set SMBIOS1 SKU string.",
1932 optional => 1,
1933 },
1934 family => {
1935 type => 'string',
1936 pattern => '[A-Za-z0-9+\/]+={0,2}',
1937 format_description => 'Base64 encoded string',
1938 description => "Set SMBIOS1 family string.",
1939 optional => 1,
1940 },
1941 base64 => {
1942 type => 'boolean',
1943 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1944 optional => 1,
1945 },
1946};
1947
1948sub parse_smbios1 {
1949 my ($data) = @_;
1950
1951 my $res = eval { parse_property_string($smbios1_fmt, $data) };
1952 warn $@ if $@;
1953 return $res;
1954}
1955
1956sub print_smbios1 {
1957 my ($smbios1) = @_;
1958 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1959}
1960
1961PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1962
1963sub parse_watchdog {
1964 my ($value) = @_;
1965
1966 return if !$value;
1967
1968 my $res = eval { parse_property_string($watchdog_fmt, $value) };
1969 warn $@ if $@;
1970 return $res;
1971}
1972
1973sub parse_guest_agent {
1974 my ($value) = @_;
1975
1976 return {} if !defined($value->{agent});
1977
1978 my $res = eval { parse_property_string($agent_fmt, $value->{agent}) };
1979 warn $@ if $@;
1980
1981 # if the agent is disabled ignore the other potentially set properties
1982 return {} if !$res->{enabled};
1983 return $res;
1984}
1985
1986sub parse_vga {
1987 my ($value) = @_;
1988
1989 return {} if !$value;
1990 my $res = eval { parse_property_string($vga_fmt, $value) };
1991 warn $@ if $@;
1992 return $res;
1993}
1994
1995sub parse_rng {
1996 my ($value) = @_;
1997
1998 return if !$value;
1999
2000 my $res = eval { parse_property_string($rng_fmt, $value) };
2001 warn $@ if $@;
2002 return $res;
2003}
2004
2005PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2006sub verify_usb_device {
2007 my ($value, $noerr) = @_;
2008
2009 return $value if parse_usb_device($value);
2010
2011 return if $noerr;
2012
2013 die "unable to parse usb device\n";
2014}
2015
2016# add JSON properties for create and set function
2017sub json_config_properties {
2018 my $prop = shift;
2019
2020 foreach my $opt (keys %$confdesc) {
2021 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2022 $opt eq 'runningmachine' || $opt eq 'runningcpu';
2023 $prop->{$opt} = $confdesc->{$opt};
2024 }
2025
2026 return $prop;
2027}
2028
2029# return copy of $confdesc_cloudinit to generate documentation
2030sub cloudinit_config_properties {
2031
2032 return dclone($confdesc_cloudinit);
2033}
2034
2035sub check_type {
2036 my ($key, $value) = @_;
2037
2038 die "unknown setting '$key'\n" if !$confdesc->{$key};
2039
2040 my $type = $confdesc->{$key}->{type};
2041
2042 if (!defined($value)) {
2043 die "got undefined value\n";
2044 }
2045
2046 if ($value =~ m/[\n\r]/) {
2047 die "property contains a line feed\n";
2048 }
2049
2050 if ($type eq 'boolean') {
2051 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2052 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2053 die "type check ('boolean') failed - got '$value'\n";
2054 } elsif ($type eq 'integer') {
2055 return int($1) if $value =~ m/^(\d+)$/;
2056 die "type check ('integer') failed - got '$value'\n";
2057 } elsif ($type eq 'number') {
2058 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2059 die "type check ('number') failed - got '$value'\n";
2060 } elsif ($type eq 'string') {
2061 if (my $fmt = $confdesc->{$key}->{format}) {
2062 PVE::JSONSchema::check_format($fmt, $value);
2063 return $value;
2064 }
2065 $value =~ s/^\"(.*)\"$/$1/;
2066 return $value;
2067 } else {
2068 die "internal error"
2069 }
2070}
2071
2072sub destroy_vm {
2073 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2074
2075 my $conf = PVE::QemuConfig->load_config($vmid);
2076
2077 PVE::QemuConfig->check_lock($conf) if !$skiplock;
2078
2079 if ($conf->{template}) {
2080 # check if any base image is still used by a linked clone
2081 PVE::QemuConfig->foreach_volume($conf, sub {
2082 my ($ds, $drive) = @_;
2083 return if drive_is_cdrom($drive);
2084
2085 my $volid = $drive->{file};
2086 return if !$volid || $volid =~ m|^/|;
2087
2088 die "base volume '$volid' is still in use by linked cloned\n"
2089 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2090
2091 });
2092 }
2093
2094 # only remove disks owned by this VM
2095 PVE::QemuConfig->foreach_volume($conf, sub {
2096 my ($ds, $drive) = @_;
2097 return if drive_is_cdrom($drive, 1);
2098
2099 my $volid = $drive->{file};
2100 return if !$volid || $volid =~ m|^/|;
2101
2102 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2103 return if !$path || !$owner || ($owner != $vmid);
2104
2105 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2106 warn "Could not remove disk '$volid', check manually: $@" if $@;
2107 });
2108
2109 # also remove unused disk
2110 my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2111 PVE::Storage::foreach_volid($vmdisks, sub {
2112 my ($volid, $sid, $volname, $d) = @_;
2113 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2114 warn $@ if $@;
2115 });
2116
2117 if (defined $replacement_conf) {
2118 PVE::QemuConfig->write_config($vmid, $replacement_conf);
2119 } else {
2120 PVE::QemuConfig->destroy_config($vmid);
2121 }
2122}
2123
2124sub parse_vm_config {
2125 my ($filename, $raw) = @_;
2126
2127 return if !defined($raw);
2128
2129 my $res = {
2130 digest => Digest::SHA::sha1_hex($raw),
2131 snapshots => {},
2132 pending => {},
2133 };
2134
2135 $filename =~ m|/qemu-server/(\d+)\.conf$|
2136 || die "got strange filename '$filename'";
2137
2138 my $vmid = $1;
2139
2140 my $conf = $res;
2141 my $descr;
2142 my $section = '';
2143
2144 my @lines = split(/\n/, $raw);
2145 foreach my $line (@lines) {
2146 next if $line =~ m/^\s*$/;
2147
2148 if ($line =~ m/^\[PENDING\]\s*$/i) {
2149 $section = 'pending';
2150 if (defined($descr)) {
2151 $descr =~ s/\s+$//;
2152 $conf->{description} = $descr;
2153 }
2154 $descr = undef;
2155 $conf = $res->{$section} = {};
2156 next;
2157
2158 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2159 $section = $1;
2160 if (defined($descr)) {
2161 $descr =~ s/\s+$//;
2162 $conf->{description} = $descr;
2163 }
2164 $descr = undef;
2165 $conf = $res->{snapshots}->{$section} = {};
2166 next;
2167 }
2168
2169 if ($line =~ m/^\#(.*)\s*$/) {
2170 $descr = '' if !defined($descr);
2171 $descr .= PVE::Tools::decode_text($1) . "\n";
2172 next;
2173 }
2174
2175 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2176 $descr = '' if !defined($descr);
2177 $descr .= PVE::Tools::decode_text($2);
2178 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2179 $conf->{snapstate} = $1;
2180 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2181 my $key = $1;
2182 my $value = $2;
2183 $conf->{$key} = $value;
2184 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2185 my $value = $1;
2186 if ($section eq 'pending') {
2187 $conf->{delete} = $value; # we parse this later
2188 } else {
2189 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2190 }
2191 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2192 my $key = $1;
2193 my $value = $2;
2194 eval { $value = check_type($key, $value); };
2195 if ($@) {
2196 warn "vm $vmid - unable to parse value of '$key' - $@";
2197 } else {
2198 $key = 'ide2' if $key eq 'cdrom';
2199 my $fmt = $confdesc->{$key}->{format};
2200 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2201 my $v = parse_drive($key, $value);
2202 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2203 $v->{file} = $volid;
2204 $value = print_drive($v);
2205 } else {
2206 warn "vm $vmid - unable to parse value of '$key'\n";
2207 next;
2208 }
2209 }
2210
2211 $conf->{$key} = $value;
2212 }
2213 }
2214 }
2215
2216 if (defined($descr)) {
2217 $descr =~ s/\s+$//;
2218 $conf->{description} = $descr;
2219 }
2220 delete $res->{snapstate}; # just to be sure
2221
2222 return $res;
2223}
2224
2225sub write_vm_config {
2226 my ($filename, $conf) = @_;
2227
2228 delete $conf->{snapstate}; # just to be sure
2229
2230 if ($conf->{cdrom}) {
2231 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2232 $conf->{ide2} = $conf->{cdrom};
2233 delete $conf->{cdrom};
2234 }
2235
2236 # we do not use 'smp' any longer
2237 if ($conf->{sockets}) {
2238 delete $conf->{smp};
2239 } elsif ($conf->{smp}) {
2240 $conf->{sockets} = $conf->{smp};
2241 delete $conf->{cores};
2242 delete $conf->{smp};
2243 }
2244
2245 my $used_volids = {};
2246
2247 my $cleanup_config = sub {
2248 my ($cref, $pending, $snapname) = @_;
2249
2250 foreach my $key (keys %$cref) {
2251 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2252 $key eq 'snapstate' || $key eq 'pending';
2253 my $value = $cref->{$key};
2254 if ($key eq 'delete') {
2255 die "propertry 'delete' is only allowed in [PENDING]\n"
2256 if !$pending;
2257 # fixme: check syntax?
2258 next;
2259 }
2260 eval { $value = check_type($key, $value); };
2261 die "unable to parse value of '$key' - $@" if $@;
2262
2263 $cref->{$key} = $value;
2264
2265 if (!$snapname && is_valid_drivename($key)) {
2266 my $drive = parse_drive($key, $value);
2267 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2268 }
2269 }
2270 };
2271
2272 &$cleanup_config($conf);
2273
2274 &$cleanup_config($conf->{pending}, 1);
2275
2276 foreach my $snapname (keys %{$conf->{snapshots}}) {
2277 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2278 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2279 }
2280
2281 # remove 'unusedX' settings if we re-add a volume
2282 foreach my $key (keys %$conf) {
2283 my $value = $conf->{$key};
2284 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2285 delete $conf->{$key};
2286 }
2287 }
2288
2289 my $generate_raw_config = sub {
2290 my ($conf, $pending) = @_;
2291
2292 my $raw = '';
2293
2294 # add description as comment to top of file
2295 if (defined(my $descr = $conf->{description})) {
2296 if ($descr) {
2297 foreach my $cl (split(/\n/, $descr)) {
2298 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2299 }
2300 } else {
2301 $raw .= "#\n" if $pending;
2302 }
2303 }
2304
2305 foreach my $key (sort keys %$conf) {
2306 next if $key =~ /^(digest|description|pending|snapshots)$/;
2307 $raw .= "$key: $conf->{$key}\n";
2308 }
2309 return $raw;
2310 };
2311
2312 my $raw = &$generate_raw_config($conf);
2313
2314 if (scalar(keys %{$conf->{pending}})){
2315 $raw .= "\n[PENDING]\n";
2316 $raw .= &$generate_raw_config($conf->{pending}, 1);
2317 }
2318
2319 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2320 $raw .= "\n[$snapname]\n";
2321 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2322 }
2323
2324 return $raw;
2325}
2326
2327sub load_defaults {
2328
2329 my $res = {};
2330
2331 # we use static defaults from our JSON schema configuration
2332 foreach my $key (keys %$confdesc) {
2333 if (defined(my $default = $confdesc->{$key}->{default})) {
2334 $res->{$key} = $default;
2335 }
2336 }
2337
2338 return $res;
2339}
2340
2341sub config_list {
2342 my $vmlist = PVE::Cluster::get_vmlist();
2343 my $res = {};
2344 return $res if !$vmlist || !$vmlist->{ids};
2345 my $ids = $vmlist->{ids};
2346 my $nodename = nodename();
2347
2348 foreach my $vmid (keys %$ids) {
2349 my $d = $ids->{$vmid};
2350 next if !$d->{node} || $d->{node} ne $nodename;
2351 next if !$d->{type} || $d->{type} ne 'qemu';
2352 $res->{$vmid}->{exists} = 1;
2353 }
2354 return $res;
2355}
2356
2357# test if VM uses local resources (to prevent migration)
2358sub check_local_resources {
2359 my ($conf, $noerr) = @_;
2360
2361 my @loc_res = ();
2362
2363 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2364 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2365
2366 push @loc_res, "ivshmem" if $conf->{ivshmem};
2367
2368 foreach my $k (keys %$conf) {
2369 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2370 # sockets are safe: they will recreated be on the target side post-migrate
2371 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2372 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2373 }
2374
2375 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2376
2377 return \@loc_res;
2378}
2379
2380# check if used storages are available on all nodes (use by migrate)
2381sub check_storage_availability {
2382 my ($storecfg, $conf, $node) = @_;
2383
2384 PVE::QemuConfig->foreach_volume($conf, sub {
2385 my ($ds, $drive) = @_;
2386
2387 my $volid = $drive->{file};
2388 return if !$volid;
2389
2390 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2391 return if !$sid;
2392
2393 # check if storage is available on both nodes
2394 my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2395 PVE::Storage::storage_check_node($storecfg, $sid, $node);
2396 });
2397}
2398
2399# list nodes where all VM images are available (used by has_feature API)
2400sub shared_nodes {
2401 my ($conf, $storecfg) = @_;
2402
2403 my $nodelist = PVE::Cluster::get_nodelist();
2404 my $nodehash = { map { $_ => 1 } @$nodelist };
2405 my $nodename = nodename();
2406
2407 PVE::QemuConfig->foreach_volume($conf, sub {
2408 my ($ds, $drive) = @_;
2409
2410 my $volid = $drive->{file};
2411 return if !$volid;
2412
2413 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2414 if ($storeid) {
2415 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2416 if ($scfg->{disable}) {
2417 $nodehash = {};
2418 } elsif (my $avail = $scfg->{nodes}) {
2419 foreach my $node (keys %$nodehash) {
2420 delete $nodehash->{$node} if !$avail->{$node};
2421 }
2422 } elsif (!$scfg->{shared}) {
2423 foreach my $node (keys %$nodehash) {
2424 delete $nodehash->{$node} if $node ne $nodename
2425 }
2426 }
2427 }
2428 });
2429
2430 return $nodehash
2431}
2432
2433sub check_local_storage_availability {
2434 my ($conf, $storecfg) = @_;
2435
2436 my $nodelist = PVE::Cluster::get_nodelist();
2437 my $nodehash = { map { $_ => {} } @$nodelist };
2438
2439 PVE::QemuConfig->foreach_volume($conf, sub {
2440 my ($ds, $drive) = @_;
2441
2442 my $volid = $drive->{file};
2443 return if !$volid;
2444
2445 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2446 if ($storeid) {
2447 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2448
2449 if ($scfg->{disable}) {
2450 foreach my $node (keys %$nodehash) {
2451 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2452 }
2453 } elsif (my $avail = $scfg->{nodes}) {
2454 foreach my $node (keys %$nodehash) {
2455 if (!$avail->{$node}) {
2456 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2457 }
2458 }
2459 }
2460 }
2461 });
2462
2463 foreach my $node (values %$nodehash) {
2464 if (my $unavail = $node->{unavailable_storages}) {
2465 $node->{unavailable_storages} = [ sort keys %$unavail ];
2466 }
2467 }
2468
2469 return $nodehash
2470}
2471
2472# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2473sub check_running {
2474 my ($vmid, $nocheck, $node) = @_;
2475
2476 PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2477 return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2478}
2479
2480sub vzlist {
2481
2482 my $vzlist = config_list();
2483
2484 my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2485
2486 while (defined(my $de = $fd->read)) {
2487 next if $de !~ m/^(\d+)\.pid$/;
2488 my $vmid = $1;
2489 next if !defined($vzlist->{$vmid});
2490 if (my $pid = check_running($vmid)) {
2491 $vzlist->{$vmid}->{pid} = $pid;
2492 }
2493 }
2494
2495 return $vzlist;
2496}
2497
2498our $vmstatus_return_properties = {
2499 vmid => get_standard_option('pve-vmid'),
2500 status => {
2501 description => "Qemu process status.",
2502 type => 'string',
2503 enum => ['stopped', 'running'],
2504 },
2505 maxmem => {
2506 description => "Maximum memory in bytes.",
2507 type => 'integer',
2508 optional => 1,
2509 renderer => 'bytes',
2510 },
2511 maxdisk => {
2512 description => "Root disk size in bytes.",
2513 type => 'integer',
2514 optional => 1,
2515 renderer => 'bytes',
2516 },
2517 name => {
2518 description => "VM name.",
2519 type => 'string',
2520 optional => 1,
2521 },
2522 qmpstatus => {
2523 description => "Qemu QMP agent status.",
2524 type => 'string',
2525 optional => 1,
2526 },
2527 pid => {
2528 description => "PID of running qemu process.",
2529 type => 'integer',
2530 optional => 1,
2531 },
2532 uptime => {
2533 description => "Uptime.",
2534 type => 'integer',
2535 optional => 1,
2536 renderer => 'duration',
2537 },
2538 cpus => {
2539 description => "Maximum usable CPUs.",
2540 type => 'number',
2541 optional => 1,
2542 },
2543 lock => {
2544 description => "The current config lock, if any.",
2545 type => 'string',
2546 optional => 1,
2547 },
2548 tags => {
2549 description => "The current configured tags, if any",
2550 type => 'string',
2551 optional => 1,
2552 },
2553};
2554
2555my $last_proc_pid_stat;
2556
2557# get VM status information
2558# This must be fast and should not block ($full == false)
2559# We only query KVM using QMP if $full == true (this can be slow)
2560sub vmstatus {
2561 my ($opt_vmid, $full) = @_;
2562
2563 my $res = {};
2564
2565 my $storecfg = PVE::Storage::config();
2566
2567 my $list = vzlist();
2568 my $defaults = load_defaults();
2569
2570 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2571
2572 my $cpucount = $cpuinfo->{cpus} || 1;
2573
2574 foreach my $vmid (keys %$list) {
2575 next if $opt_vmid && ($vmid ne $opt_vmid);
2576
2577 my $conf = PVE::QemuConfig->load_config($vmid);
2578
2579 my $d = { vmid => $vmid };
2580 $d->{pid} = $list->{$vmid}->{pid};
2581
2582 # fixme: better status?
2583 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2584
2585 my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2586 if (defined($size)) {
2587 $d->{disk} = 0; # no info available
2588 $d->{maxdisk} = $size;
2589 } else {
2590 $d->{disk} = 0;
2591 $d->{maxdisk} = 0;
2592 }
2593
2594 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2595 * ($conf->{cores} || $defaults->{cores});
2596 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2597 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2598
2599 $d->{name} = $conf->{name} || "VM $vmid";
2600 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2601 : $defaults->{memory}*(1024*1024);
2602
2603 if ($conf->{balloon}) {
2604 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2605 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2606 : $defaults->{shares};
2607 }
2608
2609 $d->{uptime} = 0;
2610 $d->{cpu} = 0;
2611 $d->{mem} = 0;
2612
2613 $d->{netout} = 0;
2614 $d->{netin} = 0;
2615
2616 $d->{diskread} = 0;
2617 $d->{diskwrite} = 0;
2618
2619 $d->{template} = PVE::QemuConfig->is_template($conf);
2620
2621 $d->{serial} = 1 if conf_has_serial($conf);
2622 $d->{lock} = $conf->{lock} if $conf->{lock};
2623 $d->{tags} = $conf->{tags} if defined($conf->{tags});
2624
2625 $res->{$vmid} = $d;
2626 }
2627
2628 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2629 foreach my $dev (keys %$netdev) {
2630 next if $dev !~ m/^tap([1-9]\d*)i/;
2631 my $vmid = $1;
2632 my $d = $res->{$vmid};
2633 next if !$d;
2634
2635 $d->{netout} += $netdev->{$dev}->{receive};
2636 $d->{netin} += $netdev->{$dev}->{transmit};
2637
2638 if ($full) {
2639 $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2640 $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2641 }
2642
2643 }
2644
2645 my $ctime = gettimeofday;
2646
2647 foreach my $vmid (keys %$list) {
2648
2649 my $d = $res->{$vmid};
2650 my $pid = $d->{pid};
2651 next if !$pid;
2652
2653 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2654 next if !$pstat; # not running
2655
2656 my $used = $pstat->{utime} + $pstat->{stime};
2657
2658 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2659
2660 if ($pstat->{vsize}) {
2661 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2662 }
2663
2664 my $old = $last_proc_pid_stat->{$pid};
2665 if (!$old) {
2666 $last_proc_pid_stat->{$pid} = {
2667 time => $ctime,
2668 used => $used,
2669 cpu => 0,
2670 };
2671 next;
2672 }
2673
2674 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2675
2676 if ($dtime > 1000) {
2677 my $dutime = $used - $old->{used};
2678
2679 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2680 $last_proc_pid_stat->{$pid} = {
2681 time => $ctime,
2682 used => $used,
2683 cpu => $d->{cpu},
2684 };
2685 } else {
2686 $d->{cpu} = $old->{cpu};
2687 }
2688 }
2689
2690 return $res if !$full;
2691
2692 my $qmpclient = PVE::QMPClient->new();
2693
2694 my $ballooncb = sub {
2695 my ($vmid, $resp) = @_;
2696
2697 my $info = $resp->{'return'};
2698 return if !$info->{max_mem};
2699
2700 my $d = $res->{$vmid};
2701
2702 # use memory assigned to VM
2703 $d->{maxmem} = $info->{max_mem};
2704 $d->{balloon} = $info->{actual};
2705
2706 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2707 $d->{mem} = $info->{total_mem} - $info->{free_mem};
2708 $d->{freemem} = $info->{free_mem};
2709 }
2710
2711 $d->{ballooninfo} = $info;
2712 };
2713
2714 my $blockstatscb = sub {
2715 my ($vmid, $resp) = @_;
2716 my $data = $resp->{'return'} || [];
2717 my $totalrdbytes = 0;
2718 my $totalwrbytes = 0;
2719
2720 for my $blockstat (@$data) {
2721 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2722 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2723
2724 $blockstat->{device} =~ s/drive-//;
2725 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2726 }
2727 $res->{$vmid}->{diskread} = $totalrdbytes;
2728 $res->{$vmid}->{diskwrite} = $totalwrbytes;
2729 };
2730
2731 my $statuscb = sub {
2732 my ($vmid, $resp) = @_;
2733
2734 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2735 # this fails if ballon driver is not loaded, so this must be
2736 # the last commnand (following command are aborted if this fails).
2737 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2738
2739 my $status = 'unknown';
2740 if (!defined($status = $resp->{'return'}->{status})) {
2741 warn "unable to get VM status\n";
2742 return;
2743 }
2744
2745 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2746 };
2747
2748 foreach my $vmid (keys %$list) {
2749 next if $opt_vmid && ($vmid ne $opt_vmid);
2750 next if !$res->{$vmid}->{pid}; # not running
2751 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2752 }
2753
2754 $qmpclient->queue_execute(undef, 2);
2755
2756 foreach my $vmid (keys %$list) {
2757 next if $opt_vmid && ($vmid ne $opt_vmid);
2758 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2759 }
2760
2761 return $res;
2762}
2763
2764sub conf_has_serial {
2765 my ($conf) = @_;
2766
2767 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2768 if ($conf->{"serial$i"}) {
2769 return 1;
2770 }
2771 }
2772
2773 return 0;
2774}
2775
2776sub conf_has_audio {
2777 my ($conf, $id) = @_;
2778
2779 $id //= 0;
2780 my $audio = $conf->{"audio$id"};
2781 return if !defined($audio);
2782
2783 my $audioproperties = parse_property_string($audio_fmt, $audio);
2784 my $audiodriver = $audioproperties->{driver} // 'spice';
2785
2786 return {
2787 dev => $audioproperties->{device},
2788 dev_id => "audiodev$id",
2789 backend => $audiodriver,
2790 backend_id => "$audiodriver-backend${id}",
2791 };
2792}
2793
2794sub audio_devs {
2795 my ($audio, $audiopciaddr, $machine_version) = @_;
2796
2797 my $devs = [];
2798
2799 my $id = $audio->{dev_id};
2800 my $audiodev = "";
2801 if (min_version($machine_version, 4, 2)) {
2802 $audiodev = ",audiodev=$audio->{backend_id}";
2803 }
2804
2805 if ($audio->{dev} eq 'AC97') {
2806 push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2807 } elsif ($audio->{dev} =~ /intel\-hda$/) {
2808 push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2809 push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2810 push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2811 } else {
2812 die "unkown audio device '$audio->{dev}', implement me!";
2813 }
2814
2815 push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2816
2817 return $devs;
2818}
2819
2820sub vga_conf_has_spice {
2821 my ($vga) = @_;
2822
2823 my $vgaconf = parse_vga($vga);
2824 my $vgatype = $vgaconf->{type};
2825 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2826
2827 return $1 || 1;
2828}
2829
2830sub is_native($) {
2831 my ($arch) = @_;
2832 return get_host_arch() eq $arch;
2833}
2834
2835sub get_vm_arch {
2836 my ($conf) = @_;
2837 return $conf->{arch} // get_host_arch();
2838}
2839
2840my $default_machines = {
2841 x86_64 => 'pc',
2842 aarch64 => 'virt',
2843};
2844
2845sub get_vm_machine {
2846 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2847
2848 my $machine = $forcemachine || $conf->{machine};
2849
2850 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2851 $arch //= 'x86_64';
2852 $machine ||= $default_machines->{$arch};
2853 if ($add_pve_version) {
2854 $kvmversion //= kvm_user_version();
2855 my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2856 $machine .= "+pve$pvever";
2857 }
2858 }
2859
2860 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2861 # for version-pinned machines that do not include a pve-version (e.g.
2862 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2863 $machine .= '+pve0';
2864 }
2865
2866 return $machine;
2867}
2868
2869sub get_ovmf_files($) {
2870 my ($arch) = @_;
2871
2872 my $ovmf = $OVMF->{$arch}
2873 or die "no OVMF images known for architecture '$arch'\n";
2874
2875 return @$ovmf;
2876}
2877
2878my $Arch2Qemu = {
2879 aarch64 => '/usr/bin/qemu-system-aarch64',
2880 x86_64 => '/usr/bin/qemu-system-x86_64',
2881};
2882sub get_command_for_arch($) {
2883 my ($arch) = @_;
2884 return '/usr/bin/kvm' if is_native($arch);
2885
2886 my $cmd = $Arch2Qemu->{$arch}
2887 or die "don't know how to emulate architecture '$arch'\n";
2888 return $cmd;
2889}
2890
2891# To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2892# to use in a QEMU command line (-cpu element), first array_intersect the result
2893# of query_supported_ with query_understood_. This is necessary because:
2894#
2895# a) query_understood_ returns flags the host cannot use and
2896# b) query_supported_ (rather the QMP call) doesn't actually return CPU
2897# flags, but CPU settings - with most of them being flags. Those settings
2898# (and some flags, curiously) cannot be specified as a "-cpu" argument.
2899#
2900# query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2901# expensive. If you need the value returned from this, you can get it much
2902# cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2903# $accel being 'kvm' or 'tcg'.
2904#
2905# pvestatd calls this function on startup and whenever the QEMU/KVM version
2906# changes, automatically populating pmxcfs.
2907#
2908# Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2909# since kvm and tcg machines support different flags
2910#
2911sub query_supported_cpu_flags {
2912 my ($arch) = @_;
2913
2914 $arch //= get_host_arch();
2915 my $default_machine = $default_machines->{$arch};
2916
2917 my $flags = {};
2918
2919 # FIXME: Once this is merged, the code below should work for ARM as well:
2920 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2921 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2922 $arch eq "aarch64";
2923
2924 my $kvm_supported = defined(kvm_version());
2925 my $qemu_cmd = get_command_for_arch($arch);
2926 my $fakevmid = -1;
2927 my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2928
2929 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2930 my $query_supported_run_qemu = sub {
2931 my ($kvm) = @_;
2932
2933 my $flags = {};
2934 my $cmd = [
2935 $qemu_cmd,
2936 '-machine', $default_machine,
2937 '-display', 'none',
2938 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2939 '-mon', 'chardev=qmp,mode=control',
2940 '-pidfile', $pidfile,
2941 '-S', '-daemonize'
2942 ];
2943
2944 if (!$kvm) {
2945 push @$cmd, '-accel', 'tcg';
2946 }
2947
2948 my $rc = run_command($cmd, noerr => 1, quiet => 0);
2949 die "QEMU flag querying VM exited with code " . $rc if $rc;
2950
2951 eval {
2952 my $cmd_result = mon_cmd(
2953 $fakevmid,
2954 'query-cpu-model-expansion',
2955 type => 'full',
2956 model => { name => 'host' }
2957 );
2958
2959 my $props = $cmd_result->{model}->{props};
2960 foreach my $prop (keys %$props) {
2961 next if $props->{$prop} ne '1';
2962 # QEMU returns some flags multiple times, with '_', '.' or '-'
2963 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2964 # We only keep those with underscores, to match /proc/cpuinfo
2965 $prop =~ s/\.|-/_/g;
2966 $flags->{$prop} = 1;
2967 }
2968 };
2969 my $err = $@;
2970
2971 # force stop with 10 sec timeout and 'nocheck'
2972 # always stop, even if QMP failed
2973 vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2974
2975 die $err if $err;
2976
2977 return [ sort keys %$flags ];
2978 };
2979
2980 # We need to query QEMU twice, since KVM and TCG have different supported flags
2981 PVE::QemuConfig->lock_config($fakevmid, sub {
2982 $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2983 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2984
2985 if ($kvm_supported) {
2986 $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2987 warn "warning: failed querying supported kvm flags: $@\n" if $@;
2988 }
2989 });
2990
2991 return $flags;
2992}
2993
2994# Understood CPU flags are written to a file at 'pve-qemu' compile time
2995my $understood_cpu_flag_dir = "/usr/share/kvm";
2996sub query_understood_cpu_flags {
2997 my $arch = get_host_arch();
2998 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2999
3000 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3001 if ! -e $filepath;
3002
3003 my $raw = file_get_contents($filepath);
3004 $raw =~ s/^\s+|\s+$//g;
3005 my @flags = split(/\s+/, $raw);
3006
3007 return \@flags;
3008}
3009
3010sub config_to_command {
3011 my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3012
3013 my $cmd = [];
3014 my $globalFlags = [];
3015 my $machineFlags = [];
3016 my $rtcFlags = [];
3017 my $devices = [];
3018 my $pciaddr = '';
3019 my $bridges = {};
3020 my $ostype = $conf->{ostype};
3021 my $winversion = windows_version($ostype);
3022 my $kvm = $conf->{kvm};
3023 my $nodename = nodename();
3024
3025 my $arch = get_vm_arch($conf);
3026 my $kvm_binary = get_command_for_arch($arch);
3027 my $kvmver = kvm_user_version($kvm_binary);
3028
3029 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3030 $kvmver //= "undefined";
3031 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3032 }
3033
3034 my $add_pve_version = min_version($kvmver, 4, 1);
3035
3036 my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3037 my $machine_version = extract_version($machine_type, $kvmver);
3038 $kvm //= 1 if is_native($arch);
3039
3040 $machine_version =~ m/(\d+)\.(\d+)/;
3041 my ($machine_major, $machine_minor) = ($1, $2);
3042
3043 if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3044 warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3045 } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3046 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3047 ." please upgrade node '$nodename'\n"
3048 } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3049 my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3050 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3051 ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3052 ." node '$nodename'\n";
3053 }
3054
3055 # if a specific +pve version is required for a feature, use $version_guard
3056 # instead of min_version to allow machines to be run with the minimum
3057 # required version
3058 my $required_pve_version = 0;
3059 my $version_guard = sub {
3060 my ($major, $minor, $pve) = @_;
3061 return 0 if !min_version($machine_version, $major, $minor, $pve);
3062 my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3063 return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3064 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3065 return 1;
3066 };
3067
3068 if ($kvm && !defined kvm_version()) {
3069 die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3070 ." or enable in BIOS.\n";
3071 }
3072
3073 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3074 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3075 my $use_old_bios_files = undef;
3076 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3077
3078 my $cpuunits = defined($conf->{cpuunits}) ?
3079 $conf->{cpuunits} : $defaults->{cpuunits};
3080
3081 push @$cmd, $kvm_binary;
3082
3083 push @$cmd, '-id', $vmid;
3084
3085 my $vmname = $conf->{name} || "vm$vmid";
3086
3087 push @$cmd, '-name', $vmname;
3088
3089 my $use_virtio = 0;
3090
3091 my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3092 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3093 push @$cmd, '-mon', "chardev=qmp,mode=control";
3094
3095 if (min_version($machine_version, 2, 12)) {
3096 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3097 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3098 }
3099
3100 push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3101
3102 push @$cmd, '-daemonize';
3103
3104 if ($conf->{smbios1}) {
3105 my $smbios_conf = parse_smbios1($conf->{smbios1});
3106 if ($smbios_conf->{base64}) {
3107 # Do not pass base64 flag to qemu
3108 delete $smbios_conf->{base64};
3109 my $smbios_string = "";
3110 foreach my $key (keys %$smbios_conf) {
3111 my $value;
3112 if ($key eq "uuid") {
3113 $value = $smbios_conf->{uuid}
3114 } else {
3115 $value = decode_base64($smbios_conf->{$key});
3116 }
3117 # qemu accepts any binary data, only commas need escaping by double comma
3118 $value =~ s/,/,,/g;
3119 $smbios_string .= "," . $key . "=" . $value if $value;
3120 }
3121 push @$cmd, '-smbios', "type=1" . $smbios_string;
3122 } else {
3123 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3124 }
3125 }
3126
3127 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3128 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3129 die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3130
3131 my ($path, $format);
3132 if (my $efidisk = $conf->{efidisk0}) {
3133 my $d = parse_drive('efidisk0', $efidisk);
3134 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3135 $format = $d->{format};
3136 if ($storeid) {
3137 $path = PVE::Storage::path($storecfg, $d->{file});
3138 if (!defined($format)) {
3139 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3140 $format = qemu_img_format($scfg, $volname);
3141 }
3142 } else {
3143 $path = $d->{file};
3144 die "efidisk format must be specified\n"
3145 if !defined($format);
3146 }
3147 } else {
3148 warn "no efidisk configured! Using temporary efivars disk.\n";
3149 $path = "/tmp/$vmid-ovmf.fd";
3150 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3151 $format = 'raw';
3152 }
3153
3154 my $size_str = "";
3155
3156 if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3157 $size_str = ",size=" . (-s $ovmf_vars);
3158 }
3159
3160 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3161 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3162 }
3163
3164 # load q35 config
3165 if ($q35) {
3166 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3167 if (min_version($machine_version, 4, 0)) {
3168 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3169 } else {
3170 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3171 }
3172 }
3173
3174 if ($conf->{vmgenid}) {
3175 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3176 }
3177
3178 # add usb controllers
3179 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3180 $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3181 push @$devices, @usbcontrollers if @usbcontrollers;
3182 my $vga = parse_vga($conf->{vga});
3183
3184 my $qxlnum = vga_conf_has_spice($conf->{vga});
3185 $vga->{type} = 'qxl' if $qxlnum;
3186
3187 if (!$vga->{type}) {
3188 if ($arch eq 'aarch64') {
3189 $vga->{type} = 'virtio';
3190 } elsif (min_version($machine_version, 2, 9)) {
3191 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3192 } else {
3193 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3194 }
3195 }
3196
3197 # enable absolute mouse coordinates (needed by vnc)
3198 my $tablet;
3199 if (defined($conf->{tablet})) {
3200 $tablet = $conf->{tablet};
3201 } else {
3202 $tablet = $defaults->{tablet};
3203 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3204 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3205 }
3206
3207 if ($tablet) {
3208 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3209 my $kbd = print_keyboarddevice_full($conf, $arch);
3210 push @$devices, '-device', $kbd if defined($kbd);
3211 }
3212
3213 my $bootorder = device_bootorder($conf);
3214
3215 # host pci device passthrough
3216 my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3217 $vmid, $conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3218
3219 # usb devices
3220 my $usb_dev_features = {};
3221 $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3222
3223 my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3224 $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3225 push @$devices, @usbdevices if @usbdevices;
3226
3227 # serial devices
3228 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3229 if (my $path = $conf->{"serial$i"}) {
3230 if ($path eq 'socket') {
3231 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3232 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3233 # On aarch64, serial0 is the UART device. Qemu only allows
3234 # connecting UART devices via the '-serial' command line, as
3235 # the device has a fixed slot on the hardware...
3236 if ($arch eq 'aarch64' && $i == 0) {
3237 push @$devices, '-serial', "chardev:serial$i";
3238 } else {
3239 push @$devices, '-device', "isa-serial,chardev=serial$i";
3240 }
3241 } else {
3242 die "no such serial device\n" if ! -c $path;
3243 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3244 push @$devices, '-device', "isa-serial,chardev=serial$i";
3245 }
3246 }
3247 }
3248
3249 # parallel devices
3250 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3251 if (my $path = $conf->{"parallel$i"}) {
3252 die "no such parallel device\n" if ! -c $path;
3253 my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3254 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3255 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3256 }
3257 }
3258
3259 if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3260 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3261 my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3262 push @$devices, @$audio_devs;
3263 }
3264
3265 my $sockets = 1;
3266 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3267 $sockets = $conf->{sockets} if $conf->{sockets};
3268
3269 my $cores = $conf->{cores} || 1;
3270
3271 my $maxcpus = $sockets * $cores;
3272
3273 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3274
3275 my $allowed_vcpus = $cpuinfo->{cpus};
3276
3277 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3278 if ($allowed_vcpus < $maxcpus);
3279
3280 if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3281
3282 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3283 for (my $i = 2; $i <= $vcpus; $i++) {
3284 my $cpustr = print_cpu_device($conf,$i);
3285 push @$cmd, '-device', $cpustr;
3286 }
3287
3288 } else {
3289
3290 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3291 }
3292 push @$cmd, '-nodefaults';
3293
3294 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3295
3296 push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3297
3298 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
3299
3300 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3301 push @$devices, '-device', print_vga_device(
3302 $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3303 my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3304 push @$cmd, '-vnc', "unix:$socket,password";
3305 } else {
3306 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3307 push @$cmd, '-nographic';
3308 }
3309
3310 # time drift fix
3311 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3312 my $useLocaltime = $conf->{localtime};
3313
3314 if ($winversion >= 5) { # windows
3315 $useLocaltime = 1 if !defined($conf->{localtime});
3316
3317 # use time drift fix when acpi is enabled
3318 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3319 $tdf = 1 if !defined($conf->{tdf});
3320 }
3321 }
3322
3323 if ($winversion >= 6) {
3324 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3325 push @$cmd, '-no-hpet';
3326 }
3327
3328 push @$rtcFlags, 'driftfix=slew' if $tdf;
3329
3330 if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3331 push @$rtcFlags, "base=$conf->{startdate}";
3332 } elsif ($useLocaltime) {
3333 push @$rtcFlags, 'base=localtime';
3334 }
3335
3336 if ($forcecpu) {
3337 push @$cmd, '-cpu', $forcecpu;
3338 } else {
3339 push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3340 }
3341
3342 PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3343
3344 push @$cmd, '-S' if $conf->{freeze};
3345
3346 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3347
3348 my $guest_agent = parse_guest_agent($conf);
3349
3350 if ($guest_agent->{enabled}) {
3351 my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3352 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3353
3354 if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3355 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3356 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3357 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3358 } elsif ($guest_agent->{type} eq 'isa') {
3359 push @$devices, '-device', "isa-serial,chardev=qga0";
3360 }
3361 }
3362
3363 my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3364 if ($rng && $version_guard->(4, 1, 2)) {
3365 check_rng_source($rng->{source});
3366
3367 my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3368 my $period = $rng->{period} // $rng_fmt->{period}->{default};
3369 my $limiter_str = "";
3370 if ($max_bytes) {
3371 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3372 }
3373
3374 my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3375 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3376 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3377 }
3378
3379 my $spice_port;
3380
3381 if ($qxlnum) {
3382 if ($qxlnum > 1) {
3383 if ($winversion){
3384 for (my $i = 1; $i < $qxlnum; $i++){
3385 push @$devices, '-device', print_vga_device(
3386 $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3387 }
3388 } else {
3389 # assume other OS works like Linux
3390 my ($ram, $vram) = ("134217728", "67108864");
3391 if ($vga->{memory}) {
3392 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3393 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3394 }
3395 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3396 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3397 }
3398 }
3399
3400 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3401
3402 my $pfamily = PVE::Tools::get_host_address_family($nodename);
3403 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3404 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3405
3406 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3407 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3408 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3409
3410 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3411 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3412
3413 my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3414 my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3415 if ($spice_enhancement->{foldersharing}) {
3416 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3417 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3418 }
3419
3420 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3421 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3422 if $spice_enhancement->{videostreaming};
3423
3424 push @$devices, '-spice', "$spice_opts";
3425 }
3426
3427 # enable balloon by default, unless explicitly disabled
3428 if (!defined($conf->{balloon}) || $conf->{balloon}) {
3429 $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3430 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3431 }
3432
3433 if ($conf->{watchdog}) {
3434 my $wdopts = parse_watchdog($conf->{watchdog});
3435 $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3436 my $watchdog = $wdopts->{model} || 'i6300esb';
3437 push @$devices, '-device', "$watchdog$pciaddr";
3438 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3439 }
3440
3441 my $vollist = [];
3442 my $scsicontroller = {};
3443 my $ahcicontroller = {};
3444 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3445
3446 # Add iscsi initiator name if available
3447 if (my $initiator = get_initiator_name()) {
3448 push @$devices, '-iscsi', "initiator-name=$initiator";
3449 }
3450
3451 PVE::QemuConfig->foreach_volume($conf, sub {
3452 my ($ds, $drive) = @_;
3453
3454 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3455 push @$vollist, $drive->{file};
3456 }
3457
3458 # ignore efidisk here, already added in bios/fw handling code above
3459 return if $drive->{interface} eq 'efidisk';
3460
3461 $use_virtio = 1 if $ds =~ m/^virtio/;
3462
3463 $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3464
3465 if ($drive->{interface} eq 'virtio'){
3466 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3467 }
3468
3469 if ($drive->{interface} eq 'scsi') {
3470
3471 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3472
3473 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3474 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3475
3476 $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3477 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3478
3479 my $iothread = '';
3480 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3481 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3482 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3483 } elsif ($drive->{iothread}) {
3484 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3485 }
3486
3487 my $queues = '';
3488 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3489 $queues = ",num_queues=$drive->{queues}";
3490 }
3491
3492 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3493 if !$scsicontroller->{$controller};
3494 $scsicontroller->{$controller}=1;
3495 }
3496
3497 if ($drive->{interface} eq 'sata') {
3498 my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3499 $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3500 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3501 if !$ahcicontroller->{$controller};
3502 $ahcicontroller->{$controller}=1;
3503 }
3504
3505 my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3506 $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3507
3508 push @$devices, '-drive',$drive_cmd;
3509 push @$devices, '-device', print_drivedevice_full(
3510 $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3511 });
3512
3513 for (my $i = 0; $i < $MAX_NETS; $i++) {
3514 my $netname = "net$i";
3515
3516 next if !$conf->{$netname};
3517 my $d = parse_net($conf->{$netname});
3518 next if !$d;
3519
3520 $use_virtio = 1 if $d->{model} eq 'virtio';
3521
3522 $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3523
3524 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3525 push @$devices, '-netdev', $netdevfull;
3526
3527 my $netdevicefull = print_netdevice_full(
3528 $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3529
3530 push @$devices, '-device', $netdevicefull;
3531 }
3532
3533 if ($conf->{ivshmem}) {
3534 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3535
3536 my $bus;
3537 if ($q35) {
3538 $bus = print_pcie_addr("ivshmem");
3539 } else {
3540 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3541 }
3542
3543 my $ivshmem_name = $ivshmem->{name} // $vmid;
3544 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3545
3546 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3547 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3548 .",size=$ivshmem->{size}M";
3549 }
3550
3551 # pci.4 is nested in pci.1
3552 $bridges->{1} = 1 if $bridges->{4};
3553
3554 if (!$q35) {
3555 # add pci bridges
3556 if (min_version($machine_version, 2, 3)) {
3557 $bridges->{1} = 1;
3558 $bridges->{2} = 1;
3559 }
3560
3561 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3562
3563 }
3564
3565 for my $k (sort {$b cmp $a} keys %$bridges) {
3566 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3567
3568 my $k_name = $k;
3569 if ($k == 2 && $legacy_igd) {
3570 $k_name = "$k-igd";
3571 }
3572 $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3573
3574 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3575 if ($q35) {
3576 # add after -readconfig pve-q35.cfg
3577 splice @$devices, 2, 0, '-device', $devstr;
3578 } else {
3579 unshift @$devices, '-device', $devstr if $k > 0;
3580 }
3581 }
3582
3583 if (!$kvm) {
3584 push @$machineFlags, 'accel=tcg';
3585 }
3586
3587 my $machine_type_min = $machine_type;
3588 if ($add_pve_version) {
3589 $machine_type_min =~ s/\+pve\d+$//;
3590 $machine_type_min .= "+pve$required_pve_version";
3591 }
3592 push @$machineFlags, "type=${machine_type_min}";
3593
3594 push @$cmd, @$devices;
3595 push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3596 push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3597 push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3598
3599 if (my $vmstate = $conf->{vmstate}) {
3600 my $statepath = PVE::Storage::path($storecfg, $vmstate);
3601 push @$vollist, $vmstate;
3602 push @$cmd, '-loadstate', $statepath;
3603 print "activating and using '$vmstate' as vmstate\n";
3604 }
3605
3606 # add custom args
3607 if ($conf->{args}) {
3608 my $aa = PVE::Tools::split_args($conf->{args});
3609 push @$cmd, @$aa;
3610 }
3611
3612 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3613}
3614
3615sub check_rng_source {
3616 my ($source) = @_;
3617
3618 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3619 die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3620 if ! -e $source;
3621
3622 my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3623 if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3624 # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3625 # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3626 die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3627 ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3628 ." to the host.\n";
3629 }
3630}
3631
3632sub spice_port {
3633 my ($vmid) = @_;
3634
3635 my $res = mon_cmd($vmid, 'query-spice');
3636
3637 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3638}
3639
3640sub vm_devices_list {
3641 my ($vmid) = @_;
3642
3643 my $res = mon_cmd($vmid, 'query-pci');
3644 my $devices_to_check = [];
3645 my $devices = {};
3646 foreach my $pcibus (@$res) {
3647 push @$devices_to_check, @{$pcibus->{devices}},
3648 }
3649
3650 while (@$devices_to_check) {
3651 my $to_check = [];
3652 for my $d (@$devices_to_check) {
3653 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3654 next if !$d->{'pci_bridge'};
3655
3656 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3657 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3658 }
3659 $devices_to_check = $to_check;
3660 }
3661
3662 my $resblock = mon_cmd($vmid, 'query-block');
3663 foreach my $block (@$resblock) {
3664 if($block->{device} =~ m/^drive-(\S+)/){
3665 $devices->{$1} = 1;
3666 }
3667 }
3668
3669 my $resmice = mon_cmd($vmid, 'query-mice');
3670 foreach my $mice (@$resmice) {
3671 if ($mice->{name} eq 'QEMU HID Tablet') {
3672 $devices->{tablet} = 1;
3673 last;
3674 }
3675 }
3676
3677 # for usb devices there is no query-usb
3678 # but we can iterate over the entries in
3679 # qom-list path=/machine/peripheral
3680 my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3681 foreach my $per (@$resperipheral) {
3682 if ($per->{name} =~ m/^usb\d+$/) {
3683 $devices->{$per->{name}} = 1;
3684 }
3685 }
3686
3687 return $devices;
3688}
3689
3690sub vm_deviceplug {
3691 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3692
3693 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3694
3695 my $devices_list = vm_devices_list($vmid);
3696 return 1 if defined($devices_list->{$deviceid});
3697
3698 # add PCI bridge if we need it for the device
3699 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3700
3701 if ($deviceid eq 'tablet') {
3702
3703 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3704
3705 } elsif ($deviceid eq 'keyboard') {
3706
3707 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3708
3709 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3710
3711 die "usb hotplug currently not reliable\n";
3712 # since we can't reliably hot unplug all added usb devices
3713 # and usb passthrough disables live migration
3714 # we disable usb hotplugging for now
3715 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3716
3717 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3718
3719 qemu_iothread_add($vmid, $deviceid, $device);
3720
3721 qemu_driveadd($storecfg, $vmid, $device);
3722 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3723
3724 qemu_deviceadd($vmid, $devicefull);
3725 eval { qemu_deviceaddverify($vmid, $deviceid); };
3726 if (my $err = $@) {
3727 eval { qemu_drivedel($vmid, $deviceid); };
3728 warn $@ if $@;
3729 die $err;
3730 }
3731
3732 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3733
3734
3735 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3736 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3737 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3738
3739 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3740
3741 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3742 qemu_iothread_add($vmid, $deviceid, $device);
3743 $devicefull .= ",iothread=iothread-$deviceid";
3744 }
3745
3746 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3747 $devicefull .= ",num_queues=$device->{queues}";
3748 }
3749
3750 qemu_deviceadd($vmid, $devicefull);
3751 qemu_deviceaddverify($vmid, $deviceid);
3752
3753 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3754
3755 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3756 qemu_driveadd($storecfg, $vmid, $device);
3757
3758 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3759 eval { qemu_deviceadd($vmid, $devicefull); };
3760 if (my $err = $@) {
3761 eval { qemu_drivedel($vmid, $deviceid); };
3762 warn $@ if $@;
3763 die $err;
3764 }
3765
3766 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3767
3768 return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3769
3770 my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3771 my $use_old_bios_files = undef;
3772 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3773
3774 my $netdevicefull = print_netdevice_full(
3775 $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3776 qemu_deviceadd($vmid, $netdevicefull);
3777 eval {
3778 qemu_deviceaddverify($vmid, $deviceid);
3779 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3780 };
3781 if (my $err = $@) {
3782 eval { qemu_netdevdel($vmid, $deviceid); };
3783 warn $@ if $@;
3784 die $err;
3785 }
3786
3787 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3788
3789 my $bridgeid = $2;
3790 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3791 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3792
3793 qemu_deviceadd($vmid, $devicefull);
3794 qemu_deviceaddverify($vmid, $deviceid);
3795
3796 } else {
3797 die "can't hotplug device '$deviceid'\n";
3798 }
3799
3800 return 1;
3801}
3802
3803# fixme: this should raise exceptions on error!
3804sub vm_deviceunplug {
3805 my ($vmid, $conf, $deviceid) = @_;
3806
3807 my $devices_list = vm_devices_list($vmid);
3808 return 1 if !defined($devices_list->{$deviceid});
3809
3810 my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3811 die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3812
3813 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3814
3815 qemu_devicedel($vmid, $deviceid);
3816
3817 } elsif ($deviceid =~ m/^usb\d+$/) {
3818
3819 die "usb hotplug currently not reliable\n";
3820 # when unplugging usb devices this way,
3821 # there may be remaining usb controllers/hubs
3822 # so we disable it for now
3823 qemu_devicedel($vmid, $deviceid);
3824 qemu_devicedelverify($vmid, $deviceid);
3825
3826 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3827
3828 qemu_devicedel($vmid, $deviceid);
3829 qemu_devicedelverify($vmid, $deviceid);
3830 qemu_drivedel($vmid, $deviceid);
3831 qemu_iothread_del($conf, $vmid, $deviceid);
3832
3833 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3834
3835 qemu_devicedel($vmid, $deviceid);
3836 qemu_devicedelverify($vmid, $deviceid);
3837 qemu_iothread_del($conf, $vmid, $deviceid);
3838
3839 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3840
3841 qemu_devicedel($vmid, $deviceid);
3842 qemu_drivedel($vmid, $deviceid);
3843 qemu_deletescsihw($conf, $vmid, $deviceid);
3844
3845 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3846
3847 qemu_devicedel($vmid, $deviceid);
3848 qemu_devicedelverify($vmid, $deviceid);
3849 qemu_netdevdel($vmid, $deviceid);
3850
3851 } else {
3852 die "can't unplug device '$deviceid'\n";
3853 }
3854
3855 return 1;
3856}
3857
3858sub qemu_deviceadd {
3859 my ($vmid, $devicefull) = @_;
3860
3861 $devicefull = "driver=".$devicefull;
3862 my %options = split(/[=,]/, $devicefull);
3863
3864 mon_cmd($vmid, "device_add" , %options);
3865}
3866
3867sub qemu_devicedel {
3868 my ($vmid, $deviceid) = @_;
3869
3870 my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3871}
3872
3873sub qemu_iothread_add {
3874 my($vmid, $deviceid, $device) = @_;
3875
3876 if ($device->{iothread}) {
3877 my $iothreads = vm_iothreads_list($vmid);
3878 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3879 }
3880}
3881
3882sub qemu_iothread_del {
3883 my($conf, $vmid, $deviceid) = @_;
3884
3885 my $confid = $deviceid;
3886 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3887 $confid = 'scsi' . $1;
3888 }
3889 my $device = parse_drive($confid, $conf->{$confid});
3890 if ($device->{iothread}) {
3891 my $iothreads = vm_iothreads_list($vmid);
3892 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3893 }
3894}
3895
3896sub qemu_objectadd {
3897 my($vmid, $objectid, $qomtype) = @_;
3898
3899 mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3900
3901 return 1;
3902}
3903
3904sub qemu_objectdel {
3905 my($vmid, $objectid) = @_;
3906
3907 mon_cmd($vmid, "object-del", id => $objectid);
3908
3909 return 1;
3910}
3911
3912sub qemu_driveadd {
3913 my ($storecfg, $vmid, $device) = @_;
3914
3915 my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3916 $drive =~ s/\\/\\\\/g;
3917 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3918
3919 # If the command succeeds qemu prints: "OK"
3920 return 1 if $ret =~ m/OK/s;
3921
3922 die "adding drive failed: $ret\n";
3923}
3924
3925sub qemu_drivedel {
3926 my($vmid, $deviceid) = @_;
3927
3928 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3929 $ret =~ s/^\s+//;
3930
3931 return 1 if $ret eq "";
3932
3933 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3934 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3935
3936 die "deleting drive $deviceid failed : $ret\n";
3937}
3938
3939sub qemu_deviceaddverify {
3940 my ($vmid, $deviceid) = @_;
3941
3942 for (my $i = 0; $i <= 5; $i++) {
3943 my $devices_list = vm_devices_list($vmid);
3944 return 1 if defined($devices_list->{$deviceid});
3945 sleep 1;
3946 }
3947
3948 die "error on hotplug device '$deviceid'\n";
3949}
3950
3951
3952sub qemu_devicedelverify {
3953 my ($vmid, $deviceid) = @_;
3954
3955 # need to verify that the device is correctly removed as device_del
3956 # is async and empty return is not reliable
3957
3958 for (my $i = 0; $i <= 5; $i++) {
3959 my $devices_list = vm_devices_list($vmid);
3960 return 1 if !defined($devices_list->{$deviceid});
3961 sleep 1;
3962 }
3963
3964 die "error on hot-unplugging device '$deviceid'\n";
3965}
3966
3967sub qemu_findorcreatescsihw {
3968 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3969
3970 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3971
3972 my $scsihwid="$controller_prefix$controller";
3973 my $devices_list = vm_devices_list($vmid);
3974
3975 if(!defined($devices_list->{$scsihwid})) {
3976 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3977 }
3978
3979 return 1;
3980}
3981
3982sub qemu_deletescsihw {
3983 my ($conf, $vmid, $opt) = @_;
3984
3985 my $device = parse_drive($opt, $conf->{$opt});
3986
3987 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3988 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3989 return 1;
3990 }
3991
3992 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3993
3994 my $devices_list = vm_devices_list($vmid);
3995 foreach my $opt (keys %{$devices_list}) {
3996 if (is_valid_drivename($opt)) {
3997 my $drive = parse_drive($opt, $conf->{$opt});
3998 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3999 return 1;
4000 }
4001 }
4002 }
4003
4004 my $scsihwid="scsihw$controller";
4005
4006 vm_deviceunplug($vmid, $conf, $scsihwid);
4007
4008 return 1;
4009}
4010
4011sub qemu_add_pci_bridge {
4012 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4013
4014 my $bridges = {};
4015
4016 my $bridgeid;
4017
4018 print_pci_addr($device, $bridges, $arch, $machine_type);
4019
4020 while (my ($k, $v) = each %$bridges) {
4021 $bridgeid = $k;
4022 }
4023 return 1 if !defined($bridgeid) || $bridgeid < 1;
4024
4025 my $bridge = "pci.$bridgeid";
4026 my $devices_list = vm_devices_list($vmid);
4027
4028 if (!defined($devices_list->{$bridge})) {
4029 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4030 }
4031
4032 return 1;
4033}
4034
4035sub qemu_set_link_status {
4036 my ($vmid, $device, $up) = @_;
4037
4038 mon_cmd($vmid, "set_link", name => $device,
4039 up => $up ? JSON::true : JSON::false);
4040}
4041
4042sub qemu_netdevadd {
4043 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4044
4045 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4046 my %options = split(/[=,]/, $netdev);
4047
4048 if (defined(my $vhost = $options{vhost})) {
4049 $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4050 }
4051
4052 if (defined(my $queues = $options{queues})) {
4053 $options{queues} = $queues + 0;
4054 }
4055
4056 mon_cmd($vmid, "netdev_add", %options);
4057 return 1;
4058}
4059
4060sub qemu_netdevdel {
4061 my ($vmid, $deviceid) = @_;
4062
4063 mon_cmd($vmid, "netdev_del", id => $deviceid);
4064}
4065
4066sub qemu_usb_hotplug {
4067 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4068
4069 return if !$device;
4070
4071 # remove the old one first
4072 vm_deviceunplug($vmid, $conf, $deviceid);
4073
4074 # check if xhci controller is necessary and available
4075 if ($device->{usb3}) {
4076
4077 my $devicelist = vm_devices_list($vmid);
4078
4079 if (!$devicelist->{xhci}) {
4080 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4081 qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4082 }
4083 }
4084 my $d = parse_usb_device($device->{host});
4085 $d->{usb3} = $device->{usb3};
4086
4087 # add the new one
4088 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4089}
4090
4091sub qemu_cpu_hotplug {
4092 my ($vmid, $conf, $vcpus) = @_;
4093
4094 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4095
4096 my $sockets = 1;
4097 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4098 $sockets = $conf->{sockets} if $conf->{sockets};
4099 my $cores = $conf->{cores} || 1;
4100 my $maxcpus = $sockets * $cores;
4101
4102 $vcpus = $maxcpus if !$vcpus;
4103
4104 die "you can't add more vcpus than maxcpus\n"
4105 if $vcpus > $maxcpus;
4106
4107 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4108
4109 if ($vcpus < $currentvcpus) {
4110
4111 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4112
4113 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4114 qemu_devicedel($vmid, "cpu$i");
4115 my $retry = 0;
4116 my $currentrunningvcpus = undef;
4117 while (1) {
4118 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4119 last if scalar(@{$currentrunningvcpus}) == $i-1;
4120 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4121 $retry++;
4122 sleep 1;
4123 }
4124 #update conf after each succesfull cpu unplug
4125 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4126 PVE::QemuConfig->write_config($vmid, $conf);
4127 }
4128 } else {
4129 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4130 }
4131
4132 return;
4133 }
4134
4135 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4136 die "vcpus in running vm does not match its configuration\n"
4137 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4138
4139 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4140
4141 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4142 my $cpustr = print_cpu_device($conf, $i);
4143 qemu_deviceadd($vmid, $cpustr);
4144
4145 my $retry = 0;
4146 my $currentrunningvcpus = undef;
4147 while (1) {
4148 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4149 last if scalar(@{$currentrunningvcpus}) == $i;
4150 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4151 sleep 1;
4152 $retry++;
4153 }
4154 #update conf after each succesfull cpu hotplug
4155 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4156 PVE::QemuConfig->write_config($vmid, $conf);
4157 }
4158 } else {
4159
4160 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4161 mon_cmd($vmid, "cpu-add", id => int($i));
4162 }
4163 }
4164}
4165
4166sub qemu_block_set_io_throttle {
4167 my ($vmid, $deviceid,
4168 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4169 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4170 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4171 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4172
4173 return if !check_running($vmid) ;
4174
4175 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4176 bps => int($bps),
4177 bps_rd => int($bps_rd),
4178 bps_wr => int($bps_wr),
4179 iops => int($iops),
4180 iops_rd => int($iops_rd),
4181 iops_wr => int($iops_wr),
4182 bps_max => int($bps_max),
4183 bps_rd_max => int($bps_rd_max),
4184 bps_wr_max => int($bps_wr_max),
4185 iops_max => int($iops_max),
4186 iops_rd_max => int($iops_rd_max),
4187 iops_wr_max => int($iops_wr_max),
4188 bps_max_length => int($bps_max_length),
4189 bps_rd_max_length => int($bps_rd_max_length),
4190 bps_wr_max_length => int($bps_wr_max_length),
4191 iops_max_length => int($iops_max_length),
4192 iops_rd_max_length => int($iops_rd_max_length),
4193 iops_wr_max_length => int($iops_wr_max_length),
4194 );
4195
4196}
4197
4198sub qemu_block_resize {
4199 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4200
4201 my $running = check_running($vmid);
4202
4203 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4204
4205 return if !$running;
4206
4207 my $padding = (1024 - $size % 1024) % 1024;
4208 $size = $size + $padding;
4209
4210 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4211
4212}
4213
4214sub qemu_volume_snapshot {
4215 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4216
4217 my $running = check_running($vmid);
4218
4219 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4220 mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4221 } else {
4222 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4223 }
4224}
4225
4226sub qemu_volume_snapshot_delete {
4227 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4228
4229 my $running = check_running($vmid);
4230
4231 if($running) {
4232
4233 $running = undef;
4234 my $conf = PVE::QemuConfig->load_config($vmid);
4235 PVE::QemuConfig->foreach_volume($conf, sub {
4236 my ($ds, $drive) = @_;
4237 $running = 1 if $drive->{file} eq $volid;
4238 });
4239 }
4240
4241 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4242 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4243 } else {
4244 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4245 }
4246}
4247
4248sub set_migration_caps {
4249 my ($vmid) = @_;
4250
4251 my $cap_ref = [];
4252
4253 my $enabled_cap = {
4254 "auto-converge" => 1,
4255 "xbzrle" => 1,
4256 "x-rdma-pin-all" => 0,
4257 "zero-blocks" => 0,
4258 "compress" => 0
4259 };
4260
4261 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4262
4263 for my $supported_capability (@$supported_capabilities) {
4264 push @$cap_ref, {
4265 capability => $supported_capability->{capability},
4266 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4267 };
4268 }
4269
4270 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4271}
4272
4273sub foreach_volid {
4274 my ($conf, $func, @param) = @_;
4275
4276 my $volhash = {};
4277
4278 my $test_volid = sub {
4279 my ($key, $drive, $snapname) = @_;
4280
4281 my $volid = $drive->{file};
4282 return if !$volid;
4283
4284 $volhash->{$volid}->{cdrom} //= 1;
4285 $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4286
4287 my $replicate = $drive->{replicate} // 1;
4288 $volhash->{$volid}->{replicate} //= 0;
4289 $volhash->{$volid}->{replicate} = 1 if $replicate;
4290
4291 $volhash->{$volid}->{shared} //= 0;
4292 $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4293
4294 $volhash->{$volid}->{referenced_in_config} //= 0;
4295 $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4296
4297 $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4298 if defined($snapname);
4299
4300 my $size = $drive->{size};
4301 $volhash->{$volid}->{size} //= $size if $size;
4302
4303 $volhash->{$volid}->{is_vmstate} //= 0;
4304 $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4305
4306 $volhash->{$volid}->{is_unused} //= 0;
4307 $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4308 };
4309
4310 my $include_opts = {
4311 extra_keys => ['vmstate'],
4312 include_unused => 1,
4313 };
4314
4315 PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4316 foreach my $snapname (keys %{$conf->{snapshots}}) {
4317 my $snap = $conf->{snapshots}->{$snapname};
4318 PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4319 }
4320
4321 foreach my $volid (keys %$volhash) {
4322 &$func($volid, $volhash->{$volid}, @param);
4323 }
4324}
4325
4326my $fast_plug_option = {
4327 'lock' => 1,
4328 'name' => 1,
4329 'onboot' => 1,
4330 'shares' => 1,
4331 'startup' => 1,
4332 'description' => 1,
4333 'protection' => 1,
4334 'vmstatestorage' => 1,
4335 'hookscript' => 1,
4336 'tags' => 1,
4337};
4338
4339# hotplug changes in [PENDING]
4340# $selection hash can be used to only apply specified options, for
4341# example: { cores => 1 } (only apply changed 'cores')
4342# $errors ref is used to return error messages
4343sub vmconfig_hotplug_pending {
4344 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4345
4346 my $defaults = load_defaults();
4347 my $arch = get_vm_arch($conf);
4348 my $machine_type = get_vm_machine($conf, undef, $arch);
4349
4350 # commit values which do not have any impact on running VM first
4351 # Note: those option cannot raise errors, we we do not care about
4352 # $selection and always apply them.
4353
4354 my $add_error = sub {
4355 my ($opt, $msg) = @_;
4356 $errors->{$opt} = "hotplug problem - $msg";
4357 };
4358
4359 my $changes = 0;
4360 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4361 if ($fast_plug_option->{$opt}) {
4362 $conf->{$opt} = $conf->{pending}->{$opt};
4363 delete $conf->{pending}->{$opt};
4364 $changes = 1;
4365 }
4366 }
4367
4368 if ($changes) {
4369 PVE::QemuConfig->write_config($vmid, $conf);
4370 }
4371
4372 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4373
4374 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4375 foreach my $opt (sort keys %$pending_delete_hash) {
4376 next if $selection && !$selection->{$opt};
4377 my $force = $pending_delete_hash->{$opt}->{force};
4378 eval {
4379 if ($opt eq 'hotplug') {
4380 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4381 } elsif ($opt eq 'tablet') {
4382 die "skip\n" if !$hotplug_features->{usb};
4383 if ($defaults->{tablet}) {
4384 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4385 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4386 if $arch eq 'aarch64';
4387 } else {
4388 vm_deviceunplug($vmid, $conf, 'tablet');
4389 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4390 }
4391 } elsif ($opt =~ m/^usb\d+/) {
4392 die "skip\n";
4393 # since we cannot reliably hot unplug usb devices
4394 # we are disabling it
4395 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4396 vm_deviceunplug($vmid, $conf, $opt);
4397 } elsif ($opt eq 'vcpus') {
4398 die "skip\n" if !$hotplug_features->{cpu};
4399 qemu_cpu_hotplug($vmid, $conf, undef);
4400 } elsif ($opt eq 'balloon') {
4401 # enable balloon device is not hotpluggable
4402 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4403 # here we reset the ballooning value to memory
4404 my $balloon = $conf->{memory} || $defaults->{memory};
4405 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4406 } elsif ($fast_plug_option->{$opt}) {
4407 # do nothing
4408 } elsif ($opt =~ m/^net(\d+)$/) {
4409 die "skip\n" if !$hotplug_features->{network};
4410 vm_deviceunplug($vmid, $conf, $opt);
4411 } elsif (is_valid_drivename($opt)) {
4412 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4413 vm_deviceunplug($vmid, $conf, $opt);
4414 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4415 } elsif ($opt =~ m/^memory$/) {
4416 die "skip\n" if !$hotplug_features->{memory};
4417 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4418 } elsif ($opt eq 'cpuunits') {
4419 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4420 } elsif ($opt eq 'cpulimit') {
4421 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4422 } else {
4423 die "skip\n";
4424 }
4425 };
4426 if (my $err = $@) {
4427 &$add_error($opt, $err) if $err ne "skip\n";
4428 } else {
4429 delete $conf->{$opt};
4430 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4431 }
4432 }
4433
4434 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4435 $apply_pending_cloudinit = sub {
4436 return if $apply_pending_cloudinit_done; # once is enough
4437 $apply_pending_cloudinit_done = 1; # once is enough
4438
4439 my ($key, $value) = @_;
4440
4441 my @cloudinit_opts = keys %$confdesc_cloudinit;
4442 foreach my $opt (keys %{$conf->{pending}}) {
4443 next if !grep { $_ eq $opt } @cloudinit_opts;
4444 $conf->{$opt} = delete $conf->{pending}->{$opt};
4445 }
4446
4447 my $new_conf = { %$conf };
4448 $new_conf->{$key} = $value;
4449 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4450 };
4451
4452 foreach my $opt (keys %{$conf->{pending}}) {
4453 next if $selection && !$selection->{$opt};
4454 my $value = $conf->{pending}->{$opt};
4455 eval {
4456 if ($opt eq 'hotplug') {
4457 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4458 } elsif ($opt eq 'tablet') {
4459 die "skip\n" if !$hotplug_features->{usb};
4460 if ($value == 1) {
4461 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4462 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4463 if $arch eq 'aarch64';
4464 } elsif ($value == 0) {
4465 vm_deviceunplug($vmid, $conf, 'tablet');
4466 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4467 }
4468 } elsif ($opt =~ m/^usb\d+$/) {
4469 die "skip\n";
4470 # since we cannot reliably hot unplug usb devices
4471 # we are disabling it
4472 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4473 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4474 die "skip\n" if !$d;
4475 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4476 } elsif ($opt eq 'vcpus') {
4477 die "skip\n" if !$hotplug_features->{cpu};
4478 qemu_cpu_hotplug($vmid, $conf, $value);
4479 } elsif ($opt eq 'balloon') {
4480 # enable/disable balloning device is not hotpluggable
4481 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4482 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4483 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4484
4485 # allow manual ballooning if shares is set to zero
4486 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4487 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4488 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4489 }
4490 } elsif ($opt =~ m/^net(\d+)$/) {
4491 # some changes can be done without hotplug
4492 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4493 $vmid, $opt, $value, $arch, $machine_type);
4494 } elsif (is_valid_drivename($opt)) {
4495 die "skip\n" if $opt eq 'efidisk0';
4496 # some changes can be done without hotplug
4497 my $drive = parse_drive($opt, $value);
4498 if (drive_is_cloudinit($drive)) {
4499 &$apply_pending_cloudinit($opt, $value);
4500 }
4501 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4502 $vmid, $opt, $value, $arch, $machine_type);
4503 } elsif ($opt =~ m/^memory$/) { #dimms
4504 die "skip\n" if !$hotplug_features->{memory};
4505 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4506 } elsif ($opt eq 'cpuunits') {
4507 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4508 } elsif ($opt eq 'cpulimit') {
4509 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4510 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4511 } else {
4512 die "skip\n"; # skip non-hot-pluggable options
4513 }
4514 };
4515 if (my $err = $@) {
4516 &$add_error($opt, $err) if $err ne "skip\n";
4517 } else {
4518 $conf->{$opt} = $value;
4519 delete $conf->{pending}->{$opt};
4520 }
4521 }
4522
4523 PVE::QemuConfig->write_config($vmid, $conf);
4524}
4525
4526sub try_deallocate_drive {
4527 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4528
4529 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4530 my $volid = $drive->{file};
4531 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4532 my $sid = PVE::Storage::parse_volume_id($volid);
4533 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4534
4535 # check if the disk is really unused
4536 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4537 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4538 PVE::Storage::vdisk_free($storecfg, $volid);
4539 return 1;
4540 } else {
4541 # If vm is not owner of this disk remove from config
4542 return 1;
4543 }
4544 }
4545
4546 return;
4547}
4548
4549sub vmconfig_delete_or_detach_drive {
4550 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4551
4552 my $drive = parse_drive($opt, $conf->{$opt});
4553
4554 my $rpcenv = PVE::RPCEnvironment::get();
4555 my $authuser = $rpcenv->get_user();
4556
4557 if ($force) {
4558 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4559 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4560 } else {
4561 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4562 }
4563}
4564
4565
4566
4567sub vmconfig_apply_pending {
4568 my ($vmid, $conf, $storecfg, $errors) = @_;
4569
4570 my $add_apply_error = sub {
4571 my ($opt, $msg) = @_;
4572 my $err_msg = "unable to apply pending change $opt : $msg";
4573 $errors->{$opt} = $err_msg;
4574 warn $err_msg;
4575 };
4576
4577 # cold plug
4578
4579 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4580 foreach my $opt (sort keys %$pending_delete_hash) {
4581 my $force = $pending_delete_hash->{$opt}->{force};
4582 eval {
4583 if ($opt =~ m/^unused/) {
4584 die "internal error";
4585 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4586 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4587 }
4588 };
4589 if (my $err = $@) {
4590 $add_apply_error->($opt, $err);
4591 } else {
4592 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4593 delete $conf->{$opt};
4594 }
4595 }
4596
4597 PVE::QemuConfig->cleanup_pending($conf);
4598
4599 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4600 next if $opt eq 'delete'; # just to be sure
4601 eval {
4602 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4603 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4604 }
4605 };
4606 if (my $err = $@) {
4607 $add_apply_error->($opt, $err);
4608 } else {
4609 $conf->{$opt} = delete $conf->{pending}->{$opt};
4610 }
4611 }
4612
4613 # write all changes at once to avoid unnecessary i/o
4614 PVE::QemuConfig->write_config($vmid, $conf);
4615}
4616
4617sub vmconfig_update_net {
4618 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4619
4620 my $newnet = parse_net($value);
4621
4622 if ($conf->{$opt}) {
4623 my $oldnet = parse_net($conf->{$opt});
4624
4625 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4626 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4627 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4628 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4629
4630 # for non online change, we try to hot-unplug
4631 die "skip\n" if !$hotplug;
4632 vm_deviceunplug($vmid, $conf, $opt);
4633 } else {
4634
4635 die "internal error" if $opt !~ m/net(\d+)/;
4636 my $iface = "tap${vmid}i$1";
4637
4638 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4639 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4640 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4641 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4642 PVE::Network::tap_unplug($iface);
4643
4644 if ($have_sdn) {
4645 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4646 } else {
4647 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4648 }
4649 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4650 # Rate can be applied on its own but any change above needs to
4651 # include the rate in tap_plug since OVS resets everything.
4652 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4653 }
4654
4655 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4656 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4657 }
4658
4659 return 1;
4660 }
4661 }
4662
4663 if ($hotplug) {
4664 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4665 } else {
4666 die "skip\n";
4667 }
4668}
4669
4670sub vmconfig_update_disk {
4671 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4672
4673 my $drive = parse_drive($opt, $value);
4674
4675 if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4676 my $media = $drive->{media} || 'disk';
4677 my $oldmedia = $old_drive->{media} || 'disk';
4678 die "unable to change media type\n" if $media ne $oldmedia;
4679
4680 if (!drive_is_cdrom($old_drive)) {
4681
4682 if ($drive->{file} ne $old_drive->{file}) {
4683
4684 die "skip\n" if !$hotplug;
4685
4686 # unplug and register as unused
4687 vm_deviceunplug($vmid, $conf, $opt);
4688 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4689
4690 } else {
4691 # update existing disk
4692
4693 # skip non hotpluggable value
4694 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4695 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4696 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4697 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4698 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4699 die "skip\n";
4700 }
4701
4702 # apply throttle
4703 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4704 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4705 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4706 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4707 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4708 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4709 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4710 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4711 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4712 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4713 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4714 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4715 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4716 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4717 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4718 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4719 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4720 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4721
4722 qemu_block_set_io_throttle(
4723 $vmid,"drive-$opt",
4724 ($drive->{mbps} || 0)*1024*1024,
4725 ($drive->{mbps_rd} || 0)*1024*1024,
4726 ($drive->{mbps_wr} || 0)*1024*1024,
4727 $drive->{iops} || 0,
4728 $drive->{iops_rd} || 0,
4729 $drive->{iops_wr} || 0,
4730 ($drive->{mbps_max} || 0)*1024*1024,
4731 ($drive->{mbps_rd_max} || 0)*1024*1024,
4732 ($drive->{mbps_wr_max} || 0)*1024*1024,
4733 $drive->{iops_max} || 0,
4734 $drive->{iops_rd_max} || 0,
4735 $drive->{iops_wr_max} || 0,
4736 $drive->{bps_max_length} || 1,
4737 $drive->{bps_rd_max_length} || 1,
4738 $drive->{bps_wr_max_length} || 1,
4739 $drive->{iops_max_length} || 1,
4740 $drive->{iops_rd_max_length} || 1,
4741 $drive->{iops_wr_max_length} || 1,
4742 );
4743
4744 }
4745
4746 return 1;
4747 }
4748
4749 } else { # cdrom
4750
4751 if ($drive->{file} eq 'none') {
4752 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4753 if (drive_is_cloudinit($old_drive)) {
4754 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4755 }
4756 } else {
4757 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4758
4759 # force eject if locked
4760 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4761
4762 if ($path) {
4763 mon_cmd($vmid, "blockdev-change-medium",
4764 id => "$opt", filename => "$path");
4765 }
4766 }
4767
4768 return 1;
4769 }
4770 }
4771
4772 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4773 # hotplug new disks
4774 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4775 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4776}
4777
4778# called in locked context by incoming migration
4779sub vm_migrate_get_nbd_disks {
4780 my ($storecfg, $conf, $replicated_volumes) = @_;
4781
4782 my $local_volumes = {};
4783 PVE::QemuConfig->foreach_volume($conf, sub {
4784 my ($ds, $drive) = @_;
4785
4786 return if drive_is_cdrom($drive);
4787
4788 my $volid = $drive->{file};
4789
4790 return if !$volid;
4791
4792 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4793
4794 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4795 return if $scfg->{shared};
4796
4797 # replicated disks re-use existing state via bitmap
4798 my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4799 $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4800 });
4801 return $local_volumes;
4802}
4803
4804# called in locked context by incoming migration
4805sub vm_migrate_alloc_nbd_disks {
4806 my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4807
4808 my $format = undef;
4809
4810 my $nbd = {};
4811 foreach my $opt (sort keys %$source_volumes) {
4812 my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4813
4814 if ($use_existing) {
4815 $nbd->{$opt}->{drivestr} = print_drive($drive);
4816 $nbd->{$opt}->{volid} = $volid;
4817 $nbd->{$opt}->{replicated} = 1;
4818 next;
4819 }
4820
4821 # If a remote storage is specified and the format of the original
4822 # volume is not available there, fall back to the default format.
4823 # Otherwise use the same format as the original.
4824 if (!$storagemap->{identity}) {
4825 $storeid = map_storage($storagemap, $storeid);
4826 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4827 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4828 my $fileFormat = qemu_img_format($scfg, $volname);
4829 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4830 } else {
4831 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4832 $format = qemu_img_format($scfg, $volname);
4833 }
4834
4835 my $size = $drive->{size} / 1024;
4836 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4837 my $newdrive = $drive;
4838 $newdrive->{format} = $format;
4839 $newdrive->{file} = $newvolid;
4840 my $drivestr = print_drive($newdrive);
4841 $nbd->{$opt}->{drivestr} = $drivestr;
4842 $nbd->{$opt}->{volid} = $newvolid;
4843 }
4844
4845 return $nbd;
4846}
4847
4848# see vm_start_nolock for parameters, additionally:
4849# migrate_opts:
4850# storagemap = parsed storage map for allocating NBD disks
4851sub vm_start {
4852 my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4853
4854 return PVE::QemuConfig->lock_config($vmid, sub {
4855 my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4856
4857 die "you can't start a vm if it's a template\n"
4858 if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4859
4860 my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4861
4862 PVE::QemuConfig->check_lock($conf)
4863 if !($params->{skiplock} || $has_suspended_lock);
4864
4865 $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4866
4867 die "VM $vmid already running\n" if check_running($vmid, undef, $migrate_opts->{migratedfrom});
4868
4869 if (my $storagemap = $migrate_opts->{storagemap}) {
4870 my $replicated = $migrate_opts->{replicated_volumes};
4871 my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4872 $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4873
4874 foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4875 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4876 }
4877 }
4878
4879 return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4880 });
4881}
4882
4883
4884# params:
4885# statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4886# skiplock => 0/1, skip checking for config lock
4887# skiptemplate => 0/1, skip checking whether VM is template
4888# forcemachine => to force Qemu machine (rollback/migration)
4889# forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4890# timeout => in seconds
4891# paused => start VM in paused state (backup)
4892# resume => resume from hibernation
4893# migrate_opts:
4894# nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4895# migratedfrom => source node
4896# spice_ticket => used for spice migration, passed via tunnel/stdin
4897# network => CIDR of migration network
4898# type => secure/insecure - tunnel over encrypted connection or plain-text
4899# nbd_proto_version => int, 0 for TCP, 1 for UNIX
4900# replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4901sub vm_start_nolock {
4902 my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4903
4904 my $statefile = $params->{statefile};
4905 my $resume = $params->{resume};
4906
4907 my $migratedfrom = $migrate_opts->{migratedfrom};
4908 my $migration_type = $migrate_opts->{type};
4909
4910 my $res = {};
4911
4912 # clean up leftover reboot request files
4913 eval { clear_reboot_request($vmid); };
4914 warn $@ if $@;
4915
4916 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4917 vmconfig_apply_pending($vmid, $conf, $storecfg);
4918 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4919 }
4920
4921 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4922
4923 my $defaults = load_defaults();
4924
4925 # set environment variable useful inside network script
4926 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4927
4928 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4929
4930 my $forcemachine = $params->{forcemachine};
4931 my $forcecpu = $params->{forcecpu};
4932 if ($resume) {
4933 # enforce machine and CPU type on suspended vm to ensure HW compatibility
4934 $forcemachine = $conf->{runningmachine};
4935 $forcecpu = $conf->{runningcpu};
4936 print "Resuming suspended VM\n";
4937 }
4938
4939 my ($cmd, $vollist, $spice_port) =
4940 config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4941
4942 my $migration_ip;
4943 my $get_migration_ip = sub {
4944 my ($nodename) = @_;
4945
4946 return $migration_ip if defined($migration_ip);
4947
4948 my $cidr = $migrate_opts->{network};
4949
4950 if (!defined($cidr)) {
4951 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4952 $cidr = $dc_conf->{migration}->{network};
4953 }
4954
4955 if (defined($cidr)) {
4956 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4957
4958 die "could not get IP: no address configured on local " .
4959 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4960
4961 die "could not get IP: multiple addresses configured on local " .
4962 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4963
4964 $migration_ip = @$ips[0];
4965 }
4966
4967 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4968 if !defined($migration_ip);
4969
4970 return $migration_ip;
4971 };
4972
4973 my $migrate_uri;
4974 if ($statefile) {
4975 if ($statefile eq 'tcp') {
4976 my $localip = "localhost";
4977 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4978 my $nodename = nodename();
4979
4980 if (!defined($migration_type)) {
4981 if (defined($datacenterconf->{migration}->{type})) {
4982 $migration_type = $datacenterconf->{migration}->{type};
4983 } else {
4984 $migration_type = 'secure';
4985 }
4986 }
4987
4988 if ($migration_type eq 'insecure') {
4989 $localip = $get_migration_ip->($nodename);
4990 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4991 }
4992
4993 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4994 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4995 $migrate_uri = "tcp:${localip}:${migrate_port}";
4996 push @$cmd, '-incoming', $migrate_uri;
4997 push @$cmd, '-S';
4998
4999 } elsif ($statefile eq 'unix') {
5000 # should be default for secure migrations as a ssh TCP forward
5001 # tunnel is not deterministic reliable ready and fails regurarly
5002 # to set up in time, so use UNIX socket forwards
5003 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5004 unlink $socket_addr;
5005
5006 $migrate_uri = "unix:$socket_addr";
5007
5008 push @$cmd, '-incoming', $migrate_uri;
5009 push @$cmd, '-S';
5010
5011 } elsif (-e $statefile) {
5012 push @$cmd, '-loadstate', $statefile;
5013 } else {
5014 my $statepath = PVE::Storage::path($storecfg, $statefile);
5015 push @$vollist, $statefile;
5016 push @$cmd, '-loadstate', $statepath;
5017 }
5018 } elsif ($params->{paused}) {
5019 push @$cmd, '-S';
5020 }
5021
5022 # host pci devices
5023 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++) {
5024 my $d = parse_hostpci($conf->{"hostpci$i"});
5025 next if !$d;
5026 my $pcidevices = $d->{pciid};
5027 foreach my $pcidevice (@$pcidevices) {
5028 my $pciid = $pcidevice->{id};
5029
5030 my $info = PVE::SysFSTools::pci_device_info("$pciid");
5031 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5032 die "no pci device info for device '$pciid'\n" if !$info;
5033
5034 if ($d->{mdev}) {
5035 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5036 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5037 } else {
5038 die "can't unbind/bind pci group to vfio '$pciid'\n"
5039 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5040 die "can't reset pci device '$pciid'\n"
5041 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5042 }
5043 }
5044 }
5045
5046 PVE::Storage::activate_volumes($storecfg, $vollist);
5047
5048 eval {
5049 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5050 outfunc => sub {}, errfunc => sub {});
5051 };
5052 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5053 # timeout should be more than enough here...
5054 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5055
5056 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5057 : $defaults->{cpuunits};
5058
5059 my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5060 my %run_params = (
5061 timeout => $statefile ? undef : $start_timeout,
5062 umask => 0077,
5063 noerr => 1,
5064 );
5065
5066 # when migrating, prefix QEMU output so other side can pick up any
5067 # errors that might occur and show the user
5068 if ($migratedfrom) {
5069 $run_params{quiet} = 1;
5070 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5071 }
5072
5073 my %properties = (
5074 Slice => 'qemu.slice',
5075 KillMode => 'none',
5076 CPUShares => $cpuunits
5077 );
5078
5079 if (my $cpulimit = $conf->{cpulimit}) {
5080 $properties{CPUQuota} = int($cpulimit * 100);
5081 }
5082 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5083
5084 my $run_qemu = sub {
5085 PVE::Tools::run_fork sub {
5086 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5087
5088 my $exitcode = run_command($cmd, %run_params);
5089 die "QEMU exited with code $exitcode\n" if $exitcode;
5090 };
5091 };
5092
5093 if ($conf->{hugepages}) {
5094
5095 my $code = sub {
5096 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5097 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5098
5099 PVE::QemuServer::Memory::hugepages_mount();
5100 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5101
5102 eval { $run_qemu->() };
5103 if (my $err = $@) {
5104 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5105 if !$conf->{keephugepages};
5106 die $err;
5107 }
5108
5109 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5110 if !$conf->{keephugepages};
5111 };
5112 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5113
5114 } else {
5115 eval { $run_qemu->() };
5116 }
5117
5118 if (my $err = $@) {
5119 # deactivate volumes if start fails
5120 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5121 die "start failed: $err";
5122 }
5123
5124 print "migration listens on $migrate_uri\n" if $migrate_uri;
5125 $res->{migrate_uri} = $migrate_uri;
5126
5127 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
5128 eval { mon_cmd($vmid, "cont"); };
5129 warn $@ if $@;
5130 }
5131
5132 #start nbd server for storage migration
5133 if (my $nbd = $migrate_opts->{nbd}) {
5134 my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5135
5136 my $migrate_storage_uri;
5137 # nbd_protocol_version > 0 for unix socket support
5138 if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5139 my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5140 mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5141 $migrate_storage_uri = "nbd:unix:$socket_path";
5142 } else {
5143 my $nodename = nodename();
5144 my $localip = $get_migration_ip->($nodename);
5145 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5146 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5147
5148 mon_cmd($vmid, "nbd-server-start", addr => {
5149 type => 'inet',
5150 data => {
5151 host => "${localip}",
5152 port => "${storage_migrate_port}",
5153 },
5154 });
5155 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5156 $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5157 }
5158
5159 $res->{migrate_storage_uri} = $migrate_storage_uri;
5160
5161 foreach my $opt (sort keys %$nbd) {
5162 my $drivestr = $nbd->{$opt}->{drivestr};
5163 my $volid = $nbd->{$opt}->{volid};
5164 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5165 my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5166 print "storage migration listens on $nbd_uri volume:$drivestr\n";
5167 print "re-using replicated volume: $opt - $volid\n"
5168 if $nbd->{$opt}->{replicated};
5169
5170 $res->{drives}->{$opt} = $nbd->{$opt};
5171 $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5172 }
5173 }
5174
5175 if ($migratedfrom) {
5176 eval {
5177 set_migration_caps($vmid);
5178 };
5179 warn $@ if $@;
5180
5181 if ($spice_port) {
5182 print "spice listens on port $spice_port\n";
5183 $res->{spice_port} = $spice_port;
5184 if ($migrate_opts->{spice_ticket}) {
5185 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5186 $migrate_opts->{spice_ticket});
5187 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5188 }
5189 }
5190
5191 } else {
5192 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5193 if !$statefile && $conf->{balloon};
5194
5195 foreach my $opt (keys %$conf) {
5196 next if $opt !~ m/^net\d+$/;
5197 my $nicconf = parse_net($conf->{$opt});
5198 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5199 }
5200 }
5201
5202 mon_cmd($vmid, 'qom-set',
5203 path => "machine/peripheral/balloon0",
5204 property => "guest-stats-polling-interval",
5205 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5206
5207 if ($resume) {
5208 print "Resumed VM, removing state\n";
5209 if (my $vmstate = $conf->{vmstate}) {
5210 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5211 PVE::Storage::vdisk_free($storecfg, $vmstate);
5212 }
5213 delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5214 PVE::QemuConfig->write_config($vmid, $conf);
5215 }
5216
5217 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5218
5219 return $res;
5220}
5221
5222sub vm_commandline {
5223 my ($storecfg, $vmid, $snapname) = @_;
5224
5225 my $conf = PVE::QemuConfig->load_config($vmid);
5226 my $forcemachine;
5227 my $forcecpu;
5228
5229 if ($snapname) {
5230 my $snapshot = $conf->{snapshots}->{$snapname};
5231 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5232
5233 # check for machine or CPU overrides in snapshot
5234 $forcemachine = $snapshot->{runningmachine};
5235 $forcecpu = $snapshot->{runningcpu};
5236
5237 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5238
5239 $conf = $snapshot;
5240 }
5241
5242 my $defaults = load_defaults();
5243
5244 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5245 $forcemachine, $forcecpu);
5246
5247 return PVE::Tools::cmd2string($cmd);
5248}
5249
5250sub vm_reset {
5251 my ($vmid, $skiplock) = @_;
5252
5253 PVE::QemuConfig->lock_config($vmid, sub {
5254
5255 my $conf = PVE::QemuConfig->load_config($vmid);
5256
5257 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5258
5259 mon_cmd($vmid, "system_reset");
5260 });
5261}
5262
5263sub get_vm_volumes {
5264 my ($conf) = @_;
5265
5266 my $vollist = [];
5267 foreach_volid($conf, sub {
5268 my ($volid, $attr) = @_;
5269
5270 return if $volid =~ m|^/|;
5271
5272 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5273 return if !$sid;
5274
5275 push @$vollist, $volid;
5276 });
5277
5278 return $vollist;
5279}
5280
5281sub vm_stop_cleanup {
5282 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5283
5284 eval {
5285
5286 if (!$keepActive) {
5287 my $vollist = get_vm_volumes($conf);
5288 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5289 }
5290
5291 foreach my $ext (qw(mon qmp pid vnc qga)) {
5292 unlink "/var/run/qemu-server/${vmid}.$ext";
5293 }
5294
5295 if ($conf->{ivshmem}) {
5296 my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5297 # just delete it for now, VMs which have this already open do not
5298 # are affected, but new VMs will get a separated one. If this
5299 # becomes an issue we either add some sort of ref-counting or just
5300 # add a "don't delete on stop" flag to the ivshmem format.
5301 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5302 }
5303
5304 foreach my $key (keys %$conf) {
5305 next if $key !~ m/^hostpci(\d+)$/;
5306 my $hostpciindex = $1;
5307 my $d = parse_hostpci($conf->{$key});
5308 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5309
5310 foreach my $pci (@{$d->{pciid}}) {
5311 my $pciid = $pci->{id};
5312 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5313 }
5314 }
5315
5316 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5317 };
5318 warn $@ if $@; # avoid errors - just warn
5319}
5320
5321# call only in locked context
5322sub _do_vm_stop {
5323 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5324
5325 my $pid = check_running($vmid, $nocheck);
5326 return if !$pid;
5327
5328 my $conf;
5329 if (!$nocheck) {
5330 $conf = PVE::QemuConfig->load_config($vmid);
5331 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5332 if (!defined($timeout) && $shutdown && $conf->{startup}) {
5333 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5334 $timeout = $opts->{down} if $opts->{down};
5335 }
5336 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5337 }
5338
5339 eval {
5340 if ($shutdown) {
5341 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5342 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5343 } else {
5344 mon_cmd($vmid, "system_powerdown");
5345 }
5346 } else {
5347 mon_cmd($vmid, "quit");
5348 }
5349 };
5350 my $err = $@;
5351
5352 if (!$err) {
5353 $timeout = 60 if !defined($timeout);
5354
5355 my $count = 0;
5356 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5357 $count++;
5358 sleep 1;
5359 }
5360
5361 if ($count >= $timeout) {
5362 if ($force) {
5363 warn "VM still running - terminating now with SIGTERM\n";
5364 kill 15, $pid;
5365 } else {
5366 die "VM quit/powerdown failed - got timeout\n";
5367 }
5368 } else {
5369 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5370 return;
5371 }
5372 } else {
5373 if (!check_running($vmid, $nocheck)) {
5374 warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5375 return;
5376 }
5377 if ($force) {
5378 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5379 kill 15, $pid;
5380 } else {
5381 die "VM quit/powerdown failed\n";
5382 }
5383 }
5384
5385 # wait again
5386 $timeout = 10;
5387
5388 my $count = 0;
5389 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5390 $count++;
5391 sleep 1;
5392 }
5393
5394 if ($count >= $timeout) {
5395 warn "VM still running - terminating now with SIGKILL\n";
5396 kill 9, $pid;
5397 sleep 1;
5398 }
5399
5400 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5401}
5402
5403# Note: use $nocheck to skip tests if VM configuration file exists.
5404# We need that when migration VMs to other nodes (files already moved)
5405# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5406sub vm_stop {
5407 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5408
5409 $force = 1 if !defined($force) && !$shutdown;
5410
5411 if ($migratedfrom){
5412 my $pid = check_running($vmid, $nocheck, $migratedfrom);
5413 kill 15, $pid if $pid;
5414 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5415 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5416 return;
5417 }
5418
5419 PVE::QemuConfig->lock_config($vmid, sub {
5420 _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5421 });
5422}
5423
5424sub vm_reboot {
5425 my ($vmid, $timeout) = @_;
5426
5427 PVE::QemuConfig->lock_config($vmid, sub {
5428 eval {
5429
5430 # only reboot if running, as qmeventd starts it again on a stop event
5431 return if !check_running($vmid);
5432
5433 create_reboot_request($vmid);
5434
5435 my $storecfg = PVE::Storage::config();
5436 _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5437
5438 };
5439 if (my $err = $@) {
5440 # avoid that the next normal shutdown will be confused for a reboot
5441 clear_reboot_request($vmid);
5442 die $err;
5443 }
5444 });
5445}
5446
5447# note: if using the statestorage parameter, the caller has to check privileges
5448sub vm_suspend {
5449 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5450
5451 my $conf;
5452 my $path;
5453 my $storecfg;
5454 my $vmstate;
5455
5456 PVE::QemuConfig->lock_config($vmid, sub {
5457
5458 $conf = PVE::QemuConfig->load_config($vmid);
5459
5460 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5461 PVE::QemuConfig->check_lock($conf)
5462 if !($skiplock || $is_backing_up);
5463
5464 die "cannot suspend to disk during backup\n"
5465 if $is_backing_up && $includestate;
5466
5467 if ($includestate) {
5468 $conf->{lock} = 'suspending';
5469 my $date = strftime("%Y-%m-%d", localtime(time()));
5470 $storecfg = PVE::Storage::config();
5471 if (!$statestorage) {
5472 $statestorage = find_vmstate_storage($conf, $storecfg);
5473 # check permissions for the storage
5474 my $rpcenv = PVE::RPCEnvironment::get();
5475 if ($rpcenv->{type} ne 'cli') {
5476 my $authuser = $rpcenv->get_user();
5477 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5478 }
5479 }
5480
5481
5482 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5483 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5484 $path = PVE::Storage::path($storecfg, $vmstate);
5485 PVE::QemuConfig->write_config($vmid, $conf);
5486 } else {
5487 mon_cmd($vmid, "stop");
5488 }
5489 });
5490
5491 if ($includestate) {
5492 # save vm state
5493 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5494
5495 eval {
5496 mon_cmd($vmid, "savevm-start", statefile => $path);
5497 for(;;) {
5498 my $state = mon_cmd($vmid, "query-savevm");
5499 if (!$state->{status}) {
5500 die "savevm not active\n";
5501 } elsif ($state->{status} eq 'active') {
5502 sleep(1);
5503 next;
5504 } elsif ($state->{status} eq 'completed') {
5505 print "State saved, quitting\n";
5506 last;
5507 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5508 die "query-savevm failed with error '$state->{error}'\n"
5509 } else {
5510 die "query-savevm returned status '$state->{status}'\n";
5511 }
5512 }
5513 };
5514 my $err = $@;
5515
5516 PVE::QemuConfig->lock_config($vmid, sub {
5517 $conf = PVE::QemuConfig->load_config($vmid);
5518 if ($err) {
5519 # cleanup, but leave suspending lock, to indicate something went wrong
5520 eval {
5521 mon_cmd($vmid, "savevm-end");
5522 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5523 PVE::Storage::vdisk_free($storecfg, $vmstate);
5524 delete $conf->@{qw(vmstate runningmachine runningcpu)};
5525 PVE::QemuConfig->write_config($vmid, $conf);
5526 };
5527 warn $@ if $@;
5528 die $err;
5529 }
5530
5531 die "lock changed unexpectedly\n"
5532 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5533
5534 mon_cmd($vmid, "quit");
5535 $conf->{lock} = 'suspended';
5536 PVE::QemuConfig->write_config($vmid, $conf);
5537 });
5538 }
5539}
5540
5541sub vm_resume {
5542 my ($vmid, $skiplock, $nocheck) = @_;
5543
5544 PVE::QemuConfig->lock_config($vmid, sub {
5545 my $res = mon_cmd($vmid, 'query-status');
5546 my $resume_cmd = 'cont';
5547
5548 if ($res->{status} && $res->{status} eq 'suspended') {
5549 $resume_cmd = 'system_wakeup';
5550 }
5551
5552 if (!$nocheck) {
5553
5554 my $conf = PVE::QemuConfig->load_config($vmid);
5555
5556 PVE::QemuConfig->check_lock($conf)
5557 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5558 }
5559
5560 mon_cmd($vmid, $resume_cmd);
5561 });
5562}
5563
5564sub vm_sendkey {
5565 my ($vmid, $skiplock, $key) = @_;
5566
5567 PVE::QemuConfig->lock_config($vmid, sub {
5568
5569 my $conf = PVE::QemuConfig->load_config($vmid);
5570
5571 # there is no qmp command, so we use the human monitor command
5572 my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5573 die $res if $res ne '';
5574 });
5575}
5576
5577# vzdump restore implementaion
5578
5579sub tar_archive_read_firstfile {
5580 my $archive = shift;
5581
5582 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5583
5584 # try to detect archive type first
5585 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5586 die "unable to open file '$archive'\n";
5587 my $firstfile = <$fh>;
5588 kill 15, $pid;
5589 close $fh;
5590
5591 die "ERROR: archive contaions no data\n" if !$firstfile;
5592 chomp $firstfile;
5593
5594 return $firstfile;
5595}
5596
5597sub tar_restore_cleanup {
5598 my ($storecfg, $statfile) = @_;
5599
5600 print STDERR "starting cleanup\n";
5601
5602 if (my $fd = IO::File->new($statfile, "r")) {
5603 while (defined(my $line = <$fd>)) {
5604 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5605 my $volid = $2;
5606 eval {
5607 if ($volid =~ m|^/|) {
5608 unlink $volid || die 'unlink failed\n';
5609 } else {
5610 PVE::Storage::vdisk_free($storecfg, $volid);
5611 }
5612 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5613 };
5614 print STDERR "unable to cleanup '$volid' - $@" if $@;
5615 } else {
5616 print STDERR "unable to parse line in statfile - $line";
5617 }
5618 }
5619 $fd->close();
5620 }
5621}
5622
5623sub restore_file_archive {
5624 my ($archive, $vmid, $user, $opts) = @_;
5625
5626 return restore_vma_archive($archive, $vmid, $user, $opts)
5627 if $archive eq '-';
5628
5629 my $info = PVE::Storage::archive_info($archive);
5630 my $format = $opts->{format} // $info->{format};
5631 my $comp = $info->{compression};
5632
5633 # try to detect archive format
5634 if ($format eq 'tar') {
5635 return restore_tar_archive($archive, $vmid, $user, $opts);
5636 } else {
5637 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5638 }
5639}
5640
5641# hepler to remove disks that will not be used after restore
5642my $restore_cleanup_oldconf = sub {
5643 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5644
5645 PVE::QemuConfig->foreach_volume($oldconf, sub {
5646 my ($ds, $drive) = @_;
5647
5648 return if drive_is_cdrom($drive, 1);
5649
5650 my $volid = $drive->{file};
5651 return if !$volid || $volid =~ m|^/|;
5652
5653 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5654 return if !$path || !$owner || ($owner != $vmid);
5655
5656 # Note: only delete disk we want to restore
5657 # other volumes will become unused
5658 if ($virtdev_hash->{$ds}) {
5659 eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5660 if (my $err = $@) {
5661 warn $err;
5662 }
5663 }
5664 });
5665
5666 # delete vmstate files, after the restore we have no snapshots anymore
5667 foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5668 my $snap = $oldconf->{snapshots}->{$snapname};
5669 if ($snap->{vmstate}) {
5670 eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5671 if (my $err = $@) {
5672 warn $err;
5673 }
5674 }
5675 }
5676};
5677
5678# Helper to parse vzdump backup device hints
5679#
5680# $rpcenv: Environment, used to ckeck storage permissions
5681# $user: User ID, to check storage permissions
5682# $storecfg: Storage configuration
5683# $fh: the file handle for reading the configuration
5684# $devinfo: should contain device sizes for all backu-up'ed devices
5685# $options: backup options (pool, default storage)
5686#
5687# Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5688my $parse_backup_hints = sub {
5689 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5690
5691 my $virtdev_hash = {};
5692
5693 while (defined(my $line = <$fh>)) {
5694 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5695 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5696 die "archive does not contain data for drive '$virtdev'\n"
5697 if !$devinfo->{$devname};
5698
5699 if (defined($options->{storage})) {
5700 $storeid = $options->{storage} || 'local';
5701 } elsif (!$storeid) {
5702 $storeid = 'local';
5703 }
5704 $format = 'raw' if !$format;
5705 $devinfo->{$devname}->{devname} = $devname;
5706 $devinfo->{$devname}->{virtdev} = $virtdev;
5707 $devinfo->{$devname}->{format} = $format;
5708 $devinfo->{$devname}->{storeid} = $storeid;
5709
5710 # check permission on storage
5711 my $pool = $options->{pool}; # todo: do we need that?
5712 if ($user ne 'root@pam') {
5713 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5714 }
5715
5716 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5717 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5718 my $virtdev = $1;
5719 my $drive = parse_drive($virtdev, $2);
5720 if (drive_is_cloudinit($drive)) {
5721 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5722 $storeid = $options->{storage} if defined ($options->{storage});
5723 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5724 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5725
5726 $virtdev_hash->{$virtdev} = {
5727 format => $format,
5728 storeid => $storeid,
5729 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5730 is_cloudinit => 1,
5731 };
5732 }
5733 }
5734 }
5735
5736 return $virtdev_hash;
5737};
5738
5739# Helper to allocate and activate all volumes required for a restore
5740#
5741# $storecfg: Storage configuration
5742# $virtdev_hash: as returned by parse_backup_hints()
5743#
5744# Returns: { $virtdev => $volid }
5745my $restore_allocate_devices = sub {
5746 my ($storecfg, $virtdev_hash, $vmid) = @_;
5747
5748 my $map = {};
5749 foreach my $virtdev (sort keys %$virtdev_hash) {
5750 my $d = $virtdev_hash->{$virtdev};
5751 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5752 my $storeid = $d->{storeid};
5753 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5754
5755 # test if requested format is supported
5756 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5757 my $supported = grep { $_ eq $d->{format} } @$validFormats;
5758 $d->{format} = $defFormat if !$supported;
5759
5760 my $name;
5761 if ($d->{is_cloudinit}) {
5762 $name = "vm-$vmid-cloudinit";
5763 $name .= ".$d->{format}" if $d->{format} ne 'raw';
5764 }
5765
5766 my $volid = PVE::Storage::vdisk_alloc(
5767 $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5768
5769 print STDERR "new volume ID is '$volid'\n";
5770 $d->{volid} = $volid;
5771
5772 PVE::Storage::activate_volumes($storecfg, [$volid]);
5773
5774 $map->{$virtdev} = $volid;
5775 }
5776
5777 return $map;
5778};
5779
5780my $restore_update_config_line = sub {
5781 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5782
5783 return if $line =~ m/^\#qmdump\#/;
5784 return if $line =~ m/^\#vzdump\#/;
5785 return if $line =~ m/^lock:/;
5786 return if $line =~ m/^unused\d+:/;
5787 return if $line =~ m/^parent:/;
5788
5789 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5790 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5791 # try to convert old 1.X settings
5792 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5793 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5794 my ($model, $macaddr) = split(/\=/, $devconfig);
5795 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5796 my $net = {
5797 model => $model,
5798 bridge => "vmbr$ind",
5799 macaddr => $macaddr,
5800 };
5801 my $netstr = print_net($net);
5802
5803 print $outfd "net$cookie->{netcount}: $netstr\n";
5804 $cookie->{netcount}++;
5805 }
5806 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5807 my ($id, $netstr) = ($1, $2);
5808 my $net = parse_net($netstr);
5809 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5810 $netstr = print_net($net);
5811 print $outfd "$id: $netstr\n";
5812 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5813 my $virtdev = $1;
5814 my $value = $3;
5815 my $di = parse_drive($virtdev, $value);
5816 if (defined($di->{backup}) && !$di->{backup}) {
5817 print $outfd "#$line";
5818 } elsif ($map->{$virtdev}) {
5819 delete $di->{format}; # format can change on restore
5820 $di->{file} = $map->{$virtdev};
5821 $value = print_drive($di);
5822 print $outfd "$virtdev: $value\n";
5823 } else {
5824 print $outfd $line;
5825 }
5826 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5827 my $vmgenid = $1;
5828 if ($vmgenid ne '0') {
5829 # always generate a new vmgenid if there was a valid one setup
5830 $vmgenid = generate_uuid();
5831 }
5832 print $outfd "vmgenid: $vmgenid\n";
5833 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5834 my ($uuid, $uuid_str);
5835 UUID::generate($uuid);
5836 UUID::unparse($uuid, $uuid_str);
5837 my $smbios1 = parse_smbios1($2);
5838 $smbios1->{uuid} = $uuid_str;
5839 print $outfd $1.print_smbios1($smbios1)."\n";
5840 } else {
5841 print $outfd $line;
5842 }
5843};
5844
5845my $restore_deactivate_volumes = sub {
5846 my ($storecfg, $devinfo) = @_;
5847
5848 my $vollist = [];
5849 foreach my $devname (keys %$devinfo) {
5850 my $volid = $devinfo->{$devname}->{volid};
5851 push @$vollist, $volid if $volid;
5852 }
5853
5854 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5855};
5856
5857my $restore_destroy_volumes = sub {
5858 my ($storecfg, $devinfo) = @_;
5859
5860 foreach my $devname (keys %$devinfo) {
5861 my $volid = $devinfo->{$devname}->{volid};
5862 next if !$volid;
5863 eval {
5864 if ($volid =~ m|^/|) {
5865 unlink $volid || die 'unlink failed\n';
5866 } else {
5867 PVE::Storage::vdisk_free($storecfg, $volid);
5868 }
5869 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5870 };
5871 print STDERR "unable to cleanup '$volid' - $@" if $@;
5872 }
5873};
5874
5875sub scan_volids {
5876 my ($cfg, $vmid) = @_;
5877
5878 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5879
5880 my $volid_hash = {};
5881 foreach my $storeid (keys %$info) {
5882 foreach my $item (@{$info->{$storeid}}) {
5883 next if !($item->{volid} && $item->{size});
5884 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5885 $volid_hash->{$item->{volid}} = $item;
5886 }
5887 }
5888
5889 return $volid_hash;
5890}
5891
5892sub update_disk_config {
5893 my ($vmid, $conf, $volid_hash) = @_;
5894
5895 my $changes;
5896 my $prefix = "VM $vmid";
5897
5898 # used and unused disks
5899 my $referenced = {};
5900
5901 # Note: it is allowed to define multiple storages with same path (alias), so
5902 # we need to check both 'volid' and real 'path' (two different volid can point
5903 # to the same path).
5904
5905 my $referencedpath = {};
5906
5907 # update size info
5908 PVE::QemuConfig->foreach_volume($conf, sub {
5909 my ($opt, $drive) = @_;
5910
5911 my $volid = $drive->{file};
5912 return if !$volid;
5913 my $volume = $volid_hash->{$volid};
5914
5915 # mark volid as "in-use" for next step
5916 $referenced->{$volid} = 1;
5917 if ($volume && (my $path = $volume->{path})) {
5918 $referencedpath->{$path} = 1;
5919 }
5920
5921 return if drive_is_cdrom($drive);
5922 return if !$volume;
5923
5924 my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5925 if (defined($updated)) {
5926 $changes = 1;
5927 $conf->{$opt} = print_drive($updated);
5928 print "$prefix ($opt): $msg\n";
5929 }
5930 });
5931
5932 # remove 'unusedX' entry if volume is used
5933 PVE::QemuConfig->foreach_unused_volume($conf, sub {
5934 my ($opt, $drive) = @_;
5935
5936 my $volid = $drive->{file};
5937 return if !$volid;
5938
5939 my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5940 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5941 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5942 $changes = 1;
5943 delete $conf->{$opt};
5944 }
5945
5946 $referenced->{$volid} = 1;
5947 $referencedpath->{$path} = 1 if $path;
5948 });
5949
5950 foreach my $volid (sort keys %$volid_hash) {
5951 next if $volid =~ m/vm-$vmid-state-/;
5952 next if $referenced->{$volid};
5953 my $path = $volid_hash->{$volid}->{path};
5954 next if !$path; # just to be sure
5955 next if $referencedpath->{$path};
5956 $changes = 1;
5957 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5958 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5959 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5960 }
5961
5962 return $changes;
5963}
5964
5965sub rescan {
5966 my ($vmid, $nolock, $dryrun) = @_;
5967
5968 my $cfg = PVE::Storage::config();
5969
5970 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5971 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5972 foreach my $stor (keys %{$cfg->{ids}}) {
5973 delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5974 }
5975
5976 print "rescan volumes...\n";
5977 my $volid_hash = scan_volids($cfg, $vmid);
5978
5979 my $updatefn = sub {
5980 my ($vmid) = @_;
5981
5982 my $conf = PVE::QemuConfig->load_config($vmid);
5983
5984 PVE::QemuConfig->check_lock($conf);
5985
5986 my $vm_volids = {};
5987 foreach my $volid (keys %$volid_hash) {
5988 my $info = $volid_hash->{$volid};
5989 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5990 }
5991
5992 my $changes = update_disk_config($vmid, $conf, $vm_volids);
5993
5994 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5995 };
5996
5997 if (defined($vmid)) {
5998 if ($nolock) {
5999 &$updatefn($vmid);
6000 } else {
6001 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6002 }
6003 } else {
6004 my $vmlist = config_list();
6005 foreach my $vmid (keys %$vmlist) {
6006 if ($nolock) {
6007 &$updatefn($vmid);
6008 } else {
6009 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6010 }
6011 }
6012 }
6013}
6014
6015sub restore_proxmox_backup_archive {
6016 my ($archive, $vmid, $user, $options) = @_;
6017
6018 my $storecfg = PVE::Storage::config();
6019
6020 my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6021 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6022
6023 my $server = $scfg->{server};
6024 my $datastore = $scfg->{datastore};
6025 my $username = $scfg->{username} // 'root@pam';
6026 my $fingerprint = $scfg->{fingerprint};
6027 my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6028
6029 my $repo = "$username\@$server:$datastore";
6030
6031 # This is only used for `pbs-restore`!
6032 my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6033 local $ENV{PBS_PASSWORD} = $password;
6034 local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6035
6036 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6037 PVE::Storage::parse_volname($storecfg, $archive);
6038
6039 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6040
6041 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6042
6043 my $tmpdir = "/var/tmp/vzdumptmp$$";
6044 rmtree $tmpdir;
6045 mkpath $tmpdir;
6046
6047 my $conffile = PVE::QemuConfig->config_file($vmid);
6048 my $tmpfn = "$conffile.$$.tmp";
6049 # disable interrupts (always do cleanups)
6050 local $SIG{INT} =
6051 local $SIG{TERM} =
6052 local $SIG{QUIT} =
6053 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6054
6055 # Note: $oldconf is undef if VM does not exists
6056 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6057 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6058
6059 my $rpcenv = PVE::RPCEnvironment::get();
6060 my $devinfo = {};
6061
6062 eval {
6063 # enable interrupts
6064 local $SIG{INT} =
6065 local $SIG{TERM} =
6066 local $SIG{QUIT} =
6067 local $SIG{HUP} =
6068 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6069
6070 my $cfgfn = "$tmpdir/qemu-server.conf";
6071 my $firewall_config_fn = "$tmpdir/fw.conf";
6072 my $index_fn = "$tmpdir/index.json";
6073
6074 my $cmd = "restore";
6075
6076 my $param = [$pbs_backup_name, "index.json", $index_fn];
6077 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6078 my $index = PVE::Tools::file_get_contents($index_fn);
6079 $index = decode_json($index);
6080
6081 # print Dumper($index);
6082 foreach my $info (@{$index->{files}}) {
6083 if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6084 my $devname = $1;
6085 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6086 $devinfo->{$devname}->{size} = $1;
6087 } else {
6088 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6089 }
6090 }
6091 }
6092
6093 my $is_qemu_server_backup = scalar(
6094 grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6095 );
6096 if (!$is_qemu_server_backup) {
6097 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6098 }
6099 my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6100
6101 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6102 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6103
6104 if ($has_firewall_config) {
6105 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6106 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6107
6108 my $pve_firewall_dir = '/etc/pve/firewall';
6109 mkdir $pve_firewall_dir; # make sure the dir exists
6110 PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6111 }
6112
6113 my $fh = IO::File->new($cfgfn, "r") ||
6114 die "unable to read qemu-server.conf - $!\n";
6115
6116 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6117
6118 # fixme: rate limit?
6119
6120 # create empty/temp config
6121 PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6122
6123 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6124
6125 # allocate volumes
6126 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6127
6128 foreach my $virtdev (sort keys %$virtdev_hash) {
6129 my $d = $virtdev_hash->{$virtdev};
6130 next if $d->{is_cloudinit}; # no need to restore cloudinit
6131
6132 my $volid = $d->{volid};
6133
6134 my $path = PVE::Storage::path($storecfg, $volid);
6135
6136 # This is the ONLY user of the PBS_ env vars set on top of this function!
6137 my $pbs_restore_cmd = [
6138 '/usr/bin/pbs-restore',
6139 '--repository', $repo,
6140 $pbs_backup_name,
6141 "$d->{devname}.img.fidx",
6142 $path,
6143 '--verbose',
6144 ];
6145
6146 push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6147 push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6148
6149 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6150 push @$pbs_restore_cmd, '--skip-zero';
6151 }
6152
6153 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6154 print "restore proxmox backup image: $dbg_cmdstring\n";
6155 run_command($pbs_restore_cmd);
6156 }
6157
6158 $fh->seek(0, 0) || die "seek failed - $!\n";
6159
6160 my $outfd = new IO::File ($tmpfn, "w") ||
6161 die "unable to write config for VM $vmid\n";
6162
6163 my $cookie = { netcount => 0 };
6164 while (defined(my $line = <$fh>)) {
6165 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6166 }
6167
6168 $fh->close();
6169 $outfd->close();
6170 };
6171 my $err = $@;
6172
6173 $restore_deactivate_volumes->($storecfg, $devinfo);
6174
6175 rmtree $tmpdir;
6176
6177 if ($err) {
6178 unlink $tmpfn;
6179 $restore_destroy_volumes->($storecfg, $devinfo);
6180 die $err;
6181 }
6182
6183 rename($tmpfn, $conffile) ||
6184 die "unable to commit configuration file '$conffile'\n";
6185
6186 PVE::Cluster::cfs_update(); # make sure we read new file
6187
6188 eval { rescan($vmid, 1); };
6189 warn $@ if $@;
6190}
6191
6192sub restore_vma_archive {
6193 my ($archive, $vmid, $user, $opts, $comp) = @_;
6194
6195 my $readfrom = $archive;
6196
6197 my $cfg = PVE::Storage::config();
6198 my $commands = [];
6199 my $bwlimit = $opts->{bwlimit};
6200
6201 my $dbg_cmdstring = '';
6202 my $add_pipe = sub {
6203 my ($cmd) = @_;
6204 push @$commands, $cmd;
6205 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6206 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6207 $readfrom = '-';
6208 };
6209
6210 my $input = undef;
6211 if ($archive eq '-') {
6212 $input = '<&STDIN';
6213 } else {
6214 # If we use a backup from a PVE defined storage we also consider that
6215 # storage's rate limit:
6216 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6217 if (defined($volid)) {
6218 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6219 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6220 if ($readlimit) {
6221 print STDERR "applying read rate limit: $readlimit\n";
6222 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6223 $add_pipe->($cstream);
6224 }
6225 }
6226 }
6227
6228 if ($comp) {
6229 my $info = PVE::Storage::decompressor_info('vma', $comp);
6230 my $cmd = $info->{decompressor};
6231 push @$cmd, $readfrom;
6232 $add_pipe->($cmd);
6233 }
6234
6235 my $tmpdir = "/var/tmp/vzdumptmp$$";
6236 rmtree $tmpdir;
6237
6238 # disable interrupts (always do cleanups)
6239 local $SIG{INT} =
6240 local $SIG{TERM} =
6241 local $SIG{QUIT} =
6242 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6243
6244 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6245 POSIX::mkfifo($mapfifo, 0600);
6246 my $fifofh;
6247
6248 my $openfifo = sub {
6249 open($fifofh, '>', $mapfifo) || die $!;
6250 };
6251
6252 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6253
6254 my $oldtimeout;
6255 my $timeout = 5;
6256
6257 my $devinfo = {};
6258
6259 my $rpcenv = PVE::RPCEnvironment::get();
6260
6261 my $conffile = PVE::QemuConfig->config_file($vmid);
6262 my $tmpfn = "$conffile.$$.tmp";
6263
6264 # Note: $oldconf is undef if VM does not exist
6265 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6266 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6267
6268 my %storage_limits;
6269
6270 my $print_devmap = sub {
6271 my $cfgfn = "$tmpdir/qemu-server.conf";
6272
6273 # we can read the config - that is already extracted
6274 my $fh = IO::File->new($cfgfn, "r") ||
6275 die "unable to read qemu-server.conf - $!\n";
6276
6277 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6278 if (-f $fwcfgfn) {
6279 my $pve_firewall_dir = '/etc/pve/firewall';
6280 mkdir $pve_firewall_dir; # make sure the dir exists
6281 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6282 }
6283
6284 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6285
6286 foreach my $key (keys %storage_limits) {
6287 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6288 next if !$limit;
6289 print STDERR "rate limit for storage $key: $limit KiB/s\n";
6290 $storage_limits{$key} = $limit * 1024;
6291 }
6292
6293 foreach my $devname (keys %$devinfo) {
6294 die "found no device mapping information for device '$devname'\n"
6295 if !$devinfo->{$devname}->{virtdev};
6296 }
6297
6298 # create empty/temp config
6299 if ($oldconf) {
6300 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6301 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6302 }
6303
6304 # allocate volumes
6305 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6306
6307 # print restore information to $fifofh
6308 foreach my $virtdev (sort keys %$virtdev_hash) {
6309 my $d = $virtdev_hash->{$virtdev};
6310 next if $d->{is_cloudinit}; # no need to restore cloudinit
6311
6312 my $storeid = $d->{storeid};
6313 my $volid = $d->{volid};
6314
6315 my $map_opts = '';
6316 if (my $limit = $storage_limits{$storeid}) {
6317 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6318 }
6319
6320 my $write_zeros = 1;
6321 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6322 $write_zeros = 0;
6323 }
6324
6325 my $path = PVE::Storage::path($cfg, $volid);
6326
6327 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6328
6329 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6330 }
6331
6332 $fh->seek(0, 0) || die "seek failed - $!\n";
6333
6334 my $outfd = new IO::File ($tmpfn, "w") ||
6335 die "unable to write config for VM $vmid\n";
6336
6337 my $cookie = { netcount => 0 };
6338 while (defined(my $line = <$fh>)) {
6339 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6340 }
6341
6342 $fh->close();
6343 $outfd->close();
6344 };
6345
6346 eval {
6347 # enable interrupts
6348 local $SIG{INT} =
6349 local $SIG{TERM} =
6350 local $SIG{QUIT} =
6351 local $SIG{HUP} =
6352 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6353 local $SIG{ALRM} = sub { die "got timeout\n"; };
6354
6355 $oldtimeout = alarm($timeout);
6356
6357 my $parser = sub {
6358 my $line = shift;
6359
6360 print "$line\n";
6361
6362 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6363 my ($dev_id, $size, $devname) = ($1, $2, $3);
6364 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6365 } elsif ($line =~ m/^CTIME: /) {
6366 # we correctly received the vma config, so we can disable
6367 # the timeout now for disk allocation (set to 10 minutes, so
6368 # that we always timeout if something goes wrong)
6369 alarm(600);
6370 &$print_devmap();
6371 print $fifofh "done\n";
6372 my $tmp = $oldtimeout || 0;
6373 $oldtimeout = undef;
6374 alarm($tmp);
6375 close($fifofh);
6376 }
6377 };
6378
6379 print "restore vma archive: $dbg_cmdstring\n";
6380 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6381 };
6382 my $err = $@;
6383
6384 alarm($oldtimeout) if $oldtimeout;
6385
6386 $restore_deactivate_volumes->($cfg, $devinfo);
6387
6388 unlink $mapfifo;
6389 rmtree $tmpdir;
6390
6391 if ($err) {
6392 unlink $tmpfn;
6393 $restore_destroy_volumes->($cfg, $devinfo);
6394 die $err;
6395 }
6396
6397 rename($tmpfn, $conffile) ||
6398 die "unable to commit configuration file '$conffile'\n";
6399
6400 PVE::Cluster::cfs_update(); # make sure we read new file
6401
6402 eval { rescan($vmid, 1); };
6403 warn $@ if $@;
6404}
6405
6406sub restore_tar_archive {
6407 my ($archive, $vmid, $user, $opts) = @_;
6408
6409 if ($archive ne '-') {
6410 my $firstfile = tar_archive_read_firstfile($archive);
6411 die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6412 if $firstfile ne 'qemu-server.conf';
6413 }
6414
6415 my $storecfg = PVE::Storage::config();
6416
6417 # avoid zombie disks when restoring over an existing VM -> cleanup first
6418 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6419 # skiplock=1 because qmrestore has set the 'create' lock itself already
6420 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6421 destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6422
6423 my $tocmd = "/usr/lib/qemu-server/qmextract";
6424
6425 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6426 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6427 $tocmd .= ' --prealloc' if $opts->{prealloc};
6428 $tocmd .= ' --info' if $opts->{info};
6429
6430 # tar option "xf" does not autodetect compression when read from STDIN,
6431 # so we pipe to zcat
6432 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6433 PVE::Tools::shellquote("--to-command=$tocmd");
6434
6435 my $tmpdir = "/var/tmp/vzdumptmp$$";
6436 mkpath $tmpdir;
6437
6438 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6439 local $ENV{VZDUMP_VMID} = $vmid;
6440 local $ENV{VZDUMP_USER} = $user;
6441
6442 my $conffile = PVE::QemuConfig->config_file($vmid);
6443 my $tmpfn = "$conffile.$$.tmp";
6444
6445 # disable interrupts (always do cleanups)
6446 local $SIG{INT} =
6447 local $SIG{TERM} =
6448 local $SIG{QUIT} =
6449 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6450
6451 eval {
6452 # enable interrupts
6453 local $SIG{INT} =
6454 local $SIG{TERM} =
6455 local $SIG{QUIT} =
6456 local $SIG{HUP} =
6457 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6458
6459 if ($archive eq '-') {
6460 print "extracting archive from STDIN\n";
6461 run_command($cmd, input => "<&STDIN");
6462 } else {
6463 print "extracting archive '$archive'\n";
6464 run_command($cmd);
6465 }
6466
6467 return if $opts->{info};
6468
6469 # read new mapping
6470 my $map = {};
6471 my $statfile = "$tmpdir/qmrestore.stat";
6472 if (my $fd = IO::File->new($statfile, "r")) {
6473 while (defined (my $line = <$fd>)) {
6474 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6475 $map->{$1} = $2 if $1;
6476 } else {
6477 print STDERR "unable to parse line in statfile - $line\n";
6478 }
6479 }
6480 $fd->close();
6481 }
6482
6483 my $confsrc = "$tmpdir/qemu-server.conf";
6484
6485 my $srcfd = new IO::File($confsrc, "r") ||
6486 die "unable to open file '$confsrc'\n";
6487
6488 my $outfd = new IO::File ($tmpfn, "w") ||
6489 die "unable to write config for VM $vmid\n";
6490
6491 my $cookie = { netcount => 0 };
6492 while (defined (my $line = <$srcfd>)) {
6493 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6494 }
6495
6496 $srcfd->close();
6497 $outfd->close();
6498 };
6499 if (my $err = $@) {
6500 unlink $tmpfn;
6501 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6502 die $err;
6503 }
6504
6505 rmtree $tmpdir;
6506
6507 rename $tmpfn, $conffile ||
6508 die "unable to commit configuration file '$conffile'\n";
6509
6510 PVE::Cluster::cfs_update(); # make sure we read new file
6511
6512 eval { rescan($vmid, 1); };
6513 warn $@ if $@;
6514};
6515
6516sub foreach_storage_used_by_vm {
6517 my ($conf, $func) = @_;
6518
6519 my $sidhash = {};
6520
6521 PVE::QemuConfig->foreach_volume($conf, sub {
6522 my ($ds, $drive) = @_;
6523 return if drive_is_cdrom($drive);
6524
6525 my $volid = $drive->{file};
6526
6527 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6528 $sidhash->{$sid} = $sid if $sid;
6529 });
6530
6531 foreach my $sid (sort keys %$sidhash) {
6532 &$func($sid);
6533 }
6534}
6535
6536my $qemu_snap_storage = {
6537 rbd => 1,
6538};
6539sub do_snapshots_with_qemu {
6540 my ($storecfg, $volid) = @_;
6541
6542 my $storage_name = PVE::Storage::parse_volume_id($volid);
6543 my $scfg = $storecfg->{ids}->{$storage_name};
6544
6545 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6546 return 1;
6547 }
6548
6549 if ($volid =~ m/\.(qcow2|qed)$/){
6550 return 1;
6551 }
6552
6553 return;
6554}
6555
6556sub qga_check_running {
6557 my ($vmid, $nowarn) = @_;
6558
6559 eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6560 if ($@) {
6561 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6562 return 0;
6563 }
6564 return 1;
6565}
6566
6567sub template_create {
6568 my ($vmid, $conf, $disk) = @_;
6569
6570 my $storecfg = PVE::Storage::config();
6571
6572 PVE::QemuConfig->foreach_volume($conf, sub {
6573 my ($ds, $drive) = @_;
6574
6575 return if drive_is_cdrom($drive);
6576 return if $disk && $ds ne $disk;
6577
6578 my $volid = $drive->{file};
6579 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6580
6581 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6582 $drive->{file} = $voliddst;
6583 $conf->{$ds} = print_drive($drive);
6584 PVE::QemuConfig->write_config($vmid, $conf);
6585 });
6586}
6587
6588sub convert_iscsi_path {
6589 my ($path) = @_;
6590
6591 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6592 my $portal = $1;
6593 my $target = $2;
6594 my $lun = $3;
6595
6596 my $initiator_name = get_initiator_name();
6597
6598 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6599 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6600 }
6601
6602 die "cannot convert iscsi path '$path', unkown format\n";
6603}
6604
6605sub qemu_img_convert {
6606 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6607
6608 my $storecfg = PVE::Storage::config();
6609 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6610 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6611
6612 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6613
6614 my $cachemode;
6615 my $src_path;
6616 my $src_is_iscsi = 0;
6617 my $src_format;
6618
6619 if ($src_storeid) {
6620 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6621 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6622 $src_format = qemu_img_format($src_scfg, $src_volname);
6623 $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6624 $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6625 $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6626 } elsif (-f $src_volid) {
6627 $src_path = $src_volid;
6628 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6629 $src_format = $1;
6630 }
6631 }
6632
6633 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6634
6635 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6636 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6637 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6638 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6639
6640 my $cmd = [];
6641 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6642 push @$cmd, '-l', "snapshot.name=$snapname"
6643 if $snapname && $src_format && $src_format eq "qcow2";
6644 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6645 push @$cmd, '-T', $cachemode if defined($cachemode);
6646
6647 if ($src_is_iscsi) {
6648 push @$cmd, '--image-opts';
6649 $src_path = convert_iscsi_path($src_path);
6650 } elsif ($src_format) {
6651 push @$cmd, '-f', $src_format;
6652 }
6653
6654 if ($dst_is_iscsi) {
6655 push @$cmd, '--target-image-opts';
6656 $dst_path = convert_iscsi_path($dst_path);
6657 } else {
6658 push @$cmd, '-O', $dst_format;
6659 }
6660
6661 push @$cmd, $src_path;
6662
6663 if (!$dst_is_iscsi && $is_zero_initialized) {
6664 push @$cmd, "zeroinit:$dst_path";
6665 } else {
6666 push @$cmd, $dst_path;
6667 }
6668
6669 my $parser = sub {
6670 my $line = shift;
6671 if($line =~ m/\((\S+)\/100\%\)/){
6672 my $percent = $1;
6673 my $transferred = int($size * $percent / 100);
6674 my $remaining = $size - $transferred;
6675
6676 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6677 }
6678
6679 };
6680
6681 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
6682 my $err = $@;
6683 die "copy failed: $err" if $err;
6684}
6685
6686sub qemu_img_format {
6687 my ($scfg, $volname) = @_;
6688
6689 if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6690 return $1;
6691 } else {
6692 return "raw";
6693 }
6694}
6695
6696sub qemu_drive_mirror {
6697 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6698
6699 $jobs = {} if !$jobs;
6700
6701 my $qemu_target;
6702 my $format;
6703 $jobs->{"drive-$drive"} = {};
6704
6705 if ($dst_volid =~ /^nbd:/) {
6706 $qemu_target = $dst_volid;
6707 $format = "nbd";
6708 } else {
6709 my $storecfg = PVE::Storage::config();
6710 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6711
6712 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6713
6714 $format = qemu_img_format($dst_scfg, $dst_volname);
6715
6716 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6717
6718 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6719 }
6720
6721 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6722 $opts->{format} = $format if $format;
6723
6724 if (defined($src_bitmap)) {
6725 $opts->{sync} = 'incremental';
6726 $opts->{bitmap} = $src_bitmap;
6727 print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6728 }
6729
6730 if (defined($bwlimit)) {
6731 $opts->{speed} = $bwlimit * 1024;
6732 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6733 } else {
6734 print "drive mirror is starting for drive-$drive\n";
6735 }
6736
6737 # if a job already runs for this device we get an error, catch it for cleanup
6738 eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6739 if (my $err = $@) {
6740 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6741 warn "$@\n" if $@;
6742 die "mirroring error: $err\n";
6743 }
6744
6745 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6746}
6747
6748# $completion can be either
6749# 'complete': wait until all jobs are ready, block-job-complete them (default)
6750# 'cancel': wait until all jobs are ready, block-job-cancel them
6751# 'skip': wait until all jobs are ready, return with block jobs in ready state
6752sub qemu_drive_mirror_monitor {
6753 my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6754
6755 $completion //= 'complete';
6756
6757 eval {
6758 my $err_complete = 0;
6759
6760 while (1) {
6761 die "storage migration timed out\n" if $err_complete > 300;
6762
6763 my $stats = mon_cmd($vmid, "query-block-jobs");
6764
6765 my $running_mirror_jobs = {};
6766 foreach my $stat (@$stats) {
6767 next if $stat->{type} ne 'mirror';
6768 $running_mirror_jobs->{$stat->{device}} = $stat;
6769 }
6770
6771 my $readycounter = 0;
6772
6773 foreach my $job (keys %$jobs) {
6774
6775 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6776 print "$job : finished\n";
6777 delete $jobs->{$job};
6778 next;
6779 }
6780
6781 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6782
6783 my $busy = $running_mirror_jobs->{$job}->{busy};
6784 my $ready = $running_mirror_jobs->{$job}->{ready};
6785 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6786 my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6787 my $remaining = $total - $transferred;
6788 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6789
6790 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6791 }
6792
6793 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6794 }
6795
6796 last if scalar(keys %$jobs) == 0;
6797
6798 if ($readycounter == scalar(keys %$jobs)) {
6799 print "all mirroring jobs are ready \n";
6800 last if $completion eq 'skip'; #do the complete later
6801
6802 if ($vmiddst && $vmiddst != $vmid) {
6803 my $agent_running = $qga && qga_check_running($vmid);
6804 if ($agent_running) {
6805 print "freeze filesystem\n";
6806 eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6807 } else {
6808 print "suspend vm\n";
6809 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6810 }
6811
6812 # if we clone a disk for a new target vm, we don't switch the disk
6813 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6814
6815 if ($agent_running) {
6816 print "unfreeze filesystem\n";
6817 eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6818 } else {
6819 print "resume vm\n";
6820 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
6821 }
6822
6823 last;
6824 } else {
6825
6826 foreach my $job (keys %$jobs) {
6827 # try to switch the disk if source and destination are on the same guest
6828 print "$job: Completing block job...\n";
6829
6830 my $op;
6831 if ($completion eq 'complete') {
6832 $op = 'block-job-complete';
6833 } elsif ($completion eq 'cancel') {
6834 $op = 'block-job-cancel';
6835 } else {
6836 die "invalid completion value: $completion\n";
6837 }
6838 eval { mon_cmd($vmid, $op, device => $job) };
6839 if ($@ =~ m/cannot be completed/) {
6840 print "$job: Block job cannot be completed, try again.\n";
6841 $err_complete++;
6842 }else {
6843 print "$job: Completed successfully.\n";
6844 $jobs->{$job}->{complete} = 1;
6845 }
6846 }
6847 }
6848 }
6849 sleep 1;
6850 }
6851 };
6852 my $err = $@;
6853
6854 if ($err) {
6855 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6856 die "mirroring error: $err";
6857 }
6858
6859}
6860
6861sub qemu_blockjobs_cancel {
6862 my ($vmid, $jobs) = @_;
6863
6864 foreach my $job (keys %$jobs) {
6865 print "$job: Cancelling block job\n";
6866 eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6867 $jobs->{$job}->{cancel} = 1;
6868 }
6869
6870 while (1) {
6871 my $stats = mon_cmd($vmid, "query-block-jobs");
6872
6873 my $running_jobs = {};
6874 foreach my $stat (@$stats) {
6875 $running_jobs->{$stat->{device}} = $stat;
6876 }
6877
6878 foreach my $job (keys %$jobs) {
6879
6880 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6881 print "$job: Done.\n";
6882 delete $jobs->{$job};
6883 }
6884 }
6885
6886 last if scalar(keys %$jobs) == 0;
6887
6888 sleep 1;
6889 }
6890}
6891
6892sub clone_disk {
6893 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6894 $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6895
6896 my $newvolid;
6897
6898 if (!$full) {
6899 print "create linked clone of drive $drivename ($drive->{file})\n";
6900 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
6901 push @$newvollist, $newvolid;
6902 } else {
6903
6904 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6905 $storeid = $storage if $storage;
6906
6907 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6908
6909 print "create full clone of drive $drivename ($drive->{file})\n";
6910 my $name = undef;
6911 my $size = undef;
6912 if (drive_is_cloudinit($drive)) {
6913 $name = "vm-$newvmid-cloudinit";
6914 $name .= ".$dst_format" if $dst_format ne 'raw';
6915 $snapname = undef;
6916 $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6917 } elsif ($drivename eq 'efidisk0') {
6918 $size = get_efivars_size($conf);
6919 } else {
6920 ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6921 }
6922 $size /= 1024;
6923 $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, $size);
6924 push @$newvollist, $newvolid;
6925
6926 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6927
6928 if (drive_is_cloudinit($drive)) {
6929 goto no_data_clone;
6930 }
6931
6932 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6933 if (!$running || $snapname) {
6934 # TODO: handle bwlimits
6935 if ($drivename eq 'efidisk0') {
6936 # the relevant data on the efidisk may be smaller than the source
6937 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6938 # that is given by the OVMF_VARS.fd
6939 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6940 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6941 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=1", "count=$size",
6942 "if=$src_path", "of=$dst_path"]);
6943 } else {
6944 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6945 }
6946 } else {
6947
6948 my $kvmver = get_running_qemu_version ($vmid);
6949 if (!min_version($kvmver, 2, 7)) {
6950 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6951 if $drive->{iothread};
6952 }
6953
6954 qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
6955 $completion, $qga, $bwlimit);
6956 }
6957 }
6958
6959no_data_clone:
6960 my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6961
6962 my $disk = $drive;
6963 $disk->{format} = undef;
6964 $disk->{file} = $newvolid;
6965 $disk->{size} = $size;
6966
6967 return $disk;
6968}
6969
6970sub get_running_qemu_version {
6971 my ($vmid) = @_;
6972 my $res = mon_cmd($vmid, "query-version");
6973 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6974}
6975
6976sub qemu_use_old_bios_files {
6977 my ($machine_type) = @_;
6978
6979 return if !$machine_type;
6980
6981 my $use_old_bios_files = undef;
6982
6983 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6984 $machine_type = $1;
6985 $use_old_bios_files = 1;
6986 } else {
6987 my $version = extract_version($machine_type, kvm_user_version());
6988 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6989 # load new efi bios files on migration. So this hack is required to allow
6990 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6991 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6992 $use_old_bios_files = !min_version($version, 2, 4);
6993 }
6994
6995 return ($use_old_bios_files, $machine_type);
6996}
6997
6998sub get_efivars_size {
6999 my ($conf) = @_;
7000 my $arch = get_vm_arch($conf);
7001 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7002 die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7003 return -s $ovmf_vars;
7004}
7005
7006sub update_efidisk_size {
7007 my ($conf) = @_;
7008
7009 return if !defined($conf->{efidisk0});
7010
7011 my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7012 $disk->{size} = get_efivars_size($conf);
7013 $conf->{efidisk0} = print_drive($disk);
7014
7015 return;
7016}
7017
7018sub create_efidisk($$$$$) {
7019 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7020
7021 my (undef, $ovmf_vars) = get_ovmf_files($arch);
7022 die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7023
7024 my $vars_size_b = -s $ovmf_vars;
7025 my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7026 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7027 PVE::Storage::activate_volumes($storecfg, [$volid]);
7028
7029 qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7030 my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7031
7032 return ($volid, $size/1024);
7033}
7034
7035sub vm_iothreads_list {
7036 my ($vmid) = @_;
7037
7038 my $res = mon_cmd($vmid, 'query-iothreads');
7039
7040 my $iothreads = {};
7041 foreach my $iothread (@$res) {
7042 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7043 }
7044
7045 return $iothreads;
7046}
7047
7048sub scsihw_infos {
7049 my ($conf, $drive) = @_;
7050
7051 my $maxdev = 0;
7052
7053 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7054 $maxdev = 7;
7055 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7056 $maxdev = 1;
7057 } else {
7058 $maxdev = 256;
7059 }
7060
7061 my $controller = int($drive->{index} / $maxdev);
7062 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7063 ? "virtioscsi"
7064 : "scsihw";
7065
7066 return ($maxdev, $controller, $controller_prefix);
7067}
7068
7069sub windows_version {
7070 my ($ostype) = @_;
7071
7072 return 0 if !$ostype;
7073
7074 my $winversion = 0;
7075
7076 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7077 $winversion = 5;
7078 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7079 $winversion = 6;
7080 } elsif ($ostype =~ m/^win(\d+)$/) {
7081 $winversion = $1;
7082 }
7083
7084 return $winversion;
7085}
7086
7087sub resolve_dst_disk_format {
7088 my ($storecfg, $storeid, $src_volname, $format) = @_;
7089 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7090
7091 if (!$format) {
7092 # if no target format is specified, use the source disk format as hint
7093 if ($src_volname) {
7094 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7095 $format = qemu_img_format($scfg, $src_volname);
7096 } else {
7097 return $defFormat;
7098 }
7099 }
7100
7101 # test if requested format is supported - else use default
7102 my $supported = grep { $_ eq $format } @$validFormats;
7103 $format = $defFormat if !$supported;
7104 return $format;
7105}
7106
7107# NOTE: if this logic changes, please update docs & possibly gui logic
7108sub find_vmstate_storage {
7109 my ($conf, $storecfg) = @_;
7110
7111 # first, return storage from conf if set
7112 return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7113
7114 my ($target, $shared, $local);
7115
7116 foreach_storage_used_by_vm($conf, sub {
7117 my ($sid) = @_;
7118 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7119 my $dst = $scfg->{shared} ? \$shared : \$local;
7120 $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7121 });
7122
7123 # second, use shared storage where VM has at least one disk
7124 # third, use local storage where VM has at least one disk
7125 # fall back to local storage
7126 $target = $shared // $local // 'local';
7127
7128 return $target;
7129}
7130
7131sub generate_uuid {
7132 my ($uuid, $uuid_str);
7133 UUID::generate($uuid);
7134 UUID::unparse($uuid, $uuid_str);
7135 return $uuid_str;
7136}
7137
7138sub generate_smbios1_uuid {
7139 return "uuid=".generate_uuid();
7140}
7141
7142sub nbd_stop {
7143 my ($vmid) = @_;
7144
7145 mon_cmd($vmid, 'nbd-server-stop');
7146}
7147
7148sub create_reboot_request {
7149 my ($vmid) = @_;
7150 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7151 or die "failed to create reboot trigger file: $!\n";
7152 close($fh);
7153}
7154
7155sub clear_reboot_request {
7156 my ($vmid) = @_;
7157 my $path = "/run/qemu-server/$vmid.reboot";
7158 my $res = 0;
7159
7160 $res = unlink($path);
7161 die "could not remove reboot request for $vmid: $!"
7162 if !$res && $! != POSIX::ENOENT;
7163
7164 return $res;
7165}
7166
7167sub bootorder_from_legacy {
7168 my ($conf, $bootcfg) = @_;
7169
7170 my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7171 my $bootindex_hash = {};
7172 my $i = 1;
7173 foreach my $o (split(//, $boot)) {
7174 $bootindex_hash->{$o} = $i*100;
7175 $i++;
7176 }
7177
7178 my $bootorder = {};
7179
7180 PVE::QemuConfig->foreach_volume($conf, sub {
7181 my ($ds, $drive) = @_;
7182
7183 if (drive_is_cdrom ($drive, 1)) {
7184 if ($bootindex_hash->{d}) {
7185 $bootorder->{$ds} = $bootindex_hash->{d};
7186 $bootindex_hash->{d} += 1;
7187 }
7188 } elsif ($bootindex_hash->{c}) {
7189 $bootorder->{$ds} = $bootindex_hash->{c}
7190 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7191 $bootindex_hash->{c} += 1;
7192 }
7193 });
7194
7195 if ($bootindex_hash->{n}) {
7196 for (my $i = 0; $i < $MAX_NETS; $i++) {
7197 my $netname = "net$i";
7198 next if !$conf->{$netname};
7199 $bootorder->{$netname} = $bootindex_hash->{n};
7200 $bootindex_hash->{n} += 1;
7201 }
7202 }
7203
7204 return $bootorder;
7205}
7206
7207# Generate default device list for 'boot: order=' property. Matches legacy
7208# default boot order, but with explicit device names. This is important, since
7209# the fallback for when neither 'order' nor the old format is specified relies
7210# on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7211sub get_default_bootdevices {
7212 my ($conf) = @_;
7213
7214 my @ret = ();
7215
7216 # harddisk
7217 my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7218 push @ret, $first if $first;
7219
7220 # cdrom
7221 $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7222 push @ret, $first if $first;
7223
7224 # network
7225 for (my $i = 0; $i < $MAX_NETS; $i++) {
7226 my $netname = "net$i";
7227 next if !$conf->{$netname};
7228 push @ret, $netname;
7229 last;
7230 }
7231
7232 return \@ret;
7233}
7234
7235sub device_bootorder {
7236 my ($conf) = @_;
7237
7238 return bootorder_from_legacy($conf) if !defined($conf->{boot});
7239
7240 my $boot = parse_property_string($boot_fmt, $conf->{boot});
7241
7242 my $bootorder = {};
7243 if (!defined($boot) || $boot->{legacy}) {
7244 $bootorder = bootorder_from_legacy($conf, $boot);
7245 } elsif ($boot->{order}) {
7246 my $i = 100; # start at 100 to allow user to insert devices before us with -args
7247 for my $dev (PVE::Tools::split_list($boot->{order})) {
7248 $bootorder->{$dev} = $i++;
7249 }
7250 }
7251
7252 return $bootorder;
7253}
7254
7255# bash completion helper
7256
7257sub complete_backup_archives {
7258 my ($cmdname, $pname, $cvalue) = @_;
7259
7260 my $cfg = PVE::Storage::config();
7261
7262 my $storeid;
7263
7264 if ($cvalue =~ m/^([^:]+):/) {
7265 $storeid = $1;
7266 }
7267
7268 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7269
7270 my $res = [];
7271 foreach my $id (keys %$data) {
7272 foreach my $item (@{$data->{$id}}) {
7273 next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7274 push @$res, $item->{volid} if defined($item->{volid});
7275 }
7276 }
7277
7278 return $res;
7279}
7280
7281my $complete_vmid_full = sub {
7282 my ($running) = @_;
7283
7284 my $idlist = vmstatus();
7285
7286 my $res = [];
7287
7288 foreach my $id (keys %$idlist) {
7289 my $d = $idlist->{$id};
7290 if (defined($running)) {
7291 next if $d->{template};
7292 next if $running && $d->{status} ne 'running';
7293 next if !$running && $d->{status} eq 'running';
7294 }
7295 push @$res, $id;
7296
7297 }
7298 return $res;
7299};
7300
7301sub complete_vmid {
7302 return &$complete_vmid_full();
7303}
7304
7305sub complete_vmid_stopped {
7306 return &$complete_vmid_full(0);
7307}
7308
7309sub complete_vmid_running {
7310 return &$complete_vmid_full(1);
7311}
7312
7313sub complete_storage {
7314
7315 my $cfg = PVE::Storage::config();
7316 my $ids = $cfg->{ids};
7317
7318 my $res = [];
7319 foreach my $sid (keys %$ids) {
7320 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7321 next if !$ids->{$sid}->{content}->{images};
7322 push @$res, $sid;
7323 }
7324
7325 return $res;
7326}
7327
7328sub complete_migration_storage {
7329 my ($cmd, $param, $current_value, $all_args) = @_;
7330
7331 my $targetnode = @$all_args[1];
7332
7333 my $cfg = PVE::Storage::config();
7334 my $ids = $cfg->{ids};
7335
7336 my $res = [];
7337 foreach my $sid (keys %$ids) {
7338 next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7339 next if !$ids->{$sid}->{content}->{images};
7340 push @$res, $sid;
7341 }
7342
7343 return $res;
7344}
7345
73461;
Virtual Machine Manager