]> git.proxmox.com Git - qemu-server.git/blame_incremental - PVE/QemuServer.pm
projects / qemu-server.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
cleanup backup code: share code between restore_vma_archive and restore_proxmox_backu...
[qemu-server.git] / PVE / QemuServer.pm
... / ...
CommitLineData
1package PVE::QemuServer;
2
3use strict;
4use warnings;
5
6use Cwd 'abs_path';
7use Digest::SHA;
8use Fcntl ':flock';
9use Fcntl;
10use File::Basename;
11use File::Copy qw(copy);
12use File::Path;
13use File::stat;
14use Getopt::Long;
15use IO::Dir;
16use IO::File;
17use IO::Handle;
18use IO::Select;
19use IO::Socket::UNIX;
20use IPC::Open3;
21use JSON;
22use MIME::Base64;
23use POSIX;
24use Storable qw(dclone);
25use Time::HiRes qw(gettimeofday);
26use URI::Escape;
27use UUID;
28
29use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
30use PVE::DataCenterConfig;
31use PVE::Exception qw(raise raise_param_exc);
32use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
33use PVE::INotify;
34use PVE::JSONSchema qw(get_standard_option);
35use PVE::ProcFSTools;
36use PVE::RPCEnvironment;
37use PVE::Storage;
38use PVE::SysFSTools;
39use PVE::Systemd;
40use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
41
42use PVE::QMPClient;
43use PVE::QemuConfig;
44use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
45use PVE::QemuServer::Cloudinit;
46use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
47use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive foreach_drive foreach_volid);
48use PVE::QemuServer::Machine;
49use PVE::QemuServer::Memory;
50use PVE::QemuServer::Monitor qw(mon_cmd);
51use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
52use PVE::QemuServer::USB qw(parse_usb_device);
53
54my $have_sdn;
55eval {
56 require PVE::Network::SDN::Zones;
57 $have_sdn = 1;
58};
59
60my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
61my $OVMF = {
62 x86_64 => [
63 "$EDK2_FW_BASE/OVMF_CODE.fd",
64 "$EDK2_FW_BASE/OVMF_VARS.fd"
65 ],
66 aarch64 => [
67 "$EDK2_FW_BASE/AAVMF_CODE.fd",
68 "$EDK2_FW_BASE/AAVMF_VARS.fd"
69 ],
70};
71
72my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
73
74# Note about locking: we use flock on the config file protect
75# against concurent actions.
76# Aditionaly, we have a 'lock' setting in the config file. This
77# can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
78# allowed when such lock is set. But you can ignore this kind of
79# lock with the --skiplock flag.
80
81cfs_register_file('/qemu-server/',
82 \&parse_vm_config,
83 \&write_vm_config);
84
85PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
86 description => "Some command save/restore state from this location.",
87 type => 'string',
88 maxLength => 128,
89 optional => 1,
90});
91
92PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
93 description => "Specifies the Qemu machine type.",
94 type => 'string',
95 pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
96 maxLength => 40,
97 optional => 1,
98});
99
100#no warnings 'redefine';
101
102sub cgroups_write {
103 my ($controller, $vmid, $option, $value) = @_;
104
105 my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
106 PVE::ProcFSTools::write_proc_entry($path, $value);
107
108}
109
110my $nodename_cache;
111sub nodename {
112 $nodename_cache //= PVE::INotify::nodename();
113 return $nodename_cache;
114}
115
116my $watchdog_fmt = {
117 model => {
118 default_key => 1,
119 type => 'string',
120 enum => [qw(i6300esb ib700)],
121 description => "Watchdog type to emulate.",
122 default => 'i6300esb',
123 optional => 1,
124 },
125 action => {
126 type => 'string',
127 enum => [qw(reset shutdown poweroff pause debug none)],
128 description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
129 optional => 1,
130 },
131};
132PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
133
134my $agent_fmt = {
135 enabled => {
136 description => "Enable/disable Qemu GuestAgent.",
137 type => 'boolean',
138 default => 0,
139 default_key => 1,
140 },
141 fstrim_cloned_disks => {
142 description => "Run fstrim after cloning/moving a disk.",
143 type => 'boolean',
144 optional => 1,
145 default => 0
146 },
147 type => {
148 description => "Select the agent type",
149 type => 'string',
150 default => 'virtio',
151 optional => 1,
152 enum => [qw(virtio isa)],
153 },
154};
155
156my $vga_fmt = {
157 type => {
158 description => "Select the VGA type.",
159 type => 'string',
160 default => 'std',
161 optional => 1,
162 default_key => 1,
163 enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
164 },
165 memory => {
166 description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
167 type => 'integer',
168 optional => 1,
169 minimum => 4,
170 maximum => 512,
171 },
172};
173
174my $ivshmem_fmt = {
175 size => {
176 type => 'integer',
177 minimum => 1,
178 description => "The size of the file in MB.",
179 },
180 name => {
181 type => 'string',
182 pattern => '[a-zA-Z0-9\-]+',
183 optional => 1,
184 format_description => 'string',
185 description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
186 },
187};
188
189my $audio_fmt = {
190 device => {
191 type => 'string',
192 enum => [qw(ich9-intel-hda intel-hda AC97)],
193 description => "Configure an audio device."
194 },
195 driver => {
196 type => 'string',
197 enum => ['spice'],
198 default => 'spice',
199 optional => 1,
200 description => "Driver backend for the audio device."
201 },
202};
203
204my $spice_enhancements_fmt = {
205 foldersharing => {
206 type => 'boolean',
207 optional => 1,
208 default => '0',
209 description => "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
210 },
211 videostreaming => {
212 type => 'string',
213 enum => ['off', 'all', 'filter'],
214 default => 'off',
215 optional => 1,
216 description => "Enable video streaming. Uses compression for detected video streams."
217 },
218};
219
220my $rng_fmt = {
221 source => {
222 type => 'string',
223 enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
224 default_key => 1,
225 description => "The file on the host to gather entropy from. In most"
226 . " cases /dev/urandom should be preferred over /dev/random"
227 . " to avoid entropy-starvation issues on the host. Using"
228 . " urandom does *not* decrease security in any meaningful"
229 . " way, as it's still seeded from real entropy, and the"
230 . " bytes provided will most likely be mixed with real"
231 . " entropy on the guest as well. /dev/hwrng can be used"
232 . " to pass through a hardware RNG from the host.",
233 },
234 max_bytes => {
235 type => 'integer',
236 description => "Maximum bytes of entropy injected into the guest every"
237 . " 'period' milliseconds. Prefer a lower value when using"
238 . " /dev/random as source. Use 0 to disable limiting"
239 . " (potentially dangerous!).",
240 optional => 1,
241
242 # default is 1 KiB/s, provides enough entropy to the guest to avoid
243 # boot-starvation issues (e.g. systemd etc...) while allowing no chance
244 # of overwhelming the host, provided we're reading from /dev/urandom
245 default => 1024,
246 },
247 period => {
248 type => 'integer',
249 description => "Every 'period' milliseconds the entropy-injection quota"
250 . " is reset, allowing the guest to retrieve another"
251 . " 'max_bytes' of entropy.",
252 optional => 1,
253 default => 1000,
254 },
255};
256
257my $confdesc = {
258 onboot => {
259 optional => 1,
260 type => 'boolean',
261 description => "Specifies whether a VM will be started during system bootup.",
262 default => 0,
263 },
264 autostart => {
265 optional => 1,
266 type => 'boolean',
267 description => "Automatic restart after crash (currently ignored).",
268 default => 0,
269 },
270 hotplug => {
271 optional => 1,
272 type => 'string', format => 'pve-hotplug-features',
273 description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
274 default => 'network,disk,usb',
275 },
276 reboot => {
277 optional => 1,
278 type => 'boolean',
279 description => "Allow reboot. If set to '0' the VM exit on reboot.",
280 default => 1,
281 },
282 lock => {
283 optional => 1,
284 type => 'string',
285 description => "Lock/unlock the VM.",
286 enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
287 },
288 cpulimit => {
289 optional => 1,
290 type => 'number',
291 description => "Limit of CPU usage.",
292 verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
293 minimum => 0,
294 maximum => 128,
295 default => 0,
296 },
297 cpuunits => {
298 optional => 1,
299 type => 'integer',
300 description => "CPU weight for a VM.",
301 verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
302 minimum => 2,
303 maximum => 262144,
304 default => 1024,
305 },
306 memory => {
307 optional => 1,
308 type => 'integer',
309 description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
310 minimum => 16,
311 default => 512,
312 },
313 balloon => {
314 optional => 1,
315 type => 'integer',
316 description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
317 minimum => 0,
318 },
319 shares => {
320 optional => 1,
321 type => 'integer',
322 description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
323 minimum => 0,
324 maximum => 50000,
325 default => 1000,
326 },
327 keyboard => {
328 optional => 1,
329 type => 'string',
330 description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
331 "It should not be necessary to set it.",
332 enum => PVE::Tools::kvmkeymaplist(),
333 default => undef,
334 },
335 name => {
336 optional => 1,
337 type => 'string', format => 'dns-name',
338 description => "Set a name for the VM. Only used on the configuration web interface.",
339 },
340 scsihw => {
341 optional => 1,
342 type => 'string',
343 description => "SCSI controller model",
344 enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
345 default => 'lsi',
346 },
347 description => {
348 optional => 1,
349 type => 'string',
350 description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
351 },
352 ostype => {
353 optional => 1,
354 type => 'string',
355 enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
356 description => "Specify guest operating system.",
357 verbose_description => <<EODESC,
358Specify guest operating system. This is used to enable special
359optimization/features for specific operating systems:
360
361[horizontal]
362other;; unspecified OS
363wxp;; Microsoft Windows XP
364w2k;; Microsoft Windows 2000
365w2k3;; Microsoft Windows 2003
366w2k8;; Microsoft Windows 2008
367wvista;; Microsoft Windows Vista
368win7;; Microsoft Windows 7
369win8;; Microsoft Windows 8/2012/2012r2
370win10;; Microsoft Windows 10/2016
371l24;; Linux 2.4 Kernel
372l26;; Linux 2.6 - 5.X Kernel
373solaris;; Solaris/OpenSolaris/OpenIndiania kernel
374EODESC
375 },
376 boot => {
377 optional => 1,
378 type => 'string',
379 description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
380 pattern => '[acdn]{1,4}',
381 default => 'cdn',
382 },
383 bootdisk => {
384 optional => 1,
385 type => 'string', format => 'pve-qm-bootdisk',
386 description => "Enable booting from specified disk.",
387 pattern => '(ide|sata|scsi|virtio)\d+',
388 },
389 smp => {
390 optional => 1,
391 type => 'integer',
392 description => "The number of CPUs. Please use option -sockets instead.",
393 minimum => 1,
394 default => 1,
395 },
396 sockets => {
397 optional => 1,
398 type => 'integer',
399 description => "The number of CPU sockets.",
400 minimum => 1,
401 default => 1,
402 },
403 cores => {
404 optional => 1,
405 type => 'integer',
406 description => "The number of cores per socket.",
407 minimum => 1,
408 default => 1,
409 },
410 numa => {
411 optional => 1,
412 type => 'boolean',
413 description => "Enable/disable NUMA.",
414 default => 0,
415 },
416 hugepages => {
417 optional => 1,
418 type => 'string',
419 description => "Enable/disable hugepages memory.",
420 enum => [qw(any 2 1024)],
421 },
422 vcpus => {
423 optional => 1,
424 type => 'integer',
425 description => "Number of hotplugged vcpus.",
426 minimum => 1,
427 default => 0,
428 },
429 acpi => {
430 optional => 1,
431 type => 'boolean',
432 description => "Enable/disable ACPI.",
433 default => 1,
434 },
435 agent => {
436 optional => 1,
437 description => "Enable/disable Qemu GuestAgent and its properties.",
438 type => 'string',
439 format => $agent_fmt,
440 },
441 kvm => {
442 optional => 1,
443 type => 'boolean',
444 description => "Enable/disable KVM hardware virtualization.",
445 default => 1,
446 },
447 tdf => {
448 optional => 1,
449 type => 'boolean',
450 description => "Enable/disable time drift fix.",
451 default => 0,
452 },
453 localtime => {
454 optional => 1,
455 type => 'boolean',
456 description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
457 },
458 freeze => {
459 optional => 1,
460 type => 'boolean',
461 description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
462 },
463 vga => {
464 optional => 1,
465 type => 'string', format => $vga_fmt,
466 description => "Configure the VGA hardware.",
467 verbose_description => "Configure the VGA Hardware. If you want to use ".
468 "high resolution modes (>= 1280x1024x16) you may need to increase " .
469 "the vga memory option. Since QEMU 2.9 the default VGA display type " .
470 "is 'std' for all OS types besides some Windows versions (XP and " .
471 "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
472 "display server. For win* OS you can select how many independent " .
473 "displays you want, Linux guests can add displays them self.\n".
474 "You can also run without any graphic card, using a serial device as terminal.",
475 },
476 watchdog => {
477 optional => 1,
478 type => 'string', format => 'pve-qm-watchdog',
479 description => "Create a virtual hardware watchdog device.",
480 verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
481 " (by a guest action), the watchdog must be periodically polled " .
482 "by an agent inside the guest or else the watchdog will reset " .
483 "the guest (or execute the respective action specified)",
484 },
485 startdate => {
486 optional => 1,
487 type => 'string',
488 typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
489 description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
490 pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
491 default => 'now',
492 },
493 startup => get_standard_option('pve-startup-order'),
494 template => {
495 optional => 1,
496 type => 'boolean',
497 description => "Enable/disable Template.",
498 default => 0,
499 },
500 args => {
501 optional => 1,
502 type => 'string',
503 description => "Arbitrary arguments passed to kvm.",
504 verbose_description => <<EODESCR,
505Arbitrary arguments passed to kvm, for example:
506
507args: -no-reboot -no-hpet
508
509NOTE: this option is for experts only.
510EODESCR
511 },
512 tablet => {
513 optional => 1,
514 type => 'boolean',
515 default => 1,
516 description => "Enable/disable the USB tablet device.",
517 verbose_description => "Enable/disable the USB tablet device. This device is " .
518 "usually needed to allow absolute mouse positioning with VNC. " .
519 "Else the mouse runs out of sync with normal VNC clients. " .
520 "If you're running lots of console-only guests on one host, " .
521 "you may consider disabling this to save some context switches. " .
522 "This is turned off by default if you use spice (-vga=qxl).",
523 },
524 migrate_speed => {
525 optional => 1,
526 type => 'integer',
527 description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
528 minimum => 0,
529 default => 0,
530 },
531 migrate_downtime => {
532 optional => 1,
533 type => 'number',
534 description => "Set maximum tolerated downtime (in seconds) for migrations.",
535 minimum => 0,
536 default => 0.1,
537 },
538 cdrom => {
539 optional => 1,
540 type => 'string', format => 'pve-qm-ide',
541 typetext => '<volume>',
542 description => "This is an alias for option -ide2",
543 },
544 cpu => {
545 optional => 1,
546 description => "Emulated CPU type.",
547 type => 'string',
548 format => $PVE::QemuServer::CPUConfig::cpu_fmt,
549 },
550 parent => get_standard_option('pve-snapshot-name', {
551 optional => 1,
552 description => "Parent snapshot name. This is used internally, and should not be modified.",
553 }),
554 snaptime => {
555 optional => 1,
556 description => "Timestamp for snapshots.",
557 type => 'integer',
558 minimum => 0,
559 },
560 vmstate => {
561 optional => 1,
562 type => 'string', format => 'pve-volume-id',
563 description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
564 },
565 vmstatestorage => get_standard_option('pve-storage-id', {
566 description => "Default storage for VM state volumes/files.",
567 optional => 1,
568 }),
569 runningmachine => get_standard_option('pve-qemu-machine', {
570 description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
571 }),
572 machine => get_standard_option('pve-qemu-machine'),
573 arch => {
574 description => "Virtual processor architecture. Defaults to the host.",
575 optional => 1,
576 type => 'string',
577 enum => [qw(x86_64 aarch64)],
578 },
579 smbios1 => {
580 description => "Specify SMBIOS type 1 fields.",
581 type => 'string', format => 'pve-qm-smbios1',
582 maxLength => 512,
583 optional => 1,
584 },
585 protection => {
586 optional => 1,
587 type => 'boolean',
588 description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
589 default => 0,
590 },
591 bios => {
592 optional => 1,
593 type => 'string',
594 enum => [ qw(seabios ovmf) ],
595 description => "Select BIOS implementation.",
596 default => 'seabios',
597 },
598 vmgenid => {
599 type => 'string',
600 pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
601 format_description => 'UUID',
602 description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
603 verbose_description => "The VM generation ID (vmgenid) device exposes a".
604 " 128-bit integer value identifier to the guest OS. This allows to".
605 " notify the guest operating system when the virtual machine is".
606 " executed with a different configuration (e.g. snapshot execution".
607 " or creation from a template). The guest operating system notices".
608 " the change, and is then able to react as appropriate by marking".
609 " its copies of distributed databases as dirty, re-initializing its".
610 " random number generator, etc.\n".
611 "Note that auto-creation only works when done throug API/CLI create".
612 " or update methods, but not when manually editing the config file.",
613 default => "1 (autogenerated)",
614 optional => 1,
615 },
616 hookscript => {
617 type => 'string',
618 format => 'pve-volume-id',
619 optional => 1,
620 description => "Script that will be executed during various steps in the vms lifetime.",
621 },
622 ivshmem => {
623 type => 'string',
624 format => $ivshmem_fmt,
625 description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
626 optional => 1,
627 },
628 audio0 => {
629 type => 'string',
630 format => $audio_fmt,
631 description => "Configure a audio device, useful in combination with QXL/Spice.",
632 optional => 1
633 },
634 spice_enhancements => {
635 type => 'string',
636 format => $spice_enhancements_fmt,
637 description => "Configure additional enhancements for SPICE.",
638 optional => 1
639 },
640 tags => {
641 type => 'string', format => 'pve-tag-list',
642 description => 'Tags of the VM. This is only meta information.',
643 optional => 1,
644 },
645 rng0 => {
646 type => 'string',
647 format => $rng_fmt,
648 description => "Configure a VirtIO-based Random Number Generator.",
649 optional => 1,
650 },
651};
652
653my $cicustom_fmt = {
654 meta => {
655 type => 'string',
656 optional => 1,
657 description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
658 format => 'pve-volume-id',
659 format_description => 'volume',
660 },
661 network => {
662 type => 'string',
663 optional => 1,
664 description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
665 format => 'pve-volume-id',
666 format_description => 'volume',
667 },
668 user => {
669 type => 'string',
670 optional => 1,
671 description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
672 format => 'pve-volume-id',
673 format_description => 'volume',
674 },
675};
676PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
677
678my $confdesc_cloudinit = {
679 citype => {
680 optional => 1,
681 type => 'string',
682 description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
683 enum => ['configdrive2', 'nocloud'],
684 },
685 ciuser => {
686 optional => 1,
687 type => 'string',
688 description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
689 },
690 cipassword => {
691 optional => 1,
692 type => 'string',
693 description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
694 },
695 cicustom => {
696 optional => 1,
697 type => 'string',
698 description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
699 format => 'pve-qm-cicustom',
700 },
701 searchdomain => {
702 optional => 1,
703 type => 'string',
704 description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
705 },
706 nameserver => {
707 optional => 1,
708 type => 'string', format => 'address-list',
709 description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
710 },
711 sshkeys => {
712 optional => 1,
713 type => 'string',
714 format => 'urlencoded',
715 description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
716 },
717};
718
719# what about other qemu settings ?
720#cpu => 'string',
721#machine => 'string',
722#fda => 'file',
723#fdb => 'file',
724#mtdblock => 'file',
725#sd => 'file',
726#pflash => 'file',
727#snapshot => 'bool',
728#bootp => 'file',
729##tftp => 'dir',
730##smb => 'dir',
731#kernel => 'file',
732#append => 'string',
733#initrd => 'file',
734##soundhw => 'string',
735
736while (my ($k, $v) = each %$confdesc) {
737 PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
738}
739
740my $MAX_USB_DEVICES = 5;
741my $MAX_NETS = 32;
742my $MAX_HOSTPCI_DEVICES = 16;
743my $MAX_SERIAL_PORTS = 4;
744my $MAX_PARALLEL_PORTS = 3;
745my $MAX_NUMA = 8;
746
747my $numa_fmt = {
748 cpus => {
749 type => "string",
750 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
751 description => "CPUs accessing this NUMA node.",
752 format_description => "id[-id];...",
753 },
754 memory => {
755 type => "number",
756 description => "Amount of memory this NUMA node provides.",
757 optional => 1,
758 },
759 hostnodes => {
760 type => "string",
761 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
762 description => "Host NUMA nodes to use.",
763 format_description => "id[-id];...",
764 optional => 1,
765 },
766 policy => {
767 type => 'string',
768 enum => [qw(preferred bind interleave)],
769 description => "NUMA allocation policy.",
770 optional => 1,
771 },
772};
773PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
774my $numadesc = {
775 optional => 1,
776 type => 'string', format => $numa_fmt,
777 description => "NUMA topology.",
778};
779PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
780
781for (my $i = 0; $i < $MAX_NUMA; $i++) {
782 $confdesc->{"numa$i"} = $numadesc;
783}
784
785my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000', 'pcnet', 'virtio',
786 'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
787 'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
788my $nic_model_list_txt = join(' ', sort @$nic_model_list);
789
790my $net_fmt_bridge_descr = <<__EOD__;
791Bridge to attach the network device to. The Proxmox VE standard bridge
792is called 'vmbr0'.
793
794If you do not specify a bridge, we create a kvm user (NATed) network
795device, which provides DHCP and DNS services. The following addresses
796are used:
797
798 10.0.2.2 Gateway
799 10.0.2.3 DNS Server
800 10.0.2.4 SMB Server
801
802The DHCP server assign addresses to the guest starting from 10.0.2.15.
803__EOD__
804
805my $net_fmt = {
806 macaddr => get_standard_option('mac-addr', {
807 description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
808 }),
809 model => {
810 type => 'string',
811 description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
812 enum => $nic_model_list,
813 default_key => 1,
814 },
815 (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
816 bridge => {
817 type => 'string',
818 description => $net_fmt_bridge_descr,
819 format_description => 'bridge',
820 optional => 1,
821 },
822 queues => {
823 type => 'integer',
824 minimum => 0, maximum => 16,
825 description => 'Number of packet queues to be used on the device.',
826 optional => 1,
827 },
828 rate => {
829 type => 'number',
830 minimum => 0,
831 description => "Rate limit in mbps (megabytes per second) as floating point number.",
832 optional => 1,
833 },
834 tag => {
835 type => 'integer',
836 minimum => 1, maximum => 4094,
837 description => 'VLAN tag to apply to packets on this interface.',
838 optional => 1,
839 },
840 trunks => {
841 type => 'string',
842 pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
843 description => 'VLAN trunks to pass through this interface.',
844 format_description => 'vlanid[;vlanid...]',
845 optional => 1,
846 },
847 firewall => {
848 type => 'boolean',
849 description => 'Whether this interface should be protected by the firewall.',
850 optional => 1,
851 },
852 link_down => {
853 type => 'boolean',
854 description => 'Whether this interface should be disconnected (like pulling the plug).',
855 optional => 1,
856 },
857};
858
859my $netdesc = {
860 optional => 1,
861 type => 'string', format => $net_fmt,
862 description => "Specify network devices.",
863};
864
865PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
866
867my $ipconfig_fmt = {
868 ip => {
869 type => 'string',
870 format => 'pve-ipv4-config',
871 format_description => 'IPv4Format/CIDR',
872 description => 'IPv4 address in CIDR format.',
873 optional => 1,
874 default => 'dhcp',
875 },
876 gw => {
877 type => 'string',
878 format => 'ipv4',
879 format_description => 'GatewayIPv4',
880 description => 'Default gateway for IPv4 traffic.',
881 optional => 1,
882 requires => 'ip',
883 },
884 ip6 => {
885 type => 'string',
886 format => 'pve-ipv6-config',
887 format_description => 'IPv6Format/CIDR',
888 description => 'IPv6 address in CIDR format.',
889 optional => 1,
890 default => 'dhcp',
891 },
892 gw6 => {
893 type => 'string',
894 format => 'ipv6',
895 format_description => 'GatewayIPv6',
896 description => 'Default gateway for IPv6 traffic.',
897 optional => 1,
898 requires => 'ip6',
899 },
900};
901PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
902my $ipconfigdesc = {
903 optional => 1,
904 type => 'string', format => 'pve-qm-ipconfig',
905 description => <<'EODESCR',
906cloud-init: Specify IP addresses and gateways for the corresponding interface.
907
908IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
909
910The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
911For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
912
913If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
914EODESCR
915};
916PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
917
918for (my $i = 0; $i < $MAX_NETS; $i++) {
919 $confdesc->{"net$i"} = $netdesc;
920 $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
921}
922
923foreach my $key (keys %$confdesc_cloudinit) {
924 $confdesc->{$key} = $confdesc_cloudinit->{$key};
925}
926
927PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
928sub verify_volume_id_or_qm_path {
929 my ($volid, $noerr) = @_;
930
931 if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
932 return $volid;
933 }
934
935 # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
936 $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
937 if ($@) {
938 return undef if $noerr;
939 die $@;
940 }
941 return $volid;
942}
943
944my $usb_fmt = {
945 host => {
946 default_key => 1,
947 type => 'string', format => 'pve-qm-usb-device',
948 format_description => 'HOSTUSBDEVICE|spice',
949 description => <<EODESCR,
950The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
951
952 'bus-port(.port)*' (decimal numbers) or
953 'vendor_id:product_id' (hexadeciaml numbers) or
954 'spice'
955
956You can use the 'lsusb -t' command to list existing usb devices.
957
958NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
959
960The value 'spice' can be used to add a usb redirection devices for spice.
961EODESCR
962 },
963 usb3 => {
964 optional => 1,
965 type => 'boolean',
966 description => "Specifies whether if given host option is a USB3 device or port.",
967 default => 0,
968 },
969};
970
971my $usbdesc = {
972 optional => 1,
973 type => 'string', format => $usb_fmt,
974 description => "Configure an USB device (n is 0 to 4).",
975};
976PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
977
978my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
979my $hostpci_fmt = {
980 host => {
981 default_key => 1,
982 type => 'string',
983 pattern => qr/$PCIRE(;$PCIRE)*/,
984 format_description => 'HOSTPCIID[;HOSTPCIID2...]',
985 description => <<EODESCR,
986Host PCI device pass through. The PCI ID of a host's PCI device or a list
987of PCI virtual functions of the host. HOSTPCIID syntax is:
988
989'bus:dev.func' (hexadecimal numbers)
990
991You can us the 'lspci' command to list existing PCI devices.
992EODESCR
993 },
994 rombar => {
995 type => 'boolean',
996 description => "Specify whether or not the device's ROM will be visible in the guest's memory map.",
997 optional => 1,
998 default => 1,
999 },
1000 romfile => {
1001 type => 'string',
1002 pattern => '[^,;]+',
1003 format_description => 'string',
1004 description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1005 optional => 1,
1006 },
1007 pcie => {
1008 type => 'boolean',
1009 description => "Choose the PCI-express bus (needs the 'q35' machine model).",
1010 optional => 1,
1011 default => 0,
1012 },
1013 'x-vga' => {
1014 type => 'boolean',
1015 description => "Enable vfio-vga device support.",
1016 optional => 1,
1017 default => 0,
1018 },
1019 'mdev' => {
1020 type => 'string',
1021 format_description => 'string',
1022 pattern => '[^/\.:]+',
1023 optional => 1,
1024 description => <<EODESCR
1025The type of mediated device to use.
1026An instance of this type will be created on startup of the VM and
1027will be cleaned up when the VM stops.
1028EODESCR
1029 }
1030};
1031PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1032
1033my $hostpcidesc = {
1034 optional => 1,
1035 type => 'string', format => 'pve-qm-hostpci',
1036 description => "Map host PCI devices into guest.",
1037 verbose_description => <<EODESCR,
1038Map host PCI devices into guest.
1039
1040NOTE: This option allows direct access to host hardware. So it is no longer
1041possible to migrate such machines - use with special care.
1042
1043CAUTION: Experimental! User reported problems with this option.
1044EODESCR
1045};
1046PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1047
1048my $serialdesc = {
1049 optional => 1,
1050 type => 'string',
1051 pattern => '(/dev/.+|socket)',
1052 description => "Create a serial device inside the VM (n is 0 to 3)",
1053 verbose_description => <<EODESCR,
1054Create a serial device inside the VM (n is 0 to 3), and pass through a
1055host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1056host side (use 'qm terminal' to open a terminal connection).
1057
1058NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1059
1060CAUTION: Experimental! User reported problems with this option.
1061EODESCR
1062};
1063
1064my $paralleldesc= {
1065 optional => 1,
1066 type => 'string',
1067 pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1068 description => "Map host parallel devices (n is 0 to 2).",
1069 verbose_description => <<EODESCR,
1070Map host parallel devices (n is 0 to 2).
1071
1072NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1073
1074CAUTION: Experimental! User reported problems with this option.
1075EODESCR
1076};
1077
1078for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
1079 $confdesc->{"parallel$i"} = $paralleldesc;
1080}
1081
1082for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
1083 $confdesc->{"serial$i"} = $serialdesc;
1084}
1085
1086for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
1087 $confdesc->{"hostpci$i"} = $hostpcidesc;
1088}
1089
1090for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1091 $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1092}
1093
1094for (my $i = 0; $i < $PVE::QemuServer::Drive::MAX_UNUSED_DISKS; $i++) {
1095 $confdesc->{"unused$i"} = $PVE::QemuServer::Drive::unuseddesc;
1096}
1097
1098for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
1099 $confdesc->{"usb$i"} = $usbdesc;
1100}
1101
1102my $kvm_api_version = 0;
1103
1104sub kvm_version {
1105 return $kvm_api_version if $kvm_api_version;
1106
1107 open my $fh, '<', '/dev/kvm'
1108 or return undef;
1109
1110 # 0xae00 => KVM_GET_API_VERSION
1111 $kvm_api_version = ioctl($fh, 0xae00, 0);
1112
1113 return $kvm_api_version;
1114}
1115
1116my $kvm_user_version = {};
1117my $kvm_mtime = {};
1118
1119sub kvm_user_version {
1120 my ($binary) = @_;
1121
1122 $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1123 my $st = stat($binary);
1124
1125 my $cachedmtime = $kvm_mtime->{$binary} // -1;
1126 return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1127 $cachedmtime == $st->mtime;
1128
1129 $kvm_user_version->{$binary} = 'unknown';
1130 $kvm_mtime->{$binary} = $st->mtime;
1131
1132 my $code = sub {
1133 my $line = shift;
1134 if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1135 $kvm_user_version->{$binary} = $2;
1136 }
1137 };
1138
1139 eval { run_command([$binary, '--version'], outfunc => $code); };
1140 warn $@ if $@;
1141
1142 return $kvm_user_version->{$binary};
1143
1144}
1145
1146sub kernel_has_vhost_net {
1147 return -c '/dev/vhost-net';
1148}
1149
1150sub option_exists {
1151 my $key = shift;
1152 return defined($confdesc->{$key});
1153}
1154
1155my $cdrom_path;
1156sub get_cdrom_path {
1157
1158 return $cdrom_path if $cdrom_path;
1159
1160 return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1161 return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1162 return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1163}
1164
1165sub get_iso_path {
1166 my ($storecfg, $vmid, $cdrom) = @_;
1167
1168 if ($cdrom eq 'cdrom') {
1169 return get_cdrom_path();
1170 } elsif ($cdrom eq 'none') {
1171 return '';
1172 } elsif ($cdrom =~ m|^/|) {
1173 return $cdrom;
1174 } else {
1175 return PVE::Storage::path($storecfg, $cdrom);
1176 }
1177}
1178
1179# try to convert old style file names to volume IDs
1180sub filename_to_volume_id {
1181 my ($vmid, $file, $media) = @_;
1182
1183 if (!($file eq 'none' || $file eq 'cdrom' ||
1184 $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1185
1186 return undef if $file =~ m|/|;
1187
1188 if ($media && $media eq 'cdrom') {
1189 $file = "local:iso/$file";
1190 } else {
1191 $file = "local:$vmid/$file";
1192 }
1193 }
1194
1195 return $file;
1196}
1197
1198sub verify_media_type {
1199 my ($opt, $vtype, $media) = @_;
1200
1201 return if !$media;
1202
1203 my $etype;
1204 if ($media eq 'disk') {
1205 $etype = 'images';
1206 } elsif ($media eq 'cdrom') {
1207 $etype = 'iso';
1208 } else {
1209 die "internal error";
1210 }
1211
1212 return if ($vtype eq $etype);
1213
1214 raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1215}
1216
1217sub cleanup_drive_path {
1218 my ($opt, $storecfg, $drive) = @_;
1219
1220 # try to convert filesystem paths to volume IDs
1221
1222 if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1223 ($drive->{file} !~ m|^/dev/.+|) &&
1224 ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1225 ($drive->{file} !~ m/^\d+$/)) {
1226 my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1227 raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1228 $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1229 verify_media_type($opt, $vtype, $drive->{media});
1230 $drive->{file} = $volid;
1231 }
1232
1233 $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1234}
1235
1236sub parse_hotplug_features {
1237 my ($data) = @_;
1238
1239 my $res = {};
1240
1241 return $res if $data eq '0';
1242
1243 $data = $confdesc->{hotplug}->{default} if $data eq '1';
1244
1245 foreach my $feature (PVE::Tools::split_list($data)) {
1246 if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1247 $res->{$1} = 1;
1248 } else {
1249 die "invalid hotplug feature '$feature'\n";
1250 }
1251 }
1252 return $res;
1253}
1254
1255PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1256sub pve_verify_hotplug_features {
1257 my ($value, $noerr) = @_;
1258
1259 return $value if parse_hotplug_features($value);
1260
1261 return undef if $noerr;
1262
1263 die "unable to parse hotplug option\n";
1264}
1265
1266sub scsi_inquiry {
1267 my($fh, $noerr) = @_;
1268
1269 my $SG_IO = 0x2285;
1270 my $SG_GET_VERSION_NUM = 0x2282;
1271
1272 my $versionbuf = "\x00" x 8;
1273 my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1274 if (!$ret) {
1275 die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1276 return undef;
1277 }
1278 my $version = unpack("I", $versionbuf);
1279 if ($version < 30000) {
1280 die "scsi generic interface too old\n" if !$noerr;
1281 return undef;
1282 }
1283
1284 my $buf = "\x00" x 36;
1285 my $sensebuf = "\x00" x 8;
1286 my $cmd = pack("C x3 C x1", 0x12, 36);
1287
1288 # see /usr/include/scsi/sg.h
1289 my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1290
1291 my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1292 length($sensebuf), 0, length($buf), $buf,
1293 $cmd, $sensebuf, 6000);
1294
1295 $ret = ioctl($fh, $SG_IO, $packet);
1296 if (!$ret) {
1297 die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1298 return undef;
1299 }
1300
1301 my @res = unpack($sg_io_hdr_t, $packet);
1302 if ($res[17] || $res[18]) {
1303 die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1304 return undef;
1305 }
1306
1307 my $res = {};
1308 (my $byte0, my $byte1, $res->{vendor},
1309 $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1310
1311 $res->{removable} = $byte1 & 128 ? 1 : 0;
1312 $res->{type} = $byte0 & 31;
1313
1314 return $res;
1315}
1316
1317sub path_is_scsi {
1318 my ($path) = @_;
1319
1320 my $fh = IO::File->new("+<$path") || return undef;
1321 my $res = scsi_inquiry($fh, 1);
1322 close($fh);
1323
1324 return $res;
1325}
1326
1327sub print_tabletdevice_full {
1328 my ($conf, $arch) = @_;
1329
1330 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1331
1332 # we use uhci for old VMs because tablet driver was buggy in older qemu
1333 my $usbbus;
1334 if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1335 $usbbus = 'ehci';
1336 } else {
1337 $usbbus = 'uhci';
1338 }
1339
1340 return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1341}
1342
1343sub print_keyboarddevice_full {
1344 my ($conf, $arch, $machine) = @_;
1345
1346 return undef if $arch ne 'aarch64';
1347
1348 return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1349}
1350
1351sub print_drivedevice_full {
1352 my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1353
1354 my $device = '';
1355 my $maxdev = 0;
1356
1357 if ($drive->{interface} eq 'virtio') {
1358 my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1359 $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1360 $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1361 } elsif ($drive->{interface} eq 'scsi') {
1362
1363 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1364 my $unit = $drive->{index} % $maxdev;
1365 my $devicetype = 'hd';
1366 my $path = '';
1367 if (drive_is_cdrom($drive)) {
1368 $devicetype = 'cd';
1369 } else {
1370 if ($drive->{file} =~ m|^/|) {
1371 $path = $drive->{file};
1372 if (my $info = path_is_scsi($path)) {
1373 if ($info->{type} == 0 && $drive->{scsiblock}) {
1374 $devicetype = 'block';
1375 } elsif ($info->{type} == 1) { # tape
1376 $devicetype = 'generic';
1377 }
1378 }
1379 } else {
1380 $path = PVE::Storage::path($storecfg, $drive->{file});
1381 }
1382
1383 # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1384 my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
1385 if ($path =~ m/^iscsi\:\/\// &&
1386 !min_version($version, 4, 1)) {
1387 $devicetype = 'generic';
1388 }
1389 }
1390
1391 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1392 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1393 } else {
1394 $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1395 }
1396
1397 if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1398 $device .= ",rotation_rate=1";
1399 }
1400 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1401
1402 } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1403 my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1404 my $controller = int($drive->{index} / $maxdev);
1405 my $unit = $drive->{index} % $maxdev;
1406 my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1407
1408 $device = "ide-$devicetype";
1409 if ($drive->{interface} eq 'ide') {
1410 $device .= ",bus=ide.$controller,unit=$unit";
1411 } else {
1412 $device .= ",bus=ahci$controller.$unit";
1413 }
1414 $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1415
1416 if ($devicetype eq 'hd') {
1417 if (my $model = $drive->{model}) {
1418 $model = URI::Escape::uri_unescape($model);
1419 $device .= ",model=$model";
1420 }
1421 if ($drive->{ssd}) {
1422 $device .= ",rotation_rate=1";
1423 }
1424 }
1425 $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1426 } elsif ($drive->{interface} eq 'usb') {
1427 die "implement me";
1428 # -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1429 } else {
1430 die "unsupported interface type";
1431 }
1432
1433 $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1434
1435 if (my $serial = $drive->{serial}) {
1436 $serial = URI::Escape::uri_unescape($serial);
1437 $device .= ",serial=$serial";
1438 }
1439
1440
1441 return $device;
1442}
1443
1444sub get_initiator_name {
1445 my $initiator;
1446
1447 my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1448 while (defined(my $line = <$fh>)) {
1449 next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1450 $initiator = $1;
1451 last;
1452 }
1453 $fh->close();
1454
1455 return $initiator;
1456}
1457
1458sub print_drive_commandline_full {
1459 my ($storecfg, $vmid, $drive) = @_;
1460
1461 my $path;
1462 my $volid = $drive->{file};
1463 my $format;
1464
1465 if (drive_is_cdrom($drive)) {
1466 $path = get_iso_path($storecfg, $vmid, $volid);
1467 } else {
1468 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1469 if ($storeid) {
1470 $path = PVE::Storage::path($storecfg, $volid);
1471 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1472 $format = qemu_img_format($scfg, $volname);
1473 } else {
1474 $path = $volid;
1475 $format = "raw";
1476 }
1477 }
1478
1479 my $opts = '';
1480 my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1481 foreach my $o (@qemu_drive_options) {
1482 $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1483 }
1484
1485 # snapshot only accepts on|off
1486 if (defined($drive->{snapshot})) {
1487 my $v = $drive->{snapshot} ? 'on' : 'off';
1488 $opts .= ",snapshot=$v";
1489 }
1490
1491 foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1492 my ($dir, $qmpname) = @$type;
1493 if (my $v = $drive->{"mbps$dir"}) {
1494 $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1495 }
1496 if (my $v = $drive->{"mbps${dir}_max"}) {
1497 $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1498 }
1499 if (my $v = $drive->{"bps${dir}_max_length"}) {
1500 $opts .= ",throttling.bps$qmpname-max-length=$v";
1501 }
1502 if (my $v = $drive->{"iops${dir}"}) {
1503 $opts .= ",throttling.iops$qmpname=$v";
1504 }
1505 if (my $v = $drive->{"iops${dir}_max"}) {
1506 $opts .= ",throttling.iops$qmpname-max=$v";
1507 }
1508 if (my $v = $drive->{"iops${dir}_max_length"}) {
1509 $opts .= ",throttling.iops$qmpname-max-length=$v";
1510 }
1511 }
1512
1513 $opts .= ",format=$format" if $format && !$drive->{format};
1514
1515 my $cache_direct = 0;
1516
1517 if (my $cache = $drive->{cache}) {
1518 $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1519 } elsif (!drive_is_cdrom($drive)) {
1520 $opts .= ",cache=none";
1521 $cache_direct = 1;
1522 }
1523
1524 # aio native works only with O_DIRECT
1525 if (!$drive->{aio}) {
1526 if($cache_direct) {
1527 $opts .= ",aio=native";
1528 } else {
1529 $opts .= ",aio=threads";
1530 }
1531 }
1532
1533 if (!drive_is_cdrom($drive)) {
1534 my $detectzeroes;
1535 if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1536 $detectzeroes = 'off';
1537 } elsif ($drive->{discard}) {
1538 $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1539 } else {
1540 # This used to be our default with discard not being specified:
1541 $detectzeroes = 'on';
1542 }
1543 $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1544 }
1545
1546 my $pathinfo = $path ? "file=$path," : '';
1547
1548 return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1549}
1550
1551sub print_netdevice_full {
1552 my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1553
1554 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1555
1556 my $device = $net->{model};
1557 if ($net->{model} eq 'virtio') {
1558 $device = 'virtio-net-pci';
1559 };
1560
1561 my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1562 my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1563 if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1564 #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1565 my $vectors = $net->{queues} * 2 + 2;
1566 $tmpstr .= ",vectors=$vectors,mq=on";
1567 }
1568 $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1569
1570 if ($use_old_bios_files) {
1571 my $romfile;
1572 if ($device eq 'virtio-net-pci') {
1573 $romfile = 'pxe-virtio.rom';
1574 } elsif ($device eq 'e1000') {
1575 $romfile = 'pxe-e1000.rom';
1576 } elsif ($device eq 'ne2k') {
1577 $romfile = 'pxe-ne2k_pci.rom';
1578 } elsif ($device eq 'pcnet') {
1579 $romfile = 'pxe-pcnet.rom';
1580 } elsif ($device eq 'rtl8139') {
1581 $romfile = 'pxe-rtl8139.rom';
1582 }
1583 $tmpstr .= ",romfile=$romfile" if $romfile;
1584 }
1585
1586 return $tmpstr;
1587}
1588
1589sub print_netdev_full {
1590 my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1591
1592 my $i = '';
1593 if ($netid =~ m/^net(\d+)$/) {
1594 $i = int($1);
1595 }
1596
1597 die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1598
1599 my $ifname = "tap${vmid}i$i";
1600
1601 # kvm uses TUNSETIFF ioctl, and that limits ifname length
1602 die "interface name '$ifname' is too long (max 15 character)\n"
1603 if length($ifname) >= 16;
1604
1605 my $vhostparam = '';
1606 if (is_native($arch)) {
1607 $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1608 }
1609
1610 my $vmname = $conf->{name} || "vm$vmid";
1611
1612 my $netdev = "";
1613 my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1614
1615 if ($net->{bridge}) {
1616 $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1617 } else {
1618 $netdev = "type=user,id=$netid,hostname=$vmname";
1619 }
1620
1621 $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1622
1623 return $netdev;
1624}
1625
1626my $vga_map = {
1627 'cirrus' => 'cirrus-vga',
1628 'std' => 'VGA',
1629 'vmware' => 'vmware-svga',
1630 'virtio' => 'virtio-vga',
1631};
1632
1633sub print_vga_device {
1634 my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1635
1636 my $type = $vga_map->{$vga->{type}};
1637 if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1638 $type = 'virtio-gpu';
1639 }
1640 my $vgamem_mb = $vga->{memory};
1641
1642 my $max_outputs = '';
1643 if ($qxlnum) {
1644 $type = $id ? 'qxl' : 'qxl-vga';
1645
1646 if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1647 # set max outputs so linux can have up to 4 qxl displays with one device
1648 if (min_version($machine_version, 4, 1)) {
1649 $max_outputs = ",max_outputs=4";
1650 }
1651 }
1652 }
1653
1654 die "no devicetype for $vga->{type}\n" if !$type;
1655
1656 my $memory = "";
1657 if ($vgamem_mb) {
1658 if ($vga->{type} eq 'virtio') {
1659 my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1660 $memory = ",max_hostmem=$bytes";
1661 } elsif ($qxlnum) {
1662 # from https://www.spice-space.org/multiple-monitors.html
1663 $memory = ",vgamem_mb=$vga->{memory}";
1664 my $ram = $vgamem_mb * 4;
1665 my $vram = $vgamem_mb * 2;
1666 $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1667 } else {
1668 $memory = ",vgamem_mb=$vga->{memory}";
1669 }
1670 } elsif ($qxlnum && $id) {
1671 $memory = ",ram_size=67108864,vram_size=33554432";
1672 }
1673
1674 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1675 my $vgaid = "vga" . ($id // '');
1676 my $pciaddr;
1677
1678 if ($q35 && $vgaid eq 'vga') {
1679 # the first display uses pcie.0 bus on q35 machines
1680 $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1681 } else {
1682 $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1683 }
1684
1685 return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}";
1686}
1687
1688sub parse_number_sets {
1689 my ($set) = @_;
1690 my $res = [];
1691 foreach my $part (split(/;/, $set)) {
1692 if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1693 die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1694 push @$res, [ $1, $2 ];
1695 } else {
1696 die "invalid range: $part\n";
1697 }
1698 }
1699 return $res;
1700}
1701
1702sub parse_numa {
1703 my ($data) = @_;
1704
1705 my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1706 $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1707 $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1708 return $res;
1709}
1710
1711sub parse_hostpci {
1712 my ($value) = @_;
1713
1714 return undef if !$value;
1715
1716 my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1717
1718 my @idlist = split(/;/, $res->{host});
1719 delete $res->{host};
1720 foreach my $id (@idlist) {
1721 my $devs = PVE::SysFSTools::lspci($id);
1722 die "no PCI device found for '$id'\n" if !scalar(@$devs);
1723 push @{$res->{pciid}}, @$devs;
1724 }
1725 return $res;
1726}
1727
1728# netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1729sub parse_net {
1730 my ($data) = @_;
1731
1732 my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1733 if ($@) {
1734 warn $@;
1735 return undef;
1736 }
1737 if (!defined($res->{macaddr})) {
1738 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1739 $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1740 }
1741 return $res;
1742}
1743
1744# ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1745sub parse_ipconfig {
1746 my ($data) = @_;
1747
1748 my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1749 if ($@) {
1750 warn $@;
1751 return undef;
1752 }
1753
1754 if ($res->{gw} && !$res->{ip}) {
1755 warn 'gateway specified without specifying an IP address';
1756 return undef;
1757 }
1758 if ($res->{gw6} && !$res->{ip6}) {
1759 warn 'IPv6 gateway specified without specifying an IPv6 address';
1760 return undef;
1761 }
1762 if ($res->{gw} && $res->{ip} eq 'dhcp') {
1763 warn 'gateway specified together with DHCP';
1764 return undef;
1765 }
1766 if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1767 # gw6 + auto/dhcp
1768 warn "IPv6 gateway specified together with $res->{ip6} address";
1769 return undef;
1770 }
1771
1772 if (!$res->{ip} && !$res->{ip6}) {
1773 return { ip => 'dhcp', ip6 => 'dhcp' };
1774 }
1775
1776 return $res;
1777}
1778
1779sub print_net {
1780 my $net = shift;
1781
1782 return PVE::JSONSchema::print_property_string($net, $net_fmt);
1783}
1784
1785sub add_random_macs {
1786 my ($settings) = @_;
1787
1788 foreach my $opt (keys %$settings) {
1789 next if $opt !~ m/^net(\d+)$/;
1790 my $net = parse_net($settings->{$opt});
1791 next if !$net;
1792 $settings->{$opt} = print_net($net);
1793 }
1794}
1795
1796sub vm_is_volid_owner {
1797 my ($storecfg, $vmid, $volid) = @_;
1798
1799 if ($volid !~ m|^/|) {
1800 my ($path, $owner);
1801 eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1802 if ($owner && ($owner == $vmid)) {
1803 return 1;
1804 }
1805 }
1806
1807 return undef;
1808}
1809
1810sub vmconfig_register_unused_drive {
1811 my ($storecfg, $vmid, $conf, $drive) = @_;
1812
1813 if (drive_is_cloudinit($drive)) {
1814 eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1815 warn $@ if $@;
1816 } elsif (!drive_is_cdrom($drive)) {
1817 my $volid = $drive->{file};
1818 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1819 PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1820 }
1821 }
1822}
1823
1824# smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1825my $smbios1_fmt = {
1826 uuid => {
1827 type => 'string',
1828 pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1829 format_description => 'UUID',
1830 description => "Set SMBIOS1 UUID.",
1831 optional => 1,
1832 },
1833 version => {
1834 type => 'string',
1835 pattern => '[A-Za-z0-9+\/]+={0,2}',
1836 format_description => 'Base64 encoded string',
1837 description => "Set SMBIOS1 version.",
1838 optional => 1,
1839 },
1840 serial => {
1841 type => 'string',
1842 pattern => '[A-Za-z0-9+\/]+={0,2}',
1843 format_description => 'Base64 encoded string',
1844 description => "Set SMBIOS1 serial number.",
1845 optional => 1,
1846 },
1847 manufacturer => {
1848 type => 'string',
1849 pattern => '[A-Za-z0-9+\/]+={0,2}',
1850 format_description => 'Base64 encoded string',
1851 description => "Set SMBIOS1 manufacturer.",
1852 optional => 1,
1853 },
1854 product => {
1855 type => 'string',
1856 pattern => '[A-Za-z0-9+\/]+={0,2}',
1857 format_description => 'Base64 encoded string',
1858 description => "Set SMBIOS1 product ID.",
1859 optional => 1,
1860 },
1861 sku => {
1862 type => 'string',
1863 pattern => '[A-Za-z0-9+\/]+={0,2}',
1864 format_description => 'Base64 encoded string',
1865 description => "Set SMBIOS1 SKU string.",
1866 optional => 1,
1867 },
1868 family => {
1869 type => 'string',
1870 pattern => '[A-Za-z0-9+\/]+={0,2}',
1871 format_description => 'Base64 encoded string',
1872 description => "Set SMBIOS1 family string.",
1873 optional => 1,
1874 },
1875 base64 => {
1876 type => 'boolean',
1877 description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1878 optional => 1,
1879 },
1880};
1881
1882sub parse_smbios1 {
1883 my ($data) = @_;
1884
1885 my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1886 warn $@ if $@;
1887 return $res;
1888}
1889
1890sub print_smbios1 {
1891 my ($smbios1) = @_;
1892 return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1893}
1894
1895PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1896
1897sub parse_watchdog {
1898 my ($value) = @_;
1899
1900 return undef if !$value;
1901
1902 my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1903 warn $@ if $@;
1904 return $res;
1905}
1906
1907sub parse_guest_agent {
1908 my ($value) = @_;
1909
1910 return {} if !defined($value->{agent});
1911
1912 my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
1913 warn $@ if $@;
1914
1915 # if the agent is disabled ignore the other potentially set properties
1916 return {} if !$res->{enabled};
1917 return $res;
1918}
1919
1920sub parse_vga {
1921 my ($value) = @_;
1922
1923 return {} if !$value;
1924 my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
1925 warn $@ if $@;
1926 return $res;
1927}
1928
1929sub parse_rng {
1930 my ($value) = @_;
1931
1932 return undef if !$value;
1933
1934 my $res = eval { PVE::JSONSchema::parse_property_string($rng_fmt, $value) };
1935 warn $@ if $@;
1936 return $res;
1937}
1938
1939PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1940sub verify_usb_device {
1941 my ($value, $noerr) = @_;
1942
1943 return $value if parse_usb_device($value);
1944
1945 return undef if $noerr;
1946
1947 die "unable to parse usb device\n";
1948}
1949
1950# add JSON properties for create and set function
1951sub json_config_properties {
1952 my $prop = shift;
1953
1954 foreach my $opt (keys %$confdesc) {
1955 next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
1956 $prop->{$opt} = $confdesc->{$opt};
1957 }
1958
1959 return $prop;
1960}
1961
1962# return copy of $confdesc_cloudinit to generate documentation
1963sub cloudinit_config_properties {
1964
1965 return dclone($confdesc_cloudinit);
1966}
1967
1968sub check_type {
1969 my ($key, $value) = @_;
1970
1971 die "unknown setting '$key'\n" if !$confdesc->{$key};
1972
1973 my $type = $confdesc->{$key}->{type};
1974
1975 if (!defined($value)) {
1976 die "got undefined value\n";
1977 }
1978
1979 if ($value =~ m/[\n\r]/) {
1980 die "property contains a line feed\n";
1981 }
1982
1983 if ($type eq 'boolean') {
1984 return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
1985 return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
1986 die "type check ('boolean') failed - got '$value'\n";
1987 } elsif ($type eq 'integer') {
1988 return int($1) if $value =~ m/^(\d+)$/;
1989 die "type check ('integer') failed - got '$value'\n";
1990 } elsif ($type eq 'number') {
1991 return $value if $value =~ m/^(\d+)(\.\d+)?$/;
1992 die "type check ('number') failed - got '$value'\n";
1993 } elsif ($type eq 'string') {
1994 if (my $fmt = $confdesc->{$key}->{format}) {
1995 PVE::JSONSchema::check_format($fmt, $value);
1996 return $value;
1997 }
1998 $value =~ s/^\"(.*)\"$/$1/;
1999 return $value;
2000 } else {
2001 die "internal error"
2002 }
2003}
2004
2005sub destroy_vm {
2006 my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2007
2008 my $conf = PVE::QemuConfig->load_config($vmid);
2009
2010 PVE::QemuConfig->check_lock($conf) if !$skiplock;
2011
2012 if ($conf->{template}) {
2013 # check if any base image is still used by a linked clone
2014 foreach_drive($conf, sub {
2015 my ($ds, $drive) = @_;
2016 return if drive_is_cdrom($drive);
2017
2018 my $volid = $drive->{file};
2019 return if !$volid || $volid =~ m|^/|;
2020
2021 die "base volume '$volid' is still in use by linked cloned\n"
2022 if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2023
2024 });
2025 }
2026
2027 # only remove disks owned by this VM
2028 foreach_drive($conf, sub {
2029 my ($ds, $drive) = @_;
2030 return if drive_is_cdrom($drive, 1);
2031
2032 my $volid = $drive->{file};
2033 return if !$volid || $volid =~ m|^/|;
2034
2035 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2036 return if !$path || !$owner || ($owner != $vmid);
2037
2038 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2039 warn "Could not remove disk '$volid', check manually: $@" if $@;
2040 });
2041
2042 # also remove unused disk
2043 my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2044 PVE::Storage::foreach_volid($vmdisks, sub {
2045 my ($volid, $sid, $volname, $d) = @_;
2046 eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2047 warn $@ if $@;
2048 });
2049
2050 if (defined $replacement_conf) {
2051 PVE::QemuConfig->write_config($vmid, $replacement_conf);
2052 } else {
2053 PVE::QemuConfig->destroy_config($vmid);
2054 }
2055}
2056
2057sub parse_vm_config {
2058 my ($filename, $raw) = @_;
2059
2060 return undef if !defined($raw);
2061
2062 my $res = {
2063 digest => Digest::SHA::sha1_hex($raw),
2064 snapshots => {},
2065 pending => {},
2066 };
2067
2068 $filename =~ m|/qemu-server/(\d+)\.conf$|
2069 || die "got strange filename '$filename'";
2070
2071 my $vmid = $1;
2072
2073 my $conf = $res;
2074 my $descr;
2075 my $section = '';
2076
2077 my @lines = split(/\n/, $raw);
2078 foreach my $line (@lines) {
2079 next if $line =~ m/^\s*$/;
2080
2081 if ($line =~ m/^\[PENDING\]\s*$/i) {
2082 $section = 'pending';
2083 if (defined($descr)) {
2084 $descr =~ s/\s+$//;
2085 $conf->{description} = $descr;
2086 }
2087 $descr = undef;
2088 $conf = $res->{$section} = {};
2089 next;
2090
2091 } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2092 $section = $1;
2093 if (defined($descr)) {
2094 $descr =~ s/\s+$//;
2095 $conf->{description} = $descr;
2096 }
2097 $descr = undef;
2098 $conf = $res->{snapshots}->{$section} = {};
2099 next;
2100 }
2101
2102 if ($line =~ m/^\#(.*)\s*$/) {
2103 $descr = '' if !defined($descr);
2104 $descr .= PVE::Tools::decode_text($1) . "\n";
2105 next;
2106 }
2107
2108 if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2109 $descr = '' if !defined($descr);
2110 $descr .= PVE::Tools::decode_text($2);
2111 } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2112 $conf->{snapstate} = $1;
2113 } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2114 my $key = $1;
2115 my $value = $2;
2116 $conf->{$key} = $value;
2117 } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2118 my $value = $1;
2119 if ($section eq 'pending') {
2120 $conf->{delete} = $value; # we parse this later
2121 } else {
2122 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2123 }
2124 } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2125 my $key = $1;
2126 my $value = $2;
2127 eval { $value = check_type($key, $value); };
2128 if ($@) {
2129 warn "vm $vmid - unable to parse value of '$key' - $@";
2130 } else {
2131 $key = 'ide2' if $key eq 'cdrom';
2132 my $fmt = $confdesc->{$key}->{format};
2133 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2134 my $v = parse_drive($key, $value);
2135 if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2136 $v->{file} = $volid;
2137 $value = print_drive($v);
2138 } else {
2139 warn "vm $vmid - unable to parse value of '$key'\n";
2140 next;
2141 }
2142 }
2143
2144 $conf->{$key} = $value;
2145 }
2146 }
2147 }
2148
2149 if (defined($descr)) {
2150 $descr =~ s/\s+$//;
2151 $conf->{description} = $descr;
2152 }
2153 delete $res->{snapstate}; # just to be sure
2154
2155 return $res;
2156}
2157
2158sub write_vm_config {
2159 my ($filename, $conf) = @_;
2160
2161 delete $conf->{snapstate}; # just to be sure
2162
2163 if ($conf->{cdrom}) {
2164 die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2165 $conf->{ide2} = $conf->{cdrom};
2166 delete $conf->{cdrom};
2167 }
2168
2169 # we do not use 'smp' any longer
2170 if ($conf->{sockets}) {
2171 delete $conf->{smp};
2172 } elsif ($conf->{smp}) {
2173 $conf->{sockets} = $conf->{smp};
2174 delete $conf->{cores};
2175 delete $conf->{smp};
2176 }
2177
2178 my $used_volids = {};
2179
2180 my $cleanup_config = sub {
2181 my ($cref, $pending, $snapname) = @_;
2182
2183 foreach my $key (keys %$cref) {
2184 next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2185 $key eq 'snapstate' || $key eq 'pending';
2186 my $value = $cref->{$key};
2187 if ($key eq 'delete') {
2188 die "propertry 'delete' is only allowed in [PENDING]\n"
2189 if !$pending;
2190 # fixme: check syntax?
2191 next;
2192 }
2193 eval { $value = check_type($key, $value); };
2194 die "unable to parse value of '$key' - $@" if $@;
2195
2196 $cref->{$key} = $value;
2197
2198 if (!$snapname && is_valid_drivename($key)) {
2199 my $drive = parse_drive($key, $value);
2200 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2201 }
2202 }
2203 };
2204
2205 &$cleanup_config($conf);
2206
2207 &$cleanup_config($conf->{pending}, 1);
2208
2209 foreach my $snapname (keys %{$conf->{snapshots}}) {
2210 die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2211 &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2212 }
2213
2214 # remove 'unusedX' settings if we re-add a volume
2215 foreach my $key (keys %$conf) {
2216 my $value = $conf->{$key};
2217 if ($key =~ m/^unused/ && $used_volids->{$value}) {
2218 delete $conf->{$key};
2219 }
2220 }
2221
2222 my $generate_raw_config = sub {
2223 my ($conf, $pending) = @_;
2224
2225 my $raw = '';
2226
2227 # add description as comment to top of file
2228 if (defined(my $descr = $conf->{description})) {
2229 if ($descr) {
2230 foreach my $cl (split(/\n/, $descr)) {
2231 $raw .= '#' . PVE::Tools::encode_text($cl) . "\n";
2232 }
2233 } else {
2234 $raw .= "#\n" if $pending;
2235 }
2236 }
2237
2238 foreach my $key (sort keys %$conf) {
2239 next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2240 $raw .= "$key: $conf->{$key}\n";
2241 }
2242 return $raw;
2243 };
2244
2245 my $raw = &$generate_raw_config($conf);
2246
2247 if (scalar(keys %{$conf->{pending}})){
2248 $raw .= "\n[PENDING]\n";
2249 $raw .= &$generate_raw_config($conf->{pending}, 1);
2250 }
2251
2252 foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2253 $raw .= "\n[$snapname]\n";
2254 $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2255 }
2256
2257 return $raw;
2258}
2259
2260sub load_defaults {
2261
2262 my $res = {};
2263
2264 # we use static defaults from our JSON schema configuration
2265 foreach my $key (keys %$confdesc) {
2266 if (defined(my $default = $confdesc->{$key}->{default})) {
2267 $res->{$key} = $default;
2268 }
2269 }
2270
2271 return $res;
2272}
2273
2274sub config_list {
2275 my $vmlist = PVE::Cluster::get_vmlist();
2276 my $res = {};
2277 return $res if !$vmlist || !$vmlist->{ids};
2278 my $ids = $vmlist->{ids};
2279 my $nodename = nodename();
2280
2281 foreach my $vmid (keys %$ids) {
2282 my $d = $ids->{$vmid};
2283 next if !$d->{node} || $d->{node} ne $nodename;
2284 next if !$d->{type} || $d->{type} ne 'qemu';
2285 $res->{$vmid}->{exists} = 1;
2286 }
2287 return $res;
2288}
2289
2290# test if VM uses local resources (to prevent migration)
2291sub check_local_resources {
2292 my ($conf, $noerr) = @_;
2293
2294 my @loc_res = ();
2295
2296 push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2297 push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2298
2299 push @loc_res, "ivshmem" if $conf->{ivshmem};
2300
2301 foreach my $k (keys %$conf) {
2302 next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2303 # sockets are safe: they will recreated be on the target side post-migrate
2304 next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2305 push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2306 }
2307
2308 die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2309
2310 return \@loc_res;
2311}
2312
2313# check if used storages are available on all nodes (use by migrate)
2314sub check_storage_availability {
2315 my ($storecfg, $conf, $node) = @_;
2316
2317 foreach_drive($conf, sub {
2318 my ($ds, $drive) = @_;
2319
2320 my $volid = $drive->{file};
2321 return if !$volid;
2322
2323 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2324 return if !$sid;
2325
2326 # check if storage is available on both nodes
2327 my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2328 PVE::Storage::storage_check_node($storecfg, $sid, $node);
2329 });
2330}
2331
2332# list nodes where all VM images are available (used by has_feature API)
2333sub shared_nodes {
2334 my ($conf, $storecfg) = @_;
2335
2336 my $nodelist = PVE::Cluster::get_nodelist();
2337 my $nodehash = { map { $_ => 1 } @$nodelist };
2338 my $nodename = nodename();
2339
2340 foreach_drive($conf, sub {
2341 my ($ds, $drive) = @_;
2342
2343 my $volid = $drive->{file};
2344 return if !$volid;
2345
2346 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2347 if ($storeid) {
2348 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2349 if ($scfg->{disable}) {
2350 $nodehash = {};
2351 } elsif (my $avail = $scfg->{nodes}) {
2352 foreach my $node (keys %$nodehash) {
2353 delete $nodehash->{$node} if !$avail->{$node};
2354 }
2355 } elsif (!$scfg->{shared}) {
2356 foreach my $node (keys %$nodehash) {
2357 delete $nodehash->{$node} if $node ne $nodename
2358 }
2359 }
2360 }
2361 });
2362
2363 return $nodehash
2364}
2365
2366sub check_local_storage_availability {
2367 my ($conf, $storecfg) = @_;
2368
2369 my $nodelist = PVE::Cluster::get_nodelist();
2370 my $nodehash = { map { $_ => {} } @$nodelist };
2371
2372 foreach_drive($conf, sub {
2373 my ($ds, $drive) = @_;
2374
2375 my $volid = $drive->{file};
2376 return if !$volid;
2377
2378 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2379 if ($storeid) {
2380 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2381
2382 if ($scfg->{disable}) {
2383 foreach my $node (keys %$nodehash) {
2384 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2385 }
2386 } elsif (my $avail = $scfg->{nodes}) {
2387 foreach my $node (keys %$nodehash) {
2388 if (!$avail->{$node}) {
2389 $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2390 }
2391 }
2392 }
2393 }
2394 });
2395
2396 foreach my $node (values %$nodehash) {
2397 if (my $unavail = $node->{unavailable_storages}) {
2398 $node->{unavailable_storages} = [ sort keys %$unavail ];
2399 }
2400 }
2401
2402 return $nodehash
2403}
2404
2405# Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2406sub check_running {
2407 my ($vmid, $nocheck, $node) = @_;
2408
2409 PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2410 return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2411}
2412
2413sub vzlist {
2414
2415 my $vzlist = config_list();
2416
2417 my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2418
2419 while (defined(my $de = $fd->read)) {
2420 next if $de !~ m/^(\d+)\.pid$/;
2421 my $vmid = $1;
2422 next if !defined($vzlist->{$vmid});
2423 if (my $pid = check_running($vmid)) {
2424 $vzlist->{$vmid}->{pid} = $pid;
2425 }
2426 }
2427
2428 return $vzlist;
2429}
2430
2431our $vmstatus_return_properties = {
2432 vmid => get_standard_option('pve-vmid'),
2433 status => {
2434 description => "Qemu process status.",
2435 type => 'string',
2436 enum => ['stopped', 'running'],
2437 },
2438 maxmem => {
2439 description => "Maximum memory in bytes.",
2440 type => 'integer',
2441 optional => 1,
2442 renderer => 'bytes',
2443 },
2444 maxdisk => {
2445 description => "Root disk size in bytes.",
2446 type => 'integer',
2447 optional => 1,
2448 renderer => 'bytes',
2449 },
2450 name => {
2451 description => "VM name.",
2452 type => 'string',
2453 optional => 1,
2454 },
2455 qmpstatus => {
2456 description => "Qemu QMP agent status.",
2457 type => 'string',
2458 optional => 1,
2459 },
2460 pid => {
2461 description => "PID of running qemu process.",
2462 type => 'integer',
2463 optional => 1,
2464 },
2465 uptime => {
2466 description => "Uptime.",
2467 type => 'integer',
2468 optional => 1,
2469 renderer => 'duration',
2470 },
2471 cpus => {
2472 description => "Maximum usable CPUs.",
2473 type => 'number',
2474 optional => 1,
2475 },
2476 lock => {
2477 description => "The current config lock, if any.",
2478 type => 'string',
2479 optional => 1,
2480 },
2481 tags => {
2482 description => "The current configured tags, if any",
2483 type => 'string',
2484 optional => 1,
2485 },
2486};
2487
2488my $last_proc_pid_stat;
2489
2490# get VM status information
2491# This must be fast and should not block ($full == false)
2492# We only query KVM using QMP if $full == true (this can be slow)
2493sub vmstatus {
2494 my ($opt_vmid, $full) = @_;
2495
2496 my $res = {};
2497
2498 my $storecfg = PVE::Storage::config();
2499
2500 my $list = vzlist();
2501 my $defaults = load_defaults();
2502
2503 my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2504
2505 my $cpucount = $cpuinfo->{cpus} || 1;
2506
2507 foreach my $vmid (keys %$list) {
2508 next if $opt_vmid && ($vmid ne $opt_vmid);
2509
2510 my $conf = PVE::QemuConfig->load_config($vmid);
2511
2512 my $d = { vmid => $vmid };
2513 $d->{pid} = $list->{$vmid}->{pid};
2514
2515 # fixme: better status?
2516 $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2517
2518 my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2519 if (defined($size)) {
2520 $d->{disk} = 0; # no info available
2521 $d->{maxdisk} = $size;
2522 } else {
2523 $d->{disk} = 0;
2524 $d->{maxdisk} = 0;
2525 }
2526
2527 $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2528 * ($conf->{cores} || $defaults->{cores});
2529 $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2530 $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2531
2532 $d->{name} = $conf->{name} || "VM $vmid";
2533 $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2534 : $defaults->{memory}*(1024*1024);
2535
2536 if ($conf->{balloon}) {
2537 $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2538 $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2539 : $defaults->{shares};
2540 }
2541
2542 $d->{uptime} = 0;
2543 $d->{cpu} = 0;
2544 $d->{mem} = 0;
2545
2546 $d->{netout} = 0;
2547 $d->{netin} = 0;
2548
2549 $d->{diskread} = 0;
2550 $d->{diskwrite} = 0;
2551
2552 $d->{template} = PVE::QemuConfig->is_template($conf);
2553
2554 $d->{serial} = 1 if conf_has_serial($conf);
2555 $d->{lock} = $conf->{lock} if $conf->{lock};
2556 $d->{tags} = $conf->{tags} if defined($conf->{tags});
2557
2558 $res->{$vmid} = $d;
2559 }
2560
2561 my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2562 foreach my $dev (keys %$netdev) {
2563 next if $dev !~ m/^tap([1-9]\d*)i/;
2564 my $vmid = $1;
2565 my $d = $res->{$vmid};
2566 next if !$d;
2567
2568 $d->{netout} += $netdev->{$dev}->{receive};
2569 $d->{netin} += $netdev->{$dev}->{transmit};
2570
2571 if ($full) {
2572 $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2573 $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2574 }
2575
2576 }
2577
2578 my $ctime = gettimeofday;
2579
2580 foreach my $vmid (keys %$list) {
2581
2582 my $d = $res->{$vmid};
2583 my $pid = $d->{pid};
2584 next if !$pid;
2585
2586 my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2587 next if !$pstat; # not running
2588
2589 my $used = $pstat->{utime} + $pstat->{stime};
2590
2591 $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2592
2593 if ($pstat->{vsize}) {
2594 $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2595 }
2596
2597 my $old = $last_proc_pid_stat->{$pid};
2598 if (!$old) {
2599 $last_proc_pid_stat->{$pid} = {
2600 time => $ctime,
2601 used => $used,
2602 cpu => 0,
2603 };
2604 next;
2605 }
2606
2607 my $dtime = ($ctime - $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2608
2609 if ($dtime > 1000) {
2610 my $dutime = $used - $old->{used};
2611
2612 $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2613 $last_proc_pid_stat->{$pid} = {
2614 time => $ctime,
2615 used => $used,
2616 cpu => $d->{cpu},
2617 };
2618 } else {
2619 $d->{cpu} = $old->{cpu};
2620 }
2621 }
2622
2623 return $res if !$full;
2624
2625 my $qmpclient = PVE::QMPClient->new();
2626
2627 my $ballooncb = sub {
2628 my ($vmid, $resp) = @_;
2629
2630 my $info = $resp->{'return'};
2631 return if !$info->{max_mem};
2632
2633 my $d = $res->{$vmid};
2634
2635 # use memory assigned to VM
2636 $d->{maxmem} = $info->{max_mem};
2637 $d->{balloon} = $info->{actual};
2638
2639 if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2640 $d->{mem} = $info->{total_mem} - $info->{free_mem};
2641 $d->{freemem} = $info->{free_mem};
2642 }
2643
2644 $d->{ballooninfo} = $info;
2645 };
2646
2647 my $blockstatscb = sub {
2648 my ($vmid, $resp) = @_;
2649 my $data = $resp->{'return'} || [];
2650 my $totalrdbytes = 0;
2651 my $totalwrbytes = 0;
2652
2653 for my $blockstat (@$data) {
2654 $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2655 $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2656
2657 $blockstat->{device} =~ s/drive-//;
2658 $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2659 }
2660 $res->{$vmid}->{diskread} = $totalrdbytes;
2661 $res->{$vmid}->{diskwrite} = $totalwrbytes;
2662 };
2663
2664 my $statuscb = sub {
2665 my ($vmid, $resp) = @_;
2666
2667 $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2668 # this fails if ballon driver is not loaded, so this must be
2669 # the last commnand (following command are aborted if this fails).
2670 $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2671
2672 my $status = 'unknown';
2673 if (!defined($status = $resp->{'return'}->{status})) {
2674 warn "unable to get VM status\n";
2675 return;
2676 }
2677
2678 $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2679 };
2680
2681 foreach my $vmid (keys %$list) {
2682 next if $opt_vmid && ($vmid ne $opt_vmid);
2683 next if !$res->{$vmid}->{pid}; # not running
2684 $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2685 }
2686
2687 $qmpclient->queue_execute(undef, 2);
2688
2689 foreach my $vmid (keys %$list) {
2690 next if $opt_vmid && ($vmid ne $opt_vmid);
2691 $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2692 }
2693
2694 return $res;
2695}
2696
2697sub conf_has_serial {
2698 my ($conf) = @_;
2699
2700 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
2701 if ($conf->{"serial$i"}) {
2702 return 1;
2703 }
2704 }
2705
2706 return 0;
2707}
2708
2709sub conf_has_audio {
2710 my ($conf, $id) = @_;
2711
2712 $id //= 0;
2713 my $audio = $conf->{"audio$id"};
2714 return undef if !defined($audio);
2715
2716 my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
2717 my $audiodriver = $audioproperties->{driver} // 'spice';
2718
2719 return {
2720 dev => $audioproperties->{device},
2721 dev_id => "audiodev$id",
2722 backend => $audiodriver,
2723 backend_id => "$audiodriver-backend${id}",
2724 };
2725}
2726
2727sub vga_conf_has_spice {
2728 my ($vga) = @_;
2729
2730 my $vgaconf = parse_vga($vga);
2731 my $vgatype = $vgaconf->{type};
2732 return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2733
2734 return $1 || 1;
2735}
2736
2737sub is_native($) {
2738 my ($arch) = @_;
2739 return get_host_arch() eq $arch;
2740}
2741
2742sub get_vm_arch {
2743 my ($conf) = @_;
2744 return $conf->{arch} // get_host_arch();
2745}
2746
2747my $default_machines = {
2748 x86_64 => 'pc',
2749 aarch64 => 'virt',
2750};
2751
2752sub get_vm_machine {
2753 my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2754
2755 my $machine = $forcemachine || $conf->{machine};
2756
2757 if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2758 $arch //= 'x86_64';
2759 $machine ||= $default_machines->{$arch};
2760 if ($add_pve_version) {
2761 $kvmversion //= kvm_user_version();
2762 my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2763 $machine .= "+pve$pvever";
2764 }
2765 }
2766
2767 if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2768 # for version-pinned machines that do not include a pve-version (e.g.
2769 # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2770 $machine .= '+pve0';
2771 }
2772
2773 return $machine;
2774}
2775
2776sub get_ovmf_files($) {
2777 my ($arch) = @_;
2778
2779 my $ovmf = $OVMF->{$arch}
2780 or die "no OVMF images known for architecture '$arch'\n";
2781
2782 return @$ovmf;
2783}
2784
2785my $Arch2Qemu = {
2786 aarch64 => '/usr/bin/qemu-system-aarch64',
2787 x86_64 => '/usr/bin/qemu-system-x86_64',
2788};
2789sub get_command_for_arch($) {
2790 my ($arch) = @_;
2791 return '/usr/bin/kvm' if is_native($arch);
2792
2793 my $cmd = $Arch2Qemu->{$arch}
2794 or die "don't know how to emulate architecture '$arch'\n";
2795 return $cmd;
2796}
2797
2798# To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2799# to use in a QEMU command line (-cpu element), first array_intersect the result
2800# of query_supported_ with query_understood_. This is necessary because:
2801#
2802# a) query_understood_ returns flags the host cannot use and
2803# b) query_supported_ (rather the QMP call) doesn't actually return CPU
2804# flags, but CPU settings - with most of them being flags. Those settings
2805# (and some flags, curiously) cannot be specified as a "-cpu" argument.
2806#
2807# query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2808# expensive. If you need the value returned from this, you can get it much
2809# cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2810# $accel being 'kvm' or 'tcg'.
2811#
2812# pvestatd calls this function on startup and whenever the QEMU/KVM version
2813# changes, automatically populating pmxcfs.
2814#
2815# Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2816# since kvm and tcg machines support different flags
2817#
2818sub query_supported_cpu_flags {
2819 my ($arch) = @_;
2820
2821 $arch //= get_host_arch();
2822 my $default_machine = $default_machines->{$arch};
2823
2824 my $flags = {};
2825
2826 # FIXME: Once this is merged, the code below should work for ARM as well:
2827 # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2828 die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2829 $arch eq "aarch64";
2830
2831 my $kvm_supported = defined(kvm_version());
2832 my $qemu_cmd = get_command_for_arch($arch);
2833 my $fakevmid = -1;
2834 my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2835
2836 # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2837 my $query_supported_run_qemu = sub {
2838 my ($kvm) = @_;
2839
2840 my $flags = {};
2841 my $cmd = [
2842 $qemu_cmd,
2843 '-machine', $default_machine,
2844 '-display', 'none',
2845 '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2846 '-mon', 'chardev=qmp,mode=control',
2847 '-pidfile', $pidfile,
2848 '-S', '-daemonize'
2849 ];
2850
2851 if (!$kvm) {
2852 push @$cmd, '-accel', 'tcg';
2853 }
2854
2855 my $rc = run_command($cmd, noerr => 1, quiet => 0);
2856 die "QEMU flag querying VM exited with code " . $rc if $rc;
2857
2858 eval {
2859 my $cmd_result = mon_cmd(
2860 $fakevmid,
2861 'query-cpu-model-expansion',
2862 type => 'full',
2863 model => { name => 'host' }
2864 );
2865
2866 my $props = $cmd_result->{model}->{props};
2867 foreach my $prop (keys %$props) {
2868 next if $props->{$prop} ne '1';
2869 # QEMU returns some flags multiple times, with '_', '.' or '-'
2870 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2871 # We only keep those with underscores, to match /proc/cpuinfo
2872 $prop =~ s/\.|-/_/g;
2873 $flags->{$prop} = 1;
2874 }
2875 };
2876 my $err = $@;
2877
2878 # force stop with 10 sec timeout and 'nocheck'
2879 # always stop, even if QMP failed
2880 vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2881
2882 die $err if $err;
2883
2884 return [ sort keys %$flags ];
2885 };
2886
2887 # We need to query QEMU twice, since KVM and TCG have different supported flags
2888 PVE::QemuConfig->lock_config($fakevmid, sub {
2889 $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2890 warn "warning: failed querying supported tcg flags: $@\n" if $@;
2891
2892 if ($kvm_supported) {
2893 $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
2894 warn "warning: failed querying supported kvm flags: $@\n" if $@;
2895 }
2896 });
2897
2898 return $flags;
2899}
2900
2901# Understood CPU flags are written to a file at 'pve-qemu' compile time
2902my $understood_cpu_flag_dir = "/usr/share/kvm";
2903sub query_understood_cpu_flags {
2904 my $arch = get_host_arch();
2905 my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
2906
2907 die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
2908 if ! -e $filepath;
2909
2910 my $raw = file_get_contents($filepath);
2911 $raw =~ s/^\s+|\s+$//g;
2912 my @flags = split(/\s+/, $raw);
2913
2914 return \@flags;
2915}
2916
2917sub config_to_command {
2918 my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2919
2920 my $cmd = [];
2921 my $globalFlags = [];
2922 my $machineFlags = [];
2923 my $rtcFlags = [];
2924 my $devices = [];
2925 my $pciaddr = '';
2926 my $bridges = {};
2927 my $ostype = $conf->{ostype};
2928 my $winversion = windows_version($ostype);
2929 my $kvm = $conf->{kvm};
2930 my $nodename = nodename();
2931
2932 my $arch = get_vm_arch($conf);
2933 my $kvm_binary = get_command_for_arch($arch);
2934 my $kvmver = kvm_user_version($kvm_binary);
2935
2936 if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
2937 $kvmver //= "undefined";
2938 die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
2939 }
2940
2941 my $add_pve_version = min_version($kvmver, 4, 1);
2942
2943 my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
2944 my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type, $kvmver);
2945 $kvm //= 1 if is_native($arch);
2946
2947 $machine_version =~ m/(\d+)\.(\d+)/;
2948 my ($machine_major, $machine_minor) = ($1, $2);
2949 die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type', please upgrade node '$nodename'\n"
2950 if !PVE::QemuServer::min_version($kvmver, $machine_major, $machine_minor);
2951
2952 if (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
2953 my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
2954 die "Installed qemu-server (max feature level for $machine_major.$machine_minor is pve$max_pve_version)"
2955 . " is too old to run machine type '$machine_type', please upgrade node '$nodename'\n";
2956 }
2957
2958 # if a specific +pve version is required for a feature, use $version_guard
2959 # instead of min_version to allow machines to be run with the minimum
2960 # required version
2961 my $required_pve_version = 0;
2962 my $version_guard = sub {
2963 my ($major, $minor, $pve) = @_;
2964 return 0 if !min_version($machine_version, $major, $minor, $pve);
2965 $required_pve_version = $pve if $pve && $pve > $required_pve_version;
2966 return 1;
2967 };
2968
2969 if ($kvm) {
2970 die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
2971 if !defined kvm_version();
2972 }
2973
2974 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2975 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2976 my $use_old_bios_files = undef;
2977 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2978
2979 my $cpuunits = defined($conf->{cpuunits}) ?
2980 $conf->{cpuunits} : $defaults->{cpuunits};
2981
2982 push @$cmd, $kvm_binary;
2983
2984 push @$cmd, '-id', $vmid;
2985
2986 my $vmname = $conf->{name} || "vm$vmid";
2987
2988 push @$cmd, '-name', $vmname;
2989
2990 my $use_virtio = 0;
2991
2992 my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
2993 push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2994 push @$cmd, '-mon', "chardev=qmp,mode=control";
2995
2996 if (min_version($machine_version, 2, 12)) {
2997 push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
2998 push @$cmd, '-mon', "chardev=qmp-event,mode=control";
2999 }
3000
3001 push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3002
3003 push @$cmd, '-daemonize';
3004
3005 if ($conf->{smbios1}) {
3006 my $smbios_conf = parse_smbios1($conf->{smbios1});
3007 if ($smbios_conf->{base64}) {
3008 # Do not pass base64 flag to qemu
3009 delete $smbios_conf->{base64};
3010 my $smbios_string = "";
3011 foreach my $key (keys %$smbios_conf) {
3012 my $value;
3013 if ($key eq "uuid") {
3014 $value = $smbios_conf->{uuid}
3015 } else {
3016 $value = decode_base64($smbios_conf->{$key});
3017 }
3018 # qemu accepts any binary data, only commas need escaping by double comma
3019 $value =~ s/,/,,/g;
3020 $smbios_string .= "," . $key . "=" . $value if $value;
3021 }
3022 push @$cmd, '-smbios', "type=1" . $smbios_string;
3023 } else {
3024 push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3025 }
3026 }
3027
3028 my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3029 if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3030 die "uefi base image not found\n" if ! -f $ovmf_code;
3031
3032 my $path;
3033 my $format;
3034 if (my $efidisk = $conf->{efidisk0}) {
3035 my $d = parse_drive('efidisk0', $efidisk);
3036 my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3037 $format = $d->{format};
3038 if ($storeid) {
3039 $path = PVE::Storage::path($storecfg, $d->{file});
3040 if (!defined($format)) {
3041 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3042 $format = qemu_img_format($scfg, $volname);
3043 }
3044 } else {
3045 $path = $d->{file};
3046 die "efidisk format must be specified\n"
3047 if !defined($format);
3048 }
3049 } else {
3050 warn "no efidisk configured! Using temporary efivars disk.\n";
3051 $path = "/tmp/$vmid-ovmf.fd";
3052 PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3053 $format = 'raw';
3054 }
3055
3056 push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3057 push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3058 }
3059
3060 # load q35 config
3061 if ($q35) {
3062 # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3063 if (min_version($machine_version, 4, 0)) {
3064 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3065 } else {
3066 push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3067 }
3068 }
3069
3070 if ($conf->{vmgenid}) {
3071 push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3072 }
3073
3074 # add usb controllers
3075 my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3076 push @$devices, @usbcontrollers if @usbcontrollers;
3077 my $vga = parse_vga($conf->{vga});
3078
3079 my $qxlnum = vga_conf_has_spice($conf->{vga});
3080 $vga->{type} = 'qxl' if $qxlnum;
3081
3082 if (!$vga->{type}) {
3083 if ($arch eq 'aarch64') {
3084 $vga->{type} = 'virtio';
3085 } elsif (min_version($machine_version, 2, 9)) {
3086 $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3087 } else {
3088 $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3089 }
3090 }
3091
3092 # enable absolute mouse coordinates (needed by vnc)
3093 my $tablet;
3094 if (defined($conf->{tablet})) {
3095 $tablet = $conf->{tablet};
3096 } else {
3097 $tablet = $defaults->{tablet};
3098 $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3099 $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3100 }
3101
3102 if ($tablet) {
3103 push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3104 my $kbd = print_keyboarddevice_full($conf, $arch);
3105 push @$devices, '-device', $kbd if defined($kbd);
3106 }
3107
3108 my $kvm_off = 0;
3109 my $gpu_passthrough;
3110
3111 # host pci devices
3112 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
3113 my $id = "hostpci$i";
3114 my $d = parse_hostpci($conf->{$id});
3115 next if !$d;
3116
3117 if (my $pcie = $d->{pcie}) {
3118 die "q35 machine model is not enabled" if !$q35;
3119 # win7 wants to have the pcie devices directly on the pcie bus
3120 # instead of in the root port
3121 if ($winversion == 7) {
3122 $pciaddr = print_pcie_addr("${id}bus0");
3123 } else {
3124 # add more root ports if needed, 4 are present by default
3125 # by pve-q35 cfgs, rest added here on demand.
3126 if ($i > 3) {
3127 push @$devices, '-device', print_pcie_root_port($i);
3128 }
3129 $pciaddr = print_pcie_addr($id);
3130 }
3131 } else {
3132 $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3133 }
3134
3135 my $xvga = '';
3136 if ($d->{'x-vga'}) {
3137 $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3138 $kvm_off = 1;
3139 $vga->{type} = 'none' if !defined($conf->{vga});
3140 $gpu_passthrough = 1;
3141 }
3142
3143 my $pcidevices = $d->{pciid};
3144 my $multifunction = 1 if @$pcidevices > 1;
3145
3146 my $sysfspath;
3147 if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3148 my $pci_id = $pcidevices->[0]->{id};
3149 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3150 $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
3151 } elsif ($d->{mdev}) {
3152 warn "ignoring mediated device '$id' with multifunction device\n";
3153 }
3154
3155 my $j=0;
3156 foreach my $pcidevice (@$pcidevices) {
3157 my $devicestr = "vfio-pci";
3158
3159 if ($sysfspath) {
3160 $devicestr .= ",sysfsdev=$sysfspath";
3161 } else {
3162 $devicestr .= ",host=$pcidevice->{id}";
3163 }
3164
3165 my $mf_addr = $multifunction ? ".$j" : '';
3166 $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3167
3168 if ($j == 0) {
3169 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3170 $devicestr .= "$xvga";
3171 $devicestr .= ",multifunction=on" if $multifunction;
3172 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3173 }
3174
3175 push @$devices, '-device', $devicestr;
3176 $j++;
3177 }
3178 }
3179
3180 # usb devices
3181 my $usb_dev_features = {};
3182 $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3183
3184 my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3185 push @$devices, @usbdevices if @usbdevices;
3186 # serial devices
3187 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++) {
3188 if (my $path = $conf->{"serial$i"}) {
3189 if ($path eq 'socket') {
3190 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3191 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3192 # On aarch64, serial0 is the UART device. Qemu only allows
3193 # connecting UART devices via the '-serial' command line, as
3194 # the device has a fixed slot on the hardware...
3195 if ($arch eq 'aarch64' && $i == 0) {
3196 push @$devices, '-serial', "chardev:serial$i";
3197 } else {
3198 push @$devices, '-device', "isa-serial,chardev=serial$i";
3199 }
3200 } else {
3201 die "no such serial device\n" if ! -c $path;
3202 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3203 push @$devices, '-device', "isa-serial,chardev=serial$i";
3204 }
3205 }
3206 }
3207
3208 # parallel devices
3209 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++) {
3210 if (my $path = $conf->{"parallel$i"}) {
3211 die "no such parallel device\n" if ! -c $path;
3212 my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3213 push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3214 push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3215 }
3216 }
3217
3218 if (my $audio = conf_has_audio($conf)) {
3219
3220 my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3221
3222 my $id = $audio->{dev_id};
3223 if ($audio->{dev} eq 'AC97') {
3224 push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3225 } elsif ($audio->{dev} =~ /intel\-hda$/) {
3226 push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3227 push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3228 push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3229 } else {
3230 die "unkown audio device '$audio->{dev}', implement me!";
3231 }
3232
3233 push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3234 }
3235
3236 my $sockets = 1;
3237 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3238 $sockets = $conf->{sockets} if $conf->{sockets};
3239
3240 my $cores = $conf->{cores} || 1;
3241
3242 my $maxcpus = $sockets * $cores;
3243
3244 my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3245
3246 my $allowed_vcpus = $cpuinfo->{cpus};
3247
3248 die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3249 if ($allowed_vcpus < $maxcpus);
3250
3251 if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3252
3253 push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3254 for (my $i = 2; $i <= $vcpus; $i++) {
3255 my $cpustr = print_cpu_device($conf,$i);
3256 push @$cmd, '-device', $cpustr;
3257 }
3258
3259 } else {
3260
3261 push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3262 }
3263 push @$cmd, '-nodefaults';
3264
3265 my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3266
3267 my $bootindex_hash = {};
3268 my $i = 1;
3269 foreach my $o (split(//, $bootorder)) {
3270 $bootindex_hash->{$o} = $i*100;
3271 $i++;
3272 }
3273
3274 push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3275
3276 push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3277
3278 push @$cmd, '-no-reboot' if defined($conf->{reboot}) && $conf->{reboot} == 0;
3279
3280 if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3281 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3282 my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3283 push @$cmd, '-vnc', "unix:$socket,password";
3284 } else {
3285 push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3286 push @$cmd, '-nographic';
3287 }
3288
3289 # time drift fix
3290 my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3291
3292 my $useLocaltime = $conf->{localtime};
3293
3294 if ($winversion >= 5) { # windows
3295 $useLocaltime = 1 if !defined($conf->{localtime});
3296
3297 # use time drift fix when acpi is enabled
3298 if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3299 $tdf = 1 if !defined($conf->{tdf});
3300 }
3301 }
3302
3303 if ($winversion >= 6) {
3304 push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3305 push @$cmd, '-no-hpet';
3306 }
3307
3308 push @$rtcFlags, 'driftfix=slew' if $tdf;
3309
3310 if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3311 push @$rtcFlags, "base=$conf->{startdate}";
3312 } elsif ($useLocaltime) {
3313 push @$rtcFlags, 'base=localtime';
3314 }
3315
3316 push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3317
3318 PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3319
3320 push @$cmd, '-S' if $conf->{freeze};
3321
3322 push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3323
3324 my $guest_agent = parse_guest_agent($conf);
3325
3326 if ($guest_agent->{enabled}) {
3327 my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3328 push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3329
3330 if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3331 my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3332 push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3333 push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3334 } elsif ($guest_agent->{type} eq 'isa') {
3335 push @$devices, '-device', "isa-serial,chardev=qga0";
3336 }
3337 }
3338
3339 my $rng = parse_rng($conf->{rng0}) if $conf->{rng0};
3340 if ($rng && &$version_guard(4, 1, 2)) {
3341 my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3342 my $period = $rng->{period} // $rng_fmt->{period}->{default};
3343
3344 my $limiter_str = "";
3345 if ($max_bytes) {
3346 $limiter_str = ",max-bytes=$max_bytes,period=$period";
3347 }
3348
3349 # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3350 die "cannot create VirtIO RNG device: source file '$rng->{source}' doesn't exist\n"
3351 if ! -e $rng->{source};
3352
3353 my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3354
3355 push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3356 push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3357 }
3358
3359 my $spice_port;
3360
3361 if ($qxlnum) {
3362 if ($qxlnum > 1) {
3363 if ($winversion){
3364 for(my $i = 1; $i < $qxlnum; $i++){
3365 push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3366 }
3367 } else {
3368 # assume other OS works like Linux
3369 my ($ram, $vram) = ("134217728", "67108864");
3370 if ($vga->{memory}) {
3371 $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3372 $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3373 }
3374 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3375 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3376 }
3377 }
3378
3379 my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3380
3381 my $pfamily = PVE::Tools::get_host_address_family($nodename);
3382 my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3383 die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3384
3385 push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3386 push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3387 push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3388
3389 my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3390 $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3391
3392 my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3393 if ($spice_enhancement->{foldersharing}) {
3394 push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3395 push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3396 }
3397
3398 my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3399 $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3400 push @$devices, '-spice', "$spice_opts";
3401 }
3402
3403 # enable balloon by default, unless explicitly disabled
3404 if (!defined($conf->{balloon}) || $conf->{balloon}) {
3405 $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3406 push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3407 }
3408
3409 if ($conf->{watchdog}) {
3410 my $wdopts = parse_watchdog($conf->{watchdog});
3411 $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3412 my $watchdog = $wdopts->{model} || 'i6300esb';
3413 push @$devices, '-device', "$watchdog$pciaddr";
3414 push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3415 }
3416
3417 my $vollist = [];
3418 my $scsicontroller = {};
3419 my $ahcicontroller = {};
3420 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3421
3422 # Add iscsi initiator name if available
3423 if (my $initiator = get_initiator_name()) {
3424 push @$devices, '-iscsi', "initiator-name=$initiator";
3425 }
3426
3427 foreach_drive($conf, sub {
3428 my ($ds, $drive) = @_;
3429
3430 if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3431 push @$vollist, $drive->{file};
3432 }
3433
3434 # ignore efidisk here, already added in bios/fw handling code above
3435 return if $drive->{interface} eq 'efidisk';
3436
3437 $use_virtio = 1 if $ds =~ m/^virtio/;
3438
3439 if (drive_is_cdrom ($drive)) {
3440 if ($bootindex_hash->{d}) {
3441 $drive->{bootindex} = $bootindex_hash->{d};
3442 $bootindex_hash->{d} += 1;
3443 }
3444 } else {
3445 if ($bootindex_hash->{c}) {
3446 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3447 $bootindex_hash->{c} += 1;
3448 }
3449 }
3450
3451 if($drive->{interface} eq 'virtio'){
3452 push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3453 }
3454
3455 if ($drive->{interface} eq 'scsi') {
3456
3457 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3458
3459 die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3460 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3461
3462 $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3463 my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3464
3465 my $iothread = '';
3466 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3467 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3468 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3469 } elsif ($drive->{iothread}) {
3470 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3471 }
3472
3473 my $queues = '';
3474 if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3475 $queues = ",num_queues=$drive->{queues}";
3476 }
3477
3478 push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3479 $scsicontroller->{$controller}=1;
3480 }
3481
3482 if ($drive->{interface} eq 'sata') {
3483 my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3484 $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3485 push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3486 $ahcicontroller->{$controller}=1;
3487 }
3488
3489 my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3490 push @$devices, '-drive',$drive_cmd;
3491 push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3492 });
3493
3494 for (my $i = 0; $i < $MAX_NETS; $i++) {
3495 next if !$conf->{"net$i"};
3496 my $d = parse_net($conf->{"net$i"});
3497 next if !$d;
3498
3499 $use_virtio = 1 if $d->{model} eq 'virtio';
3500
3501 if ($bootindex_hash->{n}) {
3502 $d->{bootindex} = $bootindex_hash->{n};
3503 $bootindex_hash->{n} += 1;
3504 }
3505
3506 my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
3507 push @$devices, '-netdev', $netdevfull;
3508
3509 my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
3510 push @$devices, '-device', $netdevicefull;
3511 }
3512
3513 if ($conf->{ivshmem}) {
3514 my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3515
3516 my $bus;
3517 if ($q35) {
3518 $bus = print_pcie_addr("ivshmem");
3519 } else {
3520 $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3521 }
3522
3523 my $ivshmem_name = $ivshmem->{name} // $vmid;
3524 my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3525
3526 push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3527 push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
3528 }
3529
3530 # pci.4 is nested in pci.1
3531 $bridges->{1} = 1 if $bridges->{4};
3532
3533 if (!$q35) {
3534 # add pci bridges
3535 if (min_version($machine_version, 2, 3)) {
3536 $bridges->{1} = 1;
3537 $bridges->{2} = 1;
3538 }
3539
3540 $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3541
3542 }
3543
3544 for my $k (sort {$b cmp $a} keys %$bridges) {
3545 next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3546 $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
3547 my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3548 if ($q35) {
3549 # add after -readconfig pve-q35.cfg
3550 splice @$devices, 2, 0, '-device', $devstr;
3551 } else {
3552 unshift @$devices, '-device', $devstr if $k > 0;
3553 }
3554 }
3555
3556 if (!$kvm) {
3557 push @$machineFlags, 'accel=tcg';
3558 }
3559
3560 my $machine_type_min = $machine_type;
3561 if ($add_pve_version) {
3562 $machine_type_min =~ s/\+pve\d+$//;
3563 $machine_type_min .= "+pve$required_pve_version";
3564 }
3565 push @$machineFlags, "type=${machine_type_min}";
3566
3567 push @$cmd, @$devices;
3568 push @$cmd, '-rtc', join(',', @$rtcFlags)
3569 if scalar(@$rtcFlags);
3570 push @$cmd, '-machine', join(',', @$machineFlags)
3571 if scalar(@$machineFlags);
3572 push @$cmd, '-global', join(',', @$globalFlags)
3573 if scalar(@$globalFlags);
3574
3575 if (my $vmstate = $conf->{vmstate}) {
3576 my $statepath = PVE::Storage::path($storecfg, $vmstate);
3577 push @$vollist, $vmstate;
3578 push @$cmd, '-loadstate', $statepath;
3579 print "activating and using '$vmstate' as vmstate\n";
3580 }
3581
3582 # add custom args
3583 if ($conf->{args}) {
3584 my $aa = PVE::Tools::split_args($conf->{args});
3585 push @$cmd, @$aa;
3586 }
3587
3588 return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3589}
3590
3591sub spice_port {
3592 my ($vmid) = @_;
3593
3594 my $res = mon_cmd($vmid, 'query-spice');
3595
3596 return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3597}
3598
3599sub vm_devices_list {
3600 my ($vmid) = @_;
3601
3602 my $res = mon_cmd($vmid, 'query-pci');
3603 my $devices_to_check = [];
3604 my $devices = {};
3605 foreach my $pcibus (@$res) {
3606 push @$devices_to_check, @{$pcibus->{devices}},
3607 }
3608
3609 while (@$devices_to_check) {
3610 my $to_check = [];
3611 for my $d (@$devices_to_check) {
3612 $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3613 next if !$d->{'pci_bridge'};
3614
3615 $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3616 push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3617 }
3618 $devices_to_check = $to_check;
3619 }
3620
3621 my $resblock = mon_cmd($vmid, 'query-block');
3622 foreach my $block (@$resblock) {
3623 if($block->{device} =~ m/^drive-(\S+)/){
3624 $devices->{$1} = 1;
3625 }
3626 }
3627
3628 my $resmice = mon_cmd($vmid, 'query-mice');
3629 foreach my $mice (@$resmice) {
3630 if ($mice->{name} eq 'QEMU HID Tablet') {
3631 $devices->{tablet} = 1;
3632 last;
3633 }
3634 }
3635
3636 # for usb devices there is no query-usb
3637 # but we can iterate over the entries in
3638 # qom-list path=/machine/peripheral
3639 my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3640 foreach my $per (@$resperipheral) {
3641 if ($per->{name} =~ m/^usb\d+$/) {
3642 $devices->{$per->{name}} = 1;
3643 }
3644 }
3645
3646 return $devices;
3647}
3648
3649sub vm_deviceplug {
3650 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3651
3652 my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3653
3654 my $devices_list = vm_devices_list($vmid);
3655 return 1 if defined($devices_list->{$deviceid});
3656
3657 qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
3658
3659 if ($deviceid eq 'tablet') {
3660
3661 qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3662
3663 } elsif ($deviceid eq 'keyboard') {
3664
3665 qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3666
3667 } elsif ($deviceid =~ m/^usb(\d+)$/) {
3668
3669 die "usb hotplug currently not reliable\n";
3670 # since we can't reliably hot unplug all added usb devices
3671 # and usb passthrough disables live migration
3672 # we disable usb hotplugging for now
3673 qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3674
3675 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3676
3677 qemu_iothread_add($vmid, $deviceid, $device);
3678
3679 qemu_driveadd($storecfg, $vmid, $device);
3680 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3681
3682 qemu_deviceadd($vmid, $devicefull);
3683 eval { qemu_deviceaddverify($vmid, $deviceid); };
3684 if (my $err = $@) {
3685 eval { qemu_drivedel($vmid, $deviceid); };
3686 warn $@ if $@;
3687 die $err;
3688 }
3689
3690 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3691
3692
3693 my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3694 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3695 my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3696
3697 my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3698
3699 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3700 qemu_iothread_add($vmid, $deviceid, $device);
3701 $devicefull .= ",iothread=iothread-$deviceid";
3702 }
3703
3704 if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3705 $devicefull .= ",num_queues=$device->{queues}";
3706 }
3707
3708 qemu_deviceadd($vmid, $devicefull);
3709 qemu_deviceaddverify($vmid, $deviceid);
3710
3711 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3712
3713 qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3714 qemu_driveadd($storecfg, $vmid, $device);
3715
3716 my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
3717 eval { qemu_deviceadd($vmid, $devicefull); };
3718 if (my $err = $@) {
3719 eval { qemu_drivedel($vmid, $deviceid); };
3720 warn $@ if $@;
3721 die $err;
3722 }
3723
3724 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3725
3726 return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3727
3728 my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3729 my $use_old_bios_files = undef;
3730 ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3731
3732 my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3733 qemu_deviceadd($vmid, $netdevicefull);
3734 eval {
3735 qemu_deviceaddverify($vmid, $deviceid);
3736 qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3737 };
3738 if (my $err = $@) {
3739 eval { qemu_netdevdel($vmid, $deviceid); };
3740 warn $@ if $@;
3741 die $err;
3742 }
3743
3744 } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3745
3746 my $bridgeid = $2;
3747 my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3748 my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3749
3750 qemu_deviceadd($vmid, $devicefull);
3751 qemu_deviceaddverify($vmid, $deviceid);
3752
3753 } else {
3754 die "can't hotplug device '$deviceid'\n";
3755 }
3756
3757 return 1;
3758}
3759
3760# fixme: this should raise exceptions on error!
3761sub vm_deviceunplug {
3762 my ($vmid, $conf, $deviceid) = @_;
3763
3764 my $devices_list = vm_devices_list($vmid);
3765 return 1 if !defined($devices_list->{$deviceid});
3766
3767 die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3768
3769 if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3770
3771 qemu_devicedel($vmid, $deviceid);
3772
3773 } elsif ($deviceid =~ m/^usb\d+$/) {
3774
3775 die "usb hotplug currently not reliable\n";
3776 # when unplugging usb devices this way,
3777 # there may be remaining usb controllers/hubs
3778 # so we disable it for now
3779 qemu_devicedel($vmid, $deviceid);
3780 qemu_devicedelverify($vmid, $deviceid);
3781
3782 } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3783
3784 qemu_devicedel($vmid, $deviceid);
3785 qemu_devicedelverify($vmid, $deviceid);
3786 qemu_drivedel($vmid, $deviceid);
3787 qemu_iothread_del($conf, $vmid, $deviceid);
3788
3789 } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3790
3791 qemu_devicedel($vmid, $deviceid);
3792 qemu_devicedelverify($vmid, $deviceid);
3793 qemu_iothread_del($conf, $vmid, $deviceid);
3794
3795 } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3796
3797 qemu_devicedel($vmid, $deviceid);
3798 qemu_drivedel($vmid, $deviceid);
3799 qemu_deletescsihw($conf, $vmid, $deviceid);
3800
3801 } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3802
3803 qemu_devicedel($vmid, $deviceid);
3804 qemu_devicedelverify($vmid, $deviceid);
3805 qemu_netdevdel($vmid, $deviceid);
3806
3807 } else {
3808 die "can't unplug device '$deviceid'\n";
3809 }
3810
3811 return 1;
3812}
3813
3814sub qemu_deviceadd {
3815 my ($vmid, $devicefull) = @_;
3816
3817 $devicefull = "driver=".$devicefull;
3818 my %options = split(/[=,]/, $devicefull);
3819
3820 mon_cmd($vmid, "device_add" , %options);
3821}
3822
3823sub qemu_devicedel {
3824 my ($vmid, $deviceid) = @_;
3825
3826 my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3827}
3828
3829sub qemu_iothread_add {
3830 my($vmid, $deviceid, $device) = @_;
3831
3832 if ($device->{iothread}) {
3833 my $iothreads = vm_iothreads_list($vmid);
3834 qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3835 }
3836}
3837
3838sub qemu_iothread_del {
3839 my($conf, $vmid, $deviceid) = @_;
3840
3841 my $confid = $deviceid;
3842 if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3843 $confid = 'scsi' . $1;
3844 }
3845 my $device = parse_drive($confid, $conf->{$confid});
3846 if ($device->{iothread}) {
3847 my $iothreads = vm_iothreads_list($vmid);
3848 qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3849 }
3850}
3851
3852sub qemu_objectadd {
3853 my($vmid, $objectid, $qomtype) = @_;
3854
3855 mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3856
3857 return 1;
3858}
3859
3860sub qemu_objectdel {
3861 my($vmid, $objectid) = @_;
3862
3863 mon_cmd($vmid, "object-del", id => $objectid);
3864
3865 return 1;
3866}
3867
3868sub qemu_driveadd {
3869 my ($storecfg, $vmid, $device) = @_;
3870
3871 my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3872 $drive =~ s/\\/\\\\/g;
3873 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3874
3875 # If the command succeeds qemu prints: "OK"
3876 return 1 if $ret =~ m/OK/s;
3877
3878 die "adding drive failed: $ret\n";
3879}
3880
3881sub qemu_drivedel {
3882 my($vmid, $deviceid) = @_;
3883
3884 my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3885 $ret =~ s/^\s+//;
3886
3887 return 1 if $ret eq "";
3888
3889 # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3890 return 1 if $ret =~ m/Device \'.*?\' not found/s;
3891
3892 die "deleting drive $deviceid failed : $ret\n";
3893}
3894
3895sub qemu_deviceaddverify {
3896 my ($vmid, $deviceid) = @_;
3897
3898 for (my $i = 0; $i <= 5; $i++) {
3899 my $devices_list = vm_devices_list($vmid);
3900 return 1 if defined($devices_list->{$deviceid});
3901 sleep 1;
3902 }
3903
3904 die "error on hotplug device '$deviceid'\n";
3905}
3906
3907
3908sub qemu_devicedelverify {
3909 my ($vmid, $deviceid) = @_;
3910
3911 # need to verify that the device is correctly removed as device_del
3912 # is async and empty return is not reliable
3913
3914 for (my $i = 0; $i <= 5; $i++) {
3915 my $devices_list = vm_devices_list($vmid);
3916 return 1 if !defined($devices_list->{$deviceid});
3917 sleep 1;
3918 }
3919
3920 die "error on hot-unplugging device '$deviceid'\n";
3921}
3922
3923sub qemu_findorcreatescsihw {
3924 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3925
3926 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3927
3928 my $scsihwid="$controller_prefix$controller";
3929 my $devices_list = vm_devices_list($vmid);
3930
3931 if(!defined($devices_list->{$scsihwid})) {
3932 vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3933 }
3934
3935 return 1;
3936}
3937
3938sub qemu_deletescsihw {
3939 my ($conf, $vmid, $opt) = @_;
3940
3941 my $device = parse_drive($opt, $conf->{$opt});
3942
3943 if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3944 vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3945 return 1;
3946 }
3947
3948 my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3949
3950 my $devices_list = vm_devices_list($vmid);
3951 foreach my $opt (keys %{$devices_list}) {
3952 if (is_valid_drivename($opt)) {
3953 my $drive = parse_drive($opt, $conf->{$opt});
3954 if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3955 return 1;
3956 }
3957 }
3958 }
3959
3960 my $scsihwid="scsihw$controller";
3961
3962 vm_deviceunplug($vmid, $conf, $scsihwid);
3963
3964 return 1;
3965}
3966
3967sub qemu_add_pci_bridge {
3968 my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3969
3970 my $bridges = {};
3971
3972 my $bridgeid;
3973
3974 print_pci_addr($device, $bridges, $arch, $machine_type);
3975
3976 while (my ($k, $v) = each %$bridges) {
3977 $bridgeid = $k;
3978 }
3979 return 1 if !defined($bridgeid) || $bridgeid < 1;
3980
3981 my $bridge = "pci.$bridgeid";
3982 my $devices_list = vm_devices_list($vmid);
3983
3984 if (!defined($devices_list->{$bridge})) {
3985 vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
3986 }
3987
3988 return 1;
3989}
3990
3991sub qemu_set_link_status {
3992 my ($vmid, $device, $up) = @_;
3993
3994 mon_cmd($vmid, "set_link", name => $device,
3995 up => $up ? JSON::true : JSON::false);
3996}
3997
3998sub qemu_netdevadd {
3999 my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4000
4001 my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4002 my %options = split(/[=,]/, $netdev);
4003
4004 mon_cmd($vmid, "netdev_add", %options);
4005 return 1;
4006}
4007
4008sub qemu_netdevdel {
4009 my ($vmid, $deviceid) = @_;
4010
4011 mon_cmd($vmid, "netdev_del", id => $deviceid);
4012}
4013
4014sub qemu_usb_hotplug {
4015 my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4016
4017 return if !$device;
4018
4019 # remove the old one first
4020 vm_deviceunplug($vmid, $conf, $deviceid);
4021
4022 # check if xhci controller is necessary and available
4023 if ($device->{usb3}) {
4024
4025 my $devicelist = vm_devices_list($vmid);
4026
4027 if (!$devicelist->{xhci}) {
4028 my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4029 qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4030 }
4031 }
4032 my $d = parse_usb_device($device->{host});
4033 $d->{usb3} = $device->{usb3};
4034
4035 # add the new one
4036 vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4037}
4038
4039sub qemu_cpu_hotplug {
4040 my ($vmid, $conf, $vcpus) = @_;
4041
4042 my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4043
4044 my $sockets = 1;
4045 $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4046 $sockets = $conf->{sockets} if $conf->{sockets};
4047 my $cores = $conf->{cores} || 1;
4048 my $maxcpus = $sockets * $cores;
4049
4050 $vcpus = $maxcpus if !$vcpus;
4051
4052 die "you can't add more vcpus than maxcpus\n"
4053 if $vcpus > $maxcpus;
4054
4055 my $currentvcpus = $conf->{vcpus} || $maxcpus;
4056
4057 if ($vcpus < $currentvcpus) {
4058
4059 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4060
4061 for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4062 qemu_devicedel($vmid, "cpu$i");
4063 my $retry = 0;
4064 my $currentrunningvcpus = undef;
4065 while (1) {
4066 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4067 last if scalar(@{$currentrunningvcpus}) == $i-1;
4068 raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4069 $retry++;
4070 sleep 1;
4071 }
4072 #update conf after each succesfull cpu unplug
4073 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4074 PVE::QemuConfig->write_config($vmid, $conf);
4075 }
4076 } else {
4077 die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4078 }
4079
4080 return;
4081 }
4082
4083 my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4084 die "vcpus in running vm does not match its configuration\n"
4085 if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4086
4087 if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4088
4089 for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4090 my $cpustr = print_cpu_device($conf, $i);
4091 qemu_deviceadd($vmid, $cpustr);
4092
4093 my $retry = 0;
4094 my $currentrunningvcpus = undef;
4095 while (1) {
4096 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4097 last if scalar(@{$currentrunningvcpus}) == $i;
4098 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4099 sleep 1;
4100 $retry++;
4101 }
4102 #update conf after each succesfull cpu hotplug
4103 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4104 PVE::QemuConfig->write_config($vmid, $conf);
4105 }
4106 } else {
4107
4108 for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4109 mon_cmd($vmid, "cpu-add", id => int($i));
4110 }
4111 }
4112}
4113
4114sub qemu_block_set_io_throttle {
4115 my ($vmid, $deviceid,
4116 $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4117 $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4118 $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4119 $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4120
4121 return if !check_running($vmid) ;
4122
4123 mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4124 bps => int($bps),
4125 bps_rd => int($bps_rd),
4126 bps_wr => int($bps_wr),
4127 iops => int($iops),
4128 iops_rd => int($iops_rd),
4129 iops_wr => int($iops_wr),
4130 bps_max => int($bps_max),
4131 bps_rd_max => int($bps_rd_max),
4132 bps_wr_max => int($bps_wr_max),
4133 iops_max => int($iops_max),
4134 iops_rd_max => int($iops_rd_max),
4135 iops_wr_max => int($iops_wr_max),
4136 bps_max_length => int($bps_max_length),
4137 bps_rd_max_length => int($bps_rd_max_length),
4138 bps_wr_max_length => int($bps_wr_max_length),
4139 iops_max_length => int($iops_max_length),
4140 iops_rd_max_length => int($iops_rd_max_length),
4141 iops_wr_max_length => int($iops_wr_max_length),
4142 );
4143
4144}
4145
4146# old code, only used to shutdown old VM after update
4147sub __read_avail {
4148 my ($fh, $timeout) = @_;
4149
4150 my $sel = new IO::Select;
4151 $sel->add($fh);
4152
4153 my $res = '';
4154 my $buf;
4155
4156 my @ready;
4157 while (scalar (@ready = $sel->can_read($timeout))) {
4158 my $count;
4159 if ($count = $fh->sysread($buf, 8192)) {
4160 if ($buf =~ /^(.*)\(qemu\) $/s) {
4161 $res .= $1;
4162 last;
4163 } else {
4164 $res .= $buf;
4165 }
4166 } else {
4167 if (!defined($count)) {
4168 die "$!\n";
4169 }
4170 last;
4171 }
4172 }
4173
4174 die "monitor read timeout\n" if !scalar(@ready);
4175
4176 return $res;
4177}
4178
4179sub qemu_block_resize {
4180 my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4181
4182 my $running = check_running($vmid);
4183
4184 $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4185
4186 return if !$running;
4187
4188 my $padding = (1024 - $size % 1024) % 1024;
4189 $size = $size + $padding;
4190
4191 mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4192
4193}
4194
4195sub qemu_volume_snapshot {
4196 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4197
4198 my $running = check_running($vmid);
4199
4200 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4201 mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4202 } else {
4203 PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4204 }
4205}
4206
4207sub qemu_volume_snapshot_delete {
4208 my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4209
4210 my $running = check_running($vmid);
4211
4212 if($running) {
4213
4214 $running = undef;
4215 my $conf = PVE::QemuConfig->load_config($vmid);
4216 foreach_drive($conf, sub {
4217 my ($ds, $drive) = @_;
4218 $running = 1 if $drive->{file} eq $volid;
4219 });
4220 }
4221
4222 if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4223 mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4224 } else {
4225 PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4226 }
4227}
4228
4229sub set_migration_caps {
4230 my ($vmid) = @_;
4231
4232 my $cap_ref = [];
4233
4234 my $enabled_cap = {
4235 "auto-converge" => 1,
4236 "xbzrle" => 1,
4237 "x-rdma-pin-all" => 0,
4238 "zero-blocks" => 0,
4239 "compress" => 0
4240 };
4241
4242 my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4243
4244 for my $supported_capability (@$supported_capabilities) {
4245 push @$cap_ref, {
4246 capability => $supported_capability->{capability},
4247 state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4248 };
4249 }
4250
4251 mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4252}
4253
4254my $fast_plug_option = {
4255 'lock' => 1,
4256 'name' => 1,
4257 'onboot' => 1,
4258 'shares' => 1,
4259 'startup' => 1,
4260 'description' => 1,
4261 'protection' => 1,
4262 'vmstatestorage' => 1,
4263 'hookscript' => 1,
4264 'tags' => 1,
4265};
4266
4267# hotplug changes in [PENDING]
4268# $selection hash can be used to only apply specified options, for
4269# example: { cores => 1 } (only apply changed 'cores')
4270# $errors ref is used to return error messages
4271sub vmconfig_hotplug_pending {
4272 my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4273
4274 my $defaults = load_defaults();
4275 my $arch = get_vm_arch($conf);
4276 my $machine_type = get_vm_machine($conf, undef, $arch);
4277
4278 # commit values which do not have any impact on running VM first
4279 # Note: those option cannot raise errors, we we do not care about
4280 # $selection and always apply them.
4281
4282 my $add_error = sub {
4283 my ($opt, $msg) = @_;
4284 $errors->{$opt} = "hotplug problem - $msg";
4285 };
4286
4287 my $changes = 0;
4288 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4289 if ($fast_plug_option->{$opt}) {
4290 $conf->{$opt} = $conf->{pending}->{$opt};
4291 delete $conf->{pending}->{$opt};
4292 $changes = 1;
4293 }
4294 }
4295
4296 if ($changes) {
4297 PVE::QemuConfig->write_config($vmid, $conf);
4298 }
4299
4300 my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4301
4302 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4303 foreach my $opt (sort keys %$pending_delete_hash) {
4304 next if $selection && !$selection->{$opt};
4305 my $force = $pending_delete_hash->{$opt}->{force};
4306 eval {
4307 if ($opt eq 'hotplug') {
4308 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4309 } elsif ($opt eq 'tablet') {
4310 die "skip\n" if !$hotplug_features->{usb};
4311 if ($defaults->{tablet}) {
4312 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4313 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4314 if $arch eq 'aarch64';
4315 } else {
4316 vm_deviceunplug($vmid, $conf, 'tablet');
4317 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4318 }
4319 } elsif ($opt =~ m/^usb\d+/) {
4320 die "skip\n";
4321 # since we cannot reliably hot unplug usb devices
4322 # we are disabling it
4323 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4324 vm_deviceunplug($vmid, $conf, $opt);
4325 } elsif ($opt eq 'vcpus') {
4326 die "skip\n" if !$hotplug_features->{cpu};
4327 qemu_cpu_hotplug($vmid, $conf, undef);
4328 } elsif ($opt eq 'balloon') {
4329 # enable balloon device is not hotpluggable
4330 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4331 # here we reset the ballooning value to memory
4332 my $balloon = $conf->{memory} || $defaults->{memory};
4333 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4334 } elsif ($fast_plug_option->{$opt}) {
4335 # do nothing
4336 } elsif ($opt =~ m/^net(\d+)$/) {
4337 die "skip\n" if !$hotplug_features->{network};
4338 vm_deviceunplug($vmid, $conf, $opt);
4339 } elsif (is_valid_drivename($opt)) {
4340 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4341 vm_deviceunplug($vmid, $conf, $opt);
4342 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4343 } elsif ($opt =~ m/^memory$/) {
4344 die "skip\n" if !$hotplug_features->{memory};
4345 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4346 } elsif ($opt eq 'cpuunits') {
4347 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4348 } elsif ($opt eq 'cpulimit') {
4349 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4350 } else {
4351 die "skip\n";
4352 }
4353 };
4354 if (my $err = $@) {
4355 &$add_error($opt, $err) if $err ne "skip\n";
4356 } else {
4357 delete $conf->{$opt};
4358 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4359 }
4360 }
4361
4362 my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4363 $apply_pending_cloudinit = sub {
4364 return if $apply_pending_cloudinit_done; # once is enough
4365 $apply_pending_cloudinit_done = 1; # once is enough
4366
4367 my ($key, $value) = @_;
4368
4369 my @cloudinit_opts = keys %$confdesc_cloudinit;
4370 foreach my $opt (keys %{$conf->{pending}}) {
4371 next if !grep { $_ eq $opt } @cloudinit_opts;
4372 $conf->{$opt} = delete $conf->{pending}->{$opt};
4373 }
4374
4375 my $new_conf = { %$conf };
4376 $new_conf->{$key} = $value;
4377 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4378 };
4379
4380 foreach my $opt (keys %{$conf->{pending}}) {
4381 next if $selection && !$selection->{$opt};
4382 my $value = $conf->{pending}->{$opt};
4383 eval {
4384 if ($opt eq 'hotplug') {
4385 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4386 } elsif ($opt eq 'tablet') {
4387 die "skip\n" if !$hotplug_features->{usb};
4388 if ($value == 1) {
4389 vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4390 vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4391 if $arch eq 'aarch64';
4392 } elsif ($value == 0) {
4393 vm_deviceunplug($vmid, $conf, 'tablet');
4394 vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4395 }
4396 } elsif ($opt =~ m/^usb\d+$/) {
4397 die "skip\n";
4398 # since we cannot reliably hot unplug usb devices
4399 # we are disabling it
4400 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4401 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4402 die "skip\n" if !$d;
4403 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4404 } elsif ($opt eq 'vcpus') {
4405 die "skip\n" if !$hotplug_features->{cpu};
4406 qemu_cpu_hotplug($vmid, $conf, $value);
4407 } elsif ($opt eq 'balloon') {
4408 # enable/disable balloning device is not hotpluggable
4409 my $old_balloon_enabled = !!(!defined($conf->{balloon}) || $conf->{balloon});
4410 my $new_balloon_enabled = !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4411 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4412
4413 # allow manual ballooning if shares is set to zero
4414 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4415 my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4416 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4417 }
4418 } elsif ($opt =~ m/^net(\d+)$/) {
4419 # some changes can be done without hotplug
4420 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4421 $vmid, $opt, $value, $arch, $machine_type);
4422 } elsif (is_valid_drivename($opt)) {
4423 die "skip\n" if $opt eq 'efidisk0';
4424 # some changes can be done without hotplug
4425 my $drive = parse_drive($opt, $value);
4426 if (drive_is_cloudinit($drive)) {
4427 &$apply_pending_cloudinit($opt, $value);
4428 }
4429 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4430 $vmid, $opt, $value, $arch, $machine_type);
4431 } elsif ($opt =~ m/^memory$/) { #dimms
4432 die "skip\n" if !$hotplug_features->{memory};
4433 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4434 } elsif ($opt eq 'cpuunits') {
4435 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4436 } elsif ($opt eq 'cpulimit') {
4437 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4438 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4439 } else {
4440 die "skip\n"; # skip non-hot-pluggable options
4441 }
4442 };
4443 if (my $err = $@) {
4444 &$add_error($opt, $err) if $err ne "skip\n";
4445 } else {
4446 $conf->{$opt} = $value;
4447 delete $conf->{pending}->{$opt};
4448 }
4449 }
4450
4451 PVE::QemuConfig->write_config($vmid, $conf);
4452}
4453
4454sub try_deallocate_drive {
4455 my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4456
4457 if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4458 my $volid = $drive->{file};
4459 if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4460 my $sid = PVE::Storage::parse_volume_id($volid);
4461 $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4462
4463 # check if the disk is really unused
4464 die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4465 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4466 PVE::Storage::vdisk_free($storecfg, $volid);
4467 return 1;
4468 } else {
4469 # If vm is not owner of this disk remove from config
4470 return 1;
4471 }
4472 }
4473
4474 return undef;
4475}
4476
4477sub vmconfig_delete_or_detach_drive {
4478 my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4479
4480 my $drive = parse_drive($opt, $conf->{$opt});
4481
4482 my $rpcenv = PVE::RPCEnvironment::get();
4483 my $authuser = $rpcenv->get_user();
4484
4485 if ($force) {
4486 $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4487 try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4488 } else {
4489 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4490 }
4491}
4492
4493
4494
4495sub vmconfig_apply_pending {
4496 my ($vmid, $conf, $storecfg, $errors) = @_;
4497
4498 my $add_apply_error = sub {
4499 my ($opt, $msg) = @_;
4500 my $err_msg = "unable to apply pending change $opt : $msg";
4501 $errors->{$opt} = $err_msg;
4502 warn $err_msg;
4503 };
4504
4505 # cold plug
4506
4507 my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4508 foreach my $opt (sort keys %$pending_delete_hash) {
4509 my $force = $pending_delete_hash->{$opt}->{force};
4510 eval {
4511 if ($opt =~ m/^unused/) {
4512 die "internal error";
4513 } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4514 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4515 }
4516 };
4517 if (my $err = $@) {
4518 $add_apply_error->($opt, $err);
4519 } else {
4520 PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4521 delete $conf->{$opt};
4522 }
4523 }
4524
4525 PVE::QemuConfig->cleanup_pending($conf);
4526
4527 foreach my $opt (keys %{$conf->{pending}}) { # add/change
4528 next if $opt eq 'delete'; # just to be sure
4529 eval {
4530 if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4531 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4532 }
4533 };
4534 if (my $err = $@) {
4535 $add_apply_error->($opt, $err);
4536 } else {
4537 $conf->{$opt} = delete $conf->{pending}->{$opt};
4538 }
4539 }
4540
4541 # write all changes at once to avoid unnecessary i/o
4542 PVE::QemuConfig->write_config($vmid, $conf);
4543}
4544
4545sub vmconfig_update_net {
4546 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4547
4548 my $newnet = parse_net($value);
4549
4550 if ($conf->{$opt}) {
4551 my $oldnet = parse_net($conf->{$opt});
4552
4553 if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4554 safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4555 safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4556 !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4557
4558 # for non online change, we try to hot-unplug
4559 die "skip\n" if !$hotplug;
4560 vm_deviceunplug($vmid, $conf, $opt);
4561 } else {
4562
4563 die "internal error" if $opt !~ m/net(\d+)/;
4564 my $iface = "tap${vmid}i$1";
4565
4566 if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4567 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4568 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4569 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4570 PVE::Network::tap_unplug($iface);
4571
4572 if ($have_sdn) {
4573 PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4574 } else {
4575 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4576 }
4577 } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4578 # Rate can be applied on its own but any change above needs to
4579 # include the rate in tap_plug since OVS resets everything.
4580 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4581 }
4582
4583 if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4584 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4585 }
4586
4587 return 1;
4588 }
4589 }
4590
4591 if ($hotplug) {
4592 vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4593 } else {
4594 die "skip\n";
4595 }
4596}
4597
4598sub vmconfig_update_disk {
4599 my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4600
4601 my $drive = parse_drive($opt, $value);
4602
4603 if ($conf->{$opt}) {
4604
4605 if (my $old_drive = parse_drive($opt, $conf->{$opt})) {
4606
4607 my $media = $drive->{media} || 'disk';
4608 my $oldmedia = $old_drive->{media} || 'disk';
4609 die "unable to change media type\n" if $media ne $oldmedia;
4610
4611 if (!drive_is_cdrom($old_drive)) {
4612
4613 if ($drive->{file} ne $old_drive->{file}) {
4614
4615 die "skip\n" if !$hotplug;
4616
4617 # unplug and register as unused
4618 vm_deviceunplug($vmid, $conf, $opt);
4619 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4620
4621 } else {
4622 # update existing disk
4623
4624 # skip non hotpluggable value
4625 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4626 safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4627 safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4628 safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4629 safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4630 die "skip\n";
4631 }
4632
4633 # apply throttle
4634 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4635 safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4636 safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4637 safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4638 safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4639 safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4640 safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4641 safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4642 safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4643 safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4644 safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4645 safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4646 safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4647 safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4648 safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4649 safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4650 safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4651 safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4652
4653 qemu_block_set_io_throttle($vmid,"drive-$opt",
4654 ($drive->{mbps} || 0)*1024*1024,
4655 ($drive->{mbps_rd} || 0)*1024*1024,
4656 ($drive->{mbps_wr} || 0)*1024*1024,
4657 $drive->{iops} || 0,
4658 $drive->{iops_rd} || 0,
4659 $drive->{iops_wr} || 0,
4660 ($drive->{mbps_max} || 0)*1024*1024,
4661 ($drive->{mbps_rd_max} || 0)*1024*1024,
4662 ($drive->{mbps_wr_max} || 0)*1024*1024,
4663 $drive->{iops_max} || 0,
4664 $drive->{iops_rd_max} || 0,
4665 $drive->{iops_wr_max} || 0,
4666 $drive->{bps_max_length} || 1,
4667 $drive->{bps_rd_max_length} || 1,
4668 $drive->{bps_wr_max_length} || 1,
4669 $drive->{iops_max_length} || 1,
4670 $drive->{iops_rd_max_length} || 1,
4671 $drive->{iops_wr_max_length} || 1);
4672
4673 }
4674
4675 return 1;
4676 }
4677
4678 } else { # cdrom
4679
4680 if ($drive->{file} eq 'none') {
4681 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4682 if (drive_is_cloudinit($old_drive)) {
4683 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4684 }
4685 } else {
4686 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4687
4688 # force eject if locked
4689 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4690
4691 if ($path) {
4692 mon_cmd($vmid, "blockdev-change-medium",
4693 id => "$opt", filename => "$path");
4694 }
4695 }
4696
4697 return 1;
4698 }
4699 }
4700 }
4701
4702 die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4703 # hotplug new disks
4704 PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4705 vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4706}
4707
4708sub vm_start {
4709 my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4710 $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage, $timeout) = @_;
4711
4712 PVE::QemuConfig->lock_config($vmid, sub {
4713 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4714
4715 die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4716
4717 my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
4718
4719 PVE::QemuConfig->check_lock($conf)
4720 if !($skiplock || $is_suspended);
4721
4722 die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4723
4724 # clean up leftover reboot request files
4725 eval { clear_reboot_request($vmid); };
4726 warn $@ if $@;
4727
4728 if (!$statefile && scalar(keys %{$conf->{pending}})) {
4729 vmconfig_apply_pending($vmid, $conf, $storecfg);
4730 $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4731 }
4732
4733 PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4734
4735 my $defaults = load_defaults();
4736
4737 # set environment variable useful inside network script
4738 $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4739
4740 my $local_volumes = {};
4741
4742 if ($targetstorage) {
4743 foreach_drive($conf, sub {
4744 my ($ds, $drive) = @_;
4745
4746 return if drive_is_cdrom($drive);
4747
4748 my $volid = $drive->{file};
4749
4750 return if !$volid;
4751
4752 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4753
4754 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4755 return if $scfg->{shared};
4756 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4757 });
4758
4759 my $format = undef;
4760
4761 foreach my $opt (sort keys %$local_volumes) {
4762
4763 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4764 my $drive = parse_drive($opt, $conf->{$opt});
4765
4766 # If a remote storage is specified and the format of the original
4767 # volume is not available there, fall back to the default format.
4768 # Otherwise use the same format as the original.
4769 if ($targetstorage && $targetstorage ne "1") {
4770 $storeid = $targetstorage;
4771 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4772 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4773 my $fileFormat = qemu_img_format($scfg, $volname);
4774 $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4775 } else {
4776 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4777 $format = qemu_img_format($scfg, $volname);
4778 }
4779
4780 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4781 my $newdrive = $drive;
4782 $newdrive->{format} = $format;
4783 $newdrive->{file} = $newvolid;
4784 my $drivestr = print_drive($newdrive);
4785 $local_volumes->{$opt} = $drivestr;
4786 #pass drive to conf for command line
4787 $conf->{$opt} = $drivestr;
4788 }
4789 }
4790
4791 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4792
4793 if ($is_suspended) {
4794 # enforce machine type on suspended vm to ensure HW compatibility
4795 $forcemachine = $conf->{runningmachine};
4796 print "Resuming suspended VM\n";
4797 }
4798
4799 my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4800
4801 my $migration_ip;
4802 my $get_migration_ip = sub {
4803 my ($cidr, $nodename) = @_;
4804
4805 return $migration_ip if defined($migration_ip);
4806
4807 if (!defined($cidr)) {
4808 my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4809 $cidr = $dc_conf->{migration}->{network};
4810 }
4811
4812 if (defined($cidr)) {
4813 my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4814
4815 die "could not get IP: no address configured on local " .
4816 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4817
4818 die "could not get IP: multiple addresses configured on local " .
4819 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4820
4821 $migration_ip = @$ips[0];
4822 }
4823
4824 $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
4825 if !defined($migration_ip);
4826
4827 return $migration_ip;
4828 };
4829
4830 my $migrate_uri;
4831 if ($statefile) {
4832 if ($statefile eq 'tcp') {
4833 my $localip = "localhost";
4834 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4835 my $nodename = nodename();
4836
4837 if (!defined($migration_type)) {
4838 if (defined($datacenterconf->{migration}->{type})) {
4839 $migration_type = $datacenterconf->{migration}->{type};
4840 } else {
4841 $migration_type = 'secure';
4842 }
4843 }
4844
4845 if ($migration_type eq 'insecure') {
4846 $localip = $get_migration_ip->($migration_network, $nodename);
4847 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4848 }
4849
4850 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4851 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4852 $migrate_uri = "tcp:${localip}:${migrate_port}";
4853 push @$cmd, '-incoming', $migrate_uri;
4854 push @$cmd, '-S';
4855
4856 } elsif ($statefile eq 'unix') {
4857 # should be default for secure migrations as a ssh TCP forward
4858 # tunnel is not deterministic reliable ready and fails regurarly
4859 # to set up in time, so use UNIX socket forwards
4860 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4861 unlink $socket_addr;
4862
4863 $migrate_uri = "unix:$socket_addr";
4864
4865 push @$cmd, '-incoming', $migrate_uri;
4866 push @$cmd, '-S';
4867
4868 } elsif (-e $statefile) {
4869 push @$cmd, '-loadstate', $statefile;
4870 } else {
4871 my $statepath = PVE::Storage::path($storecfg, $statefile);
4872 push @$vollist, $statefile;
4873 push @$cmd, '-loadstate', $statepath;
4874 }
4875 } elsif ($paused) {
4876 push @$cmd, '-S';
4877 }
4878
4879 # host pci devices
4880 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++) {
4881 my $d = parse_hostpci($conf->{"hostpci$i"});
4882 next if !$d;
4883 my $pcidevices = $d->{pciid};
4884 foreach my $pcidevice (@$pcidevices) {
4885 my $pciid = $pcidevice->{id};
4886
4887 my $info = PVE::SysFSTools::pci_device_info("$pciid");
4888 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
4889 die "no pci device info for device '$pciid'\n" if !$info;
4890
4891 if ($d->{mdev}) {
4892 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
4893 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
4894 } else {
4895 die "can't unbind/bind pci group to vfio '$pciid'\n"
4896 if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
4897 die "can't reset pci device '$pciid'\n"
4898 if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
4899 }
4900 }
4901 }
4902
4903 PVE::Storage::activate_volumes($storecfg, $vollist);
4904
4905 eval {
4906 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4907 outfunc => sub {}, errfunc => sub {});
4908 };
4909 # Issues with the above 'stop' not being fully completed are extremely rare, a very low
4910 # timeout should be more than enough here...
4911 PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
4912
4913 my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4914 : $defaults->{cpuunits};
4915
4916 my $start_timeout = $timeout // config_aware_timeout($conf, $is_suspended);
4917 my %run_params = (
4918 timeout => $statefile ? undef : $start_timeout,
4919 umask => 0077,
4920 noerr => 1,
4921 );
4922
4923 # when migrating, prefix QEMU output so other side can pick up any
4924 # errors that might occur and show the user
4925 if ($migratedfrom) {
4926 $run_params{quiet} = 1;
4927 $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
4928 }
4929
4930 my %properties = (
4931 Slice => 'qemu.slice',
4932 KillMode => 'none',
4933 CPUShares => $cpuunits
4934 );
4935
4936 if (my $cpulimit = $conf->{cpulimit}) {
4937 $properties{CPUQuota} = int($cpulimit * 100);
4938 }
4939 $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4940
4941 my $run_qemu = sub {
4942 PVE::Tools::run_fork sub {
4943 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4944
4945 my $exitcode = run_command($cmd, %run_params);
4946 die "QEMU exited with code $exitcode\n" if $exitcode;
4947 };
4948 };
4949
4950 if ($conf->{hugepages}) {
4951
4952 my $code = sub {
4953 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4954 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4955
4956 PVE::QemuServer::Memory::hugepages_mount();
4957 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4958
4959 eval { $run_qemu->() };
4960 if (my $err = $@) {
4961 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4962 die $err;
4963 }
4964
4965 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4966 };
4967 eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4968
4969 } else {
4970 eval { $run_qemu->() };
4971 }
4972
4973 if (my $err = $@) {
4974 # deactivate volumes if start fails
4975 eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4976 die "start failed: $err";
4977 }
4978
4979 print "migration listens on $migrate_uri\n" if $migrate_uri;
4980
4981 if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix') {
4982 eval { mon_cmd($vmid, "cont"); };
4983 warn $@ if $@;
4984 }
4985
4986 #start nbd server for storage migration
4987 if ($targetstorage) {
4988 my $nodename = nodename();
4989 my $localip = $get_migration_ip->($migration_network, $nodename);
4990 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4991 my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
4992
4993 mon_cmd($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
4994
4995 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4996
4997 foreach my $opt (sort keys %$local_volumes) {
4998 my $drivestr = $local_volumes->{$opt};
4999 mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5000 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5001 print "storage migration listens on $migrate_storage_uri volume:$drivestr\n";
5002 }
5003 }
5004
5005 if ($migratedfrom) {
5006 eval {
5007 set_migration_caps($vmid);
5008 };
5009 warn $@ if $@;
5010
5011 if ($spice_port) {
5012 print "spice listens on port $spice_port\n";
5013 if ($spice_ticket) {
5014 mon_cmd($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5015 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5016 }
5017 }
5018
5019 } else {
5020 mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5021 if !$statefile && $conf->{balloon};
5022
5023 foreach my $opt (keys %$conf) {
5024 next if $opt !~ m/^net\d+$/;
5025 my $nicconf = parse_net($conf->{$opt});
5026 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5027 }
5028 }
5029
5030 mon_cmd($vmid, 'qom-set',
5031 path => "machine/peripheral/balloon0",
5032 property => "guest-stats-polling-interval",
5033 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5034
5035 if ($is_suspended) {
5036 print "Resumed VM, removing state\n";
5037 if (my $vmstate = $conf->{vmstate}) {
5038 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5039 PVE::Storage::vdisk_free($storecfg, $vmstate);
5040 }
5041 delete $conf->@{qw(lock vmstate runningmachine)};
5042 PVE::QemuConfig->write_config($vmid, $conf);
5043 }
5044
5045 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5046 });
5047}
5048
5049sub vm_commandline {
5050 my ($storecfg, $vmid, $snapname) = @_;
5051
5052 my $conf = PVE::QemuConfig->load_config($vmid);
5053 my $forcemachine;
5054
5055 if ($snapname) {
5056 my $snapshot = $conf->{snapshots}->{$snapname};
5057 die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5058
5059 # check for a 'runningmachine' in snapshot
5060 $forcemachine = $snapshot->{runningmachine} if $snapshot->{runningmachine};
5061
5062 $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5063
5064 $conf = $snapshot;
5065 }
5066
5067 my $defaults = load_defaults();
5068
5069 my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5070
5071 return PVE::Tools::cmd2string($cmd);
5072}
5073
5074sub vm_reset {
5075 my ($vmid, $skiplock) = @_;
5076
5077 PVE::QemuConfig->lock_config($vmid, sub {
5078
5079 my $conf = PVE::QemuConfig->load_config($vmid);
5080
5081 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5082
5083 mon_cmd($vmid, "system_reset");
5084 });
5085}
5086
5087sub get_vm_volumes {
5088 my ($conf) = @_;
5089
5090 my $vollist = [];
5091 foreach_volid($conf, sub {
5092 my ($volid, $attr) = @_;
5093
5094 return if $volid =~ m|^/|;
5095
5096 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5097 return if !$sid;
5098
5099 push @$vollist, $volid;
5100 });
5101
5102 return $vollist;
5103}
5104
5105sub vm_stop_cleanup {
5106 my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5107
5108 eval {
5109
5110 if (!$keepActive) {
5111 my $vollist = get_vm_volumes($conf);
5112 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5113 }
5114
5115 foreach my $ext (qw(mon qmp pid vnc qga)) {
5116 unlink "/var/run/qemu-server/${vmid}.$ext";
5117 }
5118
5119 if ($conf->{ivshmem}) {
5120 my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5121 # just delete it for now, VMs which have this already open do not
5122 # are affected, but new VMs will get a separated one. If this
5123 # becomes an issue we either add some sort of ref-counting or just
5124 # add a "don't delete on stop" flag to the ivshmem format.
5125 unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5126 }
5127
5128 foreach my $key (keys %$conf) {
5129 next if $key !~ m/^hostpci(\d+)$/;
5130 my $hostpciindex = $1;
5131 my $d = parse_hostpci($conf->{$key});
5132 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5133
5134 foreach my $pci (@{$d->{pciid}}) {
5135 my $pciid = $pci->{id};
5136 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5137 }
5138 }
5139
5140 vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5141 };
5142 warn $@ if $@; # avoid errors - just warn
5143}
5144
5145# call only in locked context
5146sub _do_vm_stop {
5147 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5148
5149 my $pid = check_running($vmid, $nocheck);
5150 return if !$pid;
5151
5152 my $conf;
5153 if (!$nocheck) {
5154 $conf = PVE::QemuConfig->load_config($vmid);
5155 PVE::QemuConfig->check_lock($conf) if !$skiplock;
5156 if (!defined($timeout) && $shutdown && $conf->{startup}) {
5157 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5158 $timeout = $opts->{down} if $opts->{down};
5159 }
5160 PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5161 }
5162
5163 eval {
5164 if ($shutdown) {
5165 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5166 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5167 } else {
5168 mon_cmd($vmid, "system_powerdown");
5169 }
5170 } else {
5171 mon_cmd($vmid, "quit");
5172 }
5173 };
5174 my $err = $@;
5175
5176 if (!$err) {
5177 $timeout = 60 if !defined($timeout);
5178
5179 my $count = 0;
5180 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5181 $count++;
5182 sleep 1;
5183 }
5184
5185 if ($count >= $timeout) {
5186 if ($force) {
5187 warn "VM still running - terminating now with SIGTERM\n";
5188 kill 15, $pid;
5189 } else {
5190 die "VM quit/powerdown failed - got timeout\n";
5191 }
5192 } else {
5193 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5194 return;
5195 }
5196 } else {
5197 if ($force) {
5198 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5199 kill 15, $pid;
5200 } else {
5201 die "VM quit/powerdown failed\n";
5202 }
5203 }
5204
5205 # wait again
5206 $timeout = 10;
5207
5208 my $count = 0;
5209 while (($count < $timeout) && check_running($vmid, $nocheck)) {
5210 $count++;
5211 sleep 1;
5212 }
5213
5214 if ($count >= $timeout) {
5215 warn "VM still running - terminating now with SIGKILL\n";
5216 kill 9, $pid;
5217 sleep 1;
5218 }
5219
5220 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5221}
5222
5223# Note: use $nocheck to skip tests if VM configuration file exists.
5224# We need that when migration VMs to other nodes (files already moved)
5225# Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5226sub vm_stop {
5227 my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5228
5229 $force = 1 if !defined($force) && !$shutdown;
5230
5231 if ($migratedfrom){
5232 my $pid = check_running($vmid, $nocheck, $migratedfrom);
5233 kill 15, $pid if $pid;
5234 my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5235 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5236 return;
5237 }
5238
5239 PVE::QemuConfig->lock_config($vmid, sub {
5240 _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5241 });
5242}
5243
5244sub vm_reboot {
5245 my ($vmid, $timeout) = @_;
5246
5247 PVE::QemuConfig->lock_config($vmid, sub {
5248 eval {
5249
5250 # only reboot if running, as qmeventd starts it again on a stop event
5251 return if !check_running($vmid);
5252
5253 create_reboot_request($vmid);
5254
5255 my $storecfg = PVE::Storage::config();
5256 _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5257
5258 };
5259 if (my $err = $@) {
5260 # avoid that the next normal shutdown will be confused for a reboot
5261 clear_reboot_request($vmid);
5262 die $err;
5263 }
5264 });
5265}
5266
5267# note: if using the statestorage parameter, the caller has to check privileges
5268sub vm_suspend {
5269 my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5270
5271 my $conf;
5272 my $path;
5273 my $storecfg;
5274 my $vmstate;
5275
5276 PVE::QemuConfig->lock_config($vmid, sub {
5277
5278 $conf = PVE::QemuConfig->load_config($vmid);
5279
5280 my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5281 PVE::QemuConfig->check_lock($conf)
5282 if !($skiplock || $is_backing_up);
5283
5284 die "cannot suspend to disk during backup\n"
5285 if $is_backing_up && $includestate;
5286
5287 if ($includestate) {
5288 $conf->{lock} = 'suspending';
5289 my $date = strftime("%Y-%m-%d", localtime(time()));
5290 $storecfg = PVE::Storage::config();
5291 if (!$statestorage) {
5292 $statestorage = find_vmstate_storage($conf, $storecfg);
5293 # check permissions for the storage
5294 my $rpcenv = PVE::RPCEnvironment::get();
5295 if ($rpcenv->{type} ne 'cli') {
5296 my $authuser = $rpcenv->get_user();
5297 $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5298 }
5299 }
5300
5301
5302 $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5303 $path = PVE::Storage::path($storecfg, $vmstate);
5304 PVE::QemuConfig->write_config($vmid, $conf);
5305 } else {
5306 mon_cmd($vmid, "stop");
5307 }
5308 });
5309
5310 if ($includestate) {
5311 # save vm state
5312 PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5313
5314 eval {
5315 mon_cmd($vmid, "savevm-start", statefile => $path);
5316 for(;;) {
5317 my $state = mon_cmd($vmid, "query-savevm");
5318 if (!$state->{status}) {
5319 die "savevm not active\n";
5320 } elsif ($state->{status} eq 'active') {
5321 sleep(1);
5322 next;
5323 } elsif ($state->{status} eq 'completed') {
5324 print "State saved, quitting\n";
5325 last;
5326 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5327 die "query-savevm failed with error '$state->{error}'\n"
5328 } else {
5329 die "query-savevm returned status '$state->{status}'\n";
5330 }
5331 }
5332 };
5333 my $err = $@;
5334
5335 PVE::QemuConfig->lock_config($vmid, sub {
5336 $conf = PVE::QemuConfig->load_config($vmid);
5337 if ($err) {
5338 # cleanup, but leave suspending lock, to indicate something went wrong
5339 eval {
5340 mon_cmd($vmid, "savevm-end");
5341 PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5342 PVE::Storage::vdisk_free($storecfg, $vmstate);
5343 delete $conf->@{qw(vmstate runningmachine)};
5344 PVE::QemuConfig->write_config($vmid, $conf);
5345 };
5346 warn $@ if $@;
5347 die $err;
5348 }
5349
5350 die "lock changed unexpectedly\n"
5351 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5352
5353 mon_cmd($vmid, "quit");
5354 $conf->{lock} = 'suspended';
5355 PVE::QemuConfig->write_config($vmid, $conf);
5356 });
5357 }
5358}
5359
5360sub vm_resume {
5361 my ($vmid, $skiplock, $nocheck) = @_;
5362
5363 PVE::QemuConfig->lock_config($vmid, sub {
5364 my $res = mon_cmd($vmid, 'query-status');
5365 my $resume_cmd = 'cont';
5366
5367 if ($res->{status} && $res->{status} eq 'suspended') {
5368 $resume_cmd = 'system_wakeup';
5369 }
5370
5371 if (!$nocheck) {
5372
5373 my $conf = PVE::QemuConfig->load_config($vmid);
5374
5375 PVE::QemuConfig->check_lock($conf)
5376 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5377 }
5378
5379 mon_cmd($vmid, $resume_cmd);
5380 });
5381}
5382
5383sub vm_sendkey {
5384 my ($vmid, $skiplock, $key) = @_;
5385
5386 PVE::QemuConfig->lock_config($vmid, sub {
5387
5388 my $conf = PVE::QemuConfig->load_config($vmid);
5389
5390 # there is no qmp command, so we use the human monitor command
5391 my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5392 die $res if $res ne '';
5393 });
5394}
5395
5396# vzdump restore implementaion
5397
5398sub tar_archive_read_firstfile {
5399 my $archive = shift;
5400
5401 die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5402
5403 # try to detect archive type first
5404 my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5405 die "unable to open file '$archive'\n";
5406 my $firstfile = <$fh>;
5407 kill 15, $pid;
5408 close $fh;
5409
5410 die "ERROR: archive contaions no data\n" if !$firstfile;
5411 chomp $firstfile;
5412
5413 return $firstfile;
5414}
5415
5416sub tar_restore_cleanup {
5417 my ($storecfg, $statfile) = @_;
5418
5419 print STDERR "starting cleanup\n";
5420
5421 if (my $fd = IO::File->new($statfile, "r")) {
5422 while (defined(my $line = <$fd>)) {
5423 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5424 my $volid = $2;
5425 eval {
5426 if ($volid =~ m|^/|) {
5427 unlink $volid || die 'unlink failed\n';
5428 } else {
5429 PVE::Storage::vdisk_free($storecfg, $volid);
5430 }
5431 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5432 };
5433 print STDERR "unable to cleanup '$volid' - $@" if $@;
5434 } else {
5435 print STDERR "unable to parse line in statfile - $line";
5436 }
5437 }
5438 $fd->close();
5439 }
5440}
5441
5442sub restore_file_archive {
5443 my ($archive, $vmid, $user, $opts) = @_;
5444
5445 my $format = $opts->{format};
5446 my $comp;
5447
5448 if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5449 $format = 'tar' if !$format;
5450 $comp = 'gzip';
5451 } elsif ($archive =~ m/\.tar$/) {
5452 $format = 'tar' if !$format;
5453 } elsif ($archive =~ m/.tar.lzo$/) {
5454 $format = 'tar' if !$format;
5455 $comp = 'lzop';
5456 } elsif ($archive =~ m/\.vma$/) {
5457 $format = 'vma' if !$format;
5458 } elsif ($archive =~ m/\.vma\.gz$/) {
5459 $format = 'vma' if !$format;
5460 $comp = 'gzip';
5461 } elsif ($archive =~ m/\.vma\.lzo$/) {
5462 $format = 'vma' if !$format;
5463 $comp = 'lzop';
5464 } else {
5465 $format = 'vma' if !$format; # default
5466 }
5467
5468 # try to detect archive format
5469 if ($format eq 'tar') {
5470 return restore_tar_archive($archive, $vmid, $user, $opts);
5471 } else {
5472 return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5473 }
5474}
5475
5476# hepler to remove disks that will not be used after restore
5477my $restore_cleanup_oldconf = sub {
5478 my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5479
5480 foreach_drive($oldconf, sub {
5481 my ($ds, $drive) = @_;
5482
5483 return if drive_is_cdrom($drive, 1);
5484
5485 my $volid = $drive->{file};
5486 return if !$volid || $volid =~ m|^/|;
5487
5488 my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5489 return if !$path || !$owner || ($owner != $vmid);
5490
5491 # Note: only delete disk we want to restore
5492 # other volumes will become unused
5493 if ($virtdev_hash->{$ds}) {
5494 eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5495 if (my $err = $@) {
5496 warn $err;
5497 }
5498 }
5499 });
5500
5501 # delete vmstate files, after the restore we have no snapshots anymore
5502 foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5503 my $snap = $oldconf->{snapshots}->{$snapname};
5504 if ($snap->{vmstate}) {
5505 eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5506 if (my $err = $@) {
5507 warn $err;
5508 }
5509 }
5510 }
5511};
5512
5513# Helper to parse vzdump backup device hints
5514#
5515# $rpcenv: Environment, used to ckeck storage permissions
5516# $user: User ID, to check storage permissions
5517# $storecfg: Storage configuration
5518# $fh: the file handle for reading the configuration
5519# $devinfo: should contain device sizes for all backu-up'ed devices
5520# $options: backup options (pool, default storage)
5521#
5522# Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5523my $parse_backup_hints = sub {
5524 my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5525
5526 my $virtdev_hash = {};
5527
5528 while (defined(my $line = <$fh>)) {
5529 if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5530 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5531 die "archive does not contain data for drive '$virtdev'\n"
5532 if !$devinfo->{$devname};
5533
5534 if (defined($options->{storage})) {
5535 $storeid = $options->{storage} || 'local';
5536 } elsif (!$storeid) {
5537 $storeid = 'local';
5538 }
5539 $format = 'raw' if !$format;
5540 $devinfo->{$devname}->{devname} = $devname;
5541 $devinfo->{$devname}->{virtdev} = $virtdev;
5542 $devinfo->{$devname}->{format} = $format;
5543 $devinfo->{$devname}->{storeid} = $storeid;
5544
5545 # check permission on storage
5546 my $pool = $options->{pool}; # todo: do we need that?
5547 if ($user ne 'root@pam') {
5548 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5549 }
5550
5551 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5552 } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5553 my $virtdev = $1;
5554 my $drive = parse_drive($virtdev, $2);
5555 if (drive_is_cloudinit($drive)) {
5556 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5557 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5558 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5559
5560 $virtdev_hash->{$virtdev} = {
5561 format => $format,
5562 storeid => $options->{storage} // $storeid,
5563 size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5564 is_cloudinit => 1,
5565 };
5566 }
5567 }
5568 }
5569
5570 return $virtdev_hash;
5571};
5572
5573# Helper to allocate and activate all volumes required for a restore
5574#
5575# $storecfg: Storage configuration
5576# $virtdev_hash: as returned by parse_backup_hints()
5577#
5578# Returns: { $virtdev => $volid }
5579my $restore_allocate_devices = sub {
5580 my ($storecfg, $virtdev_hash, $vmid) = @_;
5581
5582 my $map = {};
5583 foreach my $virtdev (sort keys %$virtdev_hash) {
5584 my $d = $virtdev_hash->{$virtdev};
5585 my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5586 my $storeid = $d->{storeid};
5587 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5588
5589 # test if requested format is supported
5590 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5591 my $supported = grep { $_ eq $d->{format} } @$validFormats;
5592 $d->{format} = $defFormat if !$supported;
5593
5594 my $name;
5595 if ($d->{is_cloudinit}) {
5596 $name = "vm-$vmid-cloudinit";
5597 $name .= ".$d->{format}" if $d->{format} ne 'raw';
5598 }
5599
5600 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5601
5602 print STDERR "new volume ID is '$volid'\n";
5603 $d->{volid} = $volid;
5604
5605 PVE::Storage::activate_volumes($storecfg, [$volid]);
5606
5607 $map->{$virtdev} = $volid;
5608 }
5609
5610 return $map;
5611};
5612
5613my $restore_update_config_line = sub {
5614 my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5615
5616 return if $line =~ m/^\#qmdump\#/;
5617 return if $line =~ m/^\#vzdump\#/;
5618 return if $line =~ m/^lock:/;
5619 return if $line =~ m/^unused\d+:/;
5620 return if $line =~ m/^parent:/;
5621
5622 my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5623 if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5624 # try to convert old 1.X settings
5625 my ($id, $ind, $ethcfg) = ($1, $2, $3);
5626 foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5627 my ($model, $macaddr) = split(/\=/, $devconfig);
5628 $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5629 my $net = {
5630 model => $model,
5631 bridge => "vmbr$ind",
5632 macaddr => $macaddr,
5633 };
5634 my $netstr = print_net($net);
5635
5636 print $outfd "net$cookie->{netcount}: $netstr\n";
5637 $cookie->{netcount}++;
5638 }
5639 } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5640 my ($id, $netstr) = ($1, $2);
5641 my $net = parse_net($netstr);
5642 $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5643 $netstr = print_net($net);
5644 print $outfd "$id: $netstr\n";
5645 } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5646 my $virtdev = $1;
5647 my $value = $3;
5648 my $di = parse_drive($virtdev, $value);
5649 if (defined($di->{backup}) && !$di->{backup}) {
5650 print $outfd "#$line";
5651 } elsif ($map->{$virtdev}) {
5652 delete $di->{format}; # format can change on restore
5653 $di->{file} = $map->{$virtdev};
5654 $value = print_drive($di);
5655 print $outfd "$virtdev: $value\n";
5656 } else {
5657 print $outfd $line;
5658 }
5659 } elsif (($line =~ m/^vmgenid: (.*)/)) {
5660 my $vmgenid = $1;
5661 if ($vmgenid ne '0') {
5662 # always generate a new vmgenid if there was a valid one setup
5663 $vmgenid = generate_uuid();
5664 }
5665 print $outfd "vmgenid: $vmgenid\n";
5666 } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5667 my ($uuid, $uuid_str);
5668 UUID::generate($uuid);
5669 UUID::unparse($uuid, $uuid_str);
5670 my $smbios1 = parse_smbios1($2);
5671 $smbios1->{uuid} = $uuid_str;
5672 print $outfd $1.print_smbios1($smbios1)."\n";
5673 } else {
5674 print $outfd $line;
5675 }
5676};
5677
5678my $restore_deactivate_volumes = sub {
5679 my ($storecfg, $devinfo) = @_;
5680
5681 my $vollist = [];
5682 foreach my $devname (keys %$devinfo) {
5683 my $volid = $devinfo->{$devname}->{volid};
5684 push @$vollist, $volid if $volid;
5685 }
5686
5687 PVE::Storage::deactivate_volumes($storecfg, $vollist);
5688};
5689
5690my $restore_destroy_volumes = sub {
5691 my ($storecfg, $devinfo) = @_;
5692
5693 foreach my $devname (keys %$devinfo) {
5694 my $volid = $devinfo->{$devname}->{volid};
5695 next if !$volid;
5696 eval {
5697 if ($volid =~ m|^/|) {
5698 unlink $volid || die 'unlink failed\n';
5699 } else {
5700 PVE::Storage::vdisk_free($storecfg, $volid);
5701 }
5702 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5703 };
5704 print STDERR "unable to cleanup '$volid' - $@" if $@;
5705 }
5706};
5707
5708sub scan_volids {
5709 my ($cfg, $vmid) = @_;
5710
5711 my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5712
5713 my $volid_hash = {};
5714 foreach my $storeid (keys %$info) {
5715 foreach my $item (@{$info->{$storeid}}) {
5716 next if !($item->{volid} && $item->{size});
5717 $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5718 $volid_hash->{$item->{volid}} = $item;
5719 }
5720 }
5721
5722 return $volid_hash;
5723}
5724
5725sub update_disk_config {
5726 my ($vmid, $conf, $volid_hash) = @_;
5727
5728 my $changes;
5729 my $prefix = "VM $vmid:";
5730
5731 # used and unused disks
5732 my $referenced = {};
5733
5734 # Note: it is allowed to define multiple storages with same path (alias), so
5735 # we need to check both 'volid' and real 'path' (two different volid can point
5736 # to the same path).
5737
5738 my $referencedpath = {};
5739
5740 # update size info
5741 foreach my $opt (keys %$conf) {
5742 if (is_valid_drivename($opt)) {
5743 my $drive = parse_drive($opt, $conf->{$opt});
5744 my $volid = $drive->{file};
5745 next if !$volid;
5746
5747 # mark volid as "in-use" for next step
5748 $referenced->{$volid} = 1;
5749 if ($volid_hash->{$volid} &&
5750 (my $path = $volid_hash->{$volid}->{path})) {
5751 $referencedpath->{$path} = 1;
5752 }
5753
5754 next if drive_is_cdrom($drive);
5755 next if !$volid_hash->{$volid};
5756
5757 my ($updated, $old_size, $new_size) = PVE::QemuServer::Drive::update_disksize($drive, $volid_hash);
5758 if (defined($updated)) {
5759 $changes = 1;
5760 $conf->{$opt} = print_drive($updated);
5761 print "$prefix size of disk '$volid' ($opt) updated from $old_size to $new_size\n";
5762 }
5763 }
5764 }
5765
5766 # remove 'unusedX' entry if volume is used
5767 foreach my $opt (keys %$conf) {
5768 next if $opt !~ m/^unused\d+$/;
5769 my $volid = $conf->{$opt};
5770 my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5771 if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5772 print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5773 $changes = 1;
5774 delete $conf->{$opt};
5775 }
5776
5777 $referenced->{$volid} = 1;
5778 $referencedpath->{$path} = 1 if $path;
5779 }
5780
5781 foreach my $volid (sort keys %$volid_hash) {
5782 next if $volid =~ m/vm-$vmid-state-/;
5783 next if $referenced->{$volid};
5784 my $path = $volid_hash->{$volid}->{path};
5785 next if !$path; # just to be sure
5786 next if $referencedpath->{$path};
5787 $changes = 1;
5788 my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
5789 print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
5790 $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5791 }
5792
5793 return $changes;
5794}
5795
5796sub rescan {
5797 my ($vmid, $nolock, $dryrun) = @_;
5798
5799 my $cfg = PVE::Storage::config();
5800
5801 # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
5802 # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
5803 foreach my $stor (keys %{$cfg->{ids}}) {
5804 delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
5805 }
5806
5807 print "rescan volumes...\n";
5808 my $volid_hash = scan_volids($cfg, $vmid);
5809
5810 my $updatefn = sub {
5811 my ($vmid) = @_;
5812
5813 my $conf = PVE::QemuConfig->load_config($vmid);
5814
5815 PVE::QemuConfig->check_lock($conf);
5816
5817 my $vm_volids = {};
5818 foreach my $volid (keys %$volid_hash) {
5819 my $info = $volid_hash->{$volid};
5820 $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5821 }
5822
5823 my $changes = update_disk_config($vmid, $conf, $vm_volids);
5824
5825 PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
5826 };
5827
5828 if (defined($vmid)) {
5829 if ($nolock) {
5830 &$updatefn($vmid);
5831 } else {
5832 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5833 }
5834 } else {
5835 my $vmlist = config_list();
5836 foreach my $vmid (keys %$vmlist) {
5837 if ($nolock) {
5838 &$updatefn($vmid);
5839 } else {
5840 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5841 }
5842 }
5843 }
5844}
5845
5846sub restore_proxmox_backup_archive {
5847 my ($archive, $vmid, $user, $options) = @_;
5848
5849 my $storecfg = PVE::Storage::config();
5850
5851 my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
5852 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5853
5854 my $server = $scfg->{server};
5855 my $datastore = $scfg->{datastore};
5856 my $username = $scfg->{username} // 'root@pam';
5857 my $fingerprint = $scfg->{fingerprint};
5858
5859 my $repo = "$username\@$server:$datastore";
5860 my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
5861 local $ENV{PBS_PASSWORD} = $password;
5862 local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
5863
5864 my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
5865 PVE::Storage::parse_volname($storecfg, $archive);
5866
5867 die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
5868
5869 die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
5870
5871 my $tmpdir = "/var/tmp/vzdumptmp$$";
5872 rmtree $tmpdir;
5873 mkpath $tmpdir;
5874
5875 my $conffile = PVE::QemuConfig->config_file($vmid);
5876 my $tmpfn = "$conffile.$$.tmp";
5877 # disable interrupts (always do cleanups)
5878 local $SIG{INT} =
5879 local $SIG{TERM} =
5880 local $SIG{QUIT} =
5881 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5882
5883 # Note: $oldconf is undef if VM does not exists
5884 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5885 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5886
5887 my $rpcenv = PVE::RPCEnvironment::get();
5888 my $devinfo = {};
5889
5890 eval {
5891 # enable interrupts
5892 local $SIG{INT} =
5893 local $SIG{TERM} =
5894 local $SIG{QUIT} =
5895 local $SIG{HUP} =
5896 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5897
5898 my $cfgfn = "$tmpdir/qemu-server.conf";
5899 my $firewall_config_fn = "$tmpdir/fw.conf";
5900 my $index_fn = "$tmpdir/index.json";
5901
5902 my $cmd = "restore";
5903
5904 my $param = [$pbs_backup_name, "index.json", $index_fn];
5905 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5906 my $index = PVE::Tools::file_get_contents($index_fn);
5907 $index = decode_json($index);
5908
5909 # print Dumper($index);
5910 foreach my $info (@{$index->{files}}) {
5911 if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
5912 my $devname = $1;
5913 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
5914 $devinfo->{$devname}->{size} = $1;
5915 } else {
5916 die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
5917 }
5918 }
5919 }
5920
5921 my $is_qemu_server_backup = scalar(grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}});
5922 if (!$is_qemu_server_backup) {
5923 die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
5924 }
5925 my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
5926
5927 $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
5928 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5929
5930 if ($has_firewall_config) {
5931 $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
5932 PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
5933
5934 my $pve_firewall_dir = '/etc/pve/firewall';
5935 mkdir $pve_firewall_dir; # make sure the dir exists
5936 PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
5937 }
5938
5939 my $fh = IO::File->new($cfgfn, "r") ||
5940 "unable to read qemu-server.conf - $!\n";
5941
5942 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
5943
5944 # fixme: rate limit?
5945
5946 # create empty/temp config
5947 PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
5948
5949 $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
5950
5951 # allocate volumes
5952 my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
5953
5954 foreach my $virtdev (sort keys %$virtdev_hash) {
5955 my $d = $virtdev_hash->{$virtdev};
5956 next if $d->{is_cloudinit}; # no need to restore cloudinit
5957
5958 my $volid = $d->{volid};
5959
5960 my $path = PVE::Storage::path($storecfg, $volid);
5961 if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
5962 #$path = "zeroinit:$path"; # fixme
5963 }
5964
5965 my $pbs_restore_cmd = [
5966 '/usr/bin/proxmox-backup-client',
5967 'restore',
5968 '--repository', $repo,
5969 $pbs_backup_name,
5970 "$d->{devname}.img",
5971 '-',
5972 '--verbose',
5973 ];
5974
5975 my $import_cmd = [
5976 '/usr/bin/qemu-img',
5977 'dd', '-n', '-f', 'raw', '-O', $d->{format}, 'bs=64K',
5978 'isize=0',
5979 "osize=$d->{size}",
5980 "of=$path",
5981 ];
5982
5983 my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd) . '|' . PVE::Tools::cmd2string($import_cmd);
5984 print "restore proxmox backup image: $dbg_cmdstring\n";
5985 run_command([$pbs_restore_cmd, $import_cmd]);
5986 }
5987
5988 $fh->seek(0, 0) || die "seek failed - $!\n";
5989
5990 my $outfd = new IO::File ($tmpfn, "w") ||
5991 die "unable to write config for VM $vmid\n";
5992
5993 my $cookie = { netcount => 0 };
5994 while (defined(my $line = <$fh>)) {
5995 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
5996 }
5997
5998 $fh->close();
5999 $outfd->close();
6000 };
6001 my $err = $@;
6002
6003 $restore_deactivate_volumes->($storecfg, $devinfo);
6004
6005 rmtree $tmpdir;
6006
6007 if ($err) {
6008 unlink $tmpfn;
6009 $restore_destroy_volumes->($storecfg, $devinfo);
6010 die $err;
6011 }
6012
6013 rename($tmpfn, $conffile) ||
6014 die "unable to commit configuration file '$conffile'\n";
6015
6016 PVE::Cluster::cfs_update(); # make sure we read new file
6017
6018 eval { rescan($vmid, 1); };
6019 warn $@ if $@;
6020}
6021
6022sub restore_vma_archive {
6023 my ($archive, $vmid, $user, $opts, $comp) = @_;
6024
6025 my $readfrom = $archive;
6026
6027 my $cfg = PVE::Storage::config();
6028 my $commands = [];
6029 my $bwlimit = $opts->{bwlimit};
6030
6031 my $dbg_cmdstring = '';
6032 my $add_pipe = sub {
6033 my ($cmd) = @_;
6034 push @$commands, $cmd;
6035 $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6036 $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6037 $readfrom = '-';
6038 };
6039
6040 my $input = undef;
6041 if ($archive eq '-') {
6042 $input = '<&STDIN';
6043 } else {
6044 # If we use a backup from a PVE defined storage we also consider that
6045 # storage's rate limit:
6046 my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6047 if (defined($volid)) {
6048 my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6049 my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6050 if ($readlimit) {
6051 print STDERR "applying read rate limit: $readlimit\n";
6052 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6053 $add_pipe->($cstream);
6054 }
6055 }
6056 }
6057
6058 if ($comp) {
6059 my $cmd;
6060 if ($comp eq 'gzip') {
6061 $cmd = ['zcat', $readfrom];
6062 } elsif ($comp eq 'lzop') {
6063 $cmd = ['lzop', '-d', '-c', $readfrom];
6064 } else {
6065 die "unknown compression method '$comp'\n";
6066 }
6067 $add_pipe->($cmd);
6068 }
6069
6070 my $tmpdir = "/var/tmp/vzdumptmp$$";
6071 rmtree $tmpdir;
6072
6073 # disable interrupts (always do cleanups)
6074 local $SIG{INT} =
6075 local $SIG{TERM} =
6076 local $SIG{QUIT} =
6077 local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6078
6079 my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6080 POSIX::mkfifo($mapfifo, 0600);
6081 my $fifofh;
6082
6083 my $openfifo = sub {
6084 open($fifofh, '>', $mapfifo) || die $!;
6085 };
6086
6087 $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6088
6089 my $oldtimeout;
6090 my $timeout = 5;
6091
6092 my $devinfo = {};
6093
6094 my $rpcenv = PVE::RPCEnvironment::get();
6095
6096 my $conffile = PVE::QemuConfig->config_file($vmid);
6097 my $tmpfn = "$conffile.$$.tmp";
6098
6099 # Note: $oldconf is undef if VM does not exist
6100 my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6101 my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6102
6103 my %storage_limits;
6104
6105 my $print_devmap = sub {
6106 my $cfgfn = "$tmpdir/qemu-server.conf";
6107
6108 # we can read the config - that is already extracted
6109 my $fh = IO::File->new($cfgfn, "r") ||
6110 "unable to read qemu-server.conf - $!\n";
6111
6112 my $fwcfgfn = "$tmpdir/qemu-server.fw";
6113 if (-f $fwcfgfn) {
6114 my $pve_firewall_dir = '/etc/pve/firewall';
6115 mkdir $pve_firewall_dir; # make sure the dir exists
6116 PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6117 }
6118
6119 my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6120
6121 foreach my $key (keys %storage_limits) {
6122 my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6123 next if !$limit;
6124 print STDERR "rate limit for storage $key: $limit KiB/s\n";
6125 $storage_limits{$key} = $limit * 1024;
6126 }
6127
6128 foreach my $devname (keys %$devinfo) {
6129 die "found no device mapping information for device '$devname'\n"
6130 if !$devinfo->{$devname}->{virtdev};
6131 }
6132
6133 # create empty/temp config
6134 if ($oldconf) {
6135 PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6136 $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6137 }
6138
6139 # allocate volumes
6140 my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6141
6142 # print restore information to $fifofh
6143 foreach my $virtdev (sort keys %$virtdev_hash) {
6144 my $d = $virtdev_hash->{$virtdev};
6145 next if $d->{is_cloudinit}; # no need to restore cloudinit
6146
6147 my $storeid = $d->{storeid};
6148 my $volid = $d->{volid};
6149
6150 my $map_opts = '';
6151 if (my $limit = $storage_limits{$storeid}) {
6152 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6153 }
6154
6155 my $write_zeros = 1;
6156 if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6157 $write_zeros = 0;
6158 }
6159
6160 my $path = PVE::Storage::path($cfg, $volid);
6161
6162 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6163
6164 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6165 }
6166
6167 $fh->seek(0, 0) || die "seek failed - $!\n";
6168
6169 my $outfd = new IO::File ($tmpfn, "w") ||
6170 die "unable to write config for VM $vmid\n";
6171
6172 my $cookie = { netcount => 0 };
6173 while (defined(my $line = <$fh>)) {
6174 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6175 }
6176
6177 $fh->close();
6178 $outfd->close();
6179 };
6180
6181 eval {
6182 # enable interrupts
6183 local $SIG{INT} =
6184 local $SIG{TERM} =
6185 local $SIG{QUIT} =
6186 local $SIG{HUP} =
6187 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6188 local $SIG{ALRM} = sub { die "got timeout\n"; };
6189
6190 $oldtimeout = alarm($timeout);
6191
6192 my $parser = sub {
6193 my $line = shift;
6194
6195 print "$line\n";
6196
6197 if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6198 my ($dev_id, $size, $devname) = ($1, $2, $3);
6199 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6200 } elsif ($line =~ m/^CTIME: /) {
6201 # we correctly received the vma config, so we can disable
6202 # the timeout now for disk allocation (set to 10 minutes, so
6203 # that we always timeout if something goes wrong)
6204 alarm(600);
6205 &$print_devmap();
6206 print $fifofh "done\n";
6207 my $tmp = $oldtimeout || 0;
6208 $oldtimeout = undef;
6209 alarm($tmp);
6210 close($fifofh);
6211 }
6212 };
6213
6214 print "restore vma archive: $dbg_cmdstring\n";
6215 run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6216 };
6217 my $err = $@;
6218
6219 alarm($oldtimeout) if $oldtimeout;
6220
6221 $restore_deactivate_volumes->($cfg, $devinfo);
6222
6223 unlink $mapfifo;
6224 rmtree $tmpdir;
6225
6226 if ($err) {
6227 unlink $tmpfn;
6228 $restore_destroy_volumes->($cfg, $devinfo);
6229 die $err;
6230 }
6231
6232 rename($tmpfn, $conffile) ||
6233 die "unable to commit configuration file '$conffile'\n";
6234
6235 PVE::Cluster::cfs_update(); # make sure we read new file
6236
6237 eval { rescan($vmid, 1); };
6238 warn $@ if $@;
6239}
6240
6241sub restore_tar_archive {
6242 my ($archive, $vmid, $user, $opts) = @_;
6243
6244 if ($archive ne '-') {
6245 my $firstfile = tar_archive_read_firstfile($archive);
6246 die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6247 if $firstfile ne 'qemu-server.conf';
6248 }
6249
6250 my $storecfg = PVE::Storage::config();
6251
6252 # avoid zombie disks when restoring over an existing VM -> cleanup first
6253 # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6254 # skiplock=1 because qmrestore has set the 'create' lock itself already
6255 my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6256 destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6257
6258 my $tocmd = "/usr/lib/qemu-server/qmextract";
6259
6260 $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6261 $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6262 $tocmd .= ' --prealloc' if $opts->{prealloc};
6263 $tocmd .= ' --info' if $opts->{info};
6264
6265 # tar option "xf" does not autodetect compression when read from STDIN,
6266 # so we pipe to zcat
6267 my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6268 PVE::Tools::shellquote("--to-command=$tocmd");
6269
6270 my $tmpdir = "/var/tmp/vzdumptmp$$";
6271 mkpath $tmpdir;
6272
6273 local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6274 local $ENV{VZDUMP_VMID} = $vmid;
6275 local $ENV{VZDUMP_USER} = $user;
6276
6277 my $conffile = PVE::QemuConfig->config_file($vmid);
6278 my $tmpfn = "$conffile.$$.tmp";
6279
6280 # disable interrupts (always do cleanups)
6281 local $SIG{INT} =
6282 local $SIG{TERM} =
6283 local $SIG{QUIT} =
6284 local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6285
6286 eval {
6287 # enable interrupts
6288 local $SIG{INT} =
6289 local $SIG{TERM} =
6290 local $SIG{QUIT} =
6291 local $SIG{HUP} =
6292 local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6293
6294 if ($archive eq '-') {
6295 print "extracting archive from STDIN\n";
6296 run_command($cmd, input => "<&STDIN");
6297 } else {
6298 print "extracting archive '$archive'\n";
6299 run_command($cmd);
6300 }
6301
6302 return if $opts->{info};
6303
6304 # read new mapping
6305 my $map = {};
6306 my $statfile = "$tmpdir/qmrestore.stat";
6307 if (my $fd = IO::File->new($statfile, "r")) {
6308 while (defined (my $line = <$fd>)) {
6309 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6310 $map->{$1} = $2 if $1;
6311 } else {
6312 print STDERR "unable to parse line in statfile - $line\n";
6313 }
6314 }
6315 $fd->close();
6316 }
6317
6318 my $confsrc = "$tmpdir/qemu-server.conf";
6319
6320 my $srcfd = new IO::File($confsrc, "r") ||
6321 die "unable to open file '$confsrc'\n";
6322
6323 my $outfd = new IO::File ($tmpfn, "w") ||
6324 die "unable to write config for VM $vmid\n";
6325
6326 my $cookie = { netcount => 0 };
6327 while (defined (my $line = <$srcfd>)) {
6328 $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6329 }
6330
6331 $srcfd->close();
6332 $outfd->close();
6333 };
6334 if (my $err = $@) {
6335 unlink $tmpfn;
6336 tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6337 die $err;
6338 }
6339
6340 rmtree $tmpdir;
6341
6342 rename $tmpfn, $conffile ||
6343 die "unable to commit configuration file '$conffile'\n";
6344
6345 PVE::Cluster::cfs_update(); # make sure we read new file
6346
6347 eval { rescan($vmid, 1); };
6348 warn $@ if $@;
6349};
6350
6351sub foreach_storage_used_by_vm {
6352 my ($conf, $func) = @_;
6353
6354 my $sidhash = {};
6355
6356 foreach_drive($conf, sub {
6357 my ($ds, $drive) = @_;
6358 return if drive_is_cdrom($drive);
6359
6360 my $volid = $drive->{file};
6361
6362 my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6363 $sidhash->{$sid} = $sid if $sid;
6364 });
6365
6366 foreach my $sid (sort keys %$sidhash) {
6367 &$func($sid);
6368 }
6369}
6370
6371my $qemu_snap_storage = {
6372 rbd => 1,
6373};
6374sub do_snapshots_with_qemu {
6375 my ($storecfg, $volid) = @_;
6376
6377 my $storage_name = PVE::Storage::parse_volume_id($volid);
6378 my $scfg = $storecfg->{ids}->{$storage_name};
6379
6380 if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6381 return 1;
6382 }
6383
6384 if ($volid =~ m/\.(qcow2|qed)$/){
6385 return 1;
6386 }
6387
6388 return undef;
6389}
6390
6391sub qga_check_running {
6392 my ($vmid, $nowarn) = @_;
6393
6394 eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6395 if ($@) {
6396 warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6397 return 0;
6398 }
6399 return 1;
6400}
6401
6402sub template_create {
6403 my ($vmid, $conf, $disk) = @_;
6404
6405 my $storecfg = PVE::Storage::config();
6406
6407 foreach_drive($conf, sub {
6408 my ($ds, $drive) = @_;
6409
6410 return if drive_is_cdrom($drive);
6411 return if $disk && $ds ne $disk;
6412
6413 my $volid = $drive->{file};
6414 return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6415
6416 my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6417 $drive->{file} = $voliddst;
6418 $conf->{$ds} = print_drive($drive);
6419 PVE::QemuConfig->write_config($vmid, $conf);
6420 });
6421}
6422
6423sub convert_iscsi_path {
6424 my ($path) = @_;
6425
6426 if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6427 my $portal = $1;
6428 my $target = $2;
6429 my $lun = $3;
6430
6431 my $initiator_name = get_initiator_name();
6432
6433 return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6434 "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6435 }
6436
6437 die "cannot convert iscsi path '$path', unkown format\n";
6438}
6439
6440sub qemu_img_convert {
6441 my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6442
6443 my $storecfg = PVE::Storage::config();
6444 my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6445 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6446
6447 die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6448
6449 my $cachemode;
6450 my $src_path;
6451 my $src_is_iscsi = 0;
6452 my $src_format;
6453
6454 if ($src_storeid) {
6455 PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6456 my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6457 $src_format = qemu_img_format($src_scfg, $src_volname);
6458 $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6459 $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6460 $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6461 } elsif (-f $src_volid) {
6462 $src_path = $src_volid;
6463 if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6464 $src_format = $1;
6465 }
6466 }
6467
6468 die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6469
6470 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6471 my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6472 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6473 my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6474
6475 my $cmd = [];
6476 push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6477 push @$cmd, '-l', "snapshot.name=$snapname"
6478 if $snapname && $src_format && $src_format eq "qcow2";
6479 push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6480 push @$cmd, '-T', $cachemode if defined($cachemode);
6481
6482 if ($src_is_iscsi) {
6483 push @$cmd, '--image-opts';
6484 $src_path = convert_iscsi_path($src_path);
6485 } elsif ($src_format) {
6486 push @$cmd, '-f', $src_format;
6487 }
6488
6489 if ($dst_is_iscsi) {
6490 push @$cmd, '--target-image-opts';
6491 $dst_path = convert_iscsi_path($dst_path);
6492 } else {
6493 push @$cmd, '-O', $dst_format;
6494 }
6495
6496 push @$cmd, $src_path;
6497
6498 if (!$dst_is_iscsi && $is_zero_initialized) {
6499 push @$cmd, "zeroinit:$dst_path";
6500 } else {
6501 push @$cmd, $dst_path;
6502 }
6503
6504 my $parser = sub {
6505 my $line = shift;
6506 if($line =~ m/\((\S+)\/100\%\)/){
6507 my $percent = $1;
6508 my $transferred = int($size * $percent / 100);
6509 my $remaining = $size - $transferred;
6510
6511 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6512 }
6513
6514 };
6515
6516 eval { run_command($cmd, timeout => undef, outfunc => $parser); };
6517 my $err = $@;
6518 die "copy failed: $err" if $err;
6519}
6520
6521sub qemu_img_format {
6522 my ($scfg, $volname) = @_;
6523
6524 if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6525 return $1;
6526 } else {
6527 return "raw";
6528 }
6529}
6530
6531sub qemu_drive_mirror {
6532 my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6533
6534 $jobs = {} if !$jobs;
6535
6536 my $qemu_target;
6537 my $format;
6538 $jobs->{"drive-$drive"} = {};
6539
6540 if ($dst_volid =~ /^nbd:/) {
6541 $qemu_target = $dst_volid;
6542 $format = "nbd";
6543 } else {
6544 my $storecfg = PVE::Storage::config();
6545 my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6546
6547 my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6548
6549 $format = qemu_img_format($dst_scfg, $dst_volname);
6550
6551 my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6552
6553 $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6554 }
6555
6556 my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6557 $opts->{format} = $format if $format;
6558
6559 if (defined($bwlimit)) {
6560 $opts->{speed} = $bwlimit * 1024;
6561 print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6562 } else {
6563 print "drive mirror is starting for drive-$drive\n";
6564 }
6565
6566 # if a job already runs for this device we get an error, catch it for cleanup
6567 eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6568 if (my $err = $@) {
6569 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6570 warn "$@\n" if $@;
6571 die "mirroring error: $err\n";
6572 }
6573
6574 qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6575}
6576
6577sub qemu_drive_mirror_monitor {
6578 my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6579
6580 eval {
6581 my $err_complete = 0;
6582
6583 while (1) {
6584 die "storage migration timed out\n" if $err_complete > 300;
6585
6586 my $stats = mon_cmd($vmid, "query-block-jobs");
6587
6588 my $running_mirror_jobs = {};
6589 foreach my $stat (@$stats) {
6590 next if $stat->{type} ne 'mirror';
6591 $running_mirror_jobs->{$stat->{device}} = $stat;
6592 }
6593
6594 my $readycounter = 0;
6595
6596 foreach my $job (keys %$jobs) {
6597
6598 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6599 print "$job : finished\n";
6600 delete $jobs->{$job};
6601 next;
6602 }
6603
6604 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6605
6606 my $busy = $running_mirror_jobs->{$job}->{busy};
6607 my $ready = $running_mirror_jobs->{$job}->{ready};
6608 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6609 my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6610 my $remaining = $total - $transferred;
6611 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6612
6613 print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6614 }
6615
6616 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6617 }
6618
6619 last if scalar(keys %$jobs) == 0;
6620
6621 if ($readycounter == scalar(keys %$jobs)) {
6622 print "all mirroring jobs are ready \n";
6623 last if $skipcomplete; #do the complete later
6624
6625 if ($vmiddst && $vmiddst != $vmid) {
6626 my $agent_running = $qga && qga_check_running($vmid);
6627 if ($agent_running) {
6628 print "freeze filesystem\n";
6629 eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6630 } else {
6631 print "suspend vm\n";
6632 eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6633 }
6634
6635 # if we clone a disk for a new target vm, we don't switch the disk
6636 PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6637
6638 if ($agent_running) {
6639 print "unfreeze filesystem\n";
6640 eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6641 } else {
6642 print "resume vm\n";
6643 eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
6644 }
6645
6646 last;
6647 } else {
6648
6649 foreach my $job (keys %$jobs) {
6650 # try to switch the disk if source and destination are on the same guest
6651 print "$job: Completing block job...\n";
6652
6653 eval { mon_cmd($vmid, "block-job-complete", device => $job) };
6654 if ($@ =~ m/cannot be completed/) {
6655 print "$job: Block job cannot be completed, try again.\n";
6656 $err_complete++;
6657 }else {
6658 print "$job: Completed successfully.\n";
6659 $jobs->{$job}->{complete} = 1;
6660 }
6661 }
6662 }
6663 }
6664 sleep 1;
6665 }
6666 };
6667 my $err = $@;
6668
6669 if ($err) {
6670 eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6671 die "mirroring error: $err";
6672 }
6673
6674}
6675
6676sub qemu_blockjobs_cancel {
6677 my ($vmid, $jobs) = @_;
6678
6679 foreach my $job (keys %$jobs) {
6680 print "$job: Cancelling block job\n";
6681 eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6682 $jobs->{$job}->{cancel} = 1;
6683 }
6684
6685 while (1) {
6686 my $stats = mon_cmd($vmid, "query-block-jobs");
6687
6688 my $running_jobs = {};
6689 foreach my $stat (@$stats) {
6690 $running_jobs->{$stat->{device}} = $stat;
6691 }
6692
6693 foreach my $job (keys %$jobs) {
6694
6695 if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6696 print "$job: Done.\n";
6697 delete $jobs->{$job};
6698 }
6699 }
6700
6701 last if scalar(keys %$jobs) == 0;
6702
6703 sleep 1;
6704 }
6705}
6706
6707sub clone_disk {
6708 my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6709 $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6710
6711 my $newvolid;
6712
6713 if (!$full) {
6714 print "create linked clone of drive $drivename ($drive->{file})\n";
6715 $newvolid = PVE::Storage::vdisk_clone($storecfg, $drive->{file}, $newvmid, $snapname);
6716 push @$newvollist, $newvolid;
6717 } else {
6718
6719 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6720 $storeid = $storage if $storage;
6721
6722 my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6723 my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6724
6725 print "create full clone of drive $drivename ($drive->{file})\n";
6726 my $name = undef;
6727 if (drive_is_cloudinit($drive)) {
6728 $name = "vm-$newvmid-cloudinit";
6729 $name .= ".$dst_format" if $dst_format ne 'raw';
6730 $snapname = undef;
6731 $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6732 }
6733 $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6734 push @$newvollist, $newvolid;
6735
6736 PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6737
6738 if (drive_is_cloudinit($drive)) {
6739 goto no_data_clone;
6740 }
6741
6742 my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6743 if (!$running || $snapname) {
6744 # TODO: handle bwlimits
6745 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6746 } else {
6747
6748 my $kvmver = get_running_qemu_version ($vmid);
6749 if (!min_version($kvmver, 2, 7)) {
6750 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6751 if $drive->{iothread};
6752 }
6753
6754 qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
6755 }
6756 }
6757
6758no_data_clone:
6759 my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6760
6761 my $disk = $drive;
6762 $disk->{format} = undef;
6763 $disk->{file} = $newvolid;
6764 $disk->{size} = $size;
6765
6766 return $disk;
6767}
6768
6769sub get_running_qemu_version {
6770 my ($vmid) = @_;
6771 my $res = mon_cmd($vmid, "query-version");
6772 return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6773}
6774
6775sub qemu_use_old_bios_files {
6776 my ($machine_type) = @_;
6777
6778 return if !$machine_type;
6779
6780 my $use_old_bios_files = undef;
6781
6782 if ($machine_type =~ m/^(\S+)\.pxe$/) {
6783 $machine_type = $1;
6784 $use_old_bios_files = 1;
6785 } else {
6786 my $version = PVE::QemuServer::Machine::extract_version($machine_type, kvm_user_version());
6787 # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6788 # load new efi bios files on migration. So this hack is required to allow
6789 # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6790 # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6791 $use_old_bios_files = !min_version($version, 2, 4);
6792 }
6793
6794 return ($use_old_bios_files, $machine_type);
6795}
6796
6797sub create_efidisk($$$$$) {
6798 my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
6799
6800 my (undef, $ovmf_vars) = get_ovmf_files($arch);
6801 die "EFI vars default image not found\n" if ! -f $ovmf_vars;
6802
6803 my $vars_size_b = -s $ovmf_vars;
6804 my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
6805 my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6806 PVE::Storage::activate_volumes($storecfg, [$volid]);
6807
6808 qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
6809 my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
6810
6811 return ($volid, $size/1024);
6812}
6813
6814sub vm_iothreads_list {
6815 my ($vmid) = @_;
6816
6817 my $res = mon_cmd($vmid, 'query-iothreads');
6818
6819 my $iothreads = {};
6820 foreach my $iothread (@$res) {
6821 $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6822 }
6823
6824 return $iothreads;
6825}
6826
6827sub scsihw_infos {
6828 my ($conf, $drive) = @_;
6829
6830 my $maxdev = 0;
6831
6832 if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6833 $maxdev = 7;
6834 } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6835 $maxdev = 1;
6836 } else {
6837 $maxdev = 256;
6838 }
6839
6840 my $controller = int($drive->{index} / $maxdev);
6841 my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6842
6843 return ($maxdev, $controller, $controller_prefix);
6844}
6845
6846sub windows_version {
6847 my ($ostype) = @_;
6848
6849 return 0 if !$ostype;
6850
6851 my $winversion = 0;
6852
6853 if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6854 $winversion = 5;
6855 } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6856 $winversion = 6;
6857 } elsif ($ostype =~ m/^win(\d+)$/) {
6858 $winversion = $1;
6859 }
6860
6861 return $winversion;
6862}
6863
6864sub resolve_dst_disk_format {
6865 my ($storecfg, $storeid, $src_volname, $format) = @_;
6866 my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6867
6868 if (!$format) {
6869 # if no target format is specified, use the source disk format as hint
6870 if ($src_volname) {
6871 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6872 $format = qemu_img_format($scfg, $src_volname);
6873 } else {
6874 return $defFormat;
6875 }
6876 }
6877
6878 # test if requested format is supported - else use default
6879 my $supported = grep { $_ eq $format } @$validFormats;
6880 $format = $defFormat if !$supported;
6881 return $format;
6882}
6883
6884# NOTE: if this logic changes, please update docs & possibly gui logic
6885sub find_vmstate_storage {
6886 my ($conf, $storecfg) = @_;
6887
6888 # first, return storage from conf if set
6889 return $conf->{vmstatestorage} if $conf->{vmstatestorage};
6890
6891 my ($target, $shared, $local);
6892
6893 foreach_storage_used_by_vm($conf, sub {
6894 my ($sid) = @_;
6895 my $scfg = PVE::Storage::storage_config($storecfg, $sid);
6896 my $dst = $scfg->{shared} ? \$shared : \$local;
6897 $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
6898 });
6899
6900 # second, use shared storage where VM has at least one disk
6901 # third, use local storage where VM has at least one disk
6902 # fall back to local storage
6903 $target = $shared // $local // 'local';
6904
6905 return $target;
6906}
6907
6908sub generate_uuid {
6909 my ($uuid, $uuid_str);
6910 UUID::generate($uuid);
6911 UUID::unparse($uuid, $uuid_str);
6912 return $uuid_str;
6913}
6914
6915sub generate_smbios1_uuid {
6916 return "uuid=".generate_uuid();
6917}
6918
6919sub nbd_stop {
6920 my ($vmid) = @_;
6921
6922 mon_cmd($vmid, 'nbd-server-stop');
6923}
6924
6925sub create_reboot_request {
6926 my ($vmid) = @_;
6927 open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
6928 or die "failed to create reboot trigger file: $!\n";
6929 close($fh);
6930}
6931
6932sub clear_reboot_request {
6933 my ($vmid) = @_;
6934 my $path = "/run/qemu-server/$vmid.reboot";
6935 my $res = 0;
6936
6937 $res = unlink($path);
6938 die "could not remove reboot request for $vmid: $!"
6939 if !$res && $! != POSIX::ENOENT;
6940
6941 return $res;
6942}
6943
6944# bash completion helper
6945
6946sub complete_backup_archives {
6947 my ($cmdname, $pname, $cvalue) = @_;
6948
6949 my $cfg = PVE::Storage::config();
6950
6951 my $storeid;
6952
6953 if ($cvalue =~ m/^([^:]+):/) {
6954 $storeid = $1;
6955 }
6956
6957 my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6958
6959 my $res = [];
6960 foreach my $id (keys %$data) {
6961 foreach my $item (@{$data->{$id}}) {
6962 next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6963 push @$res, $item->{volid} if defined($item->{volid});
6964 }
6965 }
6966
6967 return $res;
6968}
6969
6970my $complete_vmid_full = sub {
6971 my ($running) = @_;
6972
6973 my $idlist = vmstatus();
6974
6975 my $res = [];
6976
6977 foreach my $id (keys %$idlist) {
6978 my $d = $idlist->{$id};
6979 if (defined($running)) {
6980 next if $d->{template};
6981 next if $running && $d->{status} ne 'running';
6982 next if !$running && $d->{status} eq 'running';
6983 }
6984 push @$res, $id;
6985
6986 }
6987 return $res;
6988};
6989
6990sub complete_vmid {
6991 return &$complete_vmid_full();
6992}
6993
6994sub complete_vmid_stopped {
6995 return &$complete_vmid_full(0);
6996}
6997
6998sub complete_vmid_running {
6999 return &$complete_vmid_full(1);
7000}
7001
7002sub complete_storage {
7003
7004 my $cfg = PVE::Storage::config();
7005 my $ids = $cfg->{ids};
7006
7007 my $res = [];
7008 foreach my $sid (keys %$ids) {
7009 next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7010 next if !$ids->{$sid}->{content}->{images};
7011 push @$res, $sid;
7012 }
7013
7014 return $res;
7015}
7016
7017sub complete_migration_storage {
7018 my ($cmd, $param, $current_value, $all_args) = @_;
7019
7020 my $targetnode = @$all_args[1];
7021
7022 my $cfg = PVE::Storage::config();
7023 my $ids = $cfg->{ids};
7024
7025 my $res = [];
7026 foreach my $sid (keys %$ids) {
7027 next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7028 next if !$ids->{$sid}->{content}->{images};
7029 push @$res, $sid;
7030 }
7031
7032 return $res;
7033}
7034
70351;
Virtual Machine Manager